diff --git a/README.md b/README.md index 7aaf10c95..40716d26a 100644 --- a/README.md +++ b/README.md @@ -672,7 +672,9 @@ Applied by the query pool to select and fail over between the nodes in the `addr | `client_id` | | Opaque client identifier surfaced server-side for observability | The ingest side also accepts store-and-forward and reconnection tuning keys (`auto_flush_*`, `initial_connect_retry`, -`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). +`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …) and +the schema-aware keys `schema_mode` (`auto`, `strict`, `off`, documented on `Sender.SchemaMode`) and +`schema_wait_millis`. `sf_max_total_bytes` caps the unacknowledged data all pooled senders buffer together — 128 MiB of memory by default, 10 GiB of disk with `sf_dir` — so a larger `sender_pool_max` adds connections, not buffer memory. The only overshoot is the minimum working set every live sender keeps: two segments (`2 × sf_max_segment_bytes`, 8 MiB by default). diff --git a/core/pom.xml b/core/pom.xml index 9a3c522e9..c47d3f9a9 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -193,7 +193,7 @@ - @@ -219,7 +219,7 @@ + so the Compat shim compiles on each JDK. --> org.codehaus.mojo build-helper-maven-plugin @@ -439,8 +439,8 @@ + Java 9+ Compat shim and module-info.class into the jar ROOT, which + breaks Java 8 consumers. Refuse to publish from any other JDK. --> org.apache.maven.plugins maven-enforcer-plugin @@ -560,13 +560,13 @@ - + org.apache.maven.plugins maven-antrun-plugin diff --git a/core/src/main/java/io/questdb/client/LineSenderSchemaException.java b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java new file mode 100644 index 000000000..888ccdc24 --- /dev/null +++ b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java @@ -0,0 +1,64 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** A schema-bound row construction failure with a stable reason. */ +public class LineSenderSchemaException extends LineSenderException { + public enum Reason { + /** + * The value cannot go into the target column: it does not parse, it is + * outside the column's range, or the column's type does not accept the + * setter's input type. + */ + INVALID_VALUE, + /** + * The sender could not obtain the table's schema: the lookup timed out, + * the connection dropped under it, or the server reported the schema as + * temporarily unavailable. A later attempt may succeed. + */ + SCHEMA_UNAVAILABLE, + /** + * The server denied this connection access to the table's schema. + */ + ACCESS_DENIED, + /** + * The client does not implement this conversion or target, or the + * server's schema is too large to return in a discovery response. + */ + UNSUPPORTED_FEATURE + } + + private final Reason reason; + + public LineSenderSchemaException(Reason reason, CharSequence message) { + super(message, false); + this.reason = reason; + } + + public Reason getReason() { + return reason; + } +} diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index d16e97804..1b017031f 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -508,7 +508,8 @@ default Sender decimalColumn(CharSequence name, CharSequence value) { * * @param timeoutMillis upper bound on the wait; {@code <= 0} returns the * current state without blocking (the flush still - * happens before the check) + * happens before the check), and {@code Long.MAX_VALUE} + * waits without a limit * @return {@code true} if the server has acknowledged every published * frame on return, {@code false} on timeout * @throws LineSenderException if the transport has latched a terminal error @@ -584,6 +585,10 @@ default long flushAndGetSequence() { * Precision is locked the first time a value is added to the column: subsequent * rows must use the same precision or a {@link LineSenderException} is thrown. * To mark the value NULL, do not call this method for the current row. + * In schema-aware mode, the server column must be a GEOHASH with exactly the + * same precision, STRING, or VARCHAR. Text targets receive a binary string + * containing exactly {@code precisionBits} characters. Legacy mode keeps the + * native GEOHASH wire representation and server-side conversion behavior. * * @param name name of the column * @param bits packed geohash; low {@code precisionBits} bits significant @@ -609,6 +614,10 @@ default Sender geoHashColumn(CharSequence name, long bits, int precisionBits) { *

* The first call locks the column at {@code value.length() * 5} bits; all * subsequent rows must supply strings of the same length. + * In schema-aware mode, the server column must be a GEOHASH with exactly that + * precision, STRING, or VARCHAR. Text targets receive the corresponding binary + * bit string. Legacy mode keeps the native GEOHASH wire representation and + * server-side conversion behavior. * * @param name name of the column * @param value base32 geohash string, 1..12 characters; must not be null or empty @@ -689,10 +698,18 @@ default Sender ipv4Column(CharSequence name, int address) { /** * Add an IPv4 column value from a dotted-quad string (e.g. "192.168.1.1"). * + * A null reference skips the column for the current row. The literal + * {@code "null"} (case-insensitive) and the exact address + * {@code "0.0.0.0"} are rejected; omit the setter or pass a null reference + * to write a missing value. + * Legacy dotted aliases such as {@code ".0.0.0.0."} are accepted and + * normalized to the IPv4 NULL value. + * * @param name name of the column - * @param address dotted-quad IPv4 address; must not be null + * @param address dotted-quad IPv4 address, or null to skip the column * @return this instance for method chaining - * @throws LineSenderException if the address fails to parse, or the + * @throws LineSenderException if the address fails to parse, represents a + * rejected NULL sentinel, or the * configured protocol version does not support IPv4 */ default Sender ipv4Column(CharSequence name, CharSequence address) { @@ -702,6 +719,10 @@ default Sender ipv4Column(CharSequence name, CharSequence address) { /** * Add a LONG256 column value, packed as four 64-bit words, least-significant first * (so the 256-bit value is {@code (l3 << 192) | (l2 << 128) | (l1 << 64) | l0}). + * In schema-aware mode, setting all four words to {@link Long#MIN_VALUE} + * writes SQL NULL. Any other combination, including a partial match, is a value. + * Legacy mode preserves its existing behavior: the four words are emitted as + * native LONG256 wire data and server-side null/conversion handling applies. * * @param name name of the column * @param l0 bits 0..63 (least significant) @@ -818,6 +839,12 @@ default Sender shortColumn(CharSequence name, short value) { * to finalize the row. You can then start a new row by calling this method again. *
* If you want to cancel the current row, you can call {@link #cancelRow()}. + * QWP WebSocket senders resolve the table's schema here when the server speaks schema-aware QWP, so the call can + * wait up to {@code schema_wait_millis}; under {@link SchemaMode#STRICT} with an asynchronous initial connect, the + * wait includes the first connection. Schema errors can therefore surface here, before the row has any values. A + * failure leaves the table selected and no row started, and calling this method again retries the resolution. + * {@link SchemaMode} decides what happens when the schema is not obtainable. When the table has no pending rows, + * for example after a flush, the first setter of the next row resolves the schema again. * * @param table name of the table * @return this instance for method chaining @@ -872,7 +899,10 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Transport failures (unreachable or + * engine until the wire is up. {@link SchemaMode} decides whether a + * row written before the first upgrade waits for schema negotiation + * or uses the legacy conversion contract. Transport failures + * (unreachable or * dropped server) retry indefinitely and are never surfaced -- the * buffered rows are safe in SF and the server may still appear. A * terminal auth, upgrade or capability rejection on the initial @@ -897,6 +927,69 @@ enum InitialConnectMode { ASYNC } + /** + * Row-encoding contract of the WebSocket (QWP) sender when the server speaks + * schema-aware QWP. It decides what a row does when its table's schema is + * not obtainable at the moment the row starts: before the first upgrade + * completes under {@link InitialConnectMode#ASYNC}, while the wire is down + * during an outage, or while a schema lookup is still pending. It is + * independent of {@link InitialConnectMode}, which only governs how + * {@code fromConfig} behaves before the first successful upgrade. + *

    + *
  • {@link #AUTO} (default) — a row uses the strict schema contract + * whenever a schema is obtainable: from the batch's pinned snapshot, + * from the cache, or from a lookup that completes within + * {@code schema_wait_millis} while the wire is up. Otherwise the row + * uses the legacy conversion contract; schema unavailability never + * fails a row. AUTO avoids network schema waits once a disconnect + * is detected. An open but unresponsive connection can still consume + * {@code schema_wait_millis} on each lookup attempt. The server, not + * the client, validates legacy rows, and values such as nanosecond + * timestamps use their legacy representation.
    • + *
  • {@link #STRICT} — once the server is known to speak schema-aware + * QWP, no row is ever encoded with the legacy contract. A row whose + * schema is not obtainable waits up to {@code schema_wait_millis}, + * across reconnects, and then fails with + * {@link LineSenderSchemaException} ({@code SCHEMA_UNAVAILABLE}). + * Under {@link InitialConnectMode#ASYNC} the first row also waits for + * the first upgrade, because the server's capability is unknown until + * then.
    • + *
  • {@link #OFF} — every row uses the legacy conversion contract, and + * the sender neither requests schema support nor looks up schemas.
    • + *
+ *

+ * A table keeps one contract while it holds pending rows: later rows of the + * table use the contract of its first pending row, and the first row after + * a flush consults the schema again. Tables with different contracts can + * share a batch. STRICT permits legacy rows only until a connection + * successfully negotiates schema support. + *

+ * The sender converts values to the column types of a cached schema. The + * server reports a schema change, such as + * {@code ALTER TABLE ... ALTER COLUMN ... TYPE}, in its reply to a batch + * that writes the table; a reply without one confirms the cached schema. + * While the connection is up, a batch looks up again a schema the server has + * not confirmed for one second, which costs it one round trip. Rows converted + * before the sender learns of a change keep the old type, for example FLOAT + * precision in a column just widened to DOUBLE: rows sent before the reply + * arrives, rows written within that second, and rows written while the + * connection is down. + *

+ * A change can also show as rows the cached schema rejects, such as + * fractional values for a column just widened from INT to DOUBLE. After two + * consecutive rejected rows for a table, the next row of that table looks the + * schema up again, waiting at most one second, and at most once a second. + * When the schema changed, the sender flushes its pending rows in the old + * types, which the server converts as it converts any frame encoded before + * the change, and converts the row to the new types. The rejected rows + * themselves are not retried. + */ + enum SchemaMode { + AUTO, + STRICT, + OFF + } + /** * Durability contract for the store-and-forward write path. Selects when * the SF segment file is fsynced; trades latency / throughput for @@ -1234,6 +1327,14 @@ public int getConnectTimeout() { private long reconnectMaxDurationMillis = PARAMETER_NOT_SET_EXPLICITLY; private int durableAckTiers = DurableAckTiers.NONE; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; + // Row-encoding contract when a schema is not obtainable (WebSocket + // only). null means "not set explicitly", which build() resolves to + // AUTO. + private SchemaMode schemaMode = null; + // Upper bound for one schema wait: the initial handshake, if still + // pending, plus one DESCRIBE round trip. Reconnects consume the same + // budget. + private long schemaWaitMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean transactional; private String senderId = DEFAULT_SENDER_ID; // Per-append deadline for SF appendBlocking spin-then-throw. Used to @@ -1798,7 +1899,7 @@ public LineSenderBuilder symbolDictResetMaxWaitMillis(long maxWaitMillis) { * {@link Sender#drain(long)} explicitly before close(). *

* Set to {@code 0} or {@code -1} to opt out — close() will not wait - * at all (fast close). Pending data is then lost in memory mode and + * at all (fast close). {@code Long.MAX_VALUE} waits without a limit. Pending data is then lost in memory mode and * recovered by the next sender in SF mode. *

* WebSocket transport only. @@ -2915,6 +3016,40 @@ public boolean isStoreAndForwardEnabled() { return sfDir != null; } + /** + * Row-encoding contract when a table's schema is not obtainable -- + * see {@link SchemaMode} for the value semantics. Default + * {@link SchemaMode#AUTO}. WebSocket transport only. + */ + public LineSenderBuilder schemaMode(SchemaMode mode) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_mode is only supported for WebSocket transport"); + } + if (mode == null) { + throw new LineSenderException("schema_mode cannot be null"); + } + this.schemaMode = mode; + return this; + } + + /** + * Per-lookup wait budget, including negotiation and reconnects. + * Default 30 s; {@code Long.MAX_VALUE} waits indefinitely. + * On timeout, {@link SchemaMode#AUTO} uses legacy encoding and + * {@link SchemaMode#STRICT} fails the row. Later lookups may wait again. + * WebSocket only. + */ + public LineSenderBuilder schemaWaitMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_wait_millis is only supported for WebSocket transport"); + } + if (millis <= 0) { + throw new LineSenderException("schema_wait_millis must be > 0: ").put(millis); + } + this.schemaWaitMillis = millis; + return this; + } + /** * Per-call deadline for {@code Sender.flush()} spinning on a full * cursor segment ring waiting for ACKs to drain space. Default @@ -3634,6 +3769,10 @@ private QwpWebSocketSender buildWebSocket(HttpTokenProvider tokenProvider) { } else { actualInitialConnectMode = InitialConnectMode.OFF; } + SchemaMode actualSchemaMode = schemaMode == null ? SchemaMode.AUTO : schemaMode; + long actualSchemaWaitMillis = schemaWaitMillis == PARAMETER_NOT_SET_EXPLICITLY + ? QwpWebSocketSender.DEFAULT_SCHEMA_WAIT_MILLIS + : schemaWaitMillis; long actualDurableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis == DURABLE_ACK_KEEPALIVE_NOT_SET ? CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS @@ -3769,7 +3908,9 @@ private QwpWebSocketSender buildWebSocket(HttpTokenProvider tokenProvider) { symbolDictReset, actualSymbolDictResetThreshold, actualSymbolDictResetMaxWaitMillis, - keepaliveIntervalMillis + keepaliveIntervalMillis, + actualSchemaMode, + actualSchemaWaitMillis ); } catch (UnreplayableSlotException e) { // The one failure build() recovers from. The slot's frames reference ids @@ -4363,6 +4504,18 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else { throw new LineSenderException("invalid initial_connect_retry [value=").put(sink).put(", allowed-values=[on, off, true, false, sync, async]]"); } + } else if (Chars.equals("schema_mode", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_mode is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "schema_mode"); + schemaMode(parseSchemaModeValue(sink)); + } else if (Chars.equals("schema_wait_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_wait_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "schema_wait_millis"); + schemaWaitMillis(parseLongValue(sink, "schema_wait_millis")); } else if (Chars.equals("sf_append_deadline_millis", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("sf_append_deadline_millis is only supported for WebSocket transport"); @@ -4652,6 +4805,15 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } + if (view.has("schema_wait_millis")) { + schemaWaitMillis(wsLong(view, v, "schema_wait_millis")); + } + s = view.getStr("schema_mode"); + if (s != null) { + v.clear(); + v.put(s); + schemaMode(parseSchemaModeValue(v)); + } if (view.has("sf_max_segment_bytes")) { storeAndForwardMaxSegmentBytes(wsSize(view, v, "sf_max_segment_bytes")); } @@ -4797,6 +4959,20 @@ private static int wsInt(ConfigView view, StringSink v, String key) { return parseIntValue(v, key); } + private static SchemaMode parseSchemaModeValue(CharSequence value) { + if (Chars.equalsIgnoreCase("auto", value)) { + return SchemaMode.AUTO; + } + if (Chars.equalsIgnoreCase("strict", value)) { + return SchemaMode.STRICT; + } + if (Chars.equalsIgnoreCase("off", value)) { + return SchemaMode.OFF; + } + throw new LineSenderException("invalid schema_mode [value=").put(value) + .put(", allowed-values=[auto, strict, off]]"); + } + private static long wsLong(ConfigView view, StringSink v, String key) { v.clear(); v.put(view.getStr(key)); @@ -4829,6 +5005,8 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("sf_max_total_bytes", sfMaxTotalBytes); m.put("sf_durability", sfDurability == null ? null : sfDurability.name()); m.put("sf_append_deadline_millis", sfAppendDeadlineMillis); + m.put("schema_mode", schemaMode == null ? null : schemaMode.name()); + m.put("schema_wait_millis", schemaWaitMillis); m.put("sf_sync_interval_millis", sfSyncIntervalMillis); m.put("close_flush_timeout_millis", closeFlushTimeoutMillis); m.put("durable_ack_keepalive_interval_millis", durableAckKeepaliveIntervalMillis); diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java index f87d6f322..1e4e4816b 100644 --- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java +++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java @@ -231,12 +231,24 @@ public static int getDecimalType(int precision, int scale) { return ((scale & 0xFF) << 18) | ((precision & 0xFF) << 8) | (DECIMAL8 + size); } + public static int getDecimalPrecision(int type) { + return (type >>> 8) & 0xFF; + } + + public static int getDecimalScale(int type) { + return (type >>> 18) & 0xFF; + } + public static int getGeoHashTypeWithBits(int bits) { assert bits > 0 && bits <= GEOLONG_MAX_BITS; // this logic relies on GeoHash type value to be clustered together return mkGeoHashType(bits, (short) (GEOBYTE + pow2SizeOfBits(bits))); } + public static int getGeoHashBits(int type) { + return (type >>> BYTE_BITS) & 0xff; + } + /** * Is an N-dimensional array type. */ @@ -244,6 +256,15 @@ public static boolean isArray(int columnType) { return ColumnType.tagOf(columnType) == ColumnType.ARRAY; } + public static boolean isDecimal(int columnType) { + short tag = tagOf(columnType); + return tag >= DECIMAL8 && tag <= DECIMAL256; + } + + public static boolean isGeoHash(int columnType) { + return (columnType & TYPE_FLAG_GEO_HASH) != 0; + } + public static boolean isNull(int columnType) { return columnType == NULL; } diff --git a/core/src/main/java/io/questdb/client/cairo/TableUtils.java b/core/src/main/java/io/questdb/client/cairo/TableUtils.java index f250e9f80..6b13bea3b 100644 --- a/core/src/main/java/io/questdb/client/cairo/TableUtils.java +++ b/core/src/main/java/io/questdb/client/cairo/TableUtils.java @@ -87,6 +87,15 @@ public static boolean isValidTableName(CharSequence tableName, int fsFileNameLim for (int i = 0; i < length; i++) { char c = tableName.charAt(i); + if (Character.isHighSurrogate(c)) { + if (++i >= length || !Character.isLowSurrogate(tableName.charAt(i))) { + return false; + } + continue; + } + if (Character.isLowSurrogate(c)) { + return false; + } switch (c) { case '.': if (i == 0 || i == length - 1 || tableName.charAt(i - 1) == '.') { diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 938d51752..da3105b4a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -84,6 +84,8 @@ public abstract class WebSocketClient implements QuietCloseable { private static final String QWP_CONTENT_ENCODING_HEADER_NAME = "X-QWP-Content-Encoding:"; private static final String QWP_DURABLE_ACK_HEADER_NAME = "X-QWP-Durable-Ack:"; private static final String QWP_MAX_BATCH_SIZE_HEADER_NAME = "X-QWP-Max-Batch-Size:"; + private static final String QWP_SCHEMA_ENABLED_VALUE = "enabled"; + private static final String QWP_SCHEMA_HEADER_NAME = "X-QWP-Schema:"; private static final String QWP_VERSION_HEADER_NAME = "X-QWP-Version:"; private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { try { @@ -148,6 +150,9 @@ public abstract class WebSocketClient implements QuietCloseable { // Requested durable-ack tier set (DurableAckTiers bitmask), sent as // X-QWP-Request-Durable-Ack: private int qwpDurableAckTiers = DurableAckTiers.NONE; + // Sent as X-QWP-Request-Schema: true. Off by default; QwpWebSocketSender + // turns it on unless schema_mode=off. + private boolean qwpRequestSchema; // Receive buffer (native memory) private long recvBufPtr; private int recvBufSize; @@ -165,6 +170,7 @@ public abstract class WebSocketClient implements QuietCloseable { // durable-ack-driven trim. Absence (after requesting tiers via // setQwpDurableAckTiers) is the early-fail signal. private boolean serverDurableAckEnabled; + private boolean serverQwpSchemaEnabled; // Server's hard cap on ingest QWP message payload bytes, extracted from // X-QWP-Max-Batch-Size on the 101 upgrade response. 0 when the server did // not advertise the header (older builds), in which case the sender falls @@ -355,6 +361,7 @@ public void connect(CharSequence host, int port) { public void disconnect() { Misc.free(socket); upgraded = false; + serverQwpSchemaEnabled = false; host = null; port = 0; recvPos = 0; @@ -494,6 +501,11 @@ public boolean isServerDurableAckEnabled() { return serverDurableAckEnabled; } + /** Returns whether a requested schema extension was confirmed by this connection. */ + public boolean isQwpSchemaEnabled() { + return serverQwpSchemaEnabled; + } + /** * Receives and processes WebSocket frames. * @@ -554,6 +566,19 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** Sends a small caller-owned byte array as one binary message. */ + public void sendBinary(byte[] data, int timeout) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + for (byte b : data) { + sendBuffer.putByte(b); + } + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends two native-memory slices as one WebSocket binary frame. The slices * are copied directly into the WebSocket send buffer and masked there, so a @@ -681,6 +706,14 @@ public void setQwpDurableAckTiers(int tiers) { this.qwpDurableAckTiers = tiers; } + /** + * Requests schema control support on subsequent WebSocket upgrades. There is + * no way to turn the request off again. + */ + public void requestQwpSchema() { + this.qwpRequestSchema = true; + } + /** * Non-blocking attempt to receive a WebSocket frame. * Returns immediately if no complete frame is available. @@ -746,6 +779,7 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe upgradeRejectWwwAuthenticate = null; upgradeRejectZone = null; upgradeStatusCode = 0; + serverQwpSchemaEnabled = false; // Generate random key byte[] keyBytes = new byte[16]; @@ -795,6 +829,9 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe sendBuffer.putAscii(DurableAckTiers.requestHeaderValue(qwpDurableAckTiers)); sendBuffer.putAscii("\r\n"); } + if (qwpRequestSchema) { + sendBuffer.putAscii("X-QWP-Request-Schema: true\r\n"); + } if (authorizationHeader != null) { sendBuffer.putAscii("Authorization: "); sendBuffer.putAscii(authorizationHeader); @@ -851,6 +888,28 @@ private static boolean excludesHeaderValue(String response, String headerName, S return true; } + private static String extractUniqueHeaderValue(String response, String headerName) { + String value = null; + int lineStart = response.indexOf("\r\n") + 2; // skip status line + while (lineStart >= 2 && lineStart < response.length()) { + int lineEnd = response.indexOf("\r\n", lineStart); + if (lineEnd < 0 || lineEnd == lineStart) { + break; + } + int colon = response.indexOf(':', lineStart); + if (colon > lineStart && colon < lineEnd + && colon - lineStart == headerName.length() - 1 + && response.regionMatches(true, lineStart, headerName, 0, headerName.length() - 1)) { + if (value != null) { + throw new HttpClientException("Duplicate X-QWP-Schema header"); + } + value = response.substring(colon + 1, lineEnd).trim(); + } + lineStart = lineEnd + 2; + } + return value; + } + /** * Extracts the zstd level the server applied from the echoed * {@code X-QWP-Content-Encoding} response header. Returns 0 when the @@ -1537,6 +1596,17 @@ private void validateUpgradeResponse(int headerEnd) { serverDurableAckEnabled = expectedDurableAckToken != null && expectedDurableAckToken.equalsIgnoreCase(serverDurableAckEcho); + // An absent confirmation means an older server. A present value is a + // protocol assertion, so a requested handshake must reject anything + // other than the single defined value instead of treating it as legacy. + if (qwpRequestSchema) { + String schemaValue = extractUniqueHeaderValue(response, QWP_SCHEMA_HEADER_NAME); + if (schemaValue != null && !schemaValue.equalsIgnoreCase(QWP_SCHEMA_ENABLED_VALUE)) { + throw new HttpClientException("Invalid X-QWP-Schema header value: ").put(schemaValue); + } + serverQwpSchemaEnabled = schemaValue != null; + } + // Extract X-QWP-Max-Batch-Size (optional). Older servers omit it; the // sender falls back to its locally configured byte budget in that case. serverMaxBatchSize = extractMaxBatchSize(response); diff --git a/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java b/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java index 10c56d5a8..026543481 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java @@ -147,6 +147,19 @@ public void clear() { memA = array.startMemoryA(); } + /** + * Returns the number of dimensions in this array. + * + * @return the array dimensionality + * @throws LineSenderException if the array is closed + */ + public int getDimensionality() { + if (closed) { + throw new LineSenderException("Cannot inspect a closed array"); + } + return array.getDimCount(); + } + /** * Closes this array and releases all associated native memory resources. *

@@ -326,4 +339,4 @@ protected int toFlatOffset(int[] coords) { } return flatOffset; } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index b2e577888..992583abb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -52,11 +52,12 @@ private void encodeColumn( long stringDataSize, int symbolDictionarySize, boolean useGorilla, - boolean useGlobalSymbols + boolean useGlobalSymbols, + boolean forceNullBitmap ) { long dataAddr = col.getDataAddress(); - writeNullHeader(col, rowCount, rowCount - valueCount); + writeNullHeader(col, rowCount, rowCount - valueCount, forceNullBitmap); switch (col.getType()) { case TYPE_BOOLEAN: @@ -239,8 +240,13 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { } } - private void writeNullHeader(QwpTableBuffer.ColumnBuffer col, int rowCount, int nullCount) { - if (nullCount > 0) { + private void writeNullHeader( + QwpTableBuffer.ColumnBuffer col, + int rowCount, + int nullCount, + boolean forceNullBitmap + ) { + if (nullCount > 0 || forceNullBitmap) { buffer.putByte((byte) 1); col.ensureNullBitmapCapacity(rowCount); long nullAddr = col.getNullBitmapAddress(); @@ -330,6 +336,10 @@ void encodeTable( boolean useGlobalSymbols, boolean useGorilla ) { + // A schema-bound GEOHASH column always ships its null bitmap, so an + // all-ones value stays distinct from NULL. + boolean isSchemaBound = tableBuffer.getSchemaBinding() != null; + writeTableHeader(tableBuffer, rowCount); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { @@ -344,7 +354,9 @@ void encodeTable( symbolDictionarySize = limitedSymbolDictionarySizes[i]; } - encodeColumn(col, rowCount, valueCount, stringDataSize, symbolDictionarySize, useGorilla, useGlobalSymbols); + encodeColumn(col, rowCount, valueCount, stringDataSize, symbolDictionarySize, + useGorilla, useGlobalSymbols, + isSchemaBound && col.getType() == TYPE_GEOHASH && col.usesNullBitmap()); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaFreshness.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaFreshness.java new file mode 100644 index 000000000..cb7f9284f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaFreshness.java @@ -0,0 +1,181 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Tracks when the server last confirmed the schema snapshot a sender caches + * for one table. The sender converts values against that snapshot, so after a + * column type change it converts to the old type until it learns of the change. + *

+ * The server reports a change in its reply to a frame that writes the table, + * and the client applies that feedback before it advances the OK watermark. + * Once the watermark covers such a frame, the cached snapshot was current when + * the frame was flushed. A DESCRIBE confirms the snapshot as of the moment + * before its request. Both times precede the server's read of the schema, so + * the computed age never understates the real one. + *

+ * One frame at a time waits for confirmation, the oldest uncovered one, since + * its reply arrives first. Frames published while it waits are not recorded. + *

+ * A re-check the server did not answer defers the next one by the age limit. + *

+ * A column type change can also show as rows the snapshot rejects. While the + * table holds pending rows, its batch keeps the snapshot, and rows that fail + * reach no {@code at()} that could flush the batch, so no reply reports the + * change. A run of {@link #REJECTIONS_BEFORE_RECHECK} rejected rows with no + * accepted one in between asks for a re-check before the next row; see + * {@link #takeRecheck}. + *

+ * The producer thread owns this object; it is not thread-safe. + */ +public final class QwpSchemaFreshness { + /** + * Consecutive rows the snapshot rejects before the next row re-checks it. + * More than one, so a single bad value in a flowing batch costs no lookup. + */ + public static final int REJECTIONS_BEFORE_RECHECK = 2; + private static final long NO_FRAME = -1L; + private long confirmedNanos; + private long failedCheckNanos; + private long frameFsn = NO_FRAME; + private long frameNanos; + private boolean hasConfirmation; + private boolean hasFailedCheck; + private boolean hasRechecked; + private boolean isConsulted; + private long recheckNanos; + // Rows the snapshot rejected since the table last accepted one. + private int rejectedRows; + + /** + * Records that the server confirmed the snapshot as of {@code nanos}. An + * earlier confirmation never replaces a later one. + */ + public void confirm(long nanos) { + if (!hasConfirmation || nanos - confirmedNanos > 0) { + confirmedNanos = nanos; + hasConfirmation = true; + } + } + + /** + * Records that a re-check ending at {@code nowNanos} got no answer, which + * defers the next re-check by the age limit. + */ + public void deferCheck(long nowNanos) { + failedCheckNanos = nowNanos; + hasFailedCheck = true; + } + + /** + * Whether the current batch of the table consulted the cached schema, so + * that the frame carrying it can confirm the cache. + */ + public boolean isConsulted() { + return isConsulted; + } + + /** + * Reports whether the snapshot needs a re-check at {@code nowNanos}: no + * confirmation and no failed re-check lies within {@code maxAgeNanos}. A + * waiting frame that {@code okFsn} covers confirms the snapshot first. + */ + public boolean isExpired(long nowNanos, long maxAgeNanos, long okFsn) { + confirmCoveredFrame(okFsn); + return (!hasConfirmation || nowNanos - confirmedNanos > maxAgeNanos) + && (!hasFailedCheck || nowNanos - failedCheckNanos > maxAgeNanos); + } + + /** + * Reports whether a run of rejected rows asks for a re-check, before the + * interval of {@link #takeRecheck} applies. + */ + public boolean isRecheckDue() { + return rejectedRows >= REJECTIONS_BEFORE_RECHECK; + } + + /** + * Marks that the current batch of the table consults the cached schema. + */ + public void markConsulted() { + isConsulted = true; + } + + /** + * Records that a frame up to {@code fsn} carried the consulted batch, + * flushed at {@code flushNanos}, and clears the consulted mark. The frame + * waits for confirmation unless an earlier frame still does. + */ + public void onFramePublished(long fsn, long flushNanos, long okFsn) { + isConsulted = false; + confirmCoveredFrame(okFsn); + if (frameFsn == NO_FRAME) { + frameFsn = fsn; + frameNanos = flushNanos; + } + } + + /** + * Records a row the table accepted, which ends a run of rejected rows. + */ + public void onRowAccepted() { + rejectedRows = 0; + } + + /** + * Records a row the snapshot rejected, for a column it knows. + */ + public void onRowRejected() { + if (rejectedRows < REJECTIONS_BEFORE_RECHECK) { + rejectedRows++; + } + } + + /** + * Reports whether to re-check the snapshot at {@code nowNanos}: a run of + * rejected rows is due, no re-check started within {@code minIntervalNanos}, + * and no re-check that got no answer ended within it; see + * {@link #deferCheck}. Taking the re-check ends the run; a run the interval + * defers stays due. + */ + public boolean takeRecheck(long nowNanos, long minIntervalNanos) { + if (!isRecheckDue() + || (hasRechecked && nowNanos - recheckNanos <= minIntervalNanos) + || (hasFailedCheck && nowNanos - failedCheckNanos <= minIntervalNanos)) { + return false; + } + rejectedRows = 0; + recheckNanos = nowNanos; + hasRechecked = true; + return true; + } + + private void confirmCoveredFrame(long okFsn) { + if (frameFsn != NO_FRAME && okFsn >= frameFsn) { + confirm(frameNanos); + frameFsn = NO_FRAME; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index f38d9e93d..1046fb1bb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -67,6 +67,7 @@ public QwpWebSocketEncoder(int bufferSize) { } public void addTable(QwpTableBuffer tableBuffer) { + validateBoundTable(tableBuffer, true); columnWriter.encodeTable(tableBuffer, true, true); } @@ -80,10 +81,7 @@ public void beginMessage( deltaStart = confirmedMaxId + 1; deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); - byte origFlags = flags; - flags = headerFlags; - writeHeader(tableCount, 0); - flags = origFlags; + writeHeader(tableCount, 0, headerFlags); payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); @@ -163,6 +161,10 @@ public int copySplitMessage( } public int encode(QwpTableBuffer tableBuffer) { + validateBoundTable(tableBuffer, false); + if (tableBuffer.getSchemaBinding() != null && tableBuffer.getRowCount() == 0) { + return 0; + } buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); @@ -179,6 +181,8 @@ public int encodeWithDeltaDict( int confirmedMaxId, int batchMaxId ) { + // Validate before beginMessage resets the staged buffer. + validateBoundTable(tableBuffer, true); beginMessage(1, globalDict, confirmedMaxId, batchMaxId); addTable(tableBuffer); return finishMessage(); @@ -190,6 +194,18 @@ public int finishMessage() { return buffer.getPosition(); } + private static void validateBoundTable(QwpTableBuffer tableBuffer, boolean additive) { + if (tableBuffer.getSchemaBinding() == null) { + return; + } + if (tableBuffer.hasInProgressRow()) { + throw new IllegalStateException("cannot encode a schema-bound table with an incomplete row"); + } + if (additive && tableBuffer.getRowCount() == 0) { + throw new IllegalStateException("cannot add an empty schema-bound table"); + } + } + public QwpBufferWriter getBuffer() { return buffer; } @@ -234,12 +250,19 @@ public void setVersion(byte version) { } public void writeHeader(int tableCount, int payloadLength) { + writeHeader(tableCount, payloadLength, flags); + } + + private void writeHeader(int tableCount, int payloadLength, byte headerFlags) { + if (tableCount < 0 || tableCount > 0xffff) { + throw new IllegalArgumentException("QWP table count is outside unsigned-short range: " + tableCount); + } buffer.putByte((byte) 'Q'); buffer.putByte((byte) 'W'); buffer.putByte((byte) 'P'); buffer.putByte((byte) '1'); buffer.putByte(version); - buffer.putByte(flags); + buffer.putByte(headerFlags); buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1fc434840..83c9f4373 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -27,6 +27,7 @@ import io.questdb.client.ClientTlsConfiguration; import io.questdb.client.ConnectionHealth; import io.questdb.client.HttpTokenProvider; +import io.questdb.client.LineSenderSchemaException; import io.questdb.client.Sender; import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; @@ -52,6 +53,9 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; @@ -82,6 +86,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -146,6 +151,10 @@ public class QwpWebSocketSender implements Sender { // Finite fallback (ms) for BACKGROUND (drainer) TCP connects when the // user left connect_timeout unset. See effectiveConnectTimeoutMs. public static final int DEFAULT_BACKGROUND_CONNECT_TIMEOUT_MS = 15_000; + // Upper bound for a row's first setter to wait on the initial handshake + // and a schema DESCRIBE round trip. A healthy lookup takes milliseconds; + // this only bounds a dead or still-connecting link. + public static final long DEFAULT_SCHEMA_WAIT_MILLIS = 30_000L; // Default for symbol_dict_reset -- periodic symbol-dictionary recycling is // on by default so a long-lived sender's dictionary does not grow without // bound. The recycle runs at a table() call that finds the backlog @@ -174,6 +183,9 @@ public class QwpWebSocketSender implements Sender { // Default for symbol_dict_reset_threshold: distinct-symbol count that // triggers a recycle once symbol_dict_reset is on. public static final int DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS = 100_000; + // Longest a batch converts against a cached table schema the server has not + // confirmed, while the wire is up; see QwpSchemaFreshness. + public static final long SCHEMA_MAX_AGE_MILLIS = 1_000L; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); @@ -188,6 +200,10 @@ public class QwpWebSocketSender implements Sender { // past any transient disk/NFS stall the deferred-close machinery exists // to survive. private static final long RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS = 30_000L; + private static final long SCHEMA_MAX_AGE_NANOS = TimeUnit.MILLISECONDS.toNanos(SCHEMA_MAX_AGE_MILLIS); + // Longest the row after a run of rejected rows waits for the re-check of its + // table's schema; see recheckRejectingSchema(). + private static final long SCHEMA_RECHECK_WAIT_MILLIS = 1_000L; private static final String WRITE_PATH = "/write/v4"; // Yields the Authorization header value presented on each WebSocket upgrade. A constant for a // fixed token or Basic credential; for an httpTokenProvider it pulls a freshly refreshed token, @@ -430,6 +446,11 @@ public class QwpWebSocketSender implements Sender { // wasEverConnected() for the whole post-recycle outage window. Volatile: // wasEverConnected() is consulted from the error-dispatcher daemon. private volatile boolean hasLoopEverConnected; + // The schema counterpart of hasLoopEverConnected. A symbol-dictionary recycle + // carries it and recycledSchemaCache into the rebuilt loop; see + // CursorWebSocketSendLoop.seedSchemaState(). Producer thread only. + private boolean hasLoopNegotiatedSchema; + private final HashMap recycledSchemaCache = new HashMap<>(); // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame this session // published, or -1 when none. Frames above it are deferred and uncommitted: // the server withholds their acks by design (their rows are rolled back on @@ -645,6 +666,12 @@ public class QwpWebSocketSender implements Sender { private long reconnectMaxDurationMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS; private int durableAckTiers; + // nanoClock reading taken before a flush publishes; see recordSchemaFramesAfterFlush(). + private long schemaFlushNanos; + // Row-encoding contract when a schema is not obtainable; see + // resolveSchemaForCurrentTable(). + private Sender.SchemaMode schemaMode = Sender.SchemaMode.AUTO; + private long schemaWaitMillis = DEFAULT_SCHEMA_WAIT_MILLIS; // Monotonic per-attempt counter snapshotted onto every connection event // fired from buildAndConnect. Counts every FOREGROUND endpoint try -- // successes and failures alike -- across this sender's lifetime. @@ -1190,6 +1217,54 @@ public static QwpWebSocketSender connectWithCredentialSupplier( CursorWebSocketSendLoop.DEFAULT_KEEPALIVE_INTERVAL_MILLIS); } + /** + * As the master overload below, with the default {@link Sender.SchemaMode#AUTO} + * schema contract and the default schema wait. Kept as its own overload so a + * caller compiled before the schema knobs moved into the master keeps linking. + */ + public static QwpWebSocketSender connectWithCredentialSupplier( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + Supplier authorizationHeaderSupplier, + int durableAckTiers, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs, + int connectTimeoutMs, + SenderConnectionListener connectionListener, + int connectionListenerInboxCapacity, + int maxFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + boolean symbolDictResetEnabled, + int symbolDictResetThresholdSymbols, + long symbolDictResetMaxWaitMillis, + long keepaliveIntervalMillis + ) { + return connectWithCredentialSupplier(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, + autoFlushIntervalNanos, authorizationHeaderSupplier, durableAckTiers, + cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, + initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, + connectionListener, connectionListenerInboxCapacity, + maxFrameRejections, poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + symbolDictResetEnabled, symbolDictResetThresholdSymbols, + symbolDictResetMaxWaitMillis, keepaliveIntervalMillis, + Sender.SchemaMode.AUTO, DEFAULT_SCHEMA_WAIT_MILLIS); + } + /** * Master connect entry point — also accepts the poison-frame detector * threshold ({@code max_frame_rejections}) and the symbol-dictionary @@ -1197,7 +1272,9 @@ public static QwpWebSocketSender connectWithCredentialSupplier( * {@code symbol_dict_reset_max_wait_millis}): consecutive server-active * rejections of the same head-of-line frame, with no ack progress in * between, before the loop escalates to a typed terminal. Also takes the - * idle keepalive PING cadence ({@code keepalive_interval_millis}). + * idle keepalive PING cadence ({@code keepalive_interval_millis}), and the + * schema contract ({@code schema_mode}) with its lookup budget + * ({@code schema_wait_millis}). *

* Named apart from {@code connect} for the reason given on * {@link #connectWithCredentialSupplier(List, ClientTlsConfiguration, int, int, long, Supplier, @@ -1242,7 +1319,9 @@ public static QwpWebSocketSender connectWithCredentialSupplier( boolean symbolDictResetEnabled, int symbolDictResetThresholdSymbols, long symbolDictResetMaxWaitMillis, - long keepaliveIntervalMillis + long keepaliveIntervalMillis, + Sender.SchemaMode schemaMode, + long schemaWaitMillis ) { if (symbolDictResetThresholdSymbols <= 0 || symbolDictResetThresholdSymbols > QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2) { @@ -1281,6 +1360,8 @@ public static QwpWebSocketSender connectWithCredentialSupplier( sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; + sender.schemaMode = schemaMode == null ? Sender.SchemaMode.AUTO : schemaMode; + sender.schemaWaitMillis = schemaWaitMillis; if (errorHandler != null) { sender.setErrorHandler(errorHandler); } @@ -1396,6 +1477,12 @@ public void at(long timestamp, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.designatedTimestamp(timestamp, unit); + sendRow(); + return; + } if (unit == ChronoUnit.NANOS) { atNanos(timestamp); } else { @@ -1403,8 +1490,7 @@ public void at(long timestamp, ChronoUnit unit) { atMicros(micros); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } } @@ -1413,11 +1499,16 @@ public void at(Instant timestamp) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.designatedTimestamp(timestamp); + sendRow(); + return; + } long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; atMicros(micros); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } } @@ -1426,11 +1517,11 @@ public void atNow() { checkNotClosed(); checkTableSelected(); try { + bindingForEffectiveWrite(); // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } } @@ -1475,7 +1566,10 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { if (timeoutMillis <= 0L) { return false; } - long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L; + // Elapsed-versus-budget, not an absolute deadline: toNanos saturates, so + // Long.MAX_VALUE waits indefinitely instead of wrapping to no wait. + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); Thread me = Thread.currentThread(); // Registered before the first check: an ack that lands after a check // unparks us (or leaves the permit), one that landed before is seen by it. @@ -1490,7 +1584,7 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { return true; } checkAwaitAckedFsnErrors(); - long remaining = deadlineNanos - System.nanoTime(); + long remaining = budgetNanos - (System.nanoTime() - startNanos); if (remaining <= 0L) { return false; } @@ -1537,26 +1631,29 @@ private void checkAwaitAckedFsnErrors() { * Adds a BINARY column value to the current row. The bytes are written * verbatim with no encoding or transformation. A {@code null} array * reference is rejected so the NULL contract stays explicit (use the null - * bitmap instead). An empty array is accepted on the wire but QuestDB's - * BINARY storage uses the same NULL sentinel for zero-length and absent - * values, so an empty payload round-trips as NULL on read. + * bitmap instead). An empty array is accepted and remains a present, + * zero-length BINARY value, distinct from an omitted/null value. */ @Override public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { checkNotClosed(); checkTableSelected(); - if (value == null) { - throw new LineSenderException( - "BINARY value cannot be null; mark the row null via the null bitmap instead"); - } try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, value); + return this; + } + if (value == null) { + throw new LineSenderException( + "BINARY value cannot be null; mark the row null via the null bitmap instead"); + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BINARY, true); if (col != null) { col.addBinary(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1571,11 +1668,20 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice slice) { checkNotClosed(); checkTableSelected(); - if (slice == null) { - throw new LineSenderException( - "BINARY slice cannot be null; mark the row null via the null bitmap instead"); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, slice); + return this; + } + if (slice == null) { + throw new LineSenderException( + "BINARY slice cannot be null; mark the row null via the null bitmap instead"); + } + return binaryColumn(columnName, slice.ptr(), slice.size()); + } catch (RuntimeException | Error e) { + throw onRowFailure(e); } - return binaryColumn(columnName, slice.ptr(), slice.size()); } /** @@ -1588,13 +1694,17 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, ptr, len); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BINARY, true); if (col != null) { col.addBinary(ptr, len); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1604,13 +1714,17 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.boolColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BOOLEAN, false); if (col != null) { col.addBoolean(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1631,13 +1745,17 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.byteColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BYTE, false); if (col != null) { col.addByte(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1664,13 +1782,17 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.charColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_CHAR, false); if (col != null) { col.addShort((short) value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2054,13 +2176,17 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL64, true); if (col != null) { col.addDecimal64(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2071,13 +2197,17 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL128, true); if (col != null) { col.addDecimal128(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2088,13 +2218,17 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2105,14 +2239,18 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { if (value == null || value.length() == 0) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value, currentDecimal256); + return this; + } currentDecimal256.ofString(value); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(currentDecimal256); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2123,13 +2261,17 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, values); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2140,13 +2282,17 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, values); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2157,13 +2303,17 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, values); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2174,13 +2324,17 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, array); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(array); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2190,13 +2344,17 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_DOUBLE, true); if (col != null) { col.addDouble(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2212,13 +2370,17 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.floatColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_FLOAT, true); if (col != null) { col.addFloat(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2357,18 +2519,22 @@ public boolean drain(long timeoutMillis) { public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int precisionBits) { checkNotClosed(); checkTableSelected(); - if (precisionBits < 1 || precisionBits > 60) { - throw new LineSenderException( - "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); - } try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.geoHashColumn(columnName, bits, precisionBits); + return this; + } + if (precisionBits < 1 || precisionBits > 60) { + throw new LineSenderException( + "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_GEOHASH, true); if (col != null) { col.addGeoHash(maskGeoHashBits(bits, precisionBits), precisionBits); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2391,6 +2557,15 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.geoHashColumn(columnName, value); + return this; + } + } catch (RuntimeException | Error e) { + throw onRowFailure(e); + } if (value == null) { throw new LineSenderException( "GEOHASH string cannot be null; mark the row null via the null bitmap instead"); @@ -2987,13 +3162,17 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.intColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_INT, true); if (col != null) { col.addInt(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3016,13 +3195,17 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.ipv4Column(columnName, address); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_IPv4, true); if (col != null) { col.addIPv4(address); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3061,6 +3244,15 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre return this; } checkTableSelected(); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.ipv4Column(columnName, address); + return this; + } + } catch (RuntimeException | Error e) { + throw onRowFailure(e); + } if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { throw new LineSenderException( "invalid IPv4 address: NULL sentinel inputs are rejected" @@ -3078,6 +3270,12 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre /** * Adds a LONG256 column value to the current row. + *

+ * In schema-aware mode, setting all four limbs to {@link Long#MIN_VALUE} + * writes SQL NULL. Any other limb combination, including a partial match, + * is a value. Legacy mode preserves its existing behavior: the four limbs + * are emitted as native LONG256 wire data and server-side null/conversion + * handling applies. * * @param columnName the column name * @param l0 the least significant 64 bits @@ -3086,17 +3284,22 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre * @param l3 the most significant 64 bits * @return this sender for method chaining */ + @Override public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.long256Column(columnName, l0, l1, l2, l3); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG256, true); if (col != null) { col.addLong256(l0, l1, l2, l3); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3107,13 +3310,18 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + // LONG_ARRAY is unimplemented; a schema refresh cannot make it valid. + throw onRowFailure(e); } return this; } @@ -3124,13 +3332,17 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3141,13 +3353,17 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3158,13 +3374,17 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(array); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3174,13 +3394,17 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.longColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG, true); if (col != null) { col.addLong(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3585,13 +3809,17 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.shortColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SHORT, false); if (col != null) { col.addShort(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3701,13 +3929,17 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.stringColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_VARCHAR, true); if (col != null) { col.addString(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3717,13 +3949,17 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.symbol(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SYMBOL, true); if (col != null) { col.addSymbol(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3737,33 +3973,34 @@ public QwpWebSocketSender table(CharSequence tableName) { } else if (resetArmed) { maybeRecycleForDictReset(); } - // Fast path: if table name matches current, skip hashmap lookup - if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { - return this; - } - // Prevent switching tables while a row is in progress - if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { - throw new LineSenderException("cannot switch tables while row is in progress" - + " [currentTable=").put(currentTableName).put(']'); - } - // Table changed - invalidate cached column references - validateTableName(tableName); - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - currentTableBuffer = tableBuffers.get(tableName); - if (currentTableBuffer != null) { - currentTableName = currentTableBuffer.getTableName(); - } else { - currentTableName = tableName.toString(); - currentTableBuffer = new QwpTableBuffer(currentTableName, this); - tableBuffers.put(currentTableName, currentTableBuffer); - } - // Re-snap so sendRow()'s delta math is anchored to this table's - // current byte count. The prior current table's bytes already match - // its last-snapped value (the in-progress-row guard above ensures - // no column setters ran on it since the last consistency point). - currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); - // Both modes accumulate rows until flush + if (currentTableName == null || !Chars.equals(tableName, currentTableName)) { + // Prevent switching tables while a row is in progress. + if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { + throw new LineSenderException("cannot switch tables while row is in progress" + + " [currentTable=").put(currentTableName).put(']'); + } + validateTableName(tableName); + // Table changed - invalidate cached column references. + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableBuffer = tableBuffers.get(tableName); + if (currentTableBuffer != null) { + currentTableName = currentTableBuffer.getTableName(); + } else { + currentTableName = tableName.toString(); + currentTableBuffer = new QwpTableBuffer(currentTableName, this); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Re-snap so sendRow()'s delta math is anchored to this table's + // current byte count. The prior current table's bytes already match + // its last-snapped value (the in-progress-row guard above ensures + // no column setters ran on it since the last consistency point). + currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); + } + // Resolve schema at the explicit table boundary. Setters retain the + // same lookup as a fallback because the selected table also starts + // implicit rows after completion, cancellation, and batch flushes. + bindingForEffectiveWrite(); return this; } @@ -3772,6 +4009,11 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.timestampColumn(columnName, value, unit); + return this; + } if (unit == ChronoUnit.NANOS) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_TIMESTAMP_NANOS, true); if (col != null) { @@ -3785,8 +4027,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C } } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3796,14 +4037,18 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.timestampColumn(columnName, value); + return this; + } long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_TIMESTAMP, true); if (col != null) { col.addLong(micros); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3820,13 +4065,17 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.uuidColumn(columnName, lo, hi); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_UUID, true); if (col != null) { col.addUuid(hi, lo); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -4256,6 +4505,9 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpDurableAckTiers(durableAckTiers); + if (schemaMode != Sender.SchemaMode.OFF) { + newClient.requestQwpSchema(); + } newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); if (cancellation != null) { // Publish the client we are about to block on so a @@ -4878,7 +5130,8 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); } } - long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(closeFlushTimeoutMillis); restoreInterrupt[0] |= Thread.interrupted(); try { Thread me = Thread.currentThread(); @@ -4904,7 +5157,8 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn } else { cursorSendLoop.checkError(); } - if (System.nanoTime() >= deadlineNanos) { + long remainingNanos = budgetNanos - (System.nanoTime() - startNanos); + if (remainingNanos <= 0L) { long acked = cursorEngine.ackedFsn(); long externalTarget = fsnEpochBase + target; long externalAcked = fsnEpochBase + acked; @@ -4926,7 +5180,7 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn ? " pending batches; data may be lost (use larger closeFlushTimeoutMillis or smaller batches)" : " pending batches; the wire is not draining: " + outage.getMessage()); } - java.util.concurrent.locks.LockSupport.parkNanos(this, deadlineNanos - System.nanoTime()); + java.util.concurrent.locks.LockSupport.parkNanos(this, remainingNanos); } } finally { ackWaiters.remove(me); @@ -5084,6 +5338,10 @@ private void ensureConnected() { // the loop no longer fires a terminal budget-exhaustion event -- it // retries indefinitely.) cursorSendLoop.setConnectionDispatcher(connectionDispatcher); + // AUTO falls back to the legacy contract when the wire drops under a + // lookup, so the lookup must fail right away rather than ride the + // reconnect. STRICT keeps it pending for the replacement connection. + cursorSendLoop.setAbandonSchemaLookupOnDisconnect(schemaMode == Sender.SchemaMode.AUTO); cursorSendLoop.setConnectionHealthTracker(healthTracker); // Seed the fresh loop's own hasEverConnected before it can observe // any endpoint-policy failure: without this, a symbol-dict @@ -5095,10 +5353,13 @@ private void ensureConnected() { if (hasLoopEverConnected) { cursorSendLoop.markEverConnected(); } + cursorSendLoop.seedSchemaState(hasLoopNegotiatedSchema, recycledSchemaCache); if (loopStartFault != null) { loopStartFault.run(); } cursorSendLoop.start(); + // The started loop owns its copy; a failed start keeps this one for the retry. + recycledSchemaCache.clear(); } catch (Throwable t) { // start() (or dispatcher construction) failed after cursorSendLoop was // assigned. Close it so a caller that retries -- re-entering @@ -5168,6 +5429,297 @@ private void ensureConnected() { hasInitialConnectRun = true; } + /** + * Selects the wire contract at an explicit table boundary or before the first + * effective value of an implicit row. + *

+ * A table pins one schema snapshot per batch. Once the table holds pending rows, + * later rows of the same batch validate against that snapshot without consulting + * the cache; the first row after a flush or reset adopts whatever the cache holds + * by then, which ACK schema feedback keeps current. While the wire is up, a cached + * snapshot the server has not confirmed for {@link #SCHEMA_MAX_AGE_MILLIS} is looked + * up again first; see {@link #resolveConfirmedSchema}. A rejection that a column + * type change could explain evicts the snapshot; see {@link #onRowFailure}. A run + * of rows the snapshot rejected makes the next row look the table up even while + * the batch holds rows; see {@link #recheckRejectingSchema}. + *

+ * When no snapshot is pinned, the configured {@link Sender.SchemaMode} decides + * between a schema lookup and the legacy contract; see + * {@link #resolveSchemaForCurrentTable()}. + */ + private QwpSchemaBinding bindingForEffectiveWrite() { + QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); + if (currentTableBuffer.hasInProgressRow()) { + return pinned; + } + ensureConnected(); + if (pinned != null && recheckRejectingSchema(pinned) && currentTableBuffer.getRowCount() > 0) { + // The pending rows carry the old column types, which the server converts + // as it converts any frame encoded before the change. Sending them frees + // the buffer to take the new schema below. + flushPendingRows(transactional); + } + // A table buffer keeps one contract while it holds rows; the first row + // after the flush consults the schema again. + if (currentTableBuffer.getRowCount() > 0) { + return pinned; + } + // AUTO uses detected connection state to decide whether to attempt lookup. + // A silent peer can leave the wire marked up. Each lookup has its own + // budget: table() and the first setter can both wait on an empty batch, + // and later batches can repeat this (up to 60 s with default settings). + // This is an accepted trade-off: transient unavailability is not cached + // or latched into legacy mode, so later attempts can adopt schema again. + QwpSchemaResponse latest = resolveSchemaForCurrentTable(); + if (latest == null) { + return selectLegacyContract(); + } + if (pinned != null && isSnapshotOf(pinned, latest)) { + return pinned; + } + return installSchemaBinding(latest); + } + + private static boolean isSnapshotOf(QwpSchemaBinding binding, QwpSchemaResponse schema) { + return binding.getTableId() == schema.getTableId() + && binding.getMetadataVersion() == schema.getMetadataVersion(); + } + + /** + * Looks the current table up again after a run of rows its pinned snapshot + * rejected, and reports whether the server's schema differs from the snapshot. + *

+ * After a column type change, a sender whose rows all fail against the old + * type never reaches the {@code at()} that could flush its batch, so no reply + * reports the change, and the batch keeps the snapshot while it holds rows. + * The re-check needs a connected wire, waits at most + * {@code SCHEMA_RECHECK_WAIT_MILLIS}, and runs at most once per + * {@link #SCHEMA_MAX_AGE_MILLIS} per table, so values that are invalid under + * the current schema cost little. A failed lookup, including one on a wire + * without schema support, says nothing about the schema, so it keeps the + * snapshot. It defers the table's next lookup from the end of its wait, as a + * failed lookup in {@link #resolveConfirmedSchema} does, so a server that never + * answers costs each table at most one wait per interval after it. + */ + private boolean recheckRejectingSchema(QwpSchemaBinding pinned) { + QwpSchemaFreshness freshness = currentTableBuffer.getSchemaFreshness(); + if (!freshness.isRecheckDue()) { + return false; + } + // The caller ran ensureConnected(), which installs a loop; the lookup after + // this re-check dereferences it the same way. + assert cursorSendLoop != null; + // A lookup on a wire that is down would wait for the reconnect. A run that + // finds the wire down stays due for the next connection. + if (!cursorSendLoop.isWireUp()) { + return false; + } + final long nowNanos = nanoClock.getAsLong(); + if (!freshness.takeRecheck(nowNanos, SCHEMA_MAX_AGE_NANOS)) { + return false; + } + final QwpSchemaResponse latest; + try { + latest = cursorSendLoop.revalidateSchema(currentTableBuffer.getSchemaKey(), + Math.min(schemaWaitMillis, SCHEMA_RECHECK_WAIT_MILLIS)); + } catch (LineSenderSchemaException e) { + freshness.deferCheck(nanoClock.getAsLong()); + return false; + } + freshness.confirm(nowNanos); + return !isSnapshotOf(pinned, latest); + } + + /** + * Resolves the current table's schema under the configured + * {@link Sender.SchemaMode}, or returns {@code null} when the row must take the + * legacy contract: the mode is OFF, the server negotiated legacy QWP, or, in + * AUTO, the schema is not obtainable. + *

+ * STRICT waits for the first upgrade and then for the lookup within one + * {@code schemaWaitMillis} budget and lets a lookup failure propagate. AUTO + * never waits on a dead wire: before the first upgrade it takes the legacy + * contract outright, while the wire is down it consults only the cache, and + * a lookup that ends in {@code SCHEMA_UNAVAILABLE} (a drop under it, a timeout, + * a server-side unavailability) or {@code UNSUPPORTED_FEATURE} (an oversized + * or unsupported schema response) selects the legacy contract instead of failing + * the row. Access denial still fails the row. + */ + private QwpSchemaResponse resolveSchemaForCurrentTable() { + QwpSchemaResponse schema = lookUpSchemaForCurrentTable(); + if (schema == null || schema.getResult() != QwpSchemaProtocol.RESULT_TOO_LARGE) { + return schema; + } + // The coordinator caches TOO_LARGE, so this costs no round trip per batch. + if (schemaMode == Sender.SchemaMode.STRICT) { + throw new LineSenderSchemaException(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + "schema lookup failed [reason=UNSUPPORTED_FEATURE, table=" + currentTableName + + ", detail=schema could not be returned within the discovery response limit]"); + } + return null; + } + + private QwpSchemaResponse lookUpSchemaForCurrentTable() { + if (schemaMode == Sender.SchemaMode.OFF) { + return null; + } + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(schemaWaitMillis); + final String key = currentTableBuffer.getSchemaKey(); + if (schemaMode == Sender.SchemaMode.STRICT) { + if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(startNanos, budgetNanos))) { + return null; + } + // STRICT writes no rows to a table it cannot describe, so no ACK feedback + // ever reports that the table shrank: ask again instead of trusting a + // cached TOO_LARGE. AUTO keeps the cache hit; its legacy rows earn feedback. + QwpSchemaResponse cached = cursorSendLoop.peekSchema(key); + if (cached != null && cached.getResult() == QwpSchemaProtocol.RESULT_TOO_LARGE) { + return cursorSendLoop.refreshSchema(key, remainingSchemaMillis(startNanos, budgetNanos)); + } + return resolveConfirmedSchema(key, cached, startNanos, budgetNanos); + } + if (!cursorSendLoop.hasEverConnected()) { + return null; + } + if (!cursorSendLoop.isWireUp()) { + // Frames of this batch reach the server after the reconnect, and their + // replies confirm the cache as of their flush. + currentTableBuffer.getSchemaFreshness().markConsulted(); + return cursorSendLoop.peekSchema(key); + } + if (!cursorSendLoop.isSchemaEnabled()) { + return null; + } + try { + return resolveConfirmedSchema(key, cursorSendLoop.peekSchema(key), startNanos, budgetNanos); + } catch (LineSenderSchemaException e) { + if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE + || e.getReason() == LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) { + return null; + } + throw e; + } + } + + /** + * Returns {@code cached} if the server confirmed it within + * {@link #SCHEMA_MAX_AGE_MILLIS} or the wire is down. Otherwise looks the table + * up, which confirms the result as of the moment before the request. + *

+ * A lookup that fails with {@code SCHEMA_UNAVAILABLE} says nothing about the + * schema: the batch keeps the cached entry, and the next re-check waits out the + * age limit, so {@code table()} and the first setter do not both wait on a + * silent server. Other failures, and any failure without a cached entry, + * propagate as for a cold lookup. + */ + private QwpSchemaResponse resolveConfirmedSchema( + String key, + QwpSchemaResponse cached, + long startNanos, + long budgetNanos + ) { + QwpSchemaFreshness freshness = currentTableBuffer.getSchemaFreshness(); + freshness.markConsulted(); + final long nowNanos = nanoClock.getAsLong(); + if (cached != null + && (!freshness.isExpired(nowNanos, SCHEMA_MAX_AGE_NANOS, schemaOkFsn()) || !cursorSendLoop.isWireUp())) { + return cached; + } + try { + QwpSchemaResponse fresh = cursorSendLoop.revalidateSchema(key, remainingSchemaMillis(startNanos, budgetNanos)); + freshness.confirm(nowNanos); + return fresh; + } catch (LineSenderSchemaException e) { + if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE) { + QwpSchemaResponse current = cursorSendLoop.peekSchema(key); + if (current != null) { + // From the end of the wait, which can outlast the age limit. + freshness.deferCheck(nanoClock.getAsLong()); + return current; + } + } + throw e; + } + } + + /** + * Commits the current table's next row to the legacy contract. A buffer still + * bound to an earlier batch's snapshot is unbound so the legacy setters lay out + * its columns afresh. + */ + private QwpSchemaBinding selectLegacyContract() { + if (currentTableBuffer.getSchemaBinding() != null) { + unbindCurrentTableBuffer(); + } + return null; + } + + /** + * Binds the row-less current table buffer to {@code latest}. A buffer that has + * been bound before, or that a flushed legacy batch left with a column layout + * ({@link QwpTableBuffer#reset()} keeps columns), cannot take a new binding in + * place; {@link QwpTableBuffer#clear()} drops both before rebinding. + */ + private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest) { + if (currentTableBuffer.getSchemaBinding() != null || currentTableBuffer.getColumnCount() != 0) { + unbindCurrentTableBuffer(); + } + return new QwpSchemaBinding(currentTableBuffer, latest); + } + + /** + * Drops the row-less current table buffer's binding and column layout so the + * next contract lays it out afresh. + */ + private void unbindCurrentTableBuffer() { + assert currentTableBuffer.getRowCount() == 0 : "table buffer unbound under pending rows"; + currentTableBuffer.clear(); + currentTableBufferSnapshotBytes = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + // Single home of the row-failure policy: roll the whole row back, then report + // the failure as raised. The sender never looks up fresh metadata from a failed + // setter. When the snapshot rejected a value for a column it knows, the column + // may have changed type since, and no frame may ever reach the server to earn + // feedback. When the snapshot type could not take the value at all, the sender + // evicts the snapshot, so the next batch looks the table up. Any such rejection + // also counts towards a run, after which the next row re-checks the schema even + // while the batch holds rows; see recheckRejectingSchema(). + private RuntimeException onRowFailure(Throwable e) { + QwpSchemaBinding binding = currentTableBuffer != null ? currentTableBuffer.getSchemaBinding() : null; + if (binding != null) { + if (binding.hasTargetTypeConflict()) { + binding.clearTargetTypeConflict(); + CursorWebSocketSendLoop loop = cursorSendLoop; + if (loop != null) { + loop.evictSchema(currentTableBuffer.getTableName(), binding.getTableId(), binding.getMetadataVersion()); + } + } + if (binding.hasRejectedKnownColumn()) { + binding.clearRejectedKnownColumn(); + currentTableBuffer.getSchemaFreshness().onRowRejected(); + } + } + rollbackRow(); + if (e instanceof Error) { + throw (Error) e; + } + return (RuntimeException) e; + } + + /** + * Milliseconds left of a saturated nanosecond budget, rounded up. Comparing + * elapsed time against the budget cannot wrap, unlike an absolute deadline, + * so a budget of Long.MAX_VALUE millis means no limit. + */ + private static long remainingSchemaMillis(long startNanos, long budgetNanos) { + long remaining = budgetNanos - (System.nanoTime() - startNanos); + return remaining <= 0 ? 0 : 1L + (remaining - 1L) / 1_000_000L; + } + private void ensureNoInProgressRow() { if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { throw new LineSenderException( @@ -5208,11 +5760,16 @@ private void flushPendingRows(boolean deferCommit) { firstPendingRowTimeNanos = 0; return; } + if (tableCount > 0xffff) { + throw new LineSenderException("too many table blocks in one QWP message [count=") + .put(tableCount).put("]"); + } if (LOG.isDebugEnabled()) { LOG.debug("Flushing pending rows [count={}, tables={}, defer={}]", pendingRowCount, tableCount, deferCommit); } + readSchemaFlushClock(tableCount); ensureActiveBufferReady(); // In full-dict mode every frame is self-sufficient: it carries the whole // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh @@ -5303,6 +5860,7 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + recordSchemaFramesAfterFlush(); resetTableBuffersAfterFlush(); } catch (Throwable t) { if (dictionaryChunksAwaitCommit) { @@ -5531,9 +6089,61 @@ private void flushPendingRowsSplit( // committed the whole group. lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + recordSchemaFramesAfterFlush(); resetTableBuffersAfterFlush(); } + /** + * Reads the clock before the flush publishes, and only when a flushed table + * consulted its schema: a sender on the legacy contract reads no clock here. + */ + private void readSchemaFlushClock(int tableCount) { + for (int i = 0; i < tableCount; i++) { + if (flushTableBuffers.getQuick(i).getSchemaFreshness().isConsulted()) { + schemaFlushNanos = nanoClock.getAsLong(); + return; + } + } + } + + /** + * Records the highest FSN this flush published against each flushed table + * that consulted its schema, as of {@link #schemaFlushNanos}. A split flush + * publishes a table's own frame earlier, but replies are cumulative, so the + * reply covering the highest FSN also delivers that frame's feedback. + */ + private void recordSchemaFramesAfterFlush() { + long frameFsn = -1L; + long okFsn = -1L; + for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { + QwpSchemaFreshness freshness = flushTableBuffers.getQuick(i).getSchemaFreshness(); + if (freshness.isConsulted()) { + if (frameFsn < 0) { + frameFsn = fsnEpochBase + cursorEngine.publishedFsn(); + okFsn = schemaOkFsn(); + } + freshness.onFramePublished(frameFsn, schemaFlushNanos, okFsn); + } + } + } + + /** + * Highest external FSN the server acknowledged at the OK level, or -1. A + * symbol-dictionary recycle restarts the engine's FSNs, and + * {@link #lastRecycleDurableFsn} covers every frame of the retired epoch. The + * producer thread runs the recycle, so it never reads a torn base and engine + * pair. The loop is null between a recycle retiring it and the next connect, + * which a flush on close can reach. + */ + private long schemaOkFsn() { + CursorWebSocketSendLoop loop = cursorSendLoop; + long ok = cursorEngine.ackedFsn(); + if (loop != null) { + ok = Math.max(ok, loop.getHighestOkFsn()); + } + return ok < 0 ? lastRecycleDurableFsn : Math.max(lastRecycleDurableFsn, fsnEpochBase + ok); + } + private void onSlotLockReleased() { slotLockReleased = true; Runnable listener = slotLockReleaseListener; @@ -6404,7 +7014,8 @@ private RuntimeException rethrowRecycleAbandoned(Throwable t, String message) { * ever-connected sticky over: close joins the I/O thread, so a connect * that landed mid-window is final here, and this is the only place an * ASYNC-initial sender's connect (observed only by the I/O thread) - * reaches {@code hasLoopEverConnected}. A loop that latched a terminal + * reaches {@code hasLoopEverConnected}. It carries the schema state over + * the same way; see {@code hasLoopNegotiatedSchema}. A loop that latched a terminal * error while it was stopping is kept instead and its error thrown, so * {@link #checkConnectionError()} throws it from every later call, as on * a sender that never recycled. @@ -6413,6 +7024,9 @@ private void retireStoppedLoop() { cursorSendLoop.checkError(); healthTracker.connectionLost(); hasLoopEverConnected |= cursorSendLoop.hasEverConnected(); + hasLoopNegotiatedSchema |= cursorSendLoop.hasNegotiatedSchema(); + recycledSchemaCache.clear(); + cursorSendLoop.exportSchemaCache(recycledSchemaCache); cursorSendLoop = null; } @@ -7158,6 +7772,7 @@ private void sendRow() { int cap = serverMaxBatchSize; long bufferedNow = currentTableBuffer.nextRow( currentTableBufferSnapshotBytes, cap > 0 ? cap : Long.MAX_VALUE); + currentTableBuffer.getSchemaFreshness().onRowAccepted(); // Interval off (0): no clock read on the row path at all. if (pendingRowCount == 0 && autoFlushIntervalNanos > 0) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 9b6cb8e24..e3431ec1d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -24,7 +24,11 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.std.LongList; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Utf8SequenceObjHashMap; @@ -32,6 +36,9 @@ import io.questdb.client.std.str.Utf8s; import org.jetbrains.annotations.TestOnly; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; /** @@ -63,6 +70,9 @@ */ public class WebSocketResponse { + public static final int SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL = 0xC0; + public static final int SCHEMA_FEEDBACK_MODE_MASK = 0xC0; + public static final int SCHEMA_FEEDBACK_MODE_UPDATES = 0x80; public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; public static final int MIN_DURABLE_ACK_SIZE = 3; // status + tableCount public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length @@ -110,9 +120,12 @@ public class WebSocketResponse { private final Utf8SequenceObjHashMap tableNameCache = new Utf8SequenceObjHashMap<>(); private final ObjList tableNames = new ObjList<>(); private final LongList tableSeqTxns = new LongList(); + private final ObjList schemaUpdateTableNames = new ObjList<>(); + private final ObjList schemaUpdates = new ObjList<>(); private String errorMessage; private int errorMessageUtf8Length; private long sequence; + private boolean schemaInvalidation; private byte status; public WebSocketResponse() { @@ -272,6 +285,26 @@ public long getTableSeqTxn(int index) { return tableSeqTxns.getQuick(index); } + public int getSchemaUpdateCount() { + return schemaUpdates.size(); + } + + public QwpSchemaResponse getSchemaUpdate(int index) { + return schemaUpdates.getQuick(index); + } + + public String getSchemaUpdateTableName(int index) { + return schemaUpdateTableNames.getQuick(index); + } + + public boolean hasSchemaUpdates() { + return schemaUpdates.size() > 0; + } + + public boolean isSchemaInvalidation() { + return schemaInvalidation; + } + /** * Returns true when this is a per-table durable-upload ACK (STATUS_DURABLE_ACK). */ @@ -308,14 +341,38 @@ public boolean isSuccess() { * @return true if successfully parsed, false if not enough data */ public boolean readFrom(long ptr, int length) { + return readFrom(ptr, length, false); + } + + /** + * Reads a response, accepting schema feedback only on a connection that + * negotiated schema support during its WebSocket upgrade. + */ + public boolean readFrom(long ptr, int length, boolean schemaNegotiated) { tableNames.clear(); tableSeqTxns.clear(); + schemaUpdateTableNames.clear(); + schemaUpdates.clear(); + schemaInvalidation = false; if (length < 1) { return false; } - status = Unsafe.getUnsafe().getByte(ptr); + int wireStatus = Unsafe.getUnsafe().getByte(ptr) & 0xff; + int feedbackMode = wireStatus & SCHEMA_FEEDBACK_MODE_MASK; + if (feedbackMode == 0x40 || (feedbackMode != 0 && !schemaNegotiated)) { + return false; + } + if (feedbackMode == SCHEMA_FEEDBACK_MODE_UPDATES && length > QwpSchemaProtocol.MAX_FRAME_SIZE) { + return false; + } + byte baseStatus = (byte) (wireStatus & ~SCHEMA_FEEDBACK_MODE_MASK); + if (baseStatus == STATUS_DURABLE_ACK && feedbackMode != 0) { + return false; + } + status = baseStatus; + int baseEnd; if (status == STATUS_OK) { if (length < MIN_OK_RESPONSE_SIZE) { @@ -324,7 +381,12 @@ public boolean readFrom(long ptr, int length) { sequence = Unsafe.getUnsafe().getLong(ptr + 1); errorMessage = null; errorMessageUtf8Length = -1; - return readTableEntries(ptr + 9, length - 9); + baseEnd = readTableEntries(ptr + 9, length - 9, feedbackMode != 0); + if (baseEnd < 0) { + return false; + } + baseEnd += 9; + return readFeedback(ptr, length, baseEnd, feedbackMode); } if (isDurableAckStatus(status)) { @@ -334,7 +396,7 @@ public boolean readFrom(long ptr, int length) { sequence = -1; errorMessage = null; errorMessageUtf8Length = -1; - return readTableEntries(ptr + 1, length - 1); + return readTableEntries(ptr + 1, length - 1, false) >= 0; } // Error response @@ -359,7 +421,8 @@ public boolean readFrom(long ptr, int length) { errorMessage = null; errorMessageUtf8Length = 0; } - return true; + baseEnd = offset + msgLen; + return readFeedback(ptr, length, baseEnd, feedbackMode); } /** @@ -428,15 +491,15 @@ public int writeTo(long ptr) { // entries, empty table names, or trailing garbage. On false, tableNames / // tableSeqTxns may hold partial state, but the caller (readFrom) clears // both lists at the start of every call so partial state never leaks. - private boolean readTableEntries(long ptr, int remaining) { + private int readTableEntries(long ptr, int remaining, boolean allowTrailing) { if (remaining < 2) { - return false; + return -1; } int tableCount = Unsafe.getUnsafe().getShort(ptr) & 0xFFFF; int offset = 2; for (int i = 0; i < tableCount; i++) { if (remaining < offset + 2) { - return false; + return -1; } int nameLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF; offset += 2; @@ -444,7 +507,7 @@ private boolean readTableEntries(long ptr, int remaining) { // table name is never zero bytes, and accepting empty names would // let a misbehaving server poison the per-table tracker with "" entries. if (nameLen == 0 || remaining < offset + nameLen + 8) { - return false; + return -1; } long nameLo = ptr + offset; long nameHi = nameLo + nameLen; @@ -454,7 +517,87 @@ private boolean readTableEntries(long ptr, int remaining) { tableNames.add(internTableName(nameLo, nameHi)); tableSeqTxns.add(seqTxn); } - return remaining == offset; + return allowTrailing || remaining == offset ? offset : -1; + } + + private boolean readFeedback(long ptr, int length, int offset, int mode) { + if (mode == 0) { + return offset == length; + } + if (mode == SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL) { + if (offset != length) { + return false; + } + schemaInvalidation = true; + return true; + } + if (length - offset < 2) { + return false; + } + int count = Unsafe.getUnsafe().getShort(ptr + offset) & 0xffff; + offset += 2; + if (count == 0) { + return false; + } + // Each entry needs nameLen + one name byte + payloadLen + the shortest + // schema payload. Reject hostile counts before sizing collections. + if (count > (length - offset) / 17) { + return false; + } + ObjList names = new ObjList<>(count); + ObjList responses = new ObjList<>(count); + LowerCaseCharSequenceIntHashMap seen = new LowerCaseCharSequenceIntHashMap(count); + try { + for (int i = 0; i < count; i++) { + if (length - offset < 2) { + return false; + } + int nameLength = Unsafe.getUnsafe().getShort(ptr + offset) & 0xffff; + offset += 2; + if (nameLength == 0 || nameLength > QwpSchemaProtocol.MAX_NAME_UTF8_LENGTH || length - offset < nameLength + 4) { + return false; + } + String name = decodeStrictUtf8(ptr + offset, nameLength); + if (!TableUtils.isValidTableName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH) || !seen.put(name, i)) { + return false; + } + offset += nameLength; + long payloadLength = Unsafe.getUnsafe().getInt(ptr + offset) & 0xffffffffL; + offset += 4; + if (payloadLength > QwpSchemaProtocol.MAX_FRAME_SIZE || payloadLength > length - offset) { + return false; + } + QwpSchemaResponse response = QwpSchemaProtocol.decodeFeedbackPayload(ptr + offset, (int) payloadLength); + offset += (int) payloadLength; + names.add(name); + responses.add(response); + } + } catch (IllegalArgumentException e) { + return false; + } + if (offset != length) { + return false; + } + for (int i = 0; i < names.size(); i++) { + schemaUpdateTableNames.add(names.getQuick(i)); + schemaUpdates.add(responses.getQuick(i)); + } + return true; + } + + private static String decodeStrictUtf8(long address, int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("invalid UTF-8 feedback table name", e); + } } private String internTableName(long lo, long hi) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 8a76edcc3..83d43a20f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.LineSenderServerException; +import io.questdb.client.LineSenderSchemaException; import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderError; import io.questdb.client.cutlass.http.client.WebSocketClient; @@ -41,6 +42,8 @@ import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; import io.questdb.client.std.CharSequenceLongHashMap; @@ -53,6 +56,7 @@ import java.util.ArrayDeque; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -326,6 +330,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final CatchUpCapGapPolicy catchUpCapGapPolicy; private final CursorSendEngine engine; private final ReconnectPolicy reconnectPolicy; + private final QwpSchemaCoordinator schemaCoordinator = new QwpSchemaCoordinator(this::wakeIoIfParked); // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -498,6 +503,17 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // foreground sender's transient auth blip becomes a producer-fatal terminal -- the // exact failure this policy exists to prevent. private volatile boolean hasEverConnected; + // STRICT row policy remembers successful schema negotiation independently of replay. + private volatile boolean hasNegotiatedSchema; + // Producer-visible wire state. True from the moment swapClient installs a + // connection until the I/O thread enters the reconnect loop for it. The + // sender's schema contract selection reads it to decide between a lookup + // (wire up) and a cache-only probe (wire down). + private volatile boolean isWireUp; + // When true, connectLoop fails a pending schema lookup immediately instead of + // letting the replacement connection re-send it. Set by a sender whose schema + // mode can fall back to the legacy contract. + private volatile boolean shouldAbandonSchemaLookupOnDisconnect; // Cause of the outage the reconnect loop is currently riding out, or null once a // connect succeeds. Written by the I/O thread, read by the producer thread so a // backpressure or drain-timeout failure can NAME the reason the wire is not draining @@ -621,8 +637,10 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // across connections. In default mode this tracks the trim watermark // exactly; in durable-ack mode it runs AHEAD of ackedFsn, which waits for // STATUS_DURABLE_ACK coverage. I/O thread only; survives reconnects. Used - // by the poison-frame detector to measure genuine acceptance progress. - private long highestOkFsn = -1L; + // by the poison-frame detector to measure genuine acceptance progress, and + // read by the producer (getHighestOkFsn) to confirm its schema cache. + // Written after the ACK's schema feedback is applied. + private volatile long highestOkFsn = -1L; // Zero-progress recycle pacer (I/O thread only; survives reconnects). // Counts consecutive strike-EXEMPT recycles -- orderly closes // (NORMAL_CLOSURE/GOING_AWAY), non-orderly closes before any send on the @@ -1034,7 +1052,9 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // already know we reached the server at least once. ASYNC startup // hands null and lets the I/O thread connect — hasEverConnected // stays false until swapClient sees its first success. + this.hasNegotiatedSchema = client != null && client.isQwpSchemaEnabled(); this.hasEverConnected = client != null; + this.isWireUp = client != null; // Adopt the engine's recovered orphaned-deferred-tail range (if any). // See the field docs on orphanSkipStartFsn/orphanSkipTipFsn; the // BackgroundDrainer path builds its loop through this same @@ -1332,6 +1352,97 @@ public void checkError() { } } + /** + * Resolves a schema through this loop's I/O thread, returning a KNOWN, MISSING or TOO_LARGE response. + * This producer-side API permits one outstanding lookup. The timeout covers the entire + * operation, including connection and I/O; zero permits only an existing cache hit. + * Failures are reported as {@link LineSenderSchemaException}. This does not mutate rows, + * store-and-forward state, or acknowledgment watermarks. + */ + public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.resolve(tableName, timeoutMillis, false); + } + + /** + * Cache-only variant of {@link #resolveSchema}: the schema the coordinator + * already holds for the table, or {@code null}. Never sends a request and + * never waits, so a producer can call it while the wire is down. + */ + public QwpSchemaResponse peekSchema(CharSequence tableName) { + return schemaCoordinator.peek(tableName); + } + + /** + * Evicts the cached KNOWN snapshot of {@code tableName} if it still has the + * given identity, so the next batch that writes the table looks it up again. + */ + public void evictSchema(CharSequence tableName, int tableId, long metadataVersion) { + schemaCoordinator.evictKnown(tableName, tableId, metadataVersion); + } + + /** + * Selects what happens to a schema lookup in flight when the connection + * drops: {@code true} fails it immediately, {@code false} (the default) keeps + * it pending with its original deadline for the replacement connection. + */ + public void setAbandonSchemaLookupOnDisconnect(boolean abandon) { + this.shouldAbandonSchemaLookupOnDisconnect = abandon; + } + + /** + * Copies this loop's schema cache into {@code sink}. Also works after {@link #close()}. + */ + public void exportSchemaCache(Map sink) { + schemaCoordinator.exportCache(sink); + } + + /** + * Seeds the schema state of a loop that replaces a retired one in a + * symbol-dictionary recycle, the schema counterpart of + * {@link #markEverConnected()}. The rebuilt loop starts without a client, + * so without this it would report that no connection ever negotiated + * schema support, and STRICT would let legacy rows through until the new + * connection comes up. The cache spares one blocking DESCRIBE per table + * and keeps AUTO on the schema contract while the new connection is + * still pending. Call before {@link #start()}. + */ + public void seedSchemaState(boolean hasNegotiatedSchema, Map cache) { + if (hasNegotiatedSchema) { + this.hasNegotiatedSchema = true; + } + schemaCoordinator.importCache(cache); + } + + /** + * Evicts the named cache entry at admission and resolves it again through the I/O thread. + * This producer-side API permits one outstanding lookup. The timeout covers the entire + * operation, including connection and I/O, and must leave time for a request. It returns + * a KNOWN, MISSING or TOO_LARGE response and reports failures as {@link LineSenderSchemaException}. + * This does not mutate rows, store-and-forward state, or acknowledgment watermarks. + */ + public QwpSchemaResponse refreshSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.resolve(tableName, timeoutMillis, true); + } + + /** + * Looks the table up through the I/O thread even when the cache holds it. + * A failed lookup leaves the cached entry in place; a denial evicts it. + * Same contract as {@link #resolveSchema} otherwise. + */ + public QwpSchemaResponse revalidateSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.revalidate(tableName, timeoutMillis); + } + + /** + * Highest engine FSN the server acknowledged at the OK level, across + * connections, or -1. The I/O thread applies an ACK's schema feedback + * before it publishes the ACK here, so a reader that sees a frame covered + * also sees that frame's feedback. + */ + public long getHighestOkFsn() { + return highestOkFsn; + } + /** * The typed rotating-credential auth terminal (401/403), or {@code null} if the loop's terminal * (if any) is a different failure class. Non-null only after {@link #checkError()} started @@ -1390,6 +1501,7 @@ public synchronized void close() { // is false and start's ioLoop will exit immediately) or entirely // after — the latch await is only skipped when the loop never ran. running = false; + schemaCoordinator.close(); Thread t = ioThread; // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets // freed on ioLoop's exit path. When t == null the loop never ran (start() @@ -1725,6 +1837,63 @@ public boolean hasEverConnected() { return hasEverConnected; } + /** + * True once any connection this loop installed negotiated schema support, + * or once {@link #seedSchemaState} carried that over from a retired loop. + * Sticky, like {@link #hasEverConnected()}. + */ + public boolean hasNegotiatedSchema() { + return hasNegotiatedSchema; + } + + public boolean isSchemaEnabled() { + WebSocketClient activeClient = client; + return activeClient != null && activeClient.isQwpSchemaEnabled(); + } + + /** + * Whether a connection is currently installed. False before the first + * upgrade completes and from the moment the I/O thread enters the reconnect + * loop until {@code swapClient} installs the replacement. + */ + public boolean isWireUp() { + return isWireUp; + } + + /** + * Waits for the first completed WebSocket upgrade and returns whether any + * installed connection has selected schema mode. A failed or still-pending + * handshake is never interpreted as a legacy peer. + */ + public boolean awaitInitialSchemaMode(long timeoutMillis) { + long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + long start = System.nanoTime(); + while (!hasEverConnected) { + checkError(); + if (!running) { + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation stopped before the first successful connection"); + } + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation was interrupted"); + } + if (timeoutNanos - (System.nanoTime() - start) <= 0) { + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation timed out before the first successful connection"); + } + LockSupport.parkNanos(Math.min(50_000L, + timeoutNanos - (System.nanoTime() - start))); + } + // Publish negotiation before hasEverConnected; recovered frames alone + // do not establish the first connection's row policy. + return hasNegotiatedSchema; + } + /** * True while the I/O thread holds a live connection: the loop is running, * not inside a connect walk (including a paced one's pre-attempt park), @@ -2080,6 +2249,11 @@ private void connectWalk(Throwable initial, String phase, long paceFirstAttemptM if (!running) { return; } + isWireUp = false; + WebSocketClient failedClient = client; + if (failedClient != null) { + schemaCoordinator.connectionLost(failedClient, shouldAbandonSchemaLookupOnDisconnect); + } if (reconnectFactory == null) { recordFatal(initial); return; @@ -2762,7 +2936,10 @@ private void ioLoop() { // Snapshot before the send attempt: a publish after this point is // caught by the re-check below, so the idle decision cannot race it. long publishedBeforeWork = engine.publishStamp(); - boolean didWork = trySendBatch(); + boolean didWork = trySendSchemaRequest(); + if (trySendBatch()) { + didWork = true; + } // 1. Try to send next frame(s). // 2. Try to receive ACKs. if (tryReceiveAcks()) { @@ -2793,14 +2970,17 @@ private void ioLoop() { } // 5. Nothing to do: block until the socket has bytes (an ACK), // a producer publishes a frame (engine rings the doorbell via - // setIoWakeup), close() rings it, or the next PING is due. + // setIoWakeup), a producer queues a schema lookup (the schema + // coordinator rings it), close() rings it, or the next PING + // is due. // No spinning: an idle connection costs zero CPU. A PING that // failed in 3/4 reconnected and rewound the cursor, which set // recycledThisIteration again: go back to trySendBatch instead. if (!didWork && running && !recycledThisIteration) { ioParked = true; try { - if (engine.publishStamp() == publishedBeforeWork) { + if (engine.publishStamp() == publishedBeforeWork + && !schemaCoordinator.hasUnsentRequest()) { awaitIdle(); } } finally { @@ -2840,6 +3020,7 @@ private void ioLoop() { } } } finally { + schemaCoordinator.close(); // Release native-segment lifetime before publishing I/O-thread // completion or running delegated engine cleanup. releaseSendingSegment(); @@ -2959,6 +3140,7 @@ private void positionCursorInSegment(MmapSegment seg, long targetFsn) { * every rethrow delivers the same instance. */ private void recordFatal(Throwable t) { + schemaCoordinator.close(); io.questdb.client.cutlass.qwp.client.QwpConnectionHealthTracker tracker = healthTracker; if (tracker != null) { tracker.failed(); @@ -3135,6 +3317,11 @@ private void snapshotReplayTarget() { private void swapClient(WebSocketClient newClient) { WebSocketClient old = this.client; this.client = newClient; + // The schema cache survives the reconnect. The server ingests by column name + // and ignores the identity a frame carries, except to decide on feedback. A + // new connection has reported nothing yet, so the first ACK for each table + // refreshes any snapshot that went stale during the outage. Clearing here + // would cost one blocking DESCRIBE per table on the next batch. recycledThisIteration = true; this.lastReconnectError = null; if (old != null) { @@ -3176,6 +3363,10 @@ private void swapClient(WebSocketClient newClient) { // dies inside the dictionary catch-up above was never fully // established, so the sender stays INITIALIZING and a subsequent // endpoint-policy rejection still latches the startup terminal. + if (newClient.isQwpSchemaEnabled()) { + this.hasNegotiatedSchema = true; + } + this.isWireUp = true; this.hasEverConnected = true; } @@ -3876,6 +4067,43 @@ private boolean tryReceiveAcks() { return any; } + private boolean trySendSchemaRequest() { + if (!schemaCoordinator.hasPendingRequest()) { + return false; + } + WebSocketClient activeClient = client; + if (activeClient == null) { + return false; + } + QwpSchemaCoordinator.Request request = schemaCoordinator.requestToSend(activeClient); + if (request == null) { + return false; + } + if (!activeClient.isQwpSchemaEnabled()) { + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "server did not negotiate schema lookup support"); + return true; + } + long remainingNanos = request.remainingNanos(); + if (remainingNanos <= 0) { + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema lookup timed out before send"); + return true; + } + int timeoutMillis = (int) Math.min(Integer.MAX_VALUE, + 1L + (remainingNanos - 1L) / 1_000_000L); + try { + activeClient.sendBinary(request.message, timeoutMillis); + return true; + } catch (Throwable t) { + if (t instanceof Error) { + throw (Error) t; + } + fail(t); + return true; + } + } + /** * Returns true if at least one frame was sent (caller skips the park). * Sends every fully published frame of the current segment in one write, @@ -4416,11 +4644,44 @@ boolean isDurableUnder(CharSequenceLongHashMap watermarks) { private final class ResponseHandler implements WebSocketFrameHandler { @Override public void onBinaryMessage(long payloadPtr, int payloadLen) { - if (!response.readFrom(payloadPtr, payloadLen)) { + if (payloadLen >= Integer.BYTES + && Unsafe.getUnsafe().getInt(payloadPtr) == QwpConstants.MAGIC_MESSAGE) { + WebSocketClient responseClient = client; + final long requestId; + try { + if (responseClient == null || !responseClient.isQwpSchemaEnabled()) { + throw new IllegalArgumentException("schema control response was not negotiated"); + } + requestId = QwpSchemaProtocol.peekResponseRequestId(payloadPtr, payloadLen); + } catch (RuntimeException e) { + schemaCoordinator.invalidResponse(responseClient, 0); + fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); + return; + } + if (!schemaCoordinator.isLiveResponse(responseClient, requestId)) { + return; + } + final QwpSchemaResponse schema; + try { + schema = QwpSchemaProtocol.decodeResponse(payloadPtr, payloadLen); + } catch (RuntimeException e) { + if (!schemaCoordinator.invalidResponse(responseClient, requestId)) { + return; + } + fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); + return; + } + schemaCoordinator.completeResponse(responseClient, schema); + return; + } + if (!response.readFrom(payloadPtr, payloadLen, client != null && client.isQwpSchemaEnabled())) { fail(new LineSenderException( "Invalid ACK response payload [length=" + payloadLen + ']')); return; } + if (response.hasSchemaUpdates() || response.isSchemaInvalidation()) { + schemaCoordinator.applyFeedback(response); + } long wireSeq = response.getSequence(); if (response.isSuccess()) { // Same sanity clamp as legacy: don't trust an ACK beyond @@ -4434,11 +4695,13 @@ public void onBinaryMessage(long payloadPtr, int payloadLen) { LOG.warn("server ACK wire seq {} outside sent range [0, {}], clamping", wireSeq, highestSent); } - counters.acks.incrementAndGet(); long okFsn = fsnAtZero + capped; if (okFsn > highestOkFsn) { highestOkFsn = okFsn; } + // After the watermark, so a reader that sees the count also sees + // the OK level, and with it the schema feedback applied above. + counters.acks.incrementAndGet(); // Clear poison-frame suspicion only when OK-level progress // reaches the suspected frame itself: acceptance at or beyond // poisonFsn proves the earlier rejections were not diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java new file mode 100644 index 000000000..01973a9f4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -0,0 +1,399 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.std.Chars; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; +import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; +import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; + +final class QwpSchemaCoordinator { + // Returns a cached entry, else asks the server. + private static final int LOOKUP_CACHED = 0; + // Evicts the cached entry, then asks the server. + private static final int LOOKUP_REFRESH = 1; + // Asks the server and keeps the cached entry unless the answer replaces it + // or denies access. + private static final int LOOKUP_REVALIDATE = 2; + // Bounded by the tables this sender writes, like its per-table buffers, + // each far heavier than an entry, so it needs no eviction. + private final HashMap cache = new HashMap<>(); + // Ends the I/O loop's idle wait; see resolve(). + private final Runnable ioWakeup; + private volatile boolean closed; + private volatile boolean hasPendingRequest; + private long nextRequestId = 1; + private Request request; + + QwpSchemaCoordinator(Runnable ioWakeup) { + this.ioWakeup = ioWakeup; + } + + QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean refresh) { + return resolve(tableName, timeoutMillis, refresh ? LOOKUP_REFRESH : LOOKUP_CACHED); + } + + /** + * Asks the server for the table's schema even when the cache holds it. A + * failed lookup leaves the cached entry in place, except that a denial + * evicts it, as for any lookup. + */ + QwpSchemaResponse revalidate(CharSequence tableName, long timeoutMillis) { + return resolve(tableName, timeoutMillis, LOOKUP_REVALIDATE); + } + + private QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, int lookup) { + final long startNanos = System.nanoTime(); + if (timeoutMillis < 0) { + throw new IllegalArgumentException("timeoutMillis must be non-negative"); + } + if (tableName == null || tableName.length() == 0 + || tableName.length() > QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH) { + throw new IllegalArgumentException("table name must contain 1 to " + + QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + " UTF-16 units"); + } + final String key = normalize(tableName); + final long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + synchronized (this) { + if (closed) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema coordinator is closed"); + } + if (lookup == LOOKUP_CACHED) { + QwpSchemaResponse cached = cache.get(key); + if (cached != null) { + return cached; + } + } else if (lookup == LOOKUP_REFRESH) { + cache.remove(key); + } + if (timeoutNanos - (System.nanoTime() - startNanos) <= 0) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); + } + if (request != null) { + throw failure(SCHEMA_UNAVAILABLE, key, "another schema lookup is already in progress"); + } + Request own = new Request(key, tableName.toString(), startNanos, timeoutNanos, + lookup == LOOKUP_REVALIDATE); + request = own; + hasPendingRequest = true; + // The I/O loop blocks without a timeout when idle. It raises its parked + // flag before it checks hasUnsentRequest(), so it either sees this + // request or gets this wake. The wake ends only the idle wait, never a + // reconnect backoff. + ioWakeup.run(); + while (!own.done) { + try { + await(own, key); + } catch (LineSenderSchemaException e) { + if (request == own) { + request = null; + hasPendingRequest = false; + notifyAll(); + } + throw e; + } + } + if (own.error != null) { + throw own.error; + } + return own.response; + } + } + + synchronized Request requestToSend(WebSocketClient client) { + if (request == null || request.done || request.sentClient != null) { + return null; + } + if (request.remainingNanos() <= 0) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema lookup timed out")); + return null; + } + // Every send attempt, first or after a reconnect, takes a fresh id, so a + // reply to an earlier attempt can never match this one. + if (request.id == 0) { + if (nextRequestId <= 0 || nextRequestId == Long.MAX_VALUE) { + complete(request, null, failure(UNSUPPORTED_FEATURE, request.key, "schema request id space is exhausted")); + return null; + } + request.id = nextRequestId++; + request.message = QwpSchemaProtocol.encodeDescribe(request.id, request.tableName); + } + request.sentClient = client; + return request; + } + + boolean hasPendingRequest() { + return hasPendingRequest; + } + + /** + * Whether a request waits for the I/O thread to send it. A request already + * on the wire does not count: its reply arrives as inbound bytes, which end + * the I/O thread's idle wait on their own. + */ + synchronized boolean hasUnsentRequest() { + return request != null && !request.done && request.sentClient == null; + } + + synchronized boolean isLiveResponse(WebSocketClient client, long requestId) { + Request current = request; + return current != null && !current.done && current.sentClient == client + && current.id == requestId && current.remainingNanos() > 0; + } + + synchronized void completeResponse(WebSocketClient client, QwpSchemaResponse response) { + Request current = request; + if (current == null || current.done || current.sentClient != client + || current.id != response.getRequestId()) { + return; + } + if (current.remainingNanos() <= 0) { + complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema lookup timed out")); + return; + } + if (isCacheable(response.getResult())) { + cache.put(current.key, response); + complete(current, response, null); + return; + } + // A revalidation keeps the entry it could not replace; access denial + // still evicts it, so the row fails as it would on a cold lookup. + if (!current.isRevalidation || response.getResult() == QwpSchemaProtocol.RESULT_DENIED) { + cache.remove(current.key); + } + switch (response.getResult()) { + case QwpSchemaProtocol.RESULT_DENIED: + complete(current, null, failure(ACCESS_DENIED, current.key, "schema access denied")); + return; + case QwpSchemaProtocol.RESULT_UNAVAILABLE: + complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema is unavailable")); + return; + default: + complete(current, null, failure(UNSUPPORTED_FEATURE, current.key, "unsupported schema result")); + } + } + + synchronized void fail(Request own, LineSenderSchemaException.Reason reason, String detail) { + if (request == own && !own.done) { + complete(own, null, failure(reason, own.key, detail)); + } + } + + void applyFeedback(WebSocketResponse feedback) { + if (feedback.isSchemaInvalidation()) { + clearCache(); + } + for (int i = 0, n = feedback.getSchemaUpdateCount(); i < n; i++) { + QwpSchemaResponse schema = feedback.getSchemaUpdate(i); + String key = normalize(feedback.getSchemaUpdateTableName(i)); + synchronized (this) { + if (closed) { + return; + } + if (isCacheable(schema.getResult())) { + cache.put(key, schema); + } else { + cache.remove(key); + } + } + } + } + + /** + * Records that {@code client} died. With {@code abandon} false the producer's + * request keeps its deadline and the replacement connection re-sends it; with + * {@code abandon} true the request fails right away, so a producer that can + * fall back to the legacy contract does not sit out the rest of its budget on + * a wire that is known to be down. + */ + synchronized void connectionLost(WebSocketClient client, boolean abandon) { + if (request == null || request.done) { + return; + } + if (abandon) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "connection lost during schema lookup")); + return; + } + if (request.sentClient == client) { + // Keep the producer's request and deadline; only its wire attempt ended. + // A fresh ID also rejects old replies delivered on the replacement connection. + request.sentClient = null; + request.id = 0; + } + } + + /** + * Cache-only lookup: the schema this coordinator already holds for the table, + * or {@code null}. Never sends a request and never waits. + */ + synchronized QwpSchemaResponse peek(CharSequence tableName) { + if (closed) { + return null; + } + return cache.get(normalize(tableName)); + } + + synchronized boolean invalidResponse(WebSocketClient client, long requestId) { + // Zero means the malformed header could not identify a request. For an + // identifiable reply, cancellation must share the liveness check's lock. + if (requestId != 0 && !isLiveResponse(client, requestId)) { + return false; + } + if (request != null && (request.sentClient == null || request.sentClient == client)) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "invalid schema control response")); + } + return true; + } + + synchronized void exportCache(Map sink) { + sink.putAll(cache); + } + + /** + * Seeds the cache with schemas another coordinator exported. ACK feedback on + * the new connection refreshes any entry that went stale meanwhile. + */ + synchronized void importCache(Map source) { + if (!closed) { + cache.putAll(source); + } + } + + synchronized void clearCache() { + if (!closed) { + cache.clear(); + } + } + + /** + * Drops the cached KNOWN snapshot of {@code tableName} when it still has the + * given identity. A newer snapshot, such as one that ACK feedback installed + * after the caller pinned its binding, stays cached. + */ + synchronized void evictKnown(CharSequence tableName, int tableId, long metadataVersion) { + if (closed) { + return; + } + String key = normalize(tableName); + QwpSchemaResponse cached = cache.get(key); + if (cached != null + && cached.getResult() == QwpSchemaProtocol.RESULT_KNOWN + && cached.getTableId() == tableId + && cached.getMetadataVersion() == metadataVersion) { + cache.remove(key); + } + } + + synchronized void close() { + // Keeps the cache: every read path checks closed first, and a + // symbol-dictionary recycle exports the entries after close. + closed = true; + if (request != null) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema coordinator is closed")); + } + notifyAll(); + } + + /** + * Results that stay true until the table's schema version changes, which + * the server then reports through ACK feedback. TOO_LARGE is a negative entry, + * like MISSING: without it, AUTO would pay a DESCRIBE round trip every batch + * for a table too wide to describe. DENIED and UNAVAILABLE may clear at any + * time, so they are never cached. + */ + private static boolean isCacheable(int result) { + return result == QwpSchemaProtocol.RESULT_KNOWN + || result == QwpSchemaProtocol.RESULT_MISSING + || result == QwpSchemaProtocol.RESULT_TOO_LARGE; + } + + private static LineSenderSchemaException failure(LineSenderSchemaException.Reason reason, String table, String detail) { + return new LineSenderSchemaException(reason, "schema lookup failed [reason=" + reason + + ", table=" + table + ", detail=" + detail + ']'); + } + + /** + * Lower-cases a table name into a cache key. A String that is already lower + * case is its own key, so the sender's pre-computed + * {@code QwpTableBuffer.getSchemaKey()} costs no allocation on a cache hit. + */ + private static String normalize(CharSequence tableName) { + if (tableName instanceof String) { + String name = (String) tableName; + for (int i = 0, n = name.length(); i < n; i++) { + char c = name.charAt(i); + if (Character.toLowerCase(c) != c) { + return Chars.toLowerCase(name); + } + } + return name; + } + return Chars.toLowerCase(tableName); + } + + private void await(Request request, String key) { + long remaining = request.remainingNanos(); + if (remaining <= 0) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); + } + try { + long millis = remaining / 1_000_000L; + int nanos = (int) (remaining % 1_000_000L); + wait(millis, nanos); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup was interrupted"); + } + } + + private void complete(Request own, QwpSchemaResponse response, LineSenderSchemaException error) { + own.response = response; + own.error = error; + own.done = true; + if (request == own) { + request = null; + hasPendingRequest = false; + } + notifyAll(); + } + + + static final class Request { + final boolean isRevalidation; + final long startNanos; + final long timeoutNanos; + long id; + final String key; + final String tableName; + byte[] message; + boolean done; + LineSenderSchemaException error; + QwpSchemaResponse response; + WebSocketClient sentClient; + + Request(String key, String tableName, long startNanos, long timeoutNanos, boolean isRevalidation) { + this.isRevalidation = isRevalidation; + this.key = key; + this.tableName = tableName; + this.startNanos = startNanos; + this.timeoutNanos = timeoutNanos; + } + + long remainingNanos() { + return timeoutNanos - (System.nanoTime() - startNanos); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 0fa7a6455..a771e509c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -255,6 +255,7 @@ public static int getFixedTypeSize(byte typeCode) { case TYPE_CHAR: return 2; case TYPE_INT: + case TYPE_IPv4: case TYPE_FLOAT: return 4; case TYPE_LONG: @@ -301,6 +302,9 @@ public static String getTypeName(byte typeCode) { case TYPE_INT: name = "INT"; break; + case TYPE_IPv4: + name = "IPv4"; + break; case TYPE_LONG: name = "LONG"; break; @@ -368,6 +372,7 @@ public static boolean isFixedWidthType(byte typeCode) { (int) typeCode == TYPE_SHORT || (int) typeCode == TYPE_CHAR || (int) typeCode == TYPE_INT || + (int) typeCode == TYPE_IPv4 || (int) typeCode == TYPE_LONG || (int) typeCode == TYPE_FLOAT || (int) typeCode == TYPE_DOUBLE || diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java new file mode 100644 index 000000000..ec024e6f7 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -0,0 +1,1992 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cairo.MicrosTimestampDriver; +import io.questdb.client.cairo.NanosTimestampDriver; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.NumericException; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.bytes.DirectByteSlice; +import io.questdb.client.std.fastdouble.FastDoubleParser; +import io.questdb.client.std.fastdouble.FastFloatParser; +import io.questdb.client.std.str.StringSink; +import io.questdb.client.std.str.Utf8StringSink; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; +import static io.questdb.client.LineSenderSchemaException.Reason.INVALID_VALUE; +import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; +import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; + +/** + * Converts supported input values against one immutable server schema result + * and appends them to a caller-owned {@link QwpTableBuffer}. A confirmed + * missing table, or a column absent from a known schema, is inferred from the + * first effective supported input value. + * + *

This binding does not own the buffer or its row lifecycle and is not + * thread-safe. The caller completes rows, encodes data and closes the buffer. + * After a setter failure, the caller must invoke both + * {@link QwpTableBuffer#cancelCurrentRow()} and + * {@link QwpTableBuffer#rollbackUncommittedColumns()} before continuing or + * completing another row. {@link QwpTableBuffer#reset()} preserves the binding; + * {@link QwpTableBuffer#clear()} invalidates it.

+ * + *

The caller must encode the buffer with the matching symbol-dictionary + * family: local dictionaries for ownerless buffers, or the owning sender's + * corresponding global dictionary for sender-backed buffers.

+ */ +public final class QwpSchemaBinding { + private static final int FIRST_DASH = 8; + private static final int SECOND_DASH = 13; + private static final int THIRD_DASH = 18; + private static final int FOURTH_DASH = 23; + private static final int UUID_LENGTH = 36; + + private final QwpTableBuffer buffer; + private final QwpSchemaResponse schema; + private final String tableName; + private boolean hasRejectedKnownColumn; + private boolean hasTargetTypeConflict; + private Decimal256 decimalTextScratch; + private StringSink charTextSink; + private StringSink floatingTextSink; + private StringSink numericTextSink; + private Utf8StringSink timestampTextSink; + private StringSink uuidTextSink; + + /** + * Attaches {@code schema} to an empty, unbound caller-owned buffer. The + * caller must correlate a + * positive request id with its DESCRIBE table, or obtain request-id-zero + * schema from the matching named write-feedback entry. + */ + public QwpSchemaBinding(QwpTableBuffer buffer, QwpSchemaResponse schema) { + if (buffer == null) { + throw new NullPointerException("buffer"); + } + String tableName = buffer.getTableName(); + if (tableName == null || !TableUtils.isValidTableName(tableName, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { + throw new IllegalArgumentException("invalid table name"); + } + if (schema == null) { + throw new NullPointerException("schema"); + } + this.tableName = tableName; + this.schema = schema; + requireUsableSchema(); + this.buffer = buffer; + buffer.attachSchemaBinding(this); + } + + /** + * Appends an owned copy of a BINARY value to an exact BINARY target. + * The caller retains the class-level row-cancellation and uncommitted-column rollback + * responsibility on failure. + */ + public QwpSchemaBinding binaryColumn(CharSequence name, byte[] value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + if (value == null) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary value cannot be null"); + } + column.addBinary(value); + return this; + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal64 value) { + return decimalColumn(name, value, "DECIMAL64", QwpConstants.TYPE_DECIMAL64); + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal128 value) { + return decimalColumn(name, value, "DECIMAL128", QwpConstants.TYPE_DECIMAL128); + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal256 value) { + return decimalColumn(name, value, "DECIMAL256", QwpConstants.TYPE_DECIMAL256); + } + + /** + * Parses the public textual-decimal overload after target selection and + * duplicate suppression, then applies the same DECIMAL256 conversion rules + * as the typed overload. The caller owns {@code scratch}; this binding does + * not retain it. + */ + public QwpSchemaBinding decimalColumn(CharSequence name, CharSequence value, Decimal256 scratch) { + buffer.requireSchemaBinding(this); + if (value == null || value.length() == 0) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DECIMAL256"); + int targetType = targetType(index, ColumnType.DECIMAL256); + if (index < 0) { + if (!isInferredWritable(existing, name, "DECIMAL256", QwpConstants.TYPE_DECIMAL256)) { + return this; + } + parseDecimalText(name, value, scratch, targetType); + // Keep the legacy public-overload contract: infer DECIMAL256 at + // the parsed natural scale. A parsed special is an effective + // null but does not pin the scale of a later finite value. + claimInferred(existing, name, QwpConstants.TYPE_DECIMAL256).addDecimal256(scratch); + return this; + } + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "DECIMAL256", index, targetType); + if (column == null) { + return this; + } + if (!ColumnType.isDecimal(targetType) + && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "DECIMAL256", targetType, "conversion is not implemented"); + } + parseDecimalText(name, value, scratch, targetType); + if (scratch.isNull()) { + if (ColumnType.isDecimal(targetType)) { + column.addSchemaDecimalNull(ColumnType.getDecimalScale(targetType)); + } else { + column.addNull(); + } + return this; + } + return appendDecimal(column, name, scratch, "DECIMAL256", targetType); + } + + /** Appends an owned copy of the bytes addressed by {@code slice}. */ + public QwpSchemaBinding binaryColumn(CharSequence name, DirectByteSlice slice) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + if (slice == null) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary slice cannot be null"); + } + appendBinary(name, column, slice.ptr(), slice.size()); + return this; + } + + /** + * Appends an owned copy of {@code [ptr, ptr + len)}. The source memory need only remain + * valid for the duration of this call. + */ + public QwpSchemaBinding binaryColumn(CharSequence name, long ptr, long len) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + appendBinary(name, column, ptr, len); + return this; + } + + public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "BOOLEAN"); + int targetType = targetType(index, ColumnType.BOOLEAN); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "BOOLEAN", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.BOOLEAN: + column.addBoolean(value); + break; + case ColumnType.BYTE: + column.addByte((byte) (value ? 1 : 0)); + break; + case ColumnType.SHORT: + column.addShort((short) (value ? 1 : 0)); + break; + case ColumnType.INT: + column.addInt(value ? 1 : 0); + break; + case ColumnType.LONG: + column.addLong(value ? 1 : 0); + break; + case ColumnType.FLOAT: + column.addFloat(value ? 1 : 0); + break; + case ColumnType.DOUBLE: + column.addDouble(value ? 1 : 0); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value ? "true" : "false"); + break; + default: + throw targetTypeConflict(name, "BOOLEAN", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 1); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[][] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 2); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[][][] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 3); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, DoubleArray array) { + buffer.requireSchemaBinding(this); + if (array == null) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DOUBLE_ARRAY"); + if (existing != null && existing.getSize() > buffer.getRowCount()) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(existing, name, index, array.getDimensionality()); + if (column != null) { + column.addDoubleArray(array); + } + return this; + } + + public QwpSchemaBinding byteColumn(CharSequence name, byte value) { + return integerColumn(name, value, "BYTE", ColumnType.BYTE, false); + } + + public QwpSchemaBinding doubleColumn(CharSequence name, double value) { + return floatingNumericColumn(name, value, "DOUBLE"); + } + + /** Appends and pins the row's designated timestamp using the schema target precision. */ + public QwpSchemaBinding designatedTimestamp(long value, ChronoUnit unit) { + buffer.requireSchemaBinding(this); + int targetType = schema.getResult() == QwpSchemaProtocol.RESULT_MISSING + ? (unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO) + : schema.getColumnType(requireDesignatedTimestamp("TIMESTAMP")); + if (unit == null) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "timestamp unit is null"); + } + if (!isSupportedTimestampUnit(unit)) { + throw unsupported(null, "TIMESTAMP", targetType, "timestamp unit is not supported: " + unit); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_MICRO + ? MicrosTimestampDriver.INSTANCE.from(value, unit) + : NanosTimestampDriver.INSTANCE.from(value, unit); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + designatedColumn(targetType, "TIMESTAMP").addLong(converted); + return this; + } + + /** Appends and pins an Instant as the row's designated timestamp. */ + public QwpSchemaBinding designatedTimestamp(Instant value) { + buffer.requireSchemaBinding(this); + int targetType = schema.getResult() == QwpSchemaProtocol.RESULT_MISSING + ? ColumnType.TIMESTAMP_MICRO + : schema.getColumnType(requireDesignatedTimestamp("TIMESTAMP")); + if (value == null) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "timestamp value is null"); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_MICRO + ? instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L) + : instantTimestamp(value, 1_000_000_000L, value.getNano()); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + designatedColumn(targetType, "TIMESTAMP").addLong(converted); + return this; + } + + private QwpTableBuffer.ColumnBuffer designatedColumn(int targetType, String inputType) { + byte wireType = timestampWireType(targetType); + if (schema.getResult() == QwpSchemaProtocol.RESULT_MISSING) { + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(""); + if (inferred != null && inferred.getType() != wireType) { + throw unsupported(null, inputType, -1, "inferred designated timestamp type conflict [inferredType=" + + QwpConstants.getTypeName(inferred.getType()) + ']'); + } + } + return buffer.getOrCreateDesignatedTimestampColumn(wireType); + } + + public QwpSchemaBinding floatColumn(CharSequence name, float value) { + return floatingNumericColumn(name, value, "FLOAT"); + } + + public QwpSchemaBinding geoHashColumn(CharSequence name, long bits, int precisionBits) { + buffer.requireSchemaBinding(this); + if (precisionBits < 1 || precisionBits > 60) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); + } + int sourceType = ColumnType.getGeoHashTypeWithBits(precisionBits); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "GEOHASH"); + int targetType = targetType(index, sourceType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "GEOHASH", index, targetType); + if (column == null) { + return this; + } + long value = bits & ((1L << precisionBits) - 1L); + if (!ColumnType.isArray(targetType) && ColumnType.isGeoHash(targetType)) { + int targetBits = ColumnType.getGeoHashBits(targetType); + if (targetBits != precisionBits) { + throw unsupported(name, "GEOHASH", targetType, + "GEOHASH precision mismatch [sourceBits=" + precisionBits + ", targetBits=" + targetBits + ']'); + } + column.addGeoHash(value, precisionBits); + } else if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + column.addString(formatGeoHash(value, precisionBits)); + } else { + throw targetTypeConflict(name, "GEOHASH", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding geoHashColumn(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + if (value == null) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "GEOHASH string cannot be null; omit the setter to write NULL"); + } + int length = value.length(); + if (length == 0) { + throw error(INVALID_VALUE, name, "GEOHASH", null, "GEOHASH string cannot be empty"); + } + if (length > 12) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "GEOHASH string exceeds 12 characters: " + length); + } + final long bits; + try { + bits = Numbers.parseGeoHashBase32(value, 0, length); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "GEOHASH", null, "invalid GEOHASH string"); + } + return geoHashColumn(name, bits, length * 5); + } + + public QwpSchemaBinding intColumn(CharSequence name, int value) { + return integerColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); + } + + public QwpSchemaBinding ipv4Column(CharSequence name, int address) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "IPv4"); + int targetType = targetType(index, ColumnType.IPv4); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "IPv4", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.IPv4 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "IPv4", targetType, "conversion is not implemented"); + } + if (address == Numbers.IPv4_NULL) { + column.addNull(); + } else if (targetType == ColumnType.IPv4) { + column.addIPv4(address); + } else { + column.addString(formatIPv4(address)); + } + return this; + } + + public QwpSchemaBinding ipv4Column(CharSequence name, CharSequence address) { + buffer.requireSchemaBinding(this); + if (address == null) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "IPv4"); + int targetType = targetType(index, ColumnType.IPv4); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "IPv4", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.IPv4 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "IPv4", targetType, "conversion is not implemented"); + } + if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { + throw error(INVALID_VALUE, name, "IPv4", targetType, + "NULL sentinel inputs are rejected; pass a null reference or omit the setter"); + } + final int packed; + try { + packed = Numbers.parseIPv4(address); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "IPv4", targetType, "invalid IPv4 address"); + } + if (packed == Numbers.IPv4_NULL) { + column.addNull(); + } else if (targetType == ColumnType.IPv4) { + column.addIPv4(packed); + } else { + column.addString(formatIPv4(packed)); + } + return this; + } + + // Mirrors integerColumn() with LONG constants. Delegating to it measured about + // 4% slower per row in QwpSchemaBindingBenchmark.schemaRow: the JIT does not + // inline the shared body, so the constants become runtime arguments. + public QwpSchemaBinding longColumn(CharSequence name, long value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "LONG"); + int targetType = targetType(index, ColumnType.LONG); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "LONG", index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType) + && !isTextTarget(targetType) + && !ColumnType.isDecimal(targetType) + && targetType != ColumnType.DATE + && targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO) { + throw targetTypeConflict(name, "LONG", targetType, "conversion is not implemented"); + } + if (ColumnType.isDecimal(targetType)) { + appendIntegerDecimal(column, name, value, "LONG", targetType, value == Long.MIN_VALUE); + return this; + } + if (value == Long.MIN_VALUE) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw invalidRange(name, "LONG", targetType); + } + column.addByte((byte) value); + break; + case ColumnType.SHORT: + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw invalidRange(name, "LONG", targetType); + } + column.addShort((short) value); + break; + case ColumnType.INT: + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw invalidRange(name, "LONG", targetType); + } + column.addInt((int) value); + break; + case ColumnType.LONG: + case ColumnType.DATE: + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + column.addLong(value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble((double) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatLong(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatLong(value)); + break; + default: + throw targetTypeConflict(name, "LONG", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding long256Column(CharSequence name, long l0, long l1, long l2, long l3) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "LONG256"); + int targetType = targetType(index, ColumnType.LONG256); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "LONG256", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.LONG256 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "LONG256", targetType, "conversion is not implemented"); + } + if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE + && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { + column.addNull(); + } else if (targetType == ColumnType.LONG256) { + column.addLong256(l0, l1, l2, l3); + } else { + column.addString(formatLong256(l0, l1, l2, l3)); + } + return this; + } + + public QwpSchemaBinding shortColumn(CharSequence name, short value) { + return integerColumn(name, value, "SHORT", ColumnType.SHORT, false); + } + + public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "STRING"); + int targetType = targetType(index, ColumnType.VARCHAR); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "STRING", index, targetType); + if (column == null) { + return this; + } + if (!ColumnType.isArray(targetType) && ColumnType.isGeoHash(targetType)) { + int precision = ColumnType.getGeoHashBits(targetType); + column.initGeoHashPrecision(precision); + if (value == null || value.length() == 0) { + column.addNull(); + } else { + appendStringGeoHash(column, name, value, precision, targetType); + } + return this; + } + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (value == null) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaStringDecimal(value, targetPrecision, targetScale)) { + throw targetValueConflict(name, "STRING", targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid decimal text"); + } + return this; + } + if (value == null) { + column.addNull(); + } else { + switch (targetType) { + case ColumnType.BYTE: + case ColumnType.SHORT: + case ColumnType.INT: + case ColumnType.LONG: + case ColumnType.FLOAT: + case ColumnType.DOUBLE: + appendStringNumeric(column, name, value, targetType); + break; + case ColumnType.BOOLEAN: + if (value.length() == 1 && value.charAt(0) == '1') { + column.addBoolean(true); + } else if (value.length() == 1 && value.charAt(0) == '0') { + column.addBoolean(false); + } else if (Chars.equalsLowerCaseAscii(value, "true")) { + column.addBoolean(true); + } else if (Chars.equalsLowerCaseAscii(value, "false")) { + column.addBoolean(false); + } else { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid BOOLEAN text"); + } + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value); + break; + case ColumnType.SYMBOL: + column.addSymbol(value); + break; + case ColumnType.BINARY: + column.addString(value); + break; + case ColumnType.CHAR: + column.addShort((short) stringToChar(value)); + break; + case ColumnType.DATE: + appendStringDate(column, name, value, targetType); + break; + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + appendStringTimestamp(column, name, value, targetType); + break; + case ColumnType.UUID: + long lo; + long hi; + try { + checkUuid(value); + lo = parseUuidLo(value); + hi = parseUuidHi(value); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid UUID text"); + } + column.addUuid(hi, lo); + break; + case ColumnType.LONG256: + appendStringLong256(column, name, value); + break; + default: + throw targetTypeConflict(name, "STRING", targetType, "conversion is not implemented"); + } + } + return this; + } + + public QwpSchemaBinding charColumn(CharSequence name, char value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "CHAR"); + int targetType = targetType(index, ColumnType.CHAR); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "CHAR", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.CHAR: + column.addShort((short) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + if (Character.isSurrogate(value)) { + throw error(INVALID_VALUE, name, "CHAR", targetType, "surrogate cannot be converted to text"); + } + StringSink sink = charTextSink; + if (sink == null) { + charTextSink = sink = new StringSink(1); + } else { + sink.clear(); + } + sink.put(value); + column.addString(sink); + break; + default: + throw targetTypeConflict(name, "CHAR", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding symbol(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "SYMBOL"); + int targetType = targetType(index, ColumnType.SYMBOL); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "SYMBOL", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value); + break; + case ColumnType.SYMBOL: + column.addSymbol(value); + break; + default: + throw targetTypeConflict(name, "SYMBOL", targetType, "conversion is not implemented"); + } + return this; + } + + /** + * Appends a non-designated timestamp supplied in an explicitly supported fixed-duration unit + * to an exact TIMESTAMP, TIMESTAMP_NS, STRING or VARCHAR schema target. Text targets normalize + * through checked microseconds and use the server's UTC millisecond presentation. Conversion + * fails on overflow; nanos converted to micros truncate toward zero. The caller retains the + * class-level row-cancellation and uncommitted-column rollback responsibility on failure. + */ + public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUnit unit) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "TIMESTAMP"); + int targetType = targetType(index, + unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "TIMESTAMP", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "TIMESTAMP", targetType, "conversion is not implemented"); + } + if (unit == null) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp unit is null"); + } + if (!isSupportedTimestampUnit(unit)) { + throw unsupported(name, "TIMESTAMP", targetType, "timestamp unit is not supported: " + unit); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_NANO + ? NanosTimestampDriver.INSTANCE.from(value, unit) + : MicrosTimestampDriver.INSTANCE.from(value, unit); + } catch (ArithmeticException e) { + throw targetValueConflict(name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + appendTimestampText(column, converted); + } else { + column.addLong(converted); + } + return this; + } + + /** + * Appends a non-designated Instant to an exact TIMESTAMP, TIMESTAMP_NS, STRING or VARCHAR + * schema target. Timestamp targets preserve their precision; text targets normalize through + * checked microseconds and use the server's UTC millisecond presentation. Final values outside + * the selected signed-long range are rejected. The caller retains the class-level rollback + * responsibility on failure. + */ + public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "TIMESTAMP"); + int targetType = targetType(index, ColumnType.TIMESTAMP_MICRO); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "TIMESTAMP", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "TIMESTAMP", targetType, "conversion is not implemented"); + } + if (value == null) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp value is null"); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_NANO + ? instantTimestamp(value, 1_000_000_000L, value.getNano()) + : instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L); + } catch (ArithmeticException e) { + throw targetValueConflict(name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + appendTimestampText(column, converted); + } else { + column.addLong(converted); + } + return this; + } + + public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "UUID"); + int targetType = targetType(index, ColumnType.UUID); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "UUID", index, targetType); + if (column == null) { + return this; + } + if (targetType == ColumnType.UUID) { + column.addUuid(hi, lo); + return this; + } + if (targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "UUID", targetType, "conversion is not implemented"); + } + if (lo == Long.MIN_VALUE && hi == Long.MIN_VALUE) { + column.addNull(); + } else { + column.addString(formatUuid(lo, hi)); + } + return this; + } + + /** Rejects an effective input family whose schema-mode conversion is not implemented. */ + public QwpSchemaBinding unsupportedColumn(CharSequence name, String inputType) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + QwpTableBuffer.ColumnBuffer inferred = index < 0 ? existing : null; + if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + return this; + } + if (index < 0) { + throw unsupported(name, inputType, -1, "input conversion is not implemented"); + } + int targetType = schema.getColumnType(index); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column != null) { + throw unsupported(name, inputType, targetType, "input conversion is not implemented"); + } + return this; + } + + private static void checkUuid(CharSequence value) throws NumericException { + if (value.length() != UUID_LENGTH + || value.charAt(FIRST_DASH) != '-' + || value.charAt(SECOND_DASH) != '-' + || value.charAt(THIRD_DASH) != '-' + || value.charAt(FOURTH_DASH) != '-') { + throw NumericException.instance(); + } + } + + private static long parseUuidHi(CharSequence value) throws NumericException { + return (Numbers.parseHexLong(value, 0, FIRST_DASH) << 32) + | (Numbers.parseHexLong(value, FIRST_DASH + 1, SECOND_DASH) << 16) + | Numbers.parseHexLong(value, SECOND_DASH + 1, THIRD_DASH); + } + + private static long parseUuidLo(CharSequence value) throws NumericException { + return (Numbers.parseHexLong(value, THIRD_DASH + 1, FOURTH_DASH) << 48) + | Numbers.parseHexLong(value, FOURTH_DASH + 1, UUID_LENGTH); + } + + public int getTableId() { + return schema.getTableId(); + } + + public long getMetadataVersion() { + return schema.getMetadataVersion(); + } + + /** + * Clears the rejection that {@link #hasRejectedKnownColumn()} reports, once + * the caller has acted on it. + */ + public void clearRejectedKnownColumn() { + hasRejectedKnownColumn = false; + } + + /** + * Clears the conflict that {@link #hasTargetTypeConflict()} reports, once the + * caller has acted on it. + */ + public void clearTargetTypeConflict() { + hasTargetTypeConflict = false; + } + + /** + * Reports whether the last rejected value targeted a column the snapshot + * knows, as an invalid value or one the snapshot type cannot take. A column + * type change after the snapshot can explain either. + */ + public boolean hasRejectedKnownColumn() { + return hasRejectedKnownColumn; + } + + /** + * Reports whether the last rejected value targeted a schema column whose + * snapshot type cannot take it. A column type change after the snapshot + * causes exactly this, so the caller evicts the snapshot from its cache. + * Input families that no column type can take never set it. + */ + public boolean hasTargetTypeConflict() { + return hasTargetTypeConflict; + } + + private LineSenderSchemaException error( + LineSenderSchemaException.Reason reason, + CharSequence column, + CharSequence inputType, + Integer targetType, + String detail + ) { + // An invalid value may suit the column's type after a change; an input that + // no column type takes, such as LONG_ARRAY, cannot. + if (reason == INVALID_VALUE && column != null && schema.getColumnIndex(column) >= 0) { + hasRejectedKnownColumn = true; + } + StringSink message = new StringSink(); + message.put("schema row error [reason=").put(reason.name()).put(", table=").putAsPrintable(tableName); + if (column != null) { + message.put(", column=").putAsPrintable(column); + } + if (inputType != null) { + message.put(", inputType=").put(inputType); + } + if (targetType != null) { + message.put(", targetType=").put(ColumnType.nameOf(targetType)).put('(').put(targetType).put(')'); + } + message.put(", detail=").put(detail).put(']'); + return new LineSenderSchemaException(reason, message); + } + + // Only a column the snapshot knows records the conflict; an inferred column's + // type comes from this batch, so a fresh snapshot could not change the outcome. + private void recordTargetTypeConflict(CharSequence name) { + if (name != null && schema.getColumnIndex(name) >= 0) { + hasTargetTypeConflict = true; + hasRejectedKnownColumn = true; + } + } + + private void requireUsableSchema() { + switch (schema.getResult()) { + case QwpSchemaProtocol.RESULT_KNOWN: + case QwpSchemaProtocol.RESULT_MISSING: + return; + case QwpSchemaProtocol.RESULT_DENIED: + throw error(ACCESS_DENIED, null, null, null, "schema access denied"); + case QwpSchemaProtocol.RESULT_UNAVAILABLE: + throw error(SCHEMA_UNAVAILABLE, null, null, null, "schema is temporarily unavailable"); + case QwpSchemaProtocol.RESULT_TOO_LARGE: + throw error(UNSUPPORTED_FEATURE, null, null, null, "schema could not be returned within the discovery response limit"); + default: + throw error(UNSUPPORTED_FEATURE, null, null, null, "unknown schema result"); + } + } + + /** Claims or creates the column for a name the schema lacks; call after {@link #isInferredWritable}. */ + private QwpTableBuffer.ColumnBuffer claimInferred( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + byte wireType + ) { + QwpTableBuffer.ColumnBuffer column = existing != null + ? buffer.claimColumn(existing) + : buffer.getOrCreateColumn(name, wireType, true); + column.setSchemaIndex(-1); + return column; + } + + /** + * Finds the buffer column for {@code name}, advancing the buffer's + * sequential cursor. An invalid name never matches, so the caller still + * rejects it through {@link #targetIndex}. + */ + private QwpTableBuffer.ColumnBuffer existingColumn(CharSequence name) { + return name == null || name.length() == 0 ? null : buffer.findColumn(name); + } + + /** + * Returns false for a duplicate write to a column the schema lacks, and + * rejects a write whose wire type conflicts with the inferred column. + */ + private boolean isInferredWritable( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + String inputType, + byte wireType + ) { + if (existing == null) { + return true; + } + if (existing.getSize() > buffer.getRowCount()) { + return false; + } + if (existing.getType() != wireType) { + throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" + + QwpConstants.getTypeName(existing.getType()) + ']'); + } + return true; + } + + private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String inputType, int expectedTargetType) { + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + int targetType = targetType(index, expectedTargetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column != null && targetType != expectedTargetType) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + return column; + } + + private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn(CharSequence name, int sourceDimensions) { + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DOUBLE_ARRAY"); + return targetDoubleArrayColumn(existing, name, index, sourceDimensions); + } + + private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + int index, + int sourceDimensions + ) { + int targetType = targetType(index, ColumnType.encodeArrayType(ColumnType.DOUBLE, sourceDimensions)); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "DOUBLE_ARRAY", index, targetType); + if (column == null) { + return null; + } + if (!ColumnType.isArray(targetType) + || ColumnType.decodeArrayElementType(targetType) != ColumnType.DOUBLE) { + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "conversion is not implemented"); + } + int targetDimensions = ColumnType.decodeWeakArrayDimensionality(targetType); + if (targetDimensions < 1 + || targetType != ColumnType.encodeArrayType(ColumnType.DOUBLE, targetDimensions)) { + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "invalid target array type"); + } + if (targetDimensions != sourceDimensions) { + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, + "array dimensionality mismatch [sourceDims=" + sourceDimensions + + ", targetDims=" + targetDimensions + ']'); + } + if (index < 0) { + int inferredDimensions = column.pinInferredArrayDimensionality(sourceDimensions); + if (inferredDimensions != sourceDimensions) { + throw unsupported(name, "DOUBLE_ARRAY", targetType, + "array dimensionality mismatch [sourceDims=" + sourceDimensions + + ", inferredDims=" + inferredDimensions + ']'); + } + } + return column; + } + + /** + * Returns the column {@code name} writes to for the in-progress row, or + * {@code null} for a duplicate write. {@code existing} is the buffer's + * column for {@code name}, found by {@link #existingColumn}. + */ + private QwpTableBuffer.ColumnBuffer targetColumn( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + String inputType, + int index, + int targetType + ) { + if (index < 0) { + final byte wireType = wireType(targetType); + if (!isInferredWritable(existing, name, inputType, wireType)) { + return null; + } + if (wireType == 0) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + return claimInferred(existing, name, wireType); + } + if (existing != null && existing.isSchemaResolved()) { + // Every check below depends only on the schema column, and passed + // when this binding created the column. + return buffer.claimColumn(existing); + } + if (index == schema.getDesignatedIndex()) { + throw unsupported(name, inputType, targetType, "designated timestamp writes are not implemented"); + } + final byte wireType = wireType(targetType); + if (wireType == 0) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + if (schema.hasColumnExtensionParameters(index)) { + throw unsupported(name, inputType, targetType, "parameterized target type"); + } + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn(name, wireType, true); + if (column != null) { + column.setSchemaIndex(index); + } + return column; + } + + private byte wireType(int targetType) { + if (ColumnType.isArray(targetType)) { + int dimensions = ColumnType.decodeWeakArrayDimensionality(targetType); + return dimensions >= 1 + && ColumnType.decodeArrayElementType(targetType) == ColumnType.DOUBLE + && targetType == ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions) + ? QwpConstants.TYPE_DOUBLE_ARRAY + : 0; + } + if (ColumnType.isGeoHash(targetType)) { + int bits = ColumnType.getGeoHashBits(targetType); + return bits >= 1 && bits <= 60 && targetType == ColumnType.getGeoHashTypeWithBits(bits) + ? QwpConstants.TYPE_GEOHASH + : 0; + } + if (ColumnType.isDecimal(targetType)) { + int precision = ColumnType.getDecimalPrecision(targetType); + int scale = ColumnType.getDecimalScale(targetType); + if (precision < 1 || precision > Decimals.MAX_PRECISION || scale > precision + || targetType != ColumnType.getDecimalType(precision, scale)) { + return 0; + } + switch (ColumnType.tagOf(targetType)) { + case ColumnType.DECIMAL8: + case ColumnType.DECIMAL16: + case ColumnType.DECIMAL32: + case ColumnType.DECIMAL64: + return QwpConstants.TYPE_DECIMAL64; + case ColumnType.DECIMAL128: + return QwpConstants.TYPE_DECIMAL128; + case ColumnType.DECIMAL256: + return QwpConstants.TYPE_DECIMAL256; + default: + return 0; + } + } + switch (targetType) { + case ColumnType.BOOLEAN: + return QwpConstants.TYPE_BOOLEAN; + case ColumnType.BYTE: + return QwpConstants.TYPE_BYTE; + case ColumnType.SHORT: + return QwpConstants.TYPE_SHORT; + case ColumnType.CHAR: + return QwpConstants.TYPE_CHAR; + case ColumnType.INT: + return QwpConstants.TYPE_INT; + case ColumnType.LONG: + return QwpConstants.TYPE_LONG; + case ColumnType.DATE: + return QwpConstants.TYPE_DATE; + case ColumnType.FLOAT: + return QwpConstants.TYPE_FLOAT; + case ColumnType.DOUBLE: + return QwpConstants.TYPE_DOUBLE; + case ColumnType.STRING: + case ColumnType.VARCHAR: + return QwpConstants.TYPE_VARCHAR; + case ColumnType.SYMBOL: + return QwpConstants.TYPE_SYMBOL; + case ColumnType.UUID: + return QwpConstants.TYPE_UUID; + case ColumnType.LONG256: + return QwpConstants.TYPE_LONG256; + case ColumnType.IPv4: + return QwpConstants.TYPE_IPv4; + case ColumnType.BINARY: + return QwpConstants.TYPE_BINARY; + case ColumnType.TIMESTAMP_MICRO: + return QwpConstants.TYPE_TIMESTAMP; + case ColumnType.TIMESTAMP_NANO: + return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: + return 0; + } + } + + private void appendBinary( + CharSequence name, + QwpTableBuffer.ColumnBuffer column, + long ptr, + long len + ) { + if (len < 0) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary length must be non-negative"); + } + if (len > Integer.MAX_VALUE) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary length exceeds the supported maximum"); + } + if (len > 0 && ptr == 0) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary pointer cannot be zero for a non-empty value"); + } + column.addBinary(ptr, len); + } + + private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String inputType, byte inferredWireType) { + buffer.requireSchemaBinding(this); + if (value == null || value.isNull()) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + if (index < 0) { + if (!isInferredWritable(existing, name, inputType, inferredWireType)) { + return this; + } + QwpTableBuffer.ColumnBuffer inferred = claimInferred(existing, name, inferredWireType); + if (value instanceof Decimal64) { + inferred.addDecimal64((Decimal64) value); + } else if (value instanceof Decimal128) { + inferred.addDecimal128((Decimal128) value); + } else { + inferred.addDecimal256((Decimal256) value); + } + return this; + } + int targetType = targetType(index, ColumnType.DECIMAL256); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column == null) { + return this; + } + return appendDecimal(column, name, value, inputType, targetType); + } + + private QwpSchemaBinding appendDecimal( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + Decimal value, + String inputType, + int targetType + ) { + if (!ColumnType.isDecimal(targetType)) { + return decimalNonDecimalColumn(column, name, value, inputType, targetType); + } + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + try { + if (!column.addSchemaDecimal(value, targetPrecision, targetScale)) { + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw targetValueConflict(name, inputType, targetType, "decimal value cannot be rescaled exactly"); + } + return this; + } + + private QwpSchemaBinding decimalNonDecimalColumn( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + Decimal value, + String inputType, + int targetType + ) { + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + Decimal256 decimal = decimalTextScratch; + if (decimal == null) { + decimalTextScratch = decimal = new Decimal256(); + } + value.toDecimal256(decimal); + decimal.setScale(decimal.getScale() & 0xff); + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(80); + } else { + sink.clear(); + } + decimal.toSink(sink); + column.addString(sink); + return this; + } + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + + private LineSenderSchemaException invalidRange(CharSequence name, String inputType, int targetType) { + return targetValueConflict(name, inputType, targetType, "value is outside target type range"); + } + + private void parseDecimalText(CharSequence name, CharSequence value, Decimal256 scratch, int targetType) { + try { + scratch.ofString(value); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "DECIMAL256", targetType, "invalid decimal text"); + } + } + + private QwpSchemaBinding integerColumn( + CharSequence name, + long value, + String inputType, + int inferredType, + boolean sourceNull + ) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + int targetType = targetType(index, inferredType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType) + && !isTextTarget(targetType) + && !ColumnType.isDecimal(targetType) + && !(inferredType == ColumnType.INT && targetType == ColumnType.IPv4) + && targetType != ColumnType.DATE + && targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + if (ColumnType.isDecimal(targetType)) { + appendIntegerDecimal(column, name, value, inputType, targetType, sourceNull); + return this; + } + if (sourceNull) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw invalidRange(name, inputType, targetType); + } + column.addByte((byte) value); + break; + case ColumnType.SHORT: + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw invalidRange(name, inputType, targetType); + } + column.addShort((short) value); + break; + case ColumnType.INT: + column.addInt((int) value); + break; + case ColumnType.IPv4: + if (value == Numbers.IPv4_NULL) { + column.addNull(); + } else { + column.addIPv4((int) value); + } + break; + case ColumnType.LONG: + case ColumnType.DATE: + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + column.addLong(value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble((double) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatLong(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatLong(value)); + break; + default: + throw new AssertionError("unsupported integer target"); + } + return this; + } + + private void appendIntegerDecimal( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + long value, + String inputType, + int targetType, + boolean sourceNull + ) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (sourceNull) { + column.addSchemaDecimalNull(targetScale); + return; + } + try { + if (!column.addSchemaLongDecimal(value, targetPrecision, targetScale)) { + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw targetValueConflict(name, inputType, targetType, "decimal value cannot be rescaled exactly"); + } + } + + private CharSequence formatLong(long value) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(20); + } else { + sink.clear(); + } + Numbers.append(sink, value, false); + return sink; + } + + private CharSequence formatLong256(long l0, long l1, long l2, long l3) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(66); + } else { + sink.clear(); + } + sink.putAscii("0x"); + if (l3 != 0) { + Numbers.appendHex(sink, l3, false); + Numbers.appendHex(sink, l2, true); + Numbers.appendHex(sink, l1, true); + Numbers.appendHex(sink, l0, true); + } else if (l2 != 0) { + Numbers.appendHex(sink, l2, false); + Numbers.appendHex(sink, l1, true); + Numbers.appendHex(sink, l0, true); + } else if (l1 != 0) { + Numbers.appendHex(sink, l1, false); + Numbers.appendHex(sink, l0, true); + } else { + Numbers.appendHex(sink, l0, false); + } + return sink; + } + + private CharSequence formatGeoHash(long value, int precisionBits) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(60); + } else { + sink.clear(); + } + for (int bit = precisionBits - 1; bit >= 0; bit--) { + sink.putAscii(((value >>> bit) & 1) == 0 ? '0' : '1'); + } + return sink; + } + + private CharSequence formatIPv4(int value) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(15); + } else { + sink.clear(); + } + Numbers.append(sink, (value >>> 24) & 0xff); + sink.put('.'); + Numbers.append(sink, (value >>> 16) & 0xff); + sink.put('.'); + Numbers.append(sink, (value >>> 8) & 0xff); + sink.put('.'); + Numbers.append(sink, value & 0xff); + return sink; + } + + private CharSequence formatFloating(double value) { + StringSink sink = floatingTextSink; + if (sink == null) { + floatingTextSink = sink = new StringSink(24); + } else { + sink.clear(); + } + Numbers.append(sink, value); + return sink; + } + + private CharSequence formatUuid(long lo, long hi) { + StringSink sink = uuidTextSink; + if (sink == null) { + uuidTextSink = sink = new StringSink(UUID_LENGTH); + } else { + sink.clear(); + } + appendUuidHex(sink, hi >>> 32, 8); + sink.put('-'); + appendUuidHex(sink, hi >>> 16, 4); + sink.put('-'); + appendUuidHex(sink, hi, 4); + sink.put('-'); + appendUuidHex(sink, lo >>> 48, 4); + sink.put('-'); + appendUuidHex(sink, lo, 12); + return sink; + } + + private static void appendUuidHex(StringSink sink, long value, int digits) { + for (int shift = (digits - 1) * 4; shift >= 0; shift -= 4) { + int nibble = (int) ((value >>> shift) & 0xf); + sink.put((char) (nibble < 10 ? '0' + nibble : 'a' + nibble - 10)); + } + } + + private LineSenderSchemaException invalidNumericText(CharSequence name, int targetType) { + return error(INVALID_VALUE, name, "STRING", targetType, "invalid numeric text"); + } + + private void appendStringNumeric( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + switch (targetType) { + case ColumnType.BYTE: + int parsed = Numbers.parseInt(value); + if (parsed < Byte.MIN_VALUE || parsed > Byte.MAX_VALUE) { + throw NumericException.instance(); + } + column.addByte((byte) parsed); + break; + case ColumnType.SHORT: + column.addShort(parseSchemaShort(value)); + break; + case ColumnType.INT: + column.addInt(Numbers.parseInt(value)); + break; + case ColumnType.LONG: + column.addLong(parseSchemaLong(value)); + break; + case ColumnType.FLOAT: + column.addFloat(parseSchemaFloat(value)); + break; + case ColumnType.DOUBLE: + column.addDouble(parseSchemaDouble(value)); + break; + default: + throw new AssertionError("unsupported numeric target"); + } + } catch (NumericException e) { + throw invalidNumericText(name, targetType); + } + } + + private void appendStringLong256( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value + ) { + int length = value.length(); + if (length < 4 || length > 66 || (length & 1) != 0 + || value.charAt(0) != '0' || value.charAt(1) != 'x') { + throw error(INVALID_VALUE, name, "STRING", (int) ColumnType.LONG256, "invalid LONG256 text"); + } + try { + int hi = length; + int lo = Math.max(2, hi - 16); + long l0 = Numbers.parseHexLong(value, lo, hi); + hi = lo; + lo = Math.max(2, hi - 16); + long l1 = hi > 2 ? Numbers.parseHexLong(value, lo, hi) : 0; + hi = lo; + lo = Math.max(2, hi - 16); + long l2 = hi > 2 ? Numbers.parseHexLong(value, lo, hi) : 0; + hi = lo; + long l3 = hi > 2 ? Numbers.parseHexLong(value, 2, hi) : 0; + if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE + && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { + throw NumericException.instance(); + } + column.addLong256(l0, l1, l2, l3); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", (int) ColumnType.LONG256, "invalid LONG256 text"); + } + } + + private void appendStringGeoHash( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int precision, + int targetType + ) { + int chars = Math.min(value.length(), 12); + if (chars * 5 < precision) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid GEOHASH text"); + } + try { + long hash = Numbers.parseGeoHashBase32(value, 0, chars); + column.addGeoHash(hash >>> (chars * 5 - precision), precision); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid GEOHASH text"); + } + } + + private void appendStringDate( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + long millis = QwpSchemaTimestampParser.parseDate(value); + if (millis == Long.MIN_VALUE) { + column.addNull(); + } else { + column.addLong(millis); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid DATE text"); + } + } + + private void appendStringTimestamp( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + long micros = QwpSchemaTimestampParser.parse(value); + if (targetType == ColumnType.TIMESTAMP_NANO) { + if (micros > Long.MAX_VALUE / 1000 || micros < Long.MIN_VALUE / 1000) { + throw NumericException.instance(); + } + column.addLong(micros * 1000); + } else { + column.addLong(micros); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid timestamp text"); + } + } + + private void appendTimestampText(QwpTableBuffer.ColumnBuffer column, long micros) { + Utf8StringSink sink = timestampTextSink; + if (sink == null) { + timestampTextSink = sink = new Utf8StringSink(32); + } else { + sink.clear(); + } + QwpSchemaTimestampFormatter.appendDateTime(sink, micros); + column.addString(sink.asAsciiCharSequence()); + } + + private static double parseSchemaDouble(CharSequence value) throws NumericException { + int length = oppositeFloatingSuffixLength(value, 'f'); + return length < 0 + ? Numbers.parseDouble(value) + : FastDoubleParser.parseDouble(value, 0, length, true); + } + + private static float parseSchemaFloat(CharSequence value) throws NumericException { + int length = oppositeFloatingSuffixLength(value, 'd'); + return length < 0 + ? FastFloatParser.parseFloat(value, true) + : FastFloatParser.parseFloat(value, 0, length, true); + } + + private static int oppositeFloatingSuffixLength(CharSequence value, char suffix) { + int index = value.length() - 1; + while (index >= 0 && value.charAt(index) <= ' ') { + index--; + } + if (index <= 0 || (value.charAt(index) | 32) != suffix) { + return -1; + } + char preceding = value.charAt(index - 1); + return (preceding >= '0' && preceding <= '9') || preceding == '.' ? index : -1; + } + + private static long parseSchemaLong(CharSequence value) throws NumericException { + int length = value.length(); + if (length == 0) { + throw NumericException.instance(); + } + boolean negative = value.charAt(0) == '-'; + int index = negative ? 1 : 0; + if (index == length) { + throw NumericException.instance(); + } + long limit = negative ? Long.MIN_VALUE : -Long.MAX_VALUE; + long multiplyLimit = limit / 10; + long result = 0; + int digitCount = 0; + for (; index < length; index++) { + char c = value.charAt(index); + if (c == '_' || c == 0x7f) { + if (digitCount == 0) { + throw NumericException.instance(); + } + digitCount = 0; + } else if (c == 'L' || c == 'l') { + if (digitCount == 0 || index + 1 != length) { + throw NumericException.instance(); + } + } else { + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + int digit = c - '0'; + if (result < multiplyLimit) { + throw NumericException.instance(); + } + result *= 10; + if (result < limit + digit) { + throw NumericException.instance(); + } + result -= digit; + digitCount++; + } + } + if (digitCount == 0) { + throw NumericException.instance(); + } + return negative ? result : -result; + } + + private static short parseSchemaShort(CharSequence value) throws NumericException { + int length = value.length(); + int index = length > 0 && value.charAt(0) == '-' ? 1 : 0; + if (index == length) { + throw NumericException.instance(); + } + for (; index < length; index++) { + char c = value.charAt(index); + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + } + int parsed = Numbers.parseInt(value); + if (parsed < Short.MIN_VALUE || parsed > Short.MAX_VALUE) { + throw NumericException.instance(); + } + return (short) parsed; + } + + private QwpSchemaBinding floatingNumericColumn(CharSequence name, double value, String inputType) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + int targetType = targetType(index, + "FLOAT".equals(inputType) ? ColumnType.FLOAT : ColumnType.DOUBLE); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType)) { + return floatingNonNumericColumn(column, name, value, inputType, targetType); + } + if (Double.isNaN(value)) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + requireWholeInRange(name, inputType, targetType, value, Byte.MIN_VALUE, Byte.MAX_VALUE); + column.addByte((byte) value); + break; + case ColumnType.SHORT: + requireWholeInRange(name, inputType, targetType, value, Short.MIN_VALUE, Short.MAX_VALUE); + column.addShort((short) value); + break; + case ColumnType.INT: + requireWholeInRange(name, inputType, targetType, value, Integer.MIN_VALUE, Integer.MAX_VALUE); + column.addInt((int) value); + break; + case ColumnType.LONG: + if (!Double.isFinite(value) || value != Math.rint(value) || value < -0x1.0p63 || value >= 0x1.0p63) { + throw error(INVALID_VALUE, name, inputType, targetType, "value cannot be represented exactly by target integer type"); + } + column.addLong((long) value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble(value); + break; + } + return this; + } + + private QwpSchemaBinding floatingNonNumericColumn( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + double value, + String inputType, + int targetType + ) { + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (!Double.isFinite(value)) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaStringDecimal(formatFloating(value), targetPrecision, targetScale)) { + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, inputType, targetType, + "floating value cannot be represented exactly by target decimal precision and scale"); + } + return this; + } + if (!isTextTarget(targetType)) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + if (Double.isNaN(value)) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatFloating(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatFloating(value)); + break; + default: + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + return this; + } + + private void requireWholeInRange( + CharSequence name, + String inputType, + int targetType, + double value, + double min, + double max + ) { + if (!Double.isFinite(value) || value != Math.rint(value) || value < min || value > max) { + throw error(INVALID_VALUE, name, inputType, targetType, "value cannot be represented exactly by target integer type"); + } + } + + private static boolean isNumericTarget(int targetType) { + return targetType == ColumnType.BYTE + || targetType == ColumnType.SHORT + || targetType == ColumnType.INT + || targetType == ColumnType.LONG + || targetType == ColumnType.FLOAT + || targetType == ColumnType.DOUBLE; + } + + private static boolean isTextTarget(int targetType) { + return targetType == ColumnType.STRING + || targetType == ColumnType.VARCHAR + || targetType == ColumnType.SYMBOL; + } + + private static boolean isSupportedTimestampUnit(ChronoUnit unit) { + return unit == ChronoUnit.NANOS + || unit == ChronoUnit.MICROS + || unit == ChronoUnit.MILLIS + || unit == ChronoUnit.SECONDS + || unit == ChronoUnit.MINUTES + || unit == ChronoUnit.HOURS + || unit == ChronoUnit.DAYS; + } + + // Matches client UTF-8 replacement followed by the server's first-BMP decoder: + // supplementary code points become NUL and malformed surrogates become '?'. + private static char stringToChar(CharSequence value) { + if (value.length() == 0) { + return 0; + } + char first = value.charAt(0); + if (Character.isHighSurrogate(first)) { + return value.length() > 1 && Character.isLowSurrogate(value.charAt(1)) ? 0 : '?'; + } + return Character.isLowSurrogate(first) ? '?' : first; + } + + private static long instantTimestamp(Instant instant, long unitsPerSecond, long fraction) { + long seconds = instant.getEpochSecond(); + if (seconds < 0) { + // Keep both terms non-positive so an intermediate underflow also means the final sum is out of range. + return Math.addExact(Math.multiplyExact(seconds + 1, unitsPerSecond), fraction - unitsPerSecond); + } + return Math.addExact(Math.multiplyExact(seconds, unitsPerSecond), fraction); + } + + private int requireDesignatedTimestamp(String inputType) { + int index = schema.getDesignatedIndex(); + if (index < 0) { + throw unsupported(null, inputType, -1, "target table has no designated timestamp"); + } + int targetType = schema.getColumnType(index); + timestampWireType(targetType); + if (schema.hasColumnExtensionParameters(index)) { + throw unsupported(null, inputType, targetType, "parameterized target type"); + } + return index; + } + + private byte timestampWireType(int targetType) { + if (targetType == ColumnType.TIMESTAMP_MICRO) { + return QwpConstants.TYPE_TIMESTAMP; + } + if (targetType == ColumnType.TIMESTAMP_NANO) { + return QwpConstants.TYPE_TIMESTAMP_NANOS; + } + throw unsupported(null, "TIMESTAMP", targetType, "designated timestamp conversion is not implemented"); + } + + /** + * Returns the schema index of {@code name}, or -1 when the schema lacks it. + * A column this binding already resolved answers without re-validating or + * re-hashing the name: it can only match a name equal to its own valid one. + */ + private int targetIndex(QwpTableBuffer.ColumnBuffer existing, CharSequence name, String inputType) { + if (existing != null && existing.isSchemaResolved()) { + return existing.getSchemaIndex(); + } + if (name == null || !TableUtils.isValidColumnName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { + throw error(INVALID_VALUE, name, inputType, null, "invalid column name"); + } + return schema.getColumnIndex(name); + } + + private int targetType(int index, int inferredType) { + return index >= 0 ? schema.getColumnType(index) : inferredType; + } + + // Rejects an input that the snapshot type of the named column cannot take. + private LineSenderSchemaException targetTypeConflict( + CharSequence name, + String inputType, + int targetType, + String detail + ) { + recordTargetTypeConflict(name); + return unsupported(name, inputType, targetType, detail); + } + + // Rejects a value that the snapshot type of the named column cannot hold, such + // as an INT overflow after the server widened the column to LONG. + private LineSenderSchemaException targetValueConflict( + CharSequence name, + String inputType, + int targetType, + String detail + ) { + recordTargetTypeConflict(name); + return error(INVALID_VALUE, name, inputType, targetType, detail); + } + + private LineSenderSchemaException unsupported(CharSequence name, String inputType, int targetType, String detail) { + return error(UNSUPPORTED_FEATURE, name, inputType, targetType, detail); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java new file mode 100644 index 000000000..a9e35dd57 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java @@ -0,0 +1,276 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; +import io.questdb.client.std.Unsafe; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Codec and wire constants for negotiated QWP schema control messages. */ +public final class QwpSchemaProtocol { + public static final byte FLAG_CONTROL = 0x20; + public static final byte KIND_DESCRIBE = 1; + public static final byte KIND_SCHEMA = 2; + public static final int MAX_COLUMN_COUNT = 2048; + public static final int MAX_FRAME_SIZE = 1024 * 1024; + public static final int MAX_NAME_UTF16_LENGTH = 127; + public static final int MAX_NAME_UTF8_LENGTH = 381; + public static final int MAX_PARAMS_LENGTH = 1024; + public static final int RESULT_DENIED = 2; + public static final int RESULT_KNOWN = 0; + public static final int RESULT_MISSING = 1; + public static final int RESULT_TOO_LARGE = 4; + public static final int RESULT_UNAVAILABLE = 3; + + private static final int KNOWN_FIXED_PAYLOAD_SIZE = 1 + 8 + 1 + 4 + 8 + 2 + 2; + private static final int MIN_PAYLOAD_SIZE = 1 + 8 + 1; + + private QwpSchemaProtocol() { + } + + public static QwpSchemaResponse decodeResponse(long address, int length) { + int payloadLength = validateResponseEnvelope(address, length, MIN_PAYLOAD_SIZE); + return decodePayload(address + QwpConstants.HEADER_SIZE, payloadLength, false); + } + + /** Validates a complete schema-response envelope and returns its positive request ID. */ + public static long peekResponseRequestId(long address, int length) { + validateResponseEnvelope(address, length, 1 + Long.BYTES); + long payloadAddress = address + QwpConstants.HEADER_SIZE; + if (getUnsignedByte(payloadAddress) != KIND_SCHEMA) { + throw malformed("unexpected control response kind"); + } + long requestId = getLong(payloadAddress + 1); + if (requestId <= 0) { + throw malformed("request id must be positive"); + } + return requestId; + } + + private static int validateResponseEnvelope(long address, int length, int minimumPayloadSize) { + if (address == 0) { + throw malformed("null frame address"); + } + if (length < QwpConstants.HEADER_SIZE + minimumPayloadSize || length > MAX_FRAME_SIZE) { + throw malformed("invalid frame length: " + length); + } + if (getInt(address) != QwpConstants.MAGIC_MESSAGE) { + throw malformed("invalid QWP magic"); + } + if (getUnsignedByte(address + 4) != 1) { + throw malformed("unsupported QWP version"); + } + if (getUnsignedByte(address + 5) != (FLAG_CONTROL & 0xff)) { + throw malformed("invalid control flags"); + } + if (getUnsignedShort(address + 6) != 0) { + throw malformed("control frame must have zero tables"); + } + int payloadLength = getInt(address + 8); + if (payloadLength != length - QwpConstants.HEADER_SIZE) { + throw malformed("payload length mismatch"); + } + return payloadLength; + } + + /** Decodes a length-delimited schema payload carried by response feedback. */ + public static QwpSchemaResponse decodeFeedbackPayload(long address, int length) { + if (address == 0) { + throw malformed("null payload address"); + } + if (length < MIN_PAYLOAD_SIZE || length > MAX_FRAME_SIZE) { + throw malformed("invalid payload length: " + length); + } + return decodePayload(address, length, true); + } + + private static QwpSchemaResponse decodePayload(long address, int length, boolean feedback) { + long p = address; + long limit = address + length; + if (getUnsignedByte(p++) != KIND_SCHEMA) { + throw malformed("unexpected control response kind"); + } + long requestId = getLong(p); + p += 8; + if (feedback ? requestId != 0 : requestId <= 0) { + throw malformed(feedback ? "feedback request id must be zero" : "request id must be positive"); + } + int result = getUnsignedByte(p++); + if (result < RESULT_KNOWN || result > RESULT_TOO_LARGE) { + throw malformed("unknown schema result: " + result); + } + if (result != RESULT_KNOWN) { + if (p != limit) { + throw malformed("trailing bytes after schema result"); + } + return new QwpSchemaResponse(requestId, result, -1, -1, -1, new QwpSchemaResponse.Column[0], + new LowerCaseCharSequenceIntHashMap()); + } + if (length < KNOWN_FIXED_PAYLOAD_SIZE) { + throw malformed("truncated known schema"); + } + require(p, 16, limit, "schema identity"); + int tableId = getInt(p); + p += 4; + long metadataVersion = getLong(p); + p += 8; + int designatedIndex = getShort(p); + p += 2; + int columnCount = getUnsignedShort(p); + p += 2; + if (tableId < 0 || metadataVersion < 0) { + throw malformed("negative schema identity"); + } + if (columnCount > MAX_COLUMN_COUNT) { + throw malformed("too many columns: " + columnCount); + } + if (designatedIndex < -1 || designatedIndex >= columnCount) { + throw malformed("invalid designated index: " + designatedIndex); + } + + QwpSchemaResponse.Column[] columns = new QwpSchemaResponse.Column[columnCount]; + LowerCaseCharSequenceIntHashMap columnNames = new LowerCaseCharSequenceIntHashMap(columnCount); + for (int i = 0; i < columnCount; i++) { + require(p, 2, limit, "column name length"); + int nameLength = getUnsignedShort(p); + p += 2; + if (nameLength == 0 || nameLength > MAX_NAME_UTF8_LENGTH) { + throw malformed("invalid column name length: " + nameLength); + } + require(p, nameLength, limit, "column name"); + String name = decodeUtf8(p, nameLength, "column name"); + if (!TableUtils.isValidColumnName(name, MAX_NAME_UTF16_LENGTH)) { + throw malformed("invalid column name"); + } + if (!columnNames.put(name, i)) { + throw malformed("duplicate case-insensitive column name"); + } + p += nameLength; + require(p, 6, limit, "column type and parameter length"); + int type = getInt(p); + p += 4; + int paramsLength = getUnsignedShort(p); + p += 2; + if (paramsLength > MAX_PARAMS_LENGTH) { + throw malformed("column parameters too large: " + paramsLength); + } + require(p, paramsLength, limit, "column parameters"); + p += paramsLength; + columns[i] = new QwpSchemaResponse.Column(name, type, paramsLength != 0); + } + if (p != limit) { + throw malformed("trailing bytes after schema"); + } + return new QwpSchemaResponse(requestId, result, tableId, metadataVersion, designatedIndex, columns, columnNames); + } + + public static byte[] encodeDescribe(long requestId, CharSequence tableName) { + if (requestId <= 0) { + throw new IllegalArgumentException("request id must be positive"); + } + if (tableName == null || !TableUtils.isValidTableName(tableName, MAX_NAME_UTF16_LENGTH)) { + throw new IllegalArgumentException("invalid table name"); + } + byte[] name = encodeName(tableName, "table name"); + int payloadLength = 1 + 8 + 2 + name.length; + ByteBuffer buffer = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(QwpConstants.MAGIC_MESSAGE); + buffer.put((byte) 1); + buffer.put(FLAG_CONTROL); + buffer.putShort((short) 0); + buffer.putInt(payloadLength); + buffer.put(KIND_DESCRIBE); + buffer.putLong(requestId); + buffer.putShort((short) name.length); + buffer.put(name); + return buffer.array(); + } + + private static byte[] copyBytes(long address, int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + return bytes; + } + + private static String decodeUtf8(long address, int length, String what) { + byte[] bytes = copyBytes(address, length); + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw malformed("invalid UTF-8 " + what); + } + } + + private static byte[] encodeName(CharSequence value, String what) { + if (value == null || value.length() == 0 || value.length() > MAX_NAME_UTF16_LENGTH) { + throw new IllegalArgumentException(what + " must contain 1 to " + MAX_NAME_UTF16_LENGTH + " UTF-16 units"); + } + byte[] bytes = value.toString().getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_NAME_UTF8_LENGTH) { + throw new IllegalArgumentException(what + " exceeds " + MAX_NAME_UTF8_LENGTH + " UTF-8 bytes"); + } + return bytes; + } + + private static int getInt(long address) { + return Unsafe.getUnsafe().getInt(address); + } + + private static long getLong(long address) { + return Unsafe.getUnsafe().getLong(address); + } + + private static short getShort(long address) { + return Unsafe.getUnsafe().getShort(address); + } + + private static int getUnsignedByte(long address) { + return Unsafe.getUnsafe().getByte(address) & 0xff; + } + + private static int getUnsignedShort(long address) { + return getShort(address) & 0xffff; + } + + private static IllegalArgumentException malformed(String message) { + return new IllegalArgumentException("Malformed QWP schema response: " + message); + } + + private static void require(long position, int needed, long limit, String what) { + if (needed < 0 || position > limit || needed > limit - position) { + throw malformed("truncated " + what); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java new file mode 100644 index 000000000..49fa5883d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java @@ -0,0 +1,122 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; + +/** + * Immutable response to a QWP schema DESCRIBE control request. The snapshot + * owns the case-insensitive column-name index that the decoder builds while + * it validates name uniqueness, so consumers never rebuild it. + */ +public final class QwpSchemaResponse { + private final LowerCaseCharSequenceIntHashMap columnIndexes; + private final Column[] columns; + private final int designatedIndex; + private final int result; + private final int tableId; + private final long metadataVersion; + private final long requestId; + + QwpSchemaResponse( + long requestId, + int result, + int tableId, + long metadataVersion, + int designatedIndex, + Column[] columns, + LowerCaseCharSequenceIntHashMap columnIndexes + ) { + this.requestId = requestId; + this.result = result; + this.tableId = tableId; + this.metadataVersion = metadataVersion; + this.designatedIndex = designatedIndex; + this.columns = columns; + this.columnIndexes = columnIndexes; + } + + public int getColumnCount() { + return columns.length; + } + + /** Returns the index of the case-insensitively named column, or -1. */ + public int getColumnIndex(CharSequence name) { + return columnIndexes.get(name); + } + + public String getColumnName(int index) { + return columns[index].name; + } + + public int getColumnType(int index) { + return columns[index].type; + } + + /** + * Returns whether the server supplied an extension parameter block for + * this column's type. The current client safely skips those bytes but must + * not interpret a parameterized type as an unparameterized one. + */ + public boolean hasColumnExtensionParameters(int index) { + return columns[index].hasTypeParameters; + } + + public int getDesignatedIndex() { + return designatedIndex; + } + + public long getMetadataVersion() { + return metadataVersion; + } + + /** Returns the positive DESCRIBE id, or zero for named write feedback. */ + public long getRequestId() { + return requestId; + } + + public int getResult() { + return result; + } + + public int getTableId() { + return tableId; + } + + public boolean hasSchema() { + return result == QwpSchemaProtocol.RESULT_KNOWN; + } + + static final class Column { + private final boolean hasTypeParameters; + private final String name; + private final int type; + + Column(String name, int type, boolean hasTypeParameters) { + this.name = name; + this.type = type; + this.hasTypeParameters = hasTypeParameters; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java new file mode 100644 index 000000000..05ff6c7ae --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java @@ -0,0 +1,187 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.Numbers; +import io.questdb.client.std.datetime.CommonUtils; +import io.questdb.client.std.str.Utf8StringSink; + +/** Schema-only UTC millisecond formatter matching the server timestamp-to-text conversion. */ +final class QwpSchemaTimestampFormatter { + private static final long DAY_MICROS = 86_400_000_000L; + private static final long AVG_YEAR_MICROS = (long) (365.2425 * DAY_MICROS); + private static final long HOUR_MICROS = 3_600_000_000L; + private static final long MILLI_MICROS = 1_000L; + private static final long MINUTE_MICROS = 60_000_000L; + private static final long SECOND_MICROS = 1_000_000L; + private static final long YEAR_MICROS_LEAP = 366 * DAY_MICROS; + private static final long YEAR_MICROS_NONLEAP = 365 * DAY_MICROS; + private static final long[] LEAP_MONTH_MICROS = new long[12]; + private static final long[] MONTH_MICROS = new long[12]; + + static { + long normal = 0; + long leap = 0; + for (int month = 0; month < 11; month++) { + normal += CommonUtils.DAYS_PER_MONTH[month] * DAY_MICROS; + leap += CommonUtils.getDaysPerMonth(month + 1, true) * DAY_MICROS; + MONTH_MICROS[month + 1] = normal; + LEAP_MONTH_MICROS[month + 1] = leap; + } + } + + private QwpSchemaTimestampFormatter() { + } + + static void appendDateTime(Utf8StringSink sink, long micros) { + if (micros == Long.MIN_VALUE) { + return; + } + int year = getYear(micros); + boolean leap = CommonUtils.isLeapYear(year); + int month = getMonthOfYear(micros, year, leap); + appendYear000(sink, year); + sink.putAscii('-'); + append0(sink, month); + sink.putAscii('-'); + append0(sink, getDayOfMonth(micros, year, month, leap)); + sink.putAscii('T'); + append0(sink, getHourOfDay(micros)); + sink.putAscii(':'); + append0(sink, getMinuteOfHour(micros)); + sink.putAscii(':'); + append0(sink, getSecondOfMinute(micros)); + sink.putAscii('.'); + append00(sink, getMillisOfSecond(micros)); + sink.putAscii('Z'); + } + + private static void append0(Utf8StringSink sink, int value) { + if (Math.abs(value) < 10) { + sink.putAscii('0'); + } + Numbers.append(sink, value); + } + + private static void append00(Utf8StringSink sink, int value) { + int absolute = Math.abs(value); + if (absolute < 10) { + sink.putAscii('0').putAscii('0'); + } else if (absolute < 100) { + sink.putAscii('0'); + } + Numbers.append(sink, value); + } + + private static void appendYear000(Utf8StringSink sink, int value) { + int absolute = Math.abs(value); + if (absolute < 10) { + sink.putAscii('0').putAscii('0').putAscii('0'); + } else if (absolute < 100) { + sink.putAscii('0').putAscii('0'); + } else if (absolute < 1000) { + sink.putAscii('0'); + } + Numbers.append(sink, value != 0 ? value : 1); + } + + private static int getDayOfMonth(long micros, int year, int month, boolean leap) { + long start = yearMicros(year, leap) + monthOfYearMicros(month, leap); + return (int) ((micros - start) / DAY_MICROS) + 1; + } + + private static int getHourOfDay(long micros) { + if (micros > -1) { + return (int) ((micros / HOUR_MICROS) % 24); + } + return 23 + (int) (((micros + 1) / HOUR_MICROS) % 24); + } + + private static int getMillisOfSecond(long micros) { + if (micros > -1) { + return (int) ((micros / MILLI_MICROS) % 1000); + } + return 999 + (int) (((micros + 1) / MILLI_MICROS) % 1000); + } + + private static int getMinuteOfHour(long micros) { + if (micros > -1) { + return (int) ((micros / MINUTE_MICROS) % 60); + } + return 59 + (int) (((micros + 1) / MINUTE_MICROS) % 60); + } + + // Keep the server's threshold-based month lookup: it is stable at the wrapped long extrema. + private static int getMonthOfYear(long micros, int year, boolean leap) { + int i = (int) (((micros - yearMicros(year, leap)) / 1000) >> 10); + if (leap) { + if (i < 182 * 84375) { + if (i < 91 * 84375) { + return i < 31 * 84375 ? 1 : i < 60 * 84375 ? 2 : 3; + } + return i < 121 * 84375 ? 4 : i < 152 * 84375 ? 5 : 6; + } + if (i < 274 * 84375) { + return i < 213 * 84375 ? 7 : i < 244 * 84375 ? 8 : 9; + } + return i < 305 * 84375 ? 10 : i < 335 * 84375 ? 11 : 12; + } + if (i < 181 * 84375) { + if (i < 90 * 84375) { + return i < 31 * 84375 ? 1 : i < 59 * 84375 ? 2 : 3; + } + return i < 120 * 84375 ? 4 : i < 151 * 84375 ? 5 : 6; + } + if (i < 273 * 84375) { + return i < 212 * 84375 ? 7 : i < 243 * 84375 ? 8 : 9; + } + return i < 304 * 84375 ? 10 : i < 334 * 84375 ? 11 : 12; + } + + private static int getSecondOfMinute(long micros) { + if (micros > -1) { + return (int) ((micros / SECOND_MICROS) % 60); + } + return 59 + (int) (((micros + 1) / SECOND_MICROS) % 60); + } + + // Source-compatible estimate and adjustment from the server Micros implementation. + private static int getYear(long micros) { + int yearsSinceEpoch = (int) (micros / AVG_YEAR_MICROS); + int estimate = 1970 + yearsSinceEpoch; + if (micros < 0 && estimate >= 1970) { + estimate = 1969; + } + boolean leap = CommonUtils.isLeapYear(estimate); + long start = yearMicros(estimate, leap); + long difference = micros - start; + if (difference < 0) { + estimate--; + } else if (difference >= (leap ? YEAR_MICROS_LEAP : YEAR_MICROS_NONLEAP)) { + estimate++; + } + return estimate; + } + + private static long monthOfYearMicros(int month, boolean leap) { + return (leap ? LEAP_MONTH_MICROS : MONTH_MICROS)[month - 1]; + } + + private static long yearMicros(int year, boolean leap) { + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + long days = year * 365L + (leapYears - 719527L); + long micros = days * DAY_MICROS; + return days < 0 && micros > 0 ? Long.MIN_VALUE : micros; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java new file mode 100644 index 000000000..ed1a57b3b --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java @@ -0,0 +1,752 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.NumericException; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.datetime.CommonUtils; + +import java.text.DateFormatSymbols; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneOffsetTransitionRule; +import java.time.zone.ZoneRules; +import java.time.zone.ZoneRulesProvider; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Source-faithful parser for the fixed formats accepted by QWP STRING-to-timestamp + * and STRING-to-DATE conversion. The grammar and computation mirror the server's + * {@code MicrosFormatUtils}, {@code DateFormatUtils} and generic format parsers; + * this is deliberately not a general date-format compiler. + */ +final class QwpSchemaTimestampParser { + private static final int DAYS_0000_TO_1970 = 719527; + private static final long GREEDY_MILLIS_FAILED = -1L; + private static final int[] MONTH_START_DAYS_COMMON = monthStartDays(false); + private static final int[] MONTH_START_DAYS_LEAP = monthStartDays(true); + private static final int REFERENCE_CENTURY = referenceCentury(); + // The server parses TIMESTAMP text with MicrosFormatUtils over UTF-8 bytes, + // and DATE text with DateFormatUtils over UTF-16. Its two parsers also differ + // in year overflow and in zone rules for negative years. + private static final Target DATE = new Target(1L, false, false, true); + private static final Target TIMESTAMP = new Target(1_000L, true, true, false); + + private QwpSchemaTimestampParser() { + } + + /** + * Parses TIMESTAMP text to epoch micros. The server parses the UTF-8 bytes, + * where no non-ASCII byte matches anything, so any non-ASCII character fails. + */ + static long parse(CharSequence value) throws NumericException { + // Only the PG layout has a space after the day, and a space there fails + // every ISO layout, so that one character selects the parser. + int n = value.length(); + int dash = firstDash(value, n); + if (dash >= 1 && dash + 6 < n && value.charAt(dash + 6) == ' ') { + return parsePg(value, dash); + } + return parseIso(value); + } + + /** Parses DATE text to epoch millis; the server parses DATE text as UTF-16. */ + static long parseDate(CharSequence value) throws NumericException { + int n = value.length(); + int dash = firstDash(value, n); + if (dash < 1) { + // Both date layouts need an inner dash and a raw number cannot hold one. + return Numbers.parseLong(value); + } + // PG ends at the day or continues with a space; only UTC has a 'T' there. + int separator = dash + 6; + if (separator >= n || value.charAt(separator) == ' ') { + return parsePgDate(value, dash); + } + if (value.charAt(separator) == 'T') { + return parseUtcDate(value); + } + throw NumericException.instance(); + } + + private static int firstDash(CharSequence value, int n) { + return indexOf(value, '-', n > 0 && value.charAt(0) == '-' ? 1 : 0, n); + } + + private static long parsePgDate(CharSequence value, int dash) throws NumericException { + int n = value.length(); + int year = greedyInt(value, 0, dash); + if (dash == 2) { + year += REFERENCE_CENTURY; + } + int pos = dash + 1; + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(DATE, year, month, day, 0, 0, 0, 0, 0); + } + require(value, pos++, n, ' '); + int tail = pos; + try { + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + int millis = 0; + if (pos < n && value.charAt(pos) == '.') { + long parsed = greedyMillis(value, pos + 1, n); + if (parsed == GREEDY_MILLIS_FAILED) { + throw NumericException.instance(); + } + millis = (int) parsed; + pos += 1 + (int) (parsed >>> 32); + } + return zoned(DATE, localTime(DATE, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + } catch (NumericException ignored) { + return zoned(DATE, localTime(DATE, year, month, day, 0, 0, 0, 0, 0), year, value, tail, n); + } + } + + private static long parseUtcDate(CharSequence value) throws NumericException { + int n = value.length(); + int pos = 0; + int year = fixedInt(value, pos, pos += 4, n); + require(value, pos++, n, '-'); + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, 'T'); + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '.'); + int millis = fixedInt(value, pos, pos += 3, n); + return zoned(DATE, localTime(DATE, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + } + + private static long parseIso(CharSequence value) throws NumericException { + int n = value.length(); + int pos = n > 0 && value.charAt(0) == '-' ? 5 : 4; + int year = pos == 5 ? -fixedInt(value, 1, pos, n) : fixedInt(value, 0, pos, n); + if (pos == n) { + return localTime(TIMESTAMP, year, 1, 1, 0, 0, 0, 0, 0); + } + require(value, pos++, n, '-'); + if (pos + 3 <= n && value.charAt(pos) == 'W') { + int week = Numbers.parseInt(value, pos + 1, pos + 3); + if (pos + 3 != n) { + throw NumericException.instance(); + } + return weekMicros(year, week); + } + int month = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(TIMESTAMP, year, month, 1, 0, 0, 0, 0, 0); + } + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(TIMESTAMP, year, month, day, 0, 0, 0, 0, 0); + } + require(value, pos++, n, 'T'); + int hour = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(TIMESTAMP, year, month, day, hour, 0, 0, 0, 0); + } + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + if (pos < n && value.charAt(pos) == '.') { + return parseIsoFraction(value, pos + 1, n, year, month, day, hour, minute, second); + } + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, 0, 0), year, value, pos, n); + } + + /** + * Tries the server's fraction layouts in its order: greedy S to SSS, then + * SSSUUU, then S followed by greedy S to SSS, then fixed SSS. A layout whose + * fraction, fields or zone fails hands over to the next one. + */ + private static long parseIsoFraction( + CharSequence value, + int pos, + int n, + int year, + int month, + int day, + int hour, + int minute, + int second + ) throws NumericException { + long parsed = greedyMillis(value, pos, n); + if (parsed != GREEDY_MILLIS_FAILED) { + try { + long local = localTime(TIMESTAMP, year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(TIMESTAMP, local, year, value, pos + (int) (parsed >>> 32), n); + } catch (NumericException ignored) { + } + } + if (pos + 6 <= n) { + try { + int millis = fixedInt(value, pos, pos + 3, n); + int micros = fixedInt(value, pos + 3, pos + 6, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, micros), year, value, pos + 6, n); + } catch (NumericException ignored) { + } + } + if (pos + 2 <= n) { + try { + fixedInt(value, pos, pos + 1, n); + parsed = greedyMillis(value, pos + 1, n); + if (parsed != GREEDY_MILLIS_FAILED) { + long local = localTime(TIMESTAMP, year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(TIMESTAMP, local, year, value, pos + 1 + (int) (parsed >>> 32), n); + } + } catch (NumericException ignored) { + } + } + // Final UTC_PATTERN fallback is fixed SSS. It intentionally follows + // the greedy S and SS forms, and therefore accepts parseInt forms + // such as +12 and 1_2 as well as a numeric-zone suffix after 3 chars. + if (pos + 3 <= n) { + try { + int millis = fixedInt(value, pos, pos + 3, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, 0), year, value, pos + 3, n); + } catch (NumericException ignored) { + } + } + throw NumericException.instance(); + } + + private static long parsePg(CharSequence value, int dash) throws NumericException { + int n = value.length(); + int year = greedyInt(value, 0, dash); + if (dash == 2) { + year += REFERENCE_CENTURY; + } + int pos = dash + 1; + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ' '); + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '.'); + int millis = fixedInt(value, pos, pos += 3, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + } + + /** + * Returns {@code (width << 32) | millis} for a greedy 1-3 digit fraction with + * an optional minus sign, or {@link #GREEDY_MILLIS_FAILED}. + */ + private static long greedyMillis(CharSequence value, int lo, int hi) { + int p = lo; + boolean negative = p < hi && value.charAt(p) == '-'; + if (negative) { + p++; + } + int digitStart = p; + int parsed = 0; + while (p < hi && value.charAt(p) >= '0' && value.charAt(p) <= '9') { + parsed = parsed * 10 + value.charAt(p++) - '0'; + } + int width = p - lo; + if (p == digitStart || width > 3) { + return GREEDY_MILLIS_FAILED; + } + while (width < 3) { + parsed *= 10; + width++; + } + return ((long) (p - lo) << 32) | ((negative ? -parsed : parsed) & 0xffffffffL); + } + + /** Local time in {@code t}'s unit, from validated fields. */ + private static long localTime( + Target t, + int year, + int month, + int day, + int hour, + int minute, + int second, + int millis, + int micros + ) throws NumericException { + boolean leap = CommonUtils.isLeapYear(year); + if (month < 1 || month > 12 || day < 1 || day > CommonUtils.getDaysPerMonth(month, leap) + || hour < 0 || hour > 24 || minute < 0 || minute > 59 + || second < 0 || second > 59) { + throw NumericException.instance(); + } + return yearStart(t, year, leap) + monthStart(t, month, leap) + (day - 1L) * t.day + + (hour % 24L) * t.hour + minute * t.minute + second * t.second + + millis * t.milli + micros; + } + + private static long weekMicros(int year, int week) throws NumericException { + if (week == -1) { + // The server's "no week" sentinel is -1, so an explicit W-1 parses + // as the bare year. + return localTime(TIMESTAMP, year, 1, 1, 0, 0, 0, 0, 0); + } + if (week < 1 || week > weeks(year)) { + throw NumericException.instance(); + } + boolean leap = CommonUtils.isLeapYear(year); + long first = yearStart(TIMESTAMP, year, leap) + (week - 1L) * 7 * TIMESTAMP.day + + isoYearDayOffset(year) * TIMESTAMP.day; + int actualYear = year(first); + int month = monthOfYear(first, actualYear, CommonUtils.isLeapYear(actualYear)); + int weekYear = year + (week == 1 && isoYearDayOffset(year) < 0 ? -1 : 0); + int day = dayOfMonth(first, weekYear, month, CommonUtils.isLeapYear(weekYear)); + // GenericMicrosFormat retains the originally parsed year's leap flag + // after an ISO week crosses a calendar-year boundary. + return yearStart(TIMESTAMP, weekYear, leap) + monthStart(TIMESTAMP, month, leap) + (day - 1L) * TIMESTAMP.day; + } + + /** Converts {@code local}, in {@code t}'s unit, to UTC using the zone text in {@code [lo, hi)}. */ + private static long zoned(Target t, long local, int year, CharSequence value, int lo, int hi) throws NumericException { + return local - ZoneMatcher.offset(t, value, lo, hi, local, year); + } + + private static int fixedInt(CharSequence s, int lo, int hi, int limit) throws NumericException { + if (hi > limit) { + throw NumericException.instance(); + } + return Numbers.parseInt(s, lo, hi); + } + + private static int greedyInt(CharSequence s, int lo, int hi) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); + } + int sign = 1; + if (s.charAt(lo) == '-') { + sign = -1; + lo++; + } + if (lo >= hi) { + throw NumericException.instance(); + } + long value = 0; + for (int i = lo; i < hi; i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + value = value * 10 + c - '0'; + if (value > (sign < 0 ? 2147483648L : Integer.MAX_VALUE)) { + throw NumericException.instance(); + } + } + return (int) (sign * value); + } + + private static int indexOf(CharSequence s, char c, int lo, int hi) { + for (int i = lo; i < hi; i++) { + if (s.charAt(i) == c) { + return i; + } + } + return -1; + } + + private static void require(CharSequence s, int pos, int hi, char c) throws NumericException { + if (pos >= hi || s.charAt(pos) != c) { + throw NumericException.instance(); + } + } + + private static int referenceCentury() { + int year = java.time.ZonedDateTime.now(ZoneOffset.UTC).getYear(); + int offset = year % 100; + return offset + 20 > 100 ? year - offset + 100 : year - offset; + } + + private static int[] monthStartDays(boolean leap) { + int[] result = new int[12]; + for (int i = 1; i < 12; i++) { + result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap); + } + return result; + } + private static long monthStart(Target t, int month, boolean leap) { + return (leap ? MONTH_START_DAYS_LEAP : MONTH_START_DAYS_COMMON)[month - 1] * t.day; + } + private static long yearStart(Target t, int year, boolean leap) { + // Preserve the server's unchecked multiplication. Only its micros + // variant saturates on negative overflow, and asymmetrically. + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + long days = year * 365L + leapYears - DAYS_0000_TO_1970; + long start = days * t.day; + return t.isYearSaturating && days < 0 && start > 0 ? Long.MIN_VALUE : start; + } + private static int dayOfWeek(Target t, long time) { + long d; + if (time > -1) { + d = time / t.day; + } else { + d = (time - (t.day - 1)) / t.day; + if (d < -3) { + return 7 + (int) ((d + 4) % 7); + } + } + return 1 + (int) ((d + 3) % 7); + } + private static int endYearDow(int year) { + return (year + Math.abs(year / 4) - Math.abs(year / 100) + Math.abs(year / 400)) % 7; + } + + private static int weeks(int year) { + return endYearDow(year) == 4 || endYearDow(year - 1) == 3 ? 53 : 52; + } + + private static int isoYearDayOffset(int year) { + int d = endYearDow(year - 1); + return (d <= 3 ? 0 : 7) - d; + } + + private static int monthOfYear(long micros, int year, boolean leap) { + long elapsed = micros - yearStart(TIMESTAMP, year, leap); + int month = 12; + while (month > 1 && elapsed < monthStart(TIMESTAMP, month, leap)) { + month--; + } + return month; + } + private static int year(long micros) { + int estimate = 1970 + (int) (micros / 31_556_952_000_000L); + if (micros < 0 && estimate >= 1970) { + estimate = 1969; + } + boolean leap = CommonUtils.isLeapYear(estimate); + long diff = micros - yearStart(TIMESTAMP, estimate, leap); + if (diff < 0) { + return estimate - 1; + } + return diff >= (leap ? 366L : 365L) * TIMESTAMP.day ? estimate + 1 : estimate; + } + private static int dayOfMonth(long micros, int year, int month, boolean leap) { + return (int) ((micros - yearStart(TIMESTAMP, year, leap) - monthStart(TIMESTAMP, month, leap)) / TIMESTAMP.day) + 1; + } + private static long nextOrSame(Target t, long time, int dow) { + int current = dayOfWeek(t, time); + return current <= dow ? time + (dow - current) * t.day : time + (7 - current + dow) * t.day; + } + private static long previousOrSame(Target t, long time, int dow) { + int current = dayOfWeek(t, time); + return current >= dow ? time - (current - dow) * t.day : time - (7 + current - dow) * t.day; + } + private static final class ZoneMatcher { + + /** + * Zone offset in {@code t}'s unit. The server matches TIMESTAMP zone text + * against UTF-8 bytes, where a non-ASCII name never matches, and DATE zone + * text case-insensitively as UTF-16. + */ + private static long offset(Target t, CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); + } + int minutes = numericOffset(s, lo, hi); + if (minutes != Integer.MIN_VALUE) { + return minutes * t.minute; + } + return NamedZones.find(s, lo, hi, t.isZoneAsciiOnly).offset(t, epoch, year); + } + + private static int numericOffset(CharSequence s, int lo, int hi) { + if (hi - lo == 1 && (s.charAt(lo) == 'Z' || s.charAt(lo) == 'z')) { + return 0; + } + int p = lo; + boolean namedPrefix = startsWithIgnoreCase(s, p, hi, "UTC") || startsWithIgnoreCase(s, p, hi, "GMT"); + if (namedPrefix) { + p += 3; + } + boolean negative = false; + if (p < hi && (s.charAt(p) == '+' || s.charAt(p) == '-')) { + negative = s.charAt(p++) == '-'; + } else if (namedPrefix) { + return Integer.MIN_VALUE; + } + if (p + 2 > hi || !digit(s.charAt(p)) || !digit(s.charAt(p + 1))) { + return Integer.MIN_VALUE; + } + int hour = (s.charAt(p) - '0') * 10 + s.charAt(p + 1) - '0'; + p += 2; + int minute = 0; + if (p < hi) { + if (s.charAt(p) == ':') { + p++; + } + if (p + 2 > hi || !digit(s.charAt(p)) || !digit(s.charAt(p + 1))) { + return Integer.MIN_VALUE; + } + minute = (s.charAt(p) - '0') * 10 + s.charAt(p + 1) - '0'; + p += 2; + } + return p == hi && hour <= 23 && minute <= 59 + ? (negative ? -1 : 1) * (hour * 60 + minute) + : Integer.MIN_VALUE; + } + + private static boolean digit(char c) { + return c >= '0' && c <= '9'; + } + + private static boolean startsWithIgnoreCase(CharSequence value, int lo, int hi, String prefix) { + if (hi - lo < prefix.length()) { + return false; + } + for (int i = 0; i < prefix.length(); i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != prefix.charAt(i)) { + return false; + } + } + return true; + } + + /** + * Zone names by case-insensitive text. A holder class, so that only a + * named-zone lookup loads every zone's rules; 'Z' and numeric offsets + * never initialize it. + */ + private static final class NamedZones { + private static final int MASK; + private static final String[] NAMES; + private static final Zone[] ZONES; + + static { + Map rules = new HashMap<>(); + for (String id : ZoneRulesProvider.getAvailableZoneIds()) { + rules.put(id, new Zone(ZoneRulesProvider.getRules(id, true))); + } + for (Map.Entry alias : ZoneId.SHORT_IDS.entrySet()) { + if (!rules.containsKey(alias.getKey())) { + Zone target = rules.get(alias.getValue()); + if (target == null) { + target = new Zone(ZoneId.of(alias.getValue()).getRules()); + } + rules.put(alias.getKey(), target); + } + } + String[][] rows = new DateFormatSymbols(Locale.ENGLISH).getZoneStrings(); + int names = 1; + for (String[] row : rows) { + names += row.length; + } + // At most half full, so a miss ends at an empty slot quickly. + int capacity = 16; + while (capacity < 2 * names) { + capacity <<= 1; + } + MASK = capacity - 1; + NAMES = new String[capacity]; + ZONES = new Zone[capacity]; + // The first definition of a name wins; UTC goes first. + define(rules.get("UTC"), "UTC"); + for (String[] row : rows) { + if (row.length == 0 || !rules.containsKey(row[0])) { + continue; + } + Zone zone = rules.get(row[0]); + for (String name : row) { + define(zone, name); + } + } + } + + private static void define(Zone zone, String name) { + if (zone == null || name == null || name.isEmpty()) { + return; + } + String text = name.toUpperCase(Locale.ROOT); + int hash = 0; + for (int i = 0, n = text.length(); i < n; i++) { + hash = 31 * hash + text.charAt(i); + } + int slot = spread(hash) & MASK; + while (NAMES[slot] != null) { + if (NAMES[slot].equals(text)) { + return; + } + slot = (slot + 1) & MASK; + } + NAMES[slot] = text; + ZONES[slot] = zone; + } + + private static Zone find(CharSequence s, int lo, int hi, boolean isAsciiOnly) throws NumericException { + int hash = 0; + for (int i = lo; i < hi; i++) { + char c = s.charAt(i); + if (isAsciiOnly && c > 0x7f) { + throw NumericException.instance(); + } + hash = 31 * hash + Character.toUpperCase(c); + } + int length = hi - lo; + for (int slot = spread(hash) & MASK; NAMES[slot] != null; slot = (slot + 1) & MASK) { + String name = NAMES[slot]; + if (name.length() == length && regionMatches(s, lo, name)) { + return ZONES[slot]; + } + } + throw NumericException.instance(); + } + + private static boolean regionMatches(CharSequence value, int lo, String name) { + for (int i = 0, n = name.length(); i < n; i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != name.charAt(i)) { + return false; + } + } + return true; + } + + private static int spread(int hash) { + return hash ^ (hash >>> 16); + } + } + + private static final class Zone { + // Seconds, or Long.MIN_VALUE when absent. + private final long cutoffSeconds; + private final long fixedOffsetSeconds; + private final long lastWallSeconds; + private final List recurring; + private final ZoneRules rules; + + private Zone(ZoneRules rules) { + this.rules = rules; + List history = rules.getTransitions(); + cutoffSeconds = history.isEmpty() ? Long.MIN_VALUE : history.get(history.size() - 1).toEpochSecond(); + recurring = rules.getTransitionRules(); + fixedOffsetSeconds = rules.isFixedOffset() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() : Long.MIN_VALUE; + lastWallSeconds = history.isEmpty() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() + : history.get(history.size() - 1).getOffsetAfter().getTotalSeconds(); + } + + private static long transitionEpoch(Target t, ZoneOffsetTransitionRule rule, int year) { + boolean leap = CommonUtils.isLeapYear(year); + int month = rule.getMonth().getValue(); + int dom = rule.getDayOfMonthIndicator(); + int dow = rule.getDayOfWeek() == null ? -1 : rule.getDayOfWeek().getValue(); + long time = rule.getLocalTime().getHour() * t.hour + rule.getLocalTime().getMinute() * t.minute + + rule.getLocalTime().getSecond() * t.second; + long timestamp; + if (dom < 0) { + timestamp = yearStart(t, year, leap) + monthStart(t, month, leap) + + (CommonUtils.getDaysPerMonth(month, leap) + dom) * t.day + time; + if (dow > -1) { + timestamp = previousOrSame(t, timestamp, dow); + } + } else { + timestamp = yearStart(t, year, leap) + monthStart(t, month, leap) + (dom - 1L) * t.day + time; + if (dow > -1) { + timestamp = nextOrSame(t, timestamp, dow); + } + } + if (rule.isMidnightEndOfDay()) { + timestamp += t.day; + } + int before = rule.getOffsetBefore().getTotalSeconds(); + if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.UTC) { + timestamp += before * t.second; + } else if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.STANDARD) { + timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * t.second; + } + return timestamp - before * t.second; + } + + private long offset(Target t, long epoch, int year) throws NumericException { + if (fixedOffsetSeconds != Long.MIN_VALUE) { + return fixedOffsetSeconds * t.second; + } + long cutoff = cutoffSeconds == Long.MIN_VALUE ? Long.MIN_VALUE : cutoffSeconds * t.second; + if (!recurring.isEmpty() && epoch > cutoff) { + if (t.isNegativeYearRuleRejected && year < 0) { + // The server's DATE recurring-rule cache rejects negative + // year keys. This is only reachable after full-width DATE + // arithmetic wraps past the cutoff. + throw NumericException.instance(); + } + // The server computes future rules with the parsed year, + // even when calendar arithmetic has wrapped the epoch. + long after = 0; + for (int i = 0, n = recurring.size(); i < n; i++) { + ZoneOffsetTransitionRule rule = recurring.get(i); + if (epoch < transitionEpoch(t, rule, year)) { + return rule.getOffsetBefore().getTotalSeconds() * t.second; + } + after = rule.getOffsetAfter().getTotalSeconds() * t.second; + } + return after; + } + if (epoch > cutoff) { + return lastWallSeconds * t.second; + } + Instant instant = Instant.ofEpochSecond(Math.floorDiv(epoch, t.second), Math.floorMod(epoch, t.second) * t.nanos); + return rules.getOffset(instant).getTotalSeconds() * t.second; + } + } + } + + /** The server's TIMESTAMP (micros) or DATE (millis) text parser family. */ + private static final class Target { + private final long day; + private final long hour; + private final boolean isNegativeYearRuleRejected; + private final boolean isYearSaturating; + private final boolean isZoneAsciiOnly; + private final long milli; + private final long minute; + // Nanos per unit. + private final long nanos; + private final long second; + + private Target( + long unitsPerMilli, + boolean isYearSaturating, + boolean isZoneAsciiOnly, + boolean isNegativeYearRuleRejected + ) { + this.milli = unitsPerMilli; + this.second = 1_000L * unitsPerMilli; + this.minute = 60_000L * unitsPerMilli; + this.hour = 3_600_000L * unitsPerMilli; + this.day = 86_400_000L * unitsPerMilli; + this.nanos = 1_000_000L / unitsPerMilli; + this.isYearSaturating = isYearSaturating; + this.isZoneAsciiOnly = isZoneAsciiOnly; + this.isNegativeYearRuleRejected = isNegativeYearRuleRejected; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 02f95b914..91ecae838 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -30,9 +30,11 @@ import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpSchemaFreshness; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -65,6 +67,11 @@ public class QwpTableBuffer implements QuietCloseable { private final LowerCaseCharSequenceIntHashMap columnNameToIndex; private final ObjList columns; private final byte[] encodedTableName; + // lower-cased table name, the schema cache key; the name itself when already lower case + private final String schemaKey; + // Survives reset() and clear(): it tracks the sender's cached schema of the + // table, not this buffer's binding. + private final QwpSchemaFreshness schemaFreshness = new QwpSchemaFreshness(); private final QwpWebSocketSender sender; private final String tableName; private QwpColumnDef[] cachedColumnDefs; @@ -78,20 +85,23 @@ public class QwpTableBuffer implements QuietCloseable { // buffer is listed once. Producer thread only. private boolean listedPending; private int rowCount; + private QwpSchemaBinding schemaBinding; public QwpTableBuffer(String tableName) { this(tableName, null); } /** - * Use this constructor overload to allow writing to a symbol column. - * {@link ColumnBuffer#addSymbol(CharSequence)} needs the sender to - * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}, registering - * the symbol in the global dictionary shared with the server. + * Use this constructor overload to encode symbol columns with the sender's + * global dictionary IDs. {@link ColumnBuffer#addSymbol(CharSequence)} calls + * {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}; buffers built + * without a sender instead maintain a local per-column symbol dictionary. */ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.tableName = tableName; this.encodedTableName = encodeName(tableName); + String lowerCaseName = Chars.toLowerCase(tableName); + this.schemaKey = tableName.equals(lowerCaseName) ? tableName : lowerCaseName; this.sender = sender; this.columns = new ObjList<>(); this.columnNameToIndex = new LowerCaseCharSequenceIntHashMap(); @@ -127,6 +137,7 @@ public void cancelCurrentRow() { * Frees all off-heap memory. */ public void clear() { + schemaBinding = null; for (int i = 0, n = columns.size(); i < n; i++) { columns.get(i).close(); } @@ -216,6 +227,44 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { return lookupColumn(name, type); } + ColumnBuffer getExistingColumnByName(CharSequence name) { + int index = columnNameToIndex.get(name); + return index >= 0 ? columns.get(index) : null; + } + + /** + * Claims {@code column}, found by {@link #findColumn}, for the in-progress + * row. Returns {@code null} when the row already holds a value for it, with + * the same first-value-wins semantics as {@link #getOrCreateColumn}. + */ + ColumnBuffer claimColumn(ColumnBuffer column) { + if (column.size > rowCount) { + return null; + } + inProgressColumnCount++; + return column; + } + + /** + * Finds a column by name without checking its type, advancing the + * sequential access cursor on a predicted hit exactly like + * {@link #getOrCreateColumn}. Returns {@code null} when absent. + */ + ColumnBuffer findColumn(CharSequence name) { + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + // Callers almost always pass the same String constant row after + // row: identity settles it without the call. + if (candidate.name == name || Chars.equalsIgnoreCase(candidate.name, name)) { + columnAccessCursor++; + return candidate; + } + } + int idx = columnNameToIndex.get(name); + return idx >= 0 ? columns.get(idx) : null; + } + /** * Gets or creates a column with the given name and type. *

@@ -237,11 +286,7 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean useN // col.size > rowCount means this column already received a value // for the in-progress row. Silently ignore the duplicate (first // value wins, same as the ILP server behaviour). - if (existing.size > rowCount) { - return null; - } - inProgressColumnCount++; - return existing; + return claimColumn(existing); } if (TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { ColumnBuffer col = createColumn(name, type, useNullBitmap); @@ -282,6 +327,39 @@ public String getTableName() { return tableName; } + public QwpSchemaBinding getSchemaBinding() { + return schemaBinding; + } + + /** + * Returns the lower-cased table name that keys the schema cache, computed + * once so a schema lookup does not allocate. + */ + public String getSchemaKey() { + return schemaKey; + } + + /** + * Returns the tracker of when the server last confirmed the sender's cached + * schema of this table. The producer thread owns it. + */ + public QwpSchemaFreshness getSchemaFreshness() { + return schemaFreshness; + } + + void attachSchemaBinding(QwpSchemaBinding binding) { + if (schemaBinding != null || rowCount != 0 || hasInProgressRow() || columns.size() != 0) { + throw new IllegalStateException("schema binding requires an empty, unbound table buffer"); + } + schemaBinding = binding; + } + + void requireSchemaBinding(QwpSchemaBinding binding) { + if (schemaBinding != binding) { + throw new IllegalStateException("schema binding is no longer attached to the table buffer"); + } + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -516,28 +594,11 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullB } private ColumnBuffer lookupColumn(CharSequence name, byte type) { - // Fast path: predict next column in sequence - int n = columns.size(); - if (columnAccessCursor < n) { - ColumnBuffer candidate = fastColumns[columnAccessCursor]; - // Callers almost always pass the same String constant row after - // row: identity settles it without the call. - if (candidate.name == name || Chars.equalsIgnoreCase(candidate.name, name)) { - columnAccessCursor++; - assertColumnType(name, type, candidate); - return candidate; - } - } - - // Slow path: hash map lookup - int idx = columnNameToIndex.get(name); - if (idx >= 0) { - ColumnBuffer existing = columns.get(idx); + ColumnBuffer existing = findColumn(name); + if (existing != null) { assertColumnType(name, type, existing); - return existing; } - - return null; + return existing; } private void rebuildColumnAccessStructures() { @@ -706,6 +767,7 @@ void reset(boolean forLong) { * operation and efficient bulk copy to network buffers. */ public static class ColumnBuffer implements QuietCloseable { + static final int SCHEMA_UNRESOLVED = Integer.MIN_VALUE; private static final long DOUBLE_ARRAY_BASE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(double[].class); final int elemSize; final String name; @@ -724,6 +786,11 @@ public static class ColumnBuffer implements QuietCloseable { private OffHeapAppendMemory dataBuffer; // Decimal storage private byte decimalScale = -1; + private boolean schemaDecimalScaleLocked; + // Schema column index the binding resolved this column to, -1 when the + // schema lacks it, or SCHEMA_UNRESOLVED before any binding resolved it. + private int schemaIndex = SCHEMA_UNRESOLVED; + private int inferredArrayDimensionality = -1; private double[] doubleArrayData; // GeoHash precision (number of bits, 1-60) private int geohashPrecision = -1; @@ -785,10 +852,17 @@ public void addBinary(byte[] value) { throw new LineSenderException( "BINARY value cannot be null; mark the row null via the null bitmap instead"); } - if (value.length > 0) { - stringData.putBytes(value, 0, value.length); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (value.length > 0) { + stringData.putBytes(value, 0, value.length); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; size++; } @@ -806,10 +880,17 @@ public void addBinary(long ptr, long len) { if (len > 0 && ptr == 0) { throw new LineSenderException("BINARY pointer cannot be 0 for a non-empty value"); } - if (len > 0) { - stringData.putBlockOfBytes(ptr, len); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (len > 0) { + stringData.putBlockOfBytes(ptr, len); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; size++; } @@ -913,6 +994,77 @@ public void addDecimal64(Decimal64 value) { size++; } + public boolean addSchemaDecimal(Decimal value, int targetPrecision, int targetScale) throws NumericException { + value.toDecimal256(rescaleTemp); + return appendSchemaDecimal(targetPrecision, targetScale); + } + + public void addSchemaDecimalNull(int targetScale) { + lockSchemaDecimalScale(targetScale); + addNull(); + } + + public boolean addSchemaLongDecimal(long value, int targetPrecision, int targetScale) throws NumericException { + rescaleTemp.ofLong(value, 0); + return appendSchemaDecimal(targetPrecision, targetScale); + } + + public boolean addSchemaStringDecimal(CharSequence value, int targetPrecision, int targetScale) throws NumericException { + rescaleTemp.ofString(value, 0, value.length(), targetPrecision, targetScale, false, false); + if (rescaleTemp.isNull()) { + addSchemaDecimalNull(targetScale); + return true; + } + return appendSchemaDecimal(targetPrecision, targetScale); + } + + private boolean appendSchemaDecimal(int targetPrecision, int targetScale) throws NumericException { + if (rescaleTemp.getScale() != targetScale) { + rescaleTemp.rescale(targetScale); + } + if (!rescaleTemp.comparePrecision(targetPrecision)) { + return false; + } + int storageSizePow2; + switch (type) { + case TYPE_DECIMAL64: + storageSizePow2 = 3; + break; + case TYPE_DECIMAL128: + storageSizePow2 = 4; + break; + case TYPE_DECIMAL256: + storageSizePow2 = 5; + break; + default: + throw new IllegalStateException("not a decimal column: " + type); + } + if (!rescaleTemp.fitsInStorageSizePow2(storageSizePow2)) { + return false; + } + lockSchemaDecimalScale(targetScale); + switch (type) { + case TYPE_DECIMAL64: + dataBuffer.putLong(rescaleTemp.getLl()); + break; + case TYPE_DECIMAL128: + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + break; + case TYPE_DECIMAL256: + dataBuffer.putLong(rescaleTemp.getHh()); + dataBuffer.putLong(rescaleTemp.getHl()); + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + break; + default: + throw new AssertionError(); + } + valueCount++; + size++; + return true; + } + public void addDouble(double value) { dataBuffer.putDouble(value); valueCount++; @@ -1032,6 +1184,13 @@ public void addFloat(float value) { * @param precision number of bits (1-60) */ public void addGeoHash(long value, int precision) { + initGeoHashPrecision(precision); + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void initGeoHashPrecision(int precision) { if (precision < 1 || precision > 60) { throw new LineSenderException("invalid GeoHash precision: " + precision + " (must be 1-60)"); } @@ -1042,9 +1201,6 @@ public void addGeoHash(long value, int precision) { "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision ); } - dataBuffer.putLong(value); - valueCount++; - size++; } public void addInt(int value) { @@ -1266,10 +1422,17 @@ public void addString(CharSequence value) { ensureNullBitmapCapacity(size + 1); markNull(size); } else { - if (value != null) { - stringData.putUtf8(value); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (value != null) { + stringData.putUtf8(value); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; } size++; @@ -1459,7 +1622,11 @@ public long getDataAddress() { } public byte getDecimalScale() { - return decimalScale; + // A decimal column containing only nulls has no natural scale yet. + // Scale -1 is internal state, not valid QWP metadata; emit the + // deterministic scale-zero representation without locking it so a + // later finite batch can still select its natural scale. + return decimalScale == -1 ? 0 : decimalScale; } public double[] getDoubleArrayData() { @@ -1490,10 +1657,25 @@ public long getNullBitmapAddress() { return nullBufPtr; } + int getSchemaIndex() { + return schemaIndex; + } + public int getSize() { return size; } + void setSchemaIndex(int schemaIndex) { + this.schemaIndex = schemaIndex; + } + + int pinInferredArrayDimensionality(int dimensionality) { + if (inferredArrayDimensionality < 0) { + inferredArrayDimensionality = dimensionality; + } + return inferredArrayDimensionality; + } + public long getStringDataAddress() { return stringData != null ? stringData.pageAddress() : 0; } @@ -1540,6 +1722,10 @@ public boolean hasSymbol(CharSequence value) { return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; } + boolean isSchemaResolved() { + return schemaIndex != SCHEMA_UNRESOLVED; + } + public boolean isNull(int index) { if (nullBufPtr == 0 || index >= nullBufCapRows) { return false; @@ -1586,7 +1772,9 @@ public void reset() { maxGlobalSymbolId = -1; arrayShapeOffset = 0; arrayDataOffset = 0; - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } // geohashPrecision is intentionally NOT cleared here. It is a // schema property, locked on first write and matched by the // server's auto-created GEOHASH(Nb) type, so preserving it across @@ -1705,7 +1893,9 @@ public void truncateTo(int newSize) { // through ColumnBuffer.reset() and intentionally preserves // geohashPrecision there to keep its server-locked value. if (newValueCount == 0) { - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } geohashPrecision = -1; maxGlobalSymbolId = -1; storeGlobalSymbolIdsOnly = false; @@ -1720,6 +1910,11 @@ public boolean usesNullBitmap() { return useNullBitmap; } + private void restoreStringAppendPositions(long dataOffset, long offsetsOffset) { + stringData.jumpTo(dataOffset); + stringOffsets.jumpTo(offsetsOffset); + } + private static int checkedElementCount(long product) { if (product > Integer.MAX_VALUE) { throw new LineSenderException("array too large: total element count exceeds int range"); @@ -1947,8 +2142,21 @@ private void markNull(int index) { hasNulls = true; } + private void lockSchemaDecimalScale(int targetScale) { + if (schemaDecimalScaleLocked) { + if ((decimalScale & 0xFF) != targetScale) { + throw new IllegalStateException("schema decimal scale changed"); + } + return; + } + decimalScale = (byte) targetScale; + schemaDecimalScaleLocked = true; + } + private void resetEmptyMetadata() { - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } geohashPrecision = -1; maxGlobalSymbolId = -1; storeGlobalSymbolIdsOnly = false; diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index 3bcbf205a..98dd2cb09 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -91,6 +91,8 @@ public final class ConfigSchema { str("reconnect_max_backoff_millis", Side.INGRESS); str("reconnect_max_duration_millis", Side.INGRESS); str("request_durable_ack", Side.INGRESS); + str("schema_mode", Side.INGRESS); + str("schema_wait_millis", Side.INGRESS); str("sender_id", Side.INGRESS); str("sf_append_deadline_millis", Side.INGRESS); str("sf_dir", Side.INGRESS); diff --git a/core/src/main/java/io/questdb/client/std/DecimalParser.java b/core/src/main/java/io/questdb/client/std/DecimalParser.java index 86fabdcbc..e3d8aee14 100644 --- a/core/src/main/java/io/questdb/client/std/DecimalParser.java +++ b/core/src/main/java/io/questdb/client/std/DecimalParser.java @@ -35,7 +35,7 @@ *

  • QuestDB decimal suffix: "123.45m" or "123.45M"
    • *
  • Floating-point suffixes: "123.45f", "123.45d" (stripped during parsing)
    • *
  • Leading zeros: "00123.45" → 123.45
    • - *
  • Leading/trailing whitespace
    • + *
  • Leading ASCII spaces
    • * * * @see Decimal @@ -54,7 +54,7 @@ public final class DecimalParser { *

    * Parsing Process: *

      - *
    1. Strip suffixes (m, M, f, d) and leading/trailing whitespace
      2. + *
    2. Strip supported suffixes (m, M, f, d) and leading ASCII spaces
      3. *
    3. Parse sign (+/-) and detect special values (NaN/Infinity)
      4. *
    4. Remove leading zeros
      5. *
    5. Extract mantissa and locate decimal point
      6. @@ -163,12 +163,21 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // We do a first pass over the literal to ensure that the format is correct (numerical and at most 1 dot) and to // measure the given precision/scale. int dot = -1; + int leadingZeroes = 0; boolean digitFound = false; + boolean hasSignificantDigit = false; int digitLo = lo; for (; lo < hi; lo++) { char c = cs.charAt(lo); if (isDigit(c)) { digitFound = true; + if (!hasSignificantDigit) { + if (c == '0') { + leadingZeroes++; + } else { + hasSignificantDigit = true; + } + } continue; } else if (c == '.' && dot == -1) { dot = lo; @@ -178,7 +187,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p } if (!digitFound) { if (skippedZeroes) { + // The stripper only walks over zeroes, so the digit it gives back is a zero. digitLo--; + leadingZeroes = 1; } else { throw NumericException.instance() .put("invalid decimal: '").put(cs) @@ -196,9 +207,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p int literalScale = dot == -1 ? 0 : (digitHi - dot - 1); int literalDigits = digitHi - digitLo - (dot == -1 ? 0 : 1); - int virtualScale = literalScale; + long virtualScale = literalScale; - int exp = 0; + long exp = 0; if (lo != hi) { // Parses exponent if ((cs.charAt(lo) | 32) == 'e') { @@ -222,7 +233,7 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // If lossy is enabled, we can strip digits after the provided scale (as long as it is provided) if (lossy && scale >= 0 && virtualScale > scale) { - int drop = virtualScale - scale; + long drop = virtualScale - scale; while (drop > 0 && digitHi > digitLo) { digitHi--; if (digitHi == dot) { @@ -237,7 +248,12 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // We still need to adjust the exponent if the user specifies a target scale. if (scale == -1) { - scale = virtualScale; + if (virtualScale > decimal.getMaxScale()) { + throw NumericException.instance() + .put("decimal '").put(cs) + .put("' exceeds maximum allowed scale of ").put(decimal.getMaxScale()); + } + scale = (int) virtualScale; } else if (virtualScale <= scale) { exp += scale - virtualScale; } else { @@ -258,8 +274,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // precision, as long as it's lower. // Compute the final precision of the decimal - int pow = literalDigits + exp; - final int finalPrecision = Math.max(pow, scale + 1); + long pow = literalDigits + exp; + final long unscaledDigits = literalDigits > leadingZeroes ? pow - leadingZeroes : 0; + final long finalPrecision = Math.max(Math.max(unscaledDigits, scale), 1); if (precision != -1 && finalPrecision > precision) { throw NumericException.instance() .put("decimal '").put(cs) @@ -277,8 +294,14 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p if (p == dot) { continue; } - - decimal.addPowerOfTenMultiple(--pow, cs.charAt(p) - '0'); + int multiplier = cs.charAt(p) - '0'; + pow--; + if (multiplier != 0) { + if (pow < 0 || pow >= decimal.getMaxPrecision()) { + throw NumericException.instance().put("decimal '").put(cs).put("' exponent is out of range"); + } + decimal.addPowerOfTenMultiple((int) pow, multiplier); + } } decimal.setScale(scale); @@ -286,7 +309,7 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p decimal.negate(); } - return Numbers.encodeLowHighInts(finalPrecision, scale); + return Numbers.encodeLowHighInts((int) finalPrecision, scale); } /** @@ -315,9 +338,11 @@ private static boolean isDigit(char c) { * @return true if "NaN" or "Infinity" is found at position lo, false otherwise */ private static boolean isNanOrInfinite(char ch, CharSequence cs, int lo, int hi) { + // A prefix match, like the server's DecimalParser: "NaNjunk" parses as NULL on + // every server ingest path, so the client must not reject it. return (ch == 'N' && lo + 2 < hi && cs.charAt(lo + 1) == 'a' && cs.charAt(lo + 2) == 'N') || (ch == 'I' && lo + 7 < hi && cs.charAt(lo + 1) == 'n' && cs.charAt(lo + 2) == 'f' && cs.charAt(lo + 3) == 'i' && cs.charAt(lo + 4) == 'n' && cs.charAt(lo + 5) == 'i' && cs.charAt(lo + 6) == 't' && cs.charAt(lo + 7) == 'y'); } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index 78a3bf4c7..fa0189b7e 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -39,11 +39,9 @@ public final class Numbers { public static final char[] hexDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; public final static int[] hexNumbers; public static final long[] pow10; - private static final int EXP_BIAS = 1023; + private static final int[] DOUBLE_DIGIT_PAIRS = new int[100]; private static final long EXP_BIT_MASK = 0x7FF0000000000000L; private static final int EXP_SHIFT = SIGNIFICAND_WIDTH - 1; - static final long EXP_ONE = ((long) EXP_BIAS) << EXP_SHIFT; // exponent of 1.0 - private static final long FRACT_HOB = (1L << EXP_SHIFT); // assumed High-Order bit // Maps ASCII chars in [48, 122] (i.e. '0'..'z'), shifted by -48, to the // 5-bit base32 digit, or -1 if not a valid geohash character. Matches the // server-side alphabet in io.questdb.cairo.GeoHashes#base32Indexes: digits @@ -61,16 +59,11 @@ public final class Numbers { 21, 22, 23, 24, 25, 26, 27, 28, // 'p'..'w' 29, 30, 31 // 'x','y','z' }; - private static final long[] LONG_5_POW = new long[]{1L, 5L, 25L, 125L, 625L, 3125L, 15625L, 78125L, 390625L, 1953125L, 9765625L, 48828125L, 244140625L, 1220703125L, 6103515625L, 30517578125L, 152587890625L, 762939453125L, 3814697265625L, 19073486328125L, 95367431640625L, 476837158203125L, 2384185791015625L, 11920928955078125L, 59604644775390625L, 298023223876953125L, 1490116119384765625L}; - private static final int MAX_SMALL_BIN_EXP = 62; - private static final int MIN_SMALL_BIN_EXP = -(63 / 3); - private static final int[] N_5_BITS = new int[]{0, 3, 5, 7, 10, 12, 14, 17, 19, 21, 24, 26, 28, 31, 33, 35, 38, 40, 42, 45, 47, 49, 52, 54, 56, 59, 61}; private static final long SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; - private static final int[] SMALL_5_POW = new int[]{1, 5, 25, 125, 625, 3125, 15625, 78125, 390625, 1953125, 9765625, 48828125, 244140625, 1220703125}; - private static final int[] insignificantDigitsNumber = new int[]{0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19}; private static final LongHexAppender[] longHexAppender = new LongHexAppender[Long.SIZE + 1]; private static final LongHexAppender[] longHexAppenderPad64 = new LongHexAppender[Long.SIZE + 1]; - private final static ThreadLocal tlDoubleDigitsBuffer = new ThreadLocal<>(() -> new char[21]); + // Slot 0 holds RyuDouble.d2d()'s decimal exponent; slots 1-17 hold ASCII digits. + private final static ThreadLocal tlRyuScratch = new ThreadLocal<>(() -> new int[18]); private Numbers() { } @@ -175,47 +168,119 @@ public static void append(CharSink sink, double value) { } public static void append(CharSink sink, double value, int scale) { - final char[] digits = tlDoubleDigitsBuffer.get(); final long doubleBits = Double.doubleToRawLongBits(value); - boolean negative = (doubleBits & SIGN_BIT_MASK) != 0L; - long significantBitCount = doubleBits & SIGNIF_BIT_MASK; - int binExp = (int) ((doubleBits & EXP_BIT_MASK) >> EXP_SHIFT); - - if (binExp == 2047) { - if (significantBitCount == 0L) { - if (negative) { - sink.putAscii("-Infinity"); - } else { - sink.putAscii("Infinity"); + long ieeeMantissa = doubleBits & SIGNIF_BIT_MASK; + int ieeeExponent = (int) ((doubleBits & EXP_BIT_MASK) >> EXP_SHIFT); + + // NaN or Infinity + if (ieeeExponent == 2047) { + if (ieeeMantissa != 0) { + sink.putAscii("NaN"); + } else if ((doubleBits & SIGN_BIT_MASK) != 0) { + sink.putAscii("-Infinity"); + } else { + sink.putAscii("Infinity"); + } + return; + } + + // Negative sign (including -0.0) + if ((doubleBits & SIGN_BIT_MASK) != 0) { + sink.putAscii('-'); + } + + // Zero + if (ieeeExponent == 0 && ieeeMantissa == 0) { + sink.putAscii("0.0"); + return; + } + + // Short decimals, such as prices and readings, take an exact path that + // returns what Ryu would; Ryu decomposes the rest. + int[] scratch = tlRyuScratch.get(); + long output = shortDecimal(Math.abs(value), scratch); + if (output == 0) { + output = RyuDouble.d2d(ieeeMantissa, ieeeExponent, scratch); + } + int olength = RyuDouble.decimalLength17(output); + int decExp = scratch[0] + olength; + int digitsToWrite = olength; + if (scale < MAX_DOUBLE_SCALE && decExp > -3 && decExp < 8 && olength > decExp) { + // Do not decode fractional digits that the fixed-point scale will discard. + digitsToWrite = Math.min(olength, Math.max(Math.max(scale, 1) + decExp, 0)); + if (digitsToWrite > 0 && digitsToWrite < olength) { + output /= pow10[olength - digitsToWrite]; + } + } + int digitIndex = digitsToWrite; + for (; digitIndex > 2; digitIndex -= 2) { + long quotient = output / 100; + int pair = DOUBLE_DIGIT_PAIRS[(int) (output - quotient * 100)]; + scratch[digitIndex - 1] = pair >>> 8; + scratch[digitIndex] = pair & 0xff; + output = quotient; + } + if (digitIndex == 2) { + int pair = DOUBLE_DIGIT_PAIRS[(int) output]; + scratch[1] = pair >>> 8; + scratch[2] = pair & 0xff; + } else if (digitIndex == 1) { + scratch[1] = '0' + (int) output; + } + + if (decExp > 0 && decExp < 8) { + // Fixed-point with integer part: e.g. "1234.567" or "1234000.0" + if (olength <= decExp) { + for (int i = 0; i < olength; i++) { + sink.putAscii((char) scratch[i + 1]); + } + for (int i = olength; i < decExp; i++) { + sink.putAscii('0'); } + sink.putAscii(".0"); } else { - sink.putAscii("NaN"); + int fracDigits = olength - decExp; + if (scale < MAX_DOUBLE_SCALE && fracDigits > scale) { + fracDigits = Math.max(scale, 1); + } + for (int i = 0; i < decExp; i++) { + sink.putAscii((char) scratch[i + 1]); + } + sink.putAscii('.'); + for (int i = 0; i < fracDigits; i++) { + sink.putAscii((char) scratch[decExp + i + 1]); + } + } + } else if (decExp <= 0 && decExp > -3) { + // Leading-zero fixed-point: e.g. "0.00123" + int leadingZeros = -decExp; + int totalFrac = leadingZeros + olength; + int digitsFromOutput = olength; + if (scale < MAX_DOUBLE_SCALE && totalFrac > scale) { + totalFrac = Math.max(scale, 1); + digitsFromOutput = Math.max(totalFrac - leadingZeros, 0); + } + int zerosToWrite = Math.min(leadingZeros, totalFrac); + sink.putAscii("0."); + for (int i = 0; i < zerosToWrite; i++) { + sink.putAscii('0'); + } + for (int i = 0; i < digitsFromOutput; i++) { + sink.putAscii((char) scratch[i + 1]); } } else { - int fractionBits; - if (binExp == 0) { - if (significantBitCount == 0L) { - if (negative) { - sink.putAscii("-0.0"); - } else { - sink.putAscii("0.0"); - } - return; + // Scientific notation: e.g. "1.23E8" or "1.0E-4" + sink.putAscii((char) scratch[1]); + sink.putAscii('.'); + if (olength > 1) { + for (int i = 1; i < olength; i++) { + sink.putAscii((char) scratch[i + 1]); } - - int leadingZeros = Long.numberOfLeadingZeros(significantBitCount); - int shift = leadingZeros - (63 - EXP_SHIFT); - significantBitCount <<= shift; - binExp = 1 - shift; - fractionBits = 64 - leadingZeros; } else { - significantBitCount |= FRACT_HOB; - fractionBits = 53; + sink.putAscii('0'); } - - binExp -= EXP_BIAS; - - appendDouble0(binExp, significantBitCount, fractionBits, negative, digits, sink, scale); + sink.putAscii('E'); + append(sink, decExp - 1); } } @@ -406,7 +471,10 @@ public static int parseIPv4_0(CharSequence sequence, final int p, int lim) throw if (sign == '.') { do { lo++; - } while (sequence.charAt(lo) == '.'); + } while (lo < lim && sequence.charAt(lo) == '.'); + if (lo == lim) { + throw NumericException.instance().put("IPv4 address must have 4 octets, found: 1"); + } } else { throw NumericException.instance().put("invalid IPv4 address: ").put(sequence); } @@ -477,347 +545,6 @@ public static long parseLong(Utf8Sequence sequence) throws NumericException { return parseLong0(sequence.asAsciiCharSequence(), sequence.size()); } - private static void appendDouble0( - int binExp, - long fractionBits, - int significantBitCount, - boolean negative, - char[] digits, - CharSink out, - int outScale - ) { - assert fractionBits > 0L; - assert (fractionBits & FRACT_HOB) != 0L; - - final int tailZeroes = Long.numberOfTrailingZeros(fractionBits); - final int fractBitCount = EXP_SHIFT + 1 - tailZeroes; - int decExp; - int firstDigitIndex; - int nDigits; - - final int tinyBitCount = Math.max(0, fractBitCount - binExp - 1); - if (binExp < MAX_SMALL_BIN_EXP + 1 && binExp > MIN_SMALL_BIN_EXP - 1 && tinyBitCount < LONG_5_POW.length && fractBitCount + N_5_BITS[tinyBitCount] < 64 && tinyBitCount == 0) { - int insignificant; - if (binExp > significantBitCount) { - insignificant = insignificantDigitsForPow2(binExp - significantBitCount - 1); - } else { - insignificant = 0; - } - - if (binExp >= EXP_SHIFT) { - fractionBits <<= binExp - EXP_SHIFT; - } else { - fractionBits >>>= EXP_SHIFT - binExp; - } - - // - int binExp2 = 0; - if (insignificant != 0) { - long pow10 = LONG_5_POW[insignificant] << insignificant; - long residue = fractionBits % pow10; - fractionBits /= pow10; - binExp2 += insignificant; - if (residue >= pow10 >> 1) { - ++fractionBits; - } - } - - int digitIndex = digits.length - 1; - int digit; - if (fractionBits <= Integer.MAX_VALUE) { - assert fractionBits > 0L : fractionBits; - - int fractRemaining = (int) fractionBits; - digit = fractRemaining % 10; - - for (fractRemaining /= 10; digit == 0; fractRemaining /= 10) { - ++binExp2; - digit = fractRemaining % 10; - } - - while (fractRemaining != 0) { - digits[digitIndex--] = (char) (digit + '0'); - ++binExp2; - digit = fractRemaining % 10; - fractRemaining /= 10; - } - - } else { - digit = (int) (fractionBits % 10L); - - for (fractionBits /= 10L; digit == 0; fractionBits /= 10L) { - ++binExp2; - digit = (int) (fractionBits % 10L); - } - - while (fractionBits != 0L) { - digits[digitIndex--] = (char) (digit + '0'); - ++binExp2; - digit = (int) (fractionBits % 10L); - fractionBits /= 10L; - } - - } - digits[digitIndex] = (char) (digit + '0'); - - decExp = binExp2 + 1; - firstDigitIndex = digitIndex; - nDigits = digits.length - digitIndex; - } else { - int estDecExp = estimateDecExpDouble(fractionBits, binExp); - int B5 = Math.max(0, -estDecExp); - int B2 = B5 + tinyBitCount + binExp; - int S5 = Math.max(0, estDecExp); - int S2 = S5 + tinyBitCount; - int M2 = B2 - significantBitCount; - fractionBits >>>= tailZeroes; - B2 -= fractBitCount - 1; - int common2factor = Math.min(B2, S2); - B2 -= common2factor; - S2 -= common2factor; - M2 -= common2factor; - if (fractBitCount == 1) { - --M2; - } - - if (M2 < 0) { - B2 -= M2; - S2 -= M2; - M2 = 0; - } - - int bBits = fractBitCount + B2 + (B5 < N_5_BITS.length ? N_5_BITS[B5] : B5 * 3); - int tenBits = S2 + 1 + (S5 + 1 < N_5_BITS.length ? N_5_BITS[S5 + 1] : (S5 + 1) * 3); - boolean low; - boolean high; - long lowDigitDifference; - int q; - int digitIndex; - if (bBits < 64 && tenBits < 64) { - if (bBits < 32 && tenBits < 32) { - int b = (int) fractionBits * SMALL_5_POW[B5] << B2; - int s = SMALL_5_POW[S5] << S2; - int m = SMALL_5_POW[B5] << M2; - int tens = s * 10; - digitIndex = 0; - q = b / s; - b = 10 * (b % s); - m *= 10; - low = b < m; - high = b + m > tens; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - for (; !low && !high; digits[digitIndex++] = (char) ('0' + q)) { - q = b / s; - b = 10 * (b % s); - m *= 10; - - assert q < 10 : q; - - if ((long) m > 0L) { - low = b < m; - high = b + m > tens; - } else { - low = true; - high = true; - } - } - - lowDigitDifference = ((long) b << 1) - tens; - } else { - long b = fractionBits * LONG_5_POW[B5] << B2; - long s = LONG_5_POW[S5] << S2; - long m = LONG_5_POW[B5] << M2; - long tens = s * 10L; - digitIndex = 0; - q = (int) (b / s); - b = 10L * (b % s); - m *= 10L; - low = b < m; - high = b + m > tens; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - for (; !low && !high; digits[digitIndex++] = (char) ('0' + q)) { - q = (int) (b / s); - b = 10L * (b % s); - m *= 10L; - - assert q < 10 : q; - - if (m > 0L) { - low = b < m; - high = b + m > tens; - } else { - low = true; - high = true; - } - } - lowDigitDifference = (b << 1) - tens; - } - } else { - FdBig sVal = FdBig.valueOfPow52(S5, S2); - final int shiftBias = sVal.getNormalizationBias(); - sVal = sVal.leftShift(shiftBias); - FdBig bVal = FdBig.valueOfMulPow52(fractionBits, B5, B2 + shiftBias); - FdBig mVal = FdBig.valueOfPow52(B5 + 1, M2 + shiftBias + 1); - FdBig tensVal = FdBig.valueOfPow52(S5 + 1, S2 + shiftBias + 1); - digitIndex = 0; - q = bVal.quoRemIteration(sVal); - low = bVal.cmp(mVal) < 0; - high = tensVal.addAndCmp(bVal, mVal) <= 0; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - while (!low && !high) { - q = bVal.quoRemIteration(sVal); - - assert q < 10 : q; - - mVal = mVal.multBy10(); - low = bVal.cmp(mVal) < 0; - high = tensVal.addAndCmp(bVal, mVal) <= 0; - digits[digitIndex++] = (char) ('0' + q); - } - - if (high && low) { - bVal = bVal.leftShift(1); - lowDigitDifference = bVal.cmp(tensVal); - } else { - lowDigitDifference = 0L; - } - } - - decExp = estDecExp + 1; - firstDigitIndex = 0; - nDigits = digitIndex; - if (high) { - if (low) { - if (lowDigitDifference == 0L) { - if ((digits[firstDigitIndex + nDigits - 1] & 1) != 0) { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } else if (lowDigitDifference > 0L) { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } else { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } - } - - appendDouble00(digits, firstDigitIndex, nDigits, negative, decExp, out, outScale); - } - - private static void appendDouble00( - char[] digits, - int firstDigitIndex, - int nDigits, - boolean isNegative, - int decExp, - CharSink sink, - int outScale - ) { - assert nDigits <= MAX_DOUBLE_SCALE : nDigits; - if (isNegative) { - sink.putAscii('-'); - } - - int exp; - if (decExp > 0 && decExp < 8) { - exp = Math.min(nDigits, decExp); - sink.putAscii(digits, firstDigitIndex, exp); - if (exp < decExp) { - exp = decExp - exp; - sink.fillAscii('0', exp); - sink.putAscii('.'); - sink.putAscii('0'); - } else { - sink.putAscii('.'); - if (exp < nDigits) { - sink.putAscii(digits, firstDigitIndex + exp, Math.min(nDigits - exp, outScale)); - } else { - sink.putAscii('0'); - } - } - } else if (decExp <= 0 && decExp > -3) { - sink.putAscii('0').putAscii('.'); - if (decExp != 0) { - sink.fillAscii('0', -decExp); - } - - sink.putAscii(digits, firstDigitIndex, Math.min(nDigits, outScale)); - } else { - sink.putAscii(digits[firstDigitIndex]); - sink.putAscii('.'); - if (nDigits > 1) { - sink.putAscii(digits, firstDigitIndex + 1, nDigits - 1); - } else { - sink.putAscii('0'); - } - - sink.putAscii('E'); - if (decExp <= 0) { - sink.putAscii('-'); - exp = -decExp + 1; - } else { - exp = decExp - 1; - } - - if (exp < 10) { - sink.putAscii((char) (exp + '0')); - } else if (exp < 100) { - sink.putAscii((char) (exp / 10 + '0')); - sink.putAscii((char) (exp % 10 + '0')); - } else { - sink.putAscii((char) (exp / 100 + '0')); - exp %= 100; - sink.putAscii((char) (exp / 10 + '0')); - sink.putAscii((char) (exp % 10 + '0')); - } - } - } - private static void appendInt10(CharSink sink, int i) { int c; sink.putAscii((char) ('0' + i / 1000000000)); @@ -1328,27 +1055,6 @@ private static void appendLongHexPad(CharSink sink, char hexDigit) { sink.putAscii(hexDigit); } - private static int estimateDecExpDouble(long fractBits, int binExp) { - double d2 = Double.longBitsToDouble(EXP_ONE | fractBits & SIGNIF_BIT_MASK); - double d = (d2 - 1.5D) * 0.289529654D + 0.176091259D + (double) binExp * 0.301029995663981D; - long dBits = Double.doubleToRawLongBits(d); - int exponent = (int) ((dBits & EXP_BIT_MASK) >> EXP_SHIFT) - EXP_BIAS; - final boolean isNegative = (dBits & SIGN_BIT_MASK) != 0L; - if (exponent > -1 && exponent < 52) { - final long mask = SIGNIF_BIT_MASK >> exponent; - final int r = (int) ((dBits & SIGNIF_BIT_MASK | FRACT_HOB) >> EXP_SHIFT - exponent); - return isNegative ? ((mask & dBits) == 0L ? -r : -r - 1) : r; - } else if (exponent < 0) { - return (dBits & ~SIGN_BIT_MASK) == 0L ? 0 : (isNegative ? -1 : 0); - } else { - return (int) d; - } - } - - private static int insignificantDigitsForPow2(int p2) { - return p2 > 1 && p2 < insignificantDigitsNumber.length ? insignificantDigitsNumber[p2] : 0; - } - private static int parseInt0(CharSequence sequence, final int p, int lim) throws NumericException { if (lim == p) { throw NumericException.instance().put("empty integer string"); @@ -1448,27 +1154,60 @@ private static long parseLong0(CharSequence sequence, int lim) throws NumericExc return negative ? val : -val; } - private static boolean roundupDouble(int firstDigitIndex, char[] digits, int nDigits) { - int charIndex = firstDigitIndex + nDigits - 1; - char c = digits[charIndex]; - if (c == '9') { - while (true) { - if (c != '9' || charIndex <= firstDigitIndex) { - if (c == '9') { - digits[firstDigitIndex] = '1'; - return true; - } - break; - } - - digits[charIndex] = '0'; - --charIndex; - c = digits[charIndex]; - } + /** + * Returns the shortest decimal significand of the positive finite + * {@code abs}, with its decimal exponent in {@code e10Out[0]}, when + * {@code abs} is below 10^7 and is the double nearest to a decimal with at + * most 8 fractional digits. Returns 0 otherwise, and then leaves + * {@code e10Out} alone. + *

      + * The result equals {@code RyuDouble.d2d()}'s at a fraction of its cost: Ryu + * removes one digit per pass from a 17-digit interval, so a short result + * costs it the most passes. It agrees with Ryu for these reasons: + *

        + *
      • The check {@code m / 10^8 == abs} is exact. Both operands are exact + * doubles, and IEEE division rounds correctly with ties to even, which + * accepts the interval bounds exactly when Ryu does. So it passes exactly + * when the decimal {@code m * 10^-8} rounds to {@code abs}.
        • + *
      • Below 10^7, {@code ulp(abs) * 10^8 < 1}, so the rounding interval of + * {@code abs} holds at most one decimal with 8 fractional digits.
        • + *
      • All decimals in the interval start at the same digit position, + * unless the interval holds a power of ten, which then is that one + * decimal. So the shortest decimal in the interval, which Ryu returns, + * ends no later than {@code m * 10^-8} does. It is a multiple of + * 10^-8 as well, so it is {@code m} with its trailing zeros removed.
        • + *
      + */ + private static long shortDecimal(double abs, int[] e10Out) { + if (abs >= 1e7) { + return 0; } - - digits[charIndex] = (char) (c + 1); - return false; + // abs * 1e8 < 2^50, so adding 0.5 is exact + long m = (long) (abs * 1e8 + 0.5); + // m == 0 fails here too, since abs is positive + if ((double) m / 1e8 != abs) { + return 0; + } + // m < 10^15 has at most 14 trailing zeros: strip 8, 4, 2 and 1 of them + int e10 = -8; + if (m % 100_000_000L == 0) { + m /= 100_000_000L; + e10 += 8; + } + if (m % 10_000L == 0) { + m /= 10_000L; + e10 += 4; + } + if (m % 100L == 0) { + m /= 100L; + e10 += 2; + } + if (m % 10L == 0) { + m /= 10L; + e10 += 1; + } + e10Out[0] = e10; + return m; } @FunctionalInterface @@ -1477,6 +1216,10 @@ private interface LongHexAppender { } static { + for (int i = 0; i < DOUBLE_DIGIT_PAIRS.length; i++) { + DOUBLE_DIGIT_PAIRS[i] = (('0' + i / 10) << 8) | ('0' + i % 10); + } + pow10 = new long[20]; pow10[0] = 1; for (int i = 1; i < pow10.length; i++) { diff --git a/core/src/main/java/io/questdb/client/std/RyuDouble.java b/core/src/main/java/io/questdb/client/std/RyuDouble.java new file mode 100644 index 000000000..567fded3a --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/RyuDouble.java @@ -0,0 +1,635 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +/** + * Ryu conversion of IEEE 754 doubles to their shortest decimal representation, + * ported from the server's {@code io.questdb.std.RyuDouble}. It stays + * byte-compatible with the server, so both sides format a double to the same text. + *

      + * Based on the paper "Ryu: Fast Float-to-String Conversion" by Ulf Adams (2018). + * This implementation uses 128-bit arithmetic via {@link Compat#multiplyHigh}, + * which is {@code Math.multiplyHigh} on Java 9+ and a hand-rolled equivalent on Java 8. + *

      + * The core method {@link #d2d} returns the shortest decimal significand as a {@code long} + * (1-17 digits) and writes the decimal exponent to a caller-provided array. The result + * satisfies: {@code value = output * 10^e10}. + */ +class RyuDouble { + private static final int DOUBLE_POW5_BITCOUNT = 125; + private static final int DOUBLE_POW5_INV_BITCOUNT = 125; + // ceil(2^(pow5bits(q) - 1 + 125) / 5^q) for q in [0, 291], stored as (hi64, lo64) pairs. + private static final long[] DOUBLE_POW5_INV_SPLIT = initPow5InvSplit(); + // Upper 128 bits of 5^i for i in [0, 325], stored as (hi64, lo64) pairs. + private static final long[] DOUBLE_POW5_SPLIT = initPow5Split(); + + private static long[] initPow5InvSplit() { + return new long[]{ + 0x2000000000000000L, 0x0000000000000001L, 0x1999999999999999L, 0x999999999999999AL, + 0x147AE147AE147AE1L, 0x47AE147AE147AE15L, 0x10624DD2F1A9FBE7L, 0x6C8B4395810624DEL, + 0x1A36E2EB1C432CA5L, 0x7A786C226809D496L, 0x14F8B588E368F084L, 0x61F9F01B866E43ABL, + 0x10C6F7A0B5ED8D36L, 0xB4C7F34938583622L, 0x1AD7F29ABCAF4857L, 0x87A6520EC08D236AL, + 0x15798EE2308C39DFL, 0x9FB841A566D74F88L, 0x112E0BE826D694B2L, 0xE62D01511F12A607L, + 0x1B7CDFD9D7BDBAB7L, 0xD6AE6881CB5109A4L, 0x15FD7FE17964955FL, 0xDEF1ED34A2A73AEAL, + 0x119799812DEA1119L, 0x7F27F0F6E885C8BBL, 0x1C25C268497681C2L, 0x650CB4BE40D60DF8L, + 0x16849B86A12B9B01L, 0xEA70909833DE7193L, 0x1203AF9EE756159BL, 0x21F3A6E0297EC143L, + 0x1CD2B297D889BC2BL, 0x6985D7CD0F313537L, 0x170EF54646D49689L, 0x2137DFD73F5A90F9L, + 0x12725DD1D243ABA0L, 0xE75FE645CC4873FAL, 0x1D83C94FB6D2AC34L, 0xA5663D3C7A0D865DL, + 0x179CA10C9242235DL, 0x511E976394D79EB1L, 0x12E3B40A0E9B4F7DL, 0xDA7EDF82DD794BC1L, + 0x1E392010175EE596L, 0x2A6498D1625BAC68L, 0x182DB34012B25144L, 0xEEB6E0A781E2F053L, + 0x1357C299A88EA76AL, 0x58924D52CE4F26A9L, 0x1EF2D0F5DA7DD8AAL, 0x27507BB7B07EA441L, + 0x18C240C4AECB13BBL, 0x52A6C95FC0655034L, 0x13CE9A36F23C0FC9L, 0x0EEBD44C99EAA690L, + 0x1FB0F6BE50601941L, 0xB17953ADC3110A80L, 0x195A5EFEA6B34767L, 0xC12DDC8B02740867L, + 0x14484BFEEBC29F86L, 0x3424B06F3529A052L, 0x1039D66589687F9EL, 0x901D59F290EE19DBL, + 0x19F623D5A8A73297L, 0x4CFBC31DB4B0295FL, 0x14C4E977BA1F5BACL, 0x3D9635B15D59BAB2L, + 0x109D8792FB4C4956L, 0x97AB5E277DE16228L, 0x1A95A5B7F87A0EF0L, 0xF2ABC9D8C9689D0DL, + 0x154484932D2E725AL, 0x5BBCA17A3ABA173EL, 0x11039D428A8B8EAEL, 0xAFCA1AC82EFB45CBL, + 0x1B38FB9DAA78E44AL, 0xB2DCF7A6B1920945L, 0x15C72FB1552D836EL, 0xF57D92EBC141A104L, + 0x116C262777579C58L, 0xC46475896767B403L, 0x1BE03D0BF225C6F4L, 0x6D6D88DBD8A5ECD2L, + 0x164CFDA3281E38C3L, 0x8ABE071646EB23DBL, 0x11D7314F534B609CL, 0x6EFE6C11D255B649L, + 0x1C8B821885456760L, 0xB197134FB6EF8A0EL, 0x16D601AD376AB91AL, 0x27AC0F72F8BFA1A5L, + 0x1244CE242C5560E1L, 0xB95672C260994E1EL, 0x1D3AE36D13BBCE35L, 0xF5571E03CDC21695L, + 0x17624F8A762FD82BL, 0x2AAC18030B01ABABL, 0x12B50C6EC4F31355L, 0xBBBCE0026F348956L, + 0x1DEE7A4AD4B81EEFL, 0x92C7CCD0B1EDA889L, 0x17F1FB6F10934BF2L, 0xDBD30A408E57BA07L, + 0x1327FC58DA0F6FF5L, 0x7CA8D50071DFC806L, 0x1EA6608E29B24CBBL, 0xFAA7BB33E9660CD6L, + 0x18851A0B548EA3C9L, 0x9552FC298784D711L, 0x139DAE6F76D88307L, 0xAAA8C9BAD2D0AC0EL, + 0x1F62B0B257C0D1A5L, 0xDDDADC5E1E1AACE3L, 0x191BC08EAC9A4151L, 0x7E48B04B4B488A4FL, + 0x141633A556E1CDDAL, 0xCB6D59D5D5D3A1D9L, 0x1011C2EAABE7D7E2L, 0x3C577B1177DC817BL, + 0x19B604AAACA62636L, 0xC6F25E825960CF2AL, 0x14919D5556EB51C5L, 0x6BF518684780A5BBL, + 0x10747DDDDF22A7D1L, 0x232A79ED06008496L, 0x1A53FC9631D10C81L, 0xD1DD8FE1A3340756L, + 0x150FFD44F4A73D34L, 0xA7E4731AE8F66C45L, 0x10D9976A5D52975DL, 0x531D28E253F8569EL, + 0x1AF5BF109550F22EL, 0xEB61DB03B98D5762L, 0x159165A6DDDA5B58L, 0xBC4E48CFC7A445E8L, + 0x11411E1F17E1E2ADL, 0x6371D3D96C836B20L, 0x1B9B6364F3030448L, 0x9F1C8628AD9F11CDL, + 0x1615E91D8F359D06L, 0xE5B06B53BE18DB0BL, 0x11AB20E472914A6BL, 0xEAF3890FCB4715A2L, + 0x1C45016D841BAA46L, 0x44B8DB4C7871BC37L, 0x169D9ABE03495505L, 0x03C715D6C6C1635FL, + 0x1217AEFE69077737L, 0x3638DE456BCDE919L, 0x1CF2B1970E725858L, 0x56C163A2461641C1L, + 0x17288E1271F51379L, 0xDF011C81D1AB67CEL, 0x1286D80EC190DC61L, 0x7F3416CE4155ECA5L, + 0x1DA48CE468E7C702L, 0x6520247D3556476EL, 0x17B6D71D20B96C01L, 0xEA801D30F7783925L, + 0x12F8AC174D612334L, 0xBB99B0F3F92CFA84L, 0x1E5AACF215683854L, 0x5F5C4E532847F739L, + 0x18488A5B44536043L, 0x7F7D0B75B9D32C2EL, 0x136D3B7C36A919CFL, 0x9930D5F7C7DC2358L, + 0x1F152BF9F10E8FB2L, 0x8EB4898C72F9D226L, 0x18DDBCC7F40BA628L, 0x722A07A38F2E41B8L, + 0x13E497065CD61E86L, 0xC1BB394FA5BE9AFAL, 0x1FD424D6FAF030D7L, 0x9C5EC2190930F7F6L, + 0x197683DF2F268D79L, 0x49E56814075A5FF8L, 0x145ECFE5BF520AC7L, 0x6E51201005E1E660L, + 0x104BD984990E6F05L, 0xF1DA800CD181851AL, 0x1A12F5A0F4E3E4D6L, 0x4FC400148268D4F5L, + 0x14DBF7B3F71CB711L, 0xD96999AA01ED772BL, 0x10AFF95CC5B09274L, 0xADEE1488018AC5BCL, + 0x1AB328946F80EA54L, 0x497CEDA668DE092CL, 0x155C2076BF9A5510L, 0x3ACA57B853E4D424L, + 0x1116805EFFAEAA73L, 0x623B7960431D7683L, 0x1B5733CB32B110B8L, 0x9D2BF566D1C8BD9EL, + 0x15DF5CA28EF40D60L, 0x7DBCC452416D647FL, 0x117F7D4ED8C33DE6L, 0xCAFD69DB678AB6CCL, + 0x1BFF2EE48E052FD7L, 0xAB2F0FC572778ADFL, 0x1665BF1D3E6A8CACL, 0x88F273045B92D580L, + 0x11EAFF4A98553D56L, 0xD3F528D049424466L, 0x1CAB3210F3BB9557L, 0xB988414D4203A0A3L, + 0x16EF5B40C2FC7779L, 0x6139CDD76802E6E9L, 0x125915CD68C9F92DL, 0xE761717920025254L, + 0x1D5B561574765B7CL, 0xA568B58E999D5086L, 0x177C44DDF6C515FDL, 0x5120913EE14AA6D2L, + 0x12C9D0B1923744CAL, 0xA74D40FF1AA21F0EL, 0x1E0FB44F50586E11L, 0x0BAECE64F769CB4AL, + 0x180C903F7379F1A7L, 0x3C8BD850C5EE3C3BL, 0x133D4032C2C7F485L, 0xCA0979DA37F1C9C9L, + 0x1EC866B79E0CBA6FL, 0xA9A8C2F6BFE942DBL, 0x18A0522C7E709526L, 0x2153CF2BCCBA9BE3L, + 0x13B374F06526DDB8L, 0x1AA9728970954982L, 0x1F8587E7083E2F8CL, 0xF775840F1A88759DL, + 0x19379FEC0698260AL, 0x5F9136727BA05E17L, 0x142C7FF0054684D5L, 0x1940F85B9619E4DFL, + 0x1023998CD1053710L, 0xE100C6AFAB47EA4CL, 0x19D28F47B4D524E7L, 0xCE67A44C453FDD47L, + 0x14A8729FC3DDB71FL, 0xD852E9D69DCCB106L, 0x1086C219697E2C19L, 0x79DBEE454B0A2738L, + 0x1A71368F0F30468FL, 0x295FE3A211A9D859L, 0x15275ED8D8F36BA5L, 0xBAB31C81A7BB137AL, + 0x10EC4BE0AD8F8951L, 0x6228E39AEC95A92FL, 0x1B13AC9AAF4C0EE8L, 0x9D0E38F7E0EF7517L, + 0x15A956E225D67253L, 0xB0D82D931A592A79L, 0x11544581B7DEC1DCL, 0x8D79BE0F4847552EL, + 0x1BBA08CF8C979C94L, 0x158F967EDA0BBB7CL, 0x162E6D72D6DFB076L, 0x77A611FF14D62F97L, + 0x11BEBDF578B2F391L, 0xF951A7FF43DE8C79L, 0x1C6463225AB7EC1CL, 0xC21C3FFED2FDAD8EL, + 0x16B6B5B5155FF017L, 0x01B0333242648AD8L, 0x122BC490DDE659ACL, 0x0159C28E9B83A246L, + 0x1D12D41AFCA3C2ACL, 0xCEF604175F3903A3L, 0x17424348CA1C9BBDL, 0x725E69AC4C2D9C83L, + 0x129B69070816E2FDL, 0xF5185489D68AE39CL, 0x1DC574D80CF16B2FL, 0xEE8D540FBDAB05C6L, + 0x17D12A4670C1228CL, 0xBED77672FE226B05L, 0x130DBB6B8D674ED6L, 0xFF12C528CB4EBC04L, + 0x1E7C5F127BD87E24L, 0xCB513B74787DF9A0L, 0x18637F41FCAD31B7L, 0x090DC929F9FE614DL, + 0x1382CC34CA2427C5L, 0xA0D7D42194CB810AL, 0x1F37AD21436D0C6FL, 0x67BFB9CF5478CE77L, + 0x18F9574DCF8A7059L, 0x1FCC94A5DD2D71F9L, 0x13FAAC3E3FA1F37AL, 0x7FD6DD517DBDF4C7L, + 0x1FF779FD329CB8C3L, 0xFFBE2EE8C92FEE0BL, 0x1992C7FDC216FA36L, 0x6631BF20A0F324D6L, + 0x14756CCB01ABFB5EL, 0xB827CC1A1A5C1D78L, 0x105DF0A267BCC918L, 0x935309AE7B7CE460L, + 0x1A2FE76A3F9474F4L, 0x1EEB42B0C594A099L, 0x14F31F8832DD2A5CL, 0xE58902270476E6E1L, + 0x10C27FA028B0EEB0L, 0xB7A0CE859D2BEBE7L, 0x1AD0CC33744E4AB4L, 0x59014A6F61DFDFD8L, + 0x1573D68F903EA229L, 0xE0CDD525E7E64CADL, 0x11297872D9CBB4EEL, 0x4D7177518651D6F1L, + 0x1B758D848FAC54B0L, 0x7BE8BEE8D6E957E8L, 0x15F7A46A0C89DD59L, 0xFCBA3253DF211320L, + 0x1192E9EE706E4AAEL, 0x63C8284318E74280L, 0x1C1E43171A4A1117L, 0x060D0D3827D86A66L, + 0x167E9C127B6E7412L, 0x6B3DA42CECAD21EBL, 0x11FEE341FC585CDBL, 0x88FE1CF0BD574E56L, + 0x1CCB0536608D615FL, 0x419694B462254A23L, 0x1708D0F84D3DE77FL, 0x67ABAA29E81DD4E9L, + 0x126D73F9D764B932L, 0xB95621BB2017DD87L, 0x1D7BECC2F23AC1EAL, 0xC223692B668C95A5L, + 0x179657025B6234BBL, 0xCE82BA891ED6DE1DL, 0x12DEAC01E2B4F6FCL, 0xA53562074BDF1818L, + 0x1E3113363787F194L, 0x3B889CD87964F359L, 0x18274291C6065ADCL, 0xFC6D4A46C783F5E1L, + 0x13529BA7D19EAF17L, 0x30576E9F06032B1AL, 0x1EEA92A61C311825L, 0x1A257DCB3CD1DE90L, + 0x18BBA884E35A79B7L, 0x481DFE3C30A7E540L, 0x13C9539D82AEC7C5L, 0xD34B31C9C0865100L, + 0x1FA885C8D117A609L, 0x5211E942CDA3B4CDL, 0x19539E3A40DFB807L, 0x74DB21023E1C90A4L, + 0x1442E4FB67196005L, 0xF715B401CB4A0D50L, 0x103583FC527AB337L, 0xF8DE299B09080AA7L, + 0x19EF3993B72AB859L, 0x8E304291A80CDDD7L, 0x14BF6142F8EEF9E1L, 0x3E8D020E200A4B13L, + 0x10991A9BFA58C7E7L, 0x653D9B3E80083C0FL, 0x1A8E90F9908E0CA5L, 0x6EC8F864000D2CE4L, + 0x153EDA614071A3B7L, 0x8BD3F9E999A423EAL, 0x10FF151A99F482F9L, 0x3CA994BAE1501CBBL, + 0x1B31BB5DC320D18EL, 0xC775BAC49BB3612BL, 0x15C162B168E70E0BL, 0xD2C4956A16291A89L, + 0x11678227871F3E6FL, 0xDBD0778811BA7BA1L, 0x1BD8D03F3E9863E6L, 0x2C80BF401C5D929BL, + 0x16470CFF6546B651L, 0xBD33CC3349E47549L, 0x11D270CC51055EA7L, 0xCA8FD68F6E505DD4L, + 0x1C83E7AD4E6EFDD9L, 0x4419574BE3B3C953L, 0x16CFEC8AA52597E1L, 0x0347790982F63AA9L, + 0x123FF06EEA847980L, 0xCF6C60D468C4FBBAL, 0x1D331A4B10D3F59AL, 0xE57A34870E07F92AL, + 0x175C1508DA432AE2L, 0x512E906C0B399422L, 0x12B010D3E1CF5581L, 0xDA8BA6BCD5C7A9B5L, + 0x1DE6815302E5559CL, 0x90DF712E22D90F87L, 0x17EB9AA8CF1DDE16L, 0xDA4C5A8B4F140C6CL, + 0x1322E220A5B17E78L, 0xAEA37BA2A5A9A38AL, 0x1E9E369AA2B59727L, 0x7DD25F6AA2A905A9L, + 0x187E92154EF7AC1FL, 0x97DB7F888220D154L, 0x139874DDD8C6234CL, 0x797C6606CE80A777L, + 0x1F5A549627A36BADL, 0x8F2D700AE4010BF1L, 0x191510781FB5EFBEL, 0x0C2459A25000D65AL, + 0x1410D9F9B2F7F2FEL, 0x701D1481D99A4515L, 0x100D7B2E28C65BFEL, 0xC017439B147B6A77L, + 0x19AF2B7D0E0A2CCAL, 0xCCF205C4ED9243F2L, 0x148C22CA71A1BD6FL, 0x0A5B37D0BE0E9CC2L, + 0x10701BD527B4978CL, 0x0848F973CB3EE3CEL, 0x1A4CF9550C5425ACL, 0xDA0E5BEC78649FB0L, + 0x150A6110D6A9B7BDL, 0x7B3EAFF060507FC0L, 0x10D51A73DEEE2C97L, 0x95CBBFF380406633L, + 0x1AEE90B964B04758L, 0xEFAC665266CD7052L, 0x158BA6FAB6F36C47L, 0x2623850EB8A459DBL, + 0x113C85955F29236CL, 0x1E82D0D893B6AE49L, 0x1B9408EEFEA838ACL, 0xFD9E1AF41F8AB075L, + 0x16100725988693BDL, 0x97B1AF29B2D559F7L, 0x11A66C1E139EDC97L, 0xAC8E25BAF5777B2CL, + 0x1C3D79C9B8FE2DBFL, 0x7A7D092B2258C513L, 0x169794A160CB57CCL, 0x61FDA0EF4EAD6A76L, + 0x1212DD4DE7091309L, 0xE7FE1A590BBDEEC5L, 0x1CEAFBAFD80E84DCL, 0xA6635D5B45FCB13AL, + 0x172262F3133ED0B0L, 0x851C4AAF6B308DC8L, 0x1281E8C275CBDA26L, 0xD0E36EF2BC26D7D4L, + 0x1D9CA79D894629D7L, 0xB49F17EAC6A48C86L, 0x17B08617A104EE46L, 0x2A18DFEF0550706BL, + 0x12F39E794D9D8B6BL, 0x54E0B3259DD9F389L, 0x1E5297287C2F4578L, 0x87CDEB6F62F65274L, + 0x18421286C9BF6AC6L, 0xD30B22BF825EA85DL, 0x13680ED23AFF889FL, 0x0F3C1BCC684BB9E4L, + 0x1F0CE4839198DA98L, 0x18602C7A4079296DL, 0x18D71D360E13E213L, 0x46B356C833942124L, + 0x13DF4A91A4DCB4DCL, 0x388F78A029434DB6L, 0x1FCBAA82A1612160L, 0x5A7F2766A86BAF8AL, + 0x196FBB9BB44DB44DL, 0x153285EBB9EFBFA2L, 0x145962E2F6A4903DL, 0xAA8ED189618C994EL, + 0x1047824F2BB6D9CAL, 0xEED8A7A11AD6E10CL, 0x1A0C03B1DF8AF611L, 0x7E27729B5E249B45L, + 0x14D6695B193BF80DL, 0xFE85F549181D4904L, 0x10AB877C142FF9A4L, 0xCB9E5DD4134AA0D0L, + 0x1AAC0BF9B9E65C3AL, 0xDF63C9535211014DL, 0x15566FFAFB1EB02FL, 0x191CA10F74DA6771L, + 0x1111F32F2F4BC025L, 0xADB080D92A4852C1L, 0x1B4FEB7EB212CD09L, 0x15E7348EAA0D5134L, + 0x15D98932280F0A6DL, 0xAB1F5D3EEE710DC4L, 0x117AD428200C0857L, 0xBC1917658B8DA49DL, + 0x1BF7B9D9CCE00D59L, 0x2CF4F23C127C3A94L, 0x165FC7E170B33DE0L, 0xF0C3F4FCDB969543L, + 0x11E6398126F5CB1AL, 0x5A365D9716121103L, 0x1CA38F350B22DE90L, 0x9056FC24F01CE804L, + 0x16E93F5DA2824BA6L, 0xD9DF301D8CE3ECD0L, 0x125432B14ECEA2EBL, 0xE17F59B13D8323DAL, + 0x1D53844EE47DD179L, 0x68CBC2B52F38395CL, 0x177603725064A794L, 0x53D6355DBF602DE3L, + 0x12C4CF8EA6B6EC76L, 0xA9782AB165E68B1CL, 0x1E07B27DD78B13F1L, 0x0F26AAB56FD744FAL, + 0x18062864AC6F4327L, 0x3F52222ABFDF6A62L, 0x1338205089F29C1FL, 0x65DB4E88997F884EL, + 0x1EC033B40FEA9365L, 0x6FC54A7428CC0D4AL, 0x1899C2F673220F84L, 0x596AA1F68709A43BL, + 0x13AE3591F5B4D936L, 0xADEEE7F86C07B696L, 0x1F7D228322BAF524L, 0x497E3FF3E00C5756L, + 0x1930E868E89590E9L, 0xD464FFF64CD6AC45L, 0x14272053ED4473EEL, 0x4383FFF83D7889D1L, + 0x101F4D0FF1038FF1L, 0xCF9CCCC69793A174L, 0x19CBAE7FE805B31CL, 0x7F6147A425B90252L, + 0x14A2F1FFECD15C16L, 0xCC4DD2E9B7C7350FL, 0x10825B3323DAB012L, 0x3D0B0F215FD290D9L, + 0x1A6A2B85062AB350L, 0x61AB4B689950E7C1L, 0x1521BC6A6B555C40L, 0x4E22A2BA1440B967L, + 0x10E7C9EEBC4449CDL, 0x0B4EE894DD009453L, 0x1B0C764AC6D3A948L, 0x1217DA87C800ED51L, + 0x15A391D56BDC876CL, 0xDB46486CA000BDDAL, 0x114FA7DDEFE39F8AL, 0x490506BD4CCD64AFL, + 0x1BB2A62FE638FF43L, 0xA8080AC87AE23AB1L, 0x162884F31E93FF69L, 0x5339A239FBE82EF4L, + 0x11BA03F5B20FFF87L, 0x75C7B4FB2FECF25DL, 0x1C5CD322B67FFF3FL, 0x22D92191E647EA2EL, + 0x16B0A8E891FFFF65L, 0xB57A8141850654F2L, 0x1226ED86DB3332B7L, 0xC4620101373843F5L, + 0x1D0B15A491EB8459L, 0x3A366801F1F39FEEL, 0x173C115074BC69E0L, 0xFB5EB99B27F6198BL, + 0x129674405D6387E7L, 0x2F7EFAE2865E7AD6L, 0x1DBD86CD6238D971L, 0xE597F7D0D6FD9156L, + 0x17CAD23DE82D7AC1L, 0x8479930D78CADAABL, 0x1308A831868AC89AL, 0xD06142712D6F1556L, + 0x1E74404F3DAADA91L, 0x4D686A4EAF182222L, 0x185D003F6488AEDAL, 0xA453883EF279B4E8L, + 0x137D99CC506D58AEL, 0xE9DC6CFF28615D87L, 0x1F2F5C7A1A488DE4L, 0xA960AE650D6895A4L, + 0x18F2B061AEA07183L, 0xBAB3BEB73DED4483L, 0x13F559E7BEE6C136L, 0x2EF6322C318A9D36L + }; + } + + private static long[] initPow5Split() { + return new long[]{ + 0x1000000000000000L, 0x0000000000000000L, 0x1400000000000000L, 0x0000000000000000L, + 0x1900000000000000L, 0x0000000000000000L, 0x1F40000000000000L, 0x0000000000000000L, + 0x1388000000000000L, 0x0000000000000000L, 0x186A000000000000L, 0x0000000000000000L, + 0x1E84800000000000L, 0x0000000000000000L, 0x1312D00000000000L, 0x0000000000000000L, + 0x17D7840000000000L, 0x0000000000000000L, 0x1DCD650000000000L, 0x0000000000000000L, + 0x12A05F2000000000L, 0x0000000000000000L, 0x174876E800000000L, 0x0000000000000000L, + 0x1D1A94A200000000L, 0x0000000000000000L, 0x12309CE540000000L, 0x0000000000000000L, + 0x16BCC41E90000000L, 0x0000000000000000L, 0x1C6BF52634000000L, 0x0000000000000000L, + 0x11C37937E0800000L, 0x0000000000000000L, 0x16345785D8A00000L, 0x0000000000000000L, + 0x1BC16D674EC80000L, 0x0000000000000000L, 0x1158E460913D0000L, 0x0000000000000000L, + 0x15AF1D78B58C4000L, 0x0000000000000000L, 0x1B1AE4D6E2EF5000L, 0x0000000000000000L, + 0x10F0CF064DD59200L, 0x0000000000000000L, 0x152D02C7E14AF680L, 0x0000000000000000L, + 0x1A784379D99DB420L, 0x0000000000000000L, 0x108B2A2C28029094L, 0x0000000000000000L, + 0x14ADF4B7320334B9L, 0x0000000000000000L, 0x19D971E4FE8401E7L, 0x4000000000000000L, + 0x1027E72F1F128130L, 0x8800000000000000L, 0x1431E0FAE6D7217CL, 0xAA00000000000000L, + 0x193E5939A08CE9DBL, 0xD480000000000000L, 0x1F8DEF8808B02452L, 0xC9A0000000000000L, + 0x13B8B5B5056E16B3L, 0xBE04000000000000L, 0x18A6E32246C99C60L, 0xAD85000000000000L, + 0x1ED09BEAD87C0378L, 0xD8E6400000000000L, 0x13426172C74D822BL, 0x878FE80000000000L, + 0x1812F9CF7920E2B6L, 0x6973E20000000000L, 0x1E17B84357691B64L, 0x03D0DA8000000000L, + 0x12CED32A16A1B11EL, 0x8262889000000000L, 0x178287F49C4A1D66L, 0x22FB2AB400000000L, + 0x1D6329F1C35CA4BFL, 0xABB9F56100000000L, 0x125DFA371A19E6F7L, 0xCB54395CA0000000L, + 0x16F578C4E0A060B5L, 0xBE2947B3C8000000L, 0x1CB2D6F618C878E3L, 0x2DB399A0BA000000L, + 0x11EFC659CF7D4B8DL, 0xFC90400474400000L, 0x166BB7F0435C9E71L, 0x7BB4500591500000L, + 0x1C06A5EC5433C60DL, 0xDAA16406F5A40000L, 0x118427B3B4A05BC8L, 0xA8A4DE8459868000L, + 0x15E531A0A1C872BAL, 0xD2CE16256FE82000L, 0x1B5E7E08CA3A8F69L, 0x87819BAECBE22800L, + 0x111B0EC57E6499A1L, 0xF4B1014D3F6D5900L, 0x1561D276DDFDC00AL, 0x71DD41A08F48AF40L, + 0x1ABA4714957D300DL, 0x0E549208B31ADB10L, 0x10B46C6CDD6E3E08L, 0x28F4DB456FF0C8EAL, + 0x14E1878814C9CD8AL, 0x33321216CBECFB25L, 0x1A19E96A19FC40ECL, 0xBFFE969C7EE839EEL, + 0x105031E2503DA893L, 0xF7FF1E21CF512435L, 0x14643E5AE44D12B8L, 0xF5FEE5AA43256D42L, + 0x197D4DF19D605767L, 0x337E9F14D3EEC893L, 0x1FDCA16E04B86D41L, 0x005E46DA08EA7AB7L, + 0x13E9E4E4C2F34448L, 0xA03AEC4845928CB3L, 0x18E45E1DF3B0155AL, 0xC849A75A56F72FDFL, + 0x1F1D75A5709C1AB1L, 0x7A5C1130ECB4FBD7L, 0x13726987666190AEL, 0xEC798ABE93F11D66L, + 0x184F03E93FF9F4DAL, 0xA797ED6E38ED64C0L, 0x1E62C4E38FF87211L, 0x517DE8C9C728BDF0L, + 0x12FDBB0E39FB474AL, 0xD2EEB17E1C7976B6L, 0x17BD29D1C87A191DL, 0x87AA5DDDA397D463L, + 0x1DAC74463A989F64L, 0xE994F5550C7DC97CL, 0x128BC8ABE49F639FL, 0x11FD195527CE9DEEL, + 0x172EBAD6DDC73C86L, 0xD67C5FAA71C24569L, 0x1CFA698C95390BA8L, 0x8C1B77950E32D6C3L, + 0x121C81F7DD43A749L, 0x57912ABD28DFC63AL, 0x16A3A275D494911BL, 0xAD75756C7317B7C9L, + 0x1C4C8B1349B9B562L, 0x98D2D2C78FDDA5BBL, 0x11AFD6EC0E14115DL, 0x9F83C3BCB9EA8795L, + 0x161BCCA7119915B5L, 0x0764B4ABE865297AL, 0x1BA2BFD0D5FF5B22L, 0x493DE1D6E27E73D8L, + 0x1145B7E285BF98F5L, 0x6DC6AD264D8F0867L, 0x159725DB272F7F32L, 0xC938586FE0F2CA81L, + 0x1AFCEF51F0FB5EFFL, 0x7B866E8BD92F7D21L, 0x10DE1593369D1B5FL, 0xAD34051767BDAE35L, + 0x15159AF804446237L, 0x9881065D41AD19C2L, 0x1A5B01B605557AC5L, 0x7EA147F492186033L, + 0x1078E111C3556CBBL, 0x6F24CCF8DB4F3C20L, 0x14971956342AC7EAL, 0x4AEE003712230B28L, + 0x19BCDFABC13579E4L, 0xDDA98044D6ABCDF1L, 0x10160BCB58C16C2FL, 0x0A89F02B062B60B7L, + 0x141B8EBE2EF1C73AL, 0xCD2C6C35C7B638E5L, 0x1922726DBAAE3909L, 0x8077874339A3C71EL, + 0x1F6B0F092959C74BL, 0xE0956914080CB8E5L, 0x13A2E965B9D81C8FL, 0x6C5D61AC8507F38FL, + 0x188BA3BF284E23B3L, 0x4774BA17A649F073L, 0x1EAE8CAEF261ACA0L, 0x1951E89D8FDC6C90L, + 0x132D17ED577D0BE4L, 0x0FD3316279E9C3DAL, 0x17F85DE8AD5C4EDDL, 0x13C7FDBB186434D0L, + 0x1DF67562D8B36294L, 0x58B9FD29DE7D4204L, 0x12BA095DC7701D9CL, 0xB7743E3A2B0E4943L, + 0x17688BB5394C2503L, 0xE5514DC8B5D1DB93L, 0x1D42AEA2879F2E44L, 0xDEA5A13AE3465278L, + 0x1249AD2594C37CEBL, 0x0B2784C4CE0BF38BL, 0x16DC186EF9F45C25L, 0xCDF165F6018EF06EL, + 0x1C931E8AB871732FL, 0x416DBF7381F2AC89L, 0x11DBF316B346E7FDL, 0x88E497A83137ABD6L, + 0x1652EFDC6018A1FCL, 0xEB1DBD923D8596CBL, 0x1BE7ABD3781ECA7CL, 0x25E52CF6CCE6FC7EL, + 0x1170CB642B133E8DL, 0x97AF3C1A40105DCFL, 0x15CCFE3D35D80E30L, 0xFD9B0B20D0147543L, + 0x1B403DCC834E11BDL, 0x3D01CDE904199293L, 0x1108269FD210CB16L, 0x462120B1A28FFB9CL, + 0x154A3047C694FDDBL, 0xD7A968DE0B33FA83L, 0x1A9CBC59B83A3D52L, 0xCD93C3158E00F924L, + 0x10A1F5B813246653L, 0xC07C59ED78C09BB7L, 0x14CA732617ED7FE8L, 0xB09B7068D6F0C2A4L, + 0x19FD0FEF9DE8DFE2L, 0xDCC24C830CACF34DL, 0x103E29F5C2B18BEDL, 0xC9F96FD1E7EC1810L, + 0x144DB473335DEEE9L, 0x3C77CBC661E71E14L, 0x1961219000356AA3L, 0x8B95BEB7FA60E599L, + 0x1FB969F40042C54CL, 0x6E7B2E65F8F91EFFL, 0x13D3E2388029BB4FL, 0xC50CFCFFBB9BB360L, + 0x18C8DAC6A0342A23L, 0xB6503C3FAA82A038L, 0x1EFB1178484134ACL, 0xA3E44B4F95234845L, + 0x135CEAEB2D28C0EBL, 0xE66EAF11BD360D2CL, 0x183425A5F872F126L, 0xE00A5AD62C839076L, + 0x1E412F0F768FAD70L, 0x980CF18BB7A47494L, 0x12E8BD69AA19CC66L, 0x5F0816F752C6C8DDL, + 0x17A2ECC414A03F7FL, 0xF6CA1CB527787B14L, 0x1D8BA7F519C84F5FL, 0xF47CA3E2715699D8L, + 0x127748F9301D319BL, 0xF8CDE66D86D62027L, 0x17151B377C247E02L, 0xF7016008E88BA831L, + 0x1CDA62055B2D9D83L, 0xB4C1B80B22AE923DL, 0x12087D4358FC8272L, 0x50F91306F5AD1B66L, + 0x168A9C942F3BA30EL, 0xE53757C8B3186240L, 0x1C2D43B93B0A8BD2L, 0x9E852DBADFDE7AD0L, + 0x119C4A53C4E69763L, 0xA3133C94CBEB0CC2L, 0x16035CE8B6203D3CL, 0x8BD80BB9FEE5CFF2L, + 0x1B843422E3A84C8BL, 0xAECE0EA87E9F43EFL, 0x1132A095CE492FD7L, 0x4D40C9294F238A76L, + 0x157F48BB41DB7BCDL, 0x2090FB73A2EC6D13L, 0x1ADF1AEA12525AC0L, 0x68B53A508BA78857L, + 0x10CB70D24B7378B8L, 0x417144725748B537L, 0x14FE4D06DE5056E6L, 0x51CD958EED1AE284L, + 0x1A3DE04895E46C9FL, 0xE640FAF2A8619B25L, 0x1066AC2D5DAEC3E3L, 0xEFE89CD7A93D00F8L, + 0x14805738B51A74DCL, 0xEBE2C40D938C4135L, 0x19A06D06E2611214L, 0x26DB7510F86F5182L, + 0x100444244D7CAB4CL, 0x9849292A9B4592F2L, 0x1405552D60DBD61FL, 0xBE5B73754216F7AEL, + 0x1906AA78B912CBA7L, 0xADF25052929CB599L, 0x1F485516E7577E91L, 0x996EE4673743E300L, + 0x138D352E5096AF1AL, 0xFFE54EC0828A6DE0L, 0x18708279E4BC5AE1L, 0xBFDEA270A32D0958L, + 0x1E8CA3185DEB719AL, 0x2FD64B0CCBF84BAEL, 0x1317E5EF3AB32700L, 0x5DE5EEE7FF7B2F4DL, + 0x17DDDF6B095FF0C0L, 0x755F6AA1FF59FB20L, 0x1DD55745CBB7ECF0L, 0x92B7454A7F3079E8L, + 0x12A5568B9F52F416L, 0x5BB28B4E8F7E4C31L, 0x174EAC2E8727B11BL, 0xF29F2E22335DDF3DL, + 0x1D22573A28F19D62L, 0xEF46F9AAC035570CL, 0x123576845997025DL, 0xD58C5C0AB8215668L, + 0x16C2D4256FFCC2F5L, 0x4AEF730D6629AC02L, 0x1C73892ECBFBF3B2L, 0x9DAB4FD0BFB41702L, + 0x11C835BD3F7D784FL, 0xA28B11E277D08E61L, 0x163A432C8F5CD663L, 0x8B2DD65B15C4B1FAL, + 0x1BC8D3F7B3340BFCL, 0x6DF94BF1DB35DE78L, 0x115D847AD000877DL, 0xC4BBCF772901AB0BL, + 0x15B4E5998400A95DL, 0x35EAC354F34215CEL, 0x1B221EFFE500D3B4L, 0x8365742A30129B41L, + 0x10F5535FEF208450L, 0xD21F689A5E0BA109L, 0x1532A837EAE8A565L, 0x06A742C0F58E894BL, + 0x1A7F5245E5A2CEBEL, 0x4851137132F22B9EL, 0x108F936BAF85C136L, 0xED32AC26BFD75B43L, + 0x14B378469B673184L, 0xA87F57306FCD3213L, 0x19E056584240FDE5L, 0xD29F2CFC8BC07E98L, + 0x102C35F729689EAFL, 0xA3A37C1DD7584F1FL, 0x14374374F3C2C65BL, 0x8C8C5B254D2E62E7L, + 0x1945145230B377F2L, 0x6FAF71EEA079FBA0L, 0x1F965966BCE055EFL, 0x0B9B4E6A48987A88L, + 0x13BDF7E0360C35B5L, 0x674111026D5F4C95L, 0x18AD75D8438F4322L, 0xC111554308B71FBBL, + 0x1ED8D34E547313EBL, 0x7155AA93CAE4E7A9L, 0x13478410F4C7EC73L, 0x26D58A9C5ECF10CAL, + 0x1819651531F9E78FL, 0xF08AED437682D4FCL, 0x1E1FBE5A7E786173L, 0xECADA89454238A3BL, + 0x12D3D6F88F0B3CE8L, 0x73EC895CB4963665L, 0x1788CCB6B2CE0C22L, 0x90E7ABB3E1BBC3FEL, + 0x1D6AFFE45F818F2BL, 0x352196A0DA2AB4FEL, 0x1262DFEEBBB0F97BL, 0x0134FE24885AB11FL, + 0x16FB97EA6A9D37D9L, 0xC1823DADAA715D66L, 0x1CBA7DE5054485D0L, 0x31E2CD19150DB4C0L, + 0x11F48EAF234AD3A2L, 0x1F2DC02FAD2890F8L, 0x1671B25AEC1D888AL, 0xA6F9303B9872B536L, + 0x1C0E1EF1A724EAADL, 0x50B77C4A7E8F6283L, 0x1188D357087712ACL, 0x5272ADAE8F199D92L, + 0x15EB082CCA94D757L, 0x670F591A32E004F7L, 0x1B65CA37FD3A0D2DL, 0x40D32F60BF980634L, + 0x111F9E62FE44483CL, 0x4883FD9C77BF03E1L, 0x156785FBBDD55A4BL, 0x5AA4FD0395AEC4D9L, + 0x1AC1677AAD4AB0DEL, 0x314E3C447B1A760FL, 0x10B8E0ACAC4EAE8AL, 0xDED0E5AACCF089CAL, + 0x14E718D7D7625A2DL, 0x96851F15802CAC3CL, 0x1A20DF0DCD3AF0B8L, 0xFC2666DAE037D74BL, + 0x10548B68A044D673L, 0x9D980048CC22E68FL, 0x1469AE42C8560C10L, 0x84FE005AFF2BA033L, + 0x198419D37A6B8F14L, 0xA63D8071BEF6883FL, 0x1FE52048590672D9L, 0xCFCCE08E2EB42A4FL, + 0x13EF342D37A407C8L, 0x21E00C58DD309A71L, 0x18EB0138858D09BAL, 0x2A580F6F147CC10EL, + 0x1F25C186A6F04C28L, 0xB4EE134AD99BF151L, 0x137798F428562F99L, 0x7114CC0EC80176D3L, + 0x18557F31326BBB7FL, 0xCD59FF127A01D487L, 0x1E6ADEFD7F06AA5FL, 0xC0B07ED7188249A9L, + 0x1302CB5E6F642A7BL, 0xD86E4F466F516E0AL, 0x17C37E360B3D351AL, 0xCE89E3180B25C98CL, + 0x1DB45DC38E0C8261L, 0x822C5BDE0DEF3BEFL, 0x1290BA9A38C7D17CL, 0xF15BB96AC8B58576L, + 0x1734E940C6F9C5DCL, 0x2DB2A7C57AE2E6D3L, 0x1D022390F8B83753L, 0x391F51B6D99BA087L, + 0x1221563A9B732294L, 0x03B3931248014455L, 0x16A9ABC9424FEB39L, 0x04A077D6DA01956AL, + 0x1C5416BB92E3E607L, 0x45C895CC9081FAC4L, 0x11B48E353BCE6FC4L, 0x8B9D5D9FDA513CBBL, + 0x1621B1C28AC20BB5L, 0xAE84B507D0E58BE9L, 0x1BAA1E332D728EA3L, 0x1A25E249C51EEEE4L, + 0x114A52DFFC679925L, 0xF057AD6E1B33554EL, 0x159CE797FB817F6FL, 0x6C6D98C9A2002AA2L, + 0x1B04217DFA61DF4BL, 0x4788FEFC0A80354AL, 0x10E294EEBC7D2B8FL, 0x0CB59F5D8690214FL, + 0x151B3A2A6B9C7672L, 0xCFE30734E83429A2L, 0x1A6208B50683940FL, 0x83DBC9022241340BL, + 0x107D457124123C89L, 0xB2695DA15568C087L, 0x149C96CD6D16CBACL, 0x1F03B509AAC2F0A8L, + 0x19C3BC80C85C7E97L, 0x26C4A24C1573ACD2L, 0x101A55D07D39CF1EL, 0x783AE56F8D684C04L, + 0x1420EB449C8842E6L, 0x16499ECB70C25F04L, 0x19292615C3AA539FL, 0x9BDC067E4CF2F6C5L, + 0x1F736F9B3494E887L, 0x82D3081DE02FB477L, 0x13A825C100DD1154L, 0xB1C3E512AC1DD0CAL, + 0x18922F31411455A9L, 0xDE34DE57572544FDL, 0x1EB6BAFD91596B14L, 0x55C215ED2CEE963CL, + 0x133234DE7AD7E2ECL, 0xB5994DB43C151DE6L, 0x17FEC216198DDBA7L, 0xE2FFA1214B1A655FL, + 0x1DFE729B9FF15291L, 0xDBBF89699DE0FEB7L, 0x12BF07A143F6D39BL, 0x2957B5E202AC9F32L, + 0x176EC98994F48881L, 0xF3ADA35A8357C6FFL, 0x1D4A7BEBFA31AAA2L, 0x70990C31242DB8BEL, + 0x124E8D737C5F0AA5L, 0x865FA79EB69C9377L, 0x16E230D05B76CD4EL, 0xE7F791866443B855L, + 0x1C9ABD04725480A2L, 0xA1F575E7FD54A66AL, 0x11E0B622C774D065L, 0xA53969B0FE54E802L, + 0x1658E3AB7952047FL, 0x0E87C41D3DEA2203L, 0x1BEF1C9657A6859EL, 0xD229B5248D64AA83L, + 0x117571DDF6C81383L, 0x435A1136D85EEA92L, 0x15D2CE55747A1864L, 0x143095848E76A537L, + 0x1B4781EAD1989E7DL, 0x193CBAE5B2144E84L, 0x110CB132C2FF630EL, 0x2FC5F4CF8F4CB113L, + 0x154FDD7F73BF3BD1L, 0xBBB77203731FDD57L, 0x1AA3D4DF50AF0AC6L, 0x2AA54E844FE7D4ADL, + 0x10A6650B926D66BBL, 0xDAA75112B1F0E4ECL, 0x14CFFE4E7708C06AL, 0xD15125575E6D1E27L, + 0x1A03FDE214CAF085L, 0x85A56EAD360865B1L, 0x10427EAD4CFED653L, 0x7387652C41C53F8FL, + 0x14531E58A03E8BE8L, 0x50693E7752368F72L, 0x1967E5EEC84E2EE2L, 0x64838E1526C4334FL, + 0x1FC1DF6A7A61BA9AL, 0xFDA4719A70754023L, 0x13D92BA28C7D14A0L, 0xDE86C70086494816L, + 0x18CF768B2F9C59C9L, 0x162878C0A7DB9A1BL, 0x1F03542DFB83703BL, 0x5BB296F0D1D280A2L, + 0x1362149CBD322625L, 0x194F9E5683239065L, 0x183A99C3EC7EAFAEL, 0x5FA385EC23EC747FL, + 0x1E494034E79E5B99L, 0xF78C67672CE7919EL, 0x12EDC82110C2F940L, 0x3AB7C0A07C10BB03L, + 0x17A93A2954F3B790L, 0x4965B0C89B14E9C4L, 0x1D9388B3AA30A574L, 0x5BBF1CFAC1DA2434L, + 0x127C35704A5E6768L, 0xB957721CB92856A1L, 0x171B42CC5CF60142L, 0xE7AD4EA3E7726C49L, + 0x1CE2137F74338193L, 0xA198A24CE14F075BL, 0x120D4C2FA8A030FCL, 0x44FF65700CD16499L, + 0x16909F3B92C83D3BL, 0x563F3ECC1005BDBFL, 0x1C34C70A777A4C8AL, 0x2BCF0E7F14072D2FL, + 0x11A0FC668AAC6FD6L, 0x5B61690F6C847C3EL, 0x16093B802D578BCBL, 0xF239C35347A59B4DL, + 0x1B8B8A6038AD6EBEL, 0xEEC83428198F0220L, 0x1137367C236C6537L, 0x553D20990FF96154L, + 0x1585041B2C477E85L, 0x2A8C68BF53F7B9A9L, 0x1AE64521F7595E26L, 0x752F82EF28F5A813L, + 0x10CFEB353A97DAD8L, 0x093DB1D57999890CL, 0x1503E602893DD18EL, 0x0B8D1E4AD7FFEB4FL, + 0x1A44DF832B8D45F1L, 0x8E7065DD8DFFE623L, 0x106B0BB1FB384BB6L, 0xF9063FAA78BFEFD6L, + 0x1485CE9E7A065EA4L, 0xB747CF9516EFEBCBL, 0x19A742461887F64DL, 0xE519C37A5CABE6BEL, + 0x1008896BCF54F9F0L, 0xAF301A2C79EB7037L, 0x140AABC6C32A386CL, 0xDAFC20B798664C44L, + 0x190D56B873F4C688L, 0x11BB28E57E7FDF55L, 0x1F50AC6690F1F82AL, 0x1629F31EDE1FD72BL, + 0x13926BC01A973B1AL, 0x4DDA37F34AD3E67BL, 0x187706B0213D09E0L, 0xE150C5F01D88E01AL, + 0x1E94C85C298C4C59L, 0x19A4F76C24EB1820L, 0x131CFD3999F7AFB7L, 0xB0071AA39712EF14L, + 0x17E43C8800759BA5L, 0x9C08E14C7CD7AAD9L, 0x1DDD4BAA0093028FL, 0x030B199F9C0D958FL, + 0x12AA4F4A405BE199L, 0x61E6F003C1887D7AL, 0x1754E31CD072D9FFL, 0xBA60AC04B1EA9CD8L, + 0x1D2A1BE4048F907FL, 0xA8F8D705DE65440EL, 0x123A516E82D9BA4FL, 0xC99B8663AAFF4A89L, + 0x16C8E5CA239028E3L, 0xBC0267FC95BF1D2BL, 0x1C7B1F3CAC74331CL, 0xAB0301FBBB2EE475L, + 0x11CCF385EBC89FF1L, 0xEAE1E13D54FD4ECAL, 0x1640306766BAC7EEL, 0x659A598CAA3CA27CL, + 0x1BD03C81406979E9L, 0xFF00EFEFD4CBCB1BL, 0x116225D0C841EC32L, 0x3F6095F5E4FF5EF1L, + 0x15BAAF44FA52673EL, 0xCF38BB735E3F36ADL, 0x1B295B1638E7010EL, 0x8306EA5035CF0458L, + 0x10F9D8EDE39060A9L, 0x11E4527221A162B7L, 0x15384F295C7478D3L, 0x565D670EAA09BB65L, + 0x1A8662F3B3919708L, 0x2BF4C0D2548C2A3EL, 0x1093FDD8503AFE65L, 0x1B78F88374D79A67L, + 0x14B8FD4E6449BDFEL, 0x625736A4520D8101L, 0x19E73CA1FD5C2D7DL, 0xFAED044D6690E141L, + 0x103085E53E599C6EL, 0xBCD422B0601A8CC9L, 0x143CA75E8DF0038AL, 0x6C092B5C78212FFBL, + 0x194BD136316C046DL, 0x070B763396297BF9L, 0x1F9EC583BDC70588L, 0x48CE53C07BB3DAF7L, + 0x13C33B72569C6375L, 0x2D80F4584D5068DBL, 0x18B40A4EEC437C52L, 0x78E1316E60A48311L, + }; + } + + /** + * Computes (m * (mul0:mul1)) >> shift, where mul0 is the high 64 bits and + * mul1 is the low 64 bits of a 128-bit multiplier. + *

      + * The 192-bit product is: high0 * 2^128 + (low0 + high1) * 2^64 + low1. + * We extract the upper 128 bits and right-shift by (shift - 64). + */ + // Requires shift in [65, 127] (guaranteed by the Ryu algorithm). + private static long mulShift64(long m, long mul0, long mul1, int shift) { + // Compat.multiplyHigh is signed; correct for unsigned by adding m + // when the other operand is negative (MSB set). + long high1 = Compat.multiplyHigh(m, mul1) + ((mul1 >> 63) & m); + long low0 = m * mul0; + long high0 = Compat.multiplyHigh(m, mul0) + ((mul0 >> 63) & m); + + long mid = low0 + high1; + long carry = Long.compareUnsigned(mid, low0) < 0 ? 1L : 0L; + long high = high0 + carry; + + int s = shift - 64; + return (high << (64 - s)) | (mid >>> s); + } + + // Requires p in [0, 63] because Java's << masks the shift count to 6 bits. + // Callers guarantee p < 63 via the surrounding `q < 63` guard. + private static boolean multipleOfPowerOf2(long value, int p) { + assert p >= 0 && p < 64; + return (value & ((1L << p) - 1)) == 0; + } + + private static boolean multipleOfPowerOf5(long value, int p) { + return pow5Factor(value) >= p; + } + + private static int pow5Factor(long value) { + int count = 0; + while (value > 0) { + if (value % 5 != 0) { + return count; + } + value /= 5; + count++; + } + return count; + } + + private static int pow5bits(int e) { + return ((e * 1217359) >>> 19) + 1; + } + + /** + * Converts a decoded IEEE 754 double to its shortest decimal representation. + * + * @param ieeeMantissa the 52-bit mantissa (without implicit leading 1) + * @param ieeeExponent the biased 11-bit exponent (0-2046; 0 means subnormal) + * @param e10Out single-element array; receives the decimal exponent e10 + * such that value = output * 10^e10 + * @return the decimal significand as a positive long (1-17 digits) + */ + static long d2d(long ieeeMantissa, int ieeeExponent, int[] e10Out) { + int e2; + long m2; + if (ieeeExponent == 0) { + // Subnormal: no implicit leading 1, exponent is 1 (not 0) + e2 = 1 - 1023 - 52; + m2 = ieeeMantissa; + } else { + e2 = ieeeExponent - 1023 - 52; + m2 = ieeeMantissa | (1L << 52); + } + + // Fast path for small integers: if the value is an exact integer in [1, 2^53), + // return it directly. This avoids the 4x-scaling issue in the main Ryu path. + if (e2 >= -52 && e2 <= 0) { + long mask = (1L << -e2) - 1; + if ((m2 & mask) == 0) { + long mantissa = m2 >>> -e2; + int exp = 0; + // Strip trailing decimal zeros + while (mantissa > 0) { + long q = mantissa / 10; + long r = mantissa - 10 * q; + if (r != 0) { + break; + } + mantissa = q; + exp++; + } + e10Out[0] = exp; + return mantissa; + } + } + + // Subtract 2 from e2 to account for the 4x scaling (mv = 4*m2 below). + // This ensures mv * 2^e2 == m2 * 2^(e2_original), so the decimal + // conversion produces the correct (non-4x) significand. + e2 -= 2; + + boolean acceptBounds = (m2 & 1) == 0; + + // Step 2: Determine the interval of valid decimal representations. + long mv = 4 * m2; + long mp = 4 * m2 + 2; + int mmShift = (ieeeMantissa != 0 || ieeeExponent <= 1) ? 1 : 0; + long mm = 4 * m2 - 1 - mmShift; + + // Step 3: Convert to a decimal power base using 128-bit arithmetic. + long vr, vp, vm; + int e10; + boolean vmIsTrailingZeros = false; + boolean vrIsTrailingZeros = false; + + if (e2 >= 0) { + int q = Math.max(0, ((e2 * 78913) >>> 18) - 1); + e10 = q; + int k = DOUBLE_POW5_INV_BITCOUNT + pow5bits(q) - 1; + int i = -e2 + q + k; + int idx = 2 * q; + long mul0 = DOUBLE_POW5_INV_SPLIT[idx]; + long mul1 = DOUBLE_POW5_INV_SPLIT[idx + 1]; + vr = mulShift64(mv, mul0, mul1, i); + vp = mulShift64(mp, mul0, mul1, i); + vm = mulShift64(mm, mul0, mul1, i); + + if (q <= 21) { + if (mv % 5 == 0) { + vrIsTrailingZeros = multipleOfPowerOf5(mv, q); + } else if (acceptBounds) { + vmIsTrailingZeros = multipleOfPowerOf5(mm, q); + } else { + if (multipleOfPowerOf5(mp, q)) { + vp--; + } + } + } + } else { + int q = Math.max(0, ((-e2 * 732923) >>> 20) - 1); + e10 = q + e2; + int i = -e2 - q; + int k = pow5bits(i) - DOUBLE_POW5_BITCOUNT; + int j = q - k; + int idx = 2 * i; + long mul0 = DOUBLE_POW5_SPLIT[idx]; + long mul1 = DOUBLE_POW5_SPLIT[idx + 1]; + vr = mulShift64(mv, mul0, mul1, j); + vp = mulShift64(mp, mul0, mul1, j); + vm = mulShift64(mm, mul0, mul1, j); + + if (q <= 1) { + vrIsTrailingZeros = true; + if (acceptBounds) { + vmIsTrailingZeros = mmShift == 1; + } else { + vp--; + } + } else if (q < 63) { + vrIsTrailingZeros = multipleOfPowerOf2(mv, q); + } + } + + // Step 4: Find the shortest decimal representation in the interval. + int removed = 0; + int lastRemovedDigit = 0; + + if (vmIsTrailingZeros || vrIsTrailingZeros) { + // General case with trailing zeros handling + for (; ; ) { + long vpDiv10 = vp / 10; + long vmDiv10 = vm / 10; + if (vpDiv10 <= vmDiv10) { + break; + } + long vrDiv10 = vr / 10; + boolean vmMod10IsZero = vm - 10 * vmDiv10 == 0; + vmIsTrailingZeros &= vmMod10IsZero; + vrIsTrailingZeros &= lastRemovedDigit == 0; + lastRemovedDigit = (int) (vr - 10 * vrDiv10); + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + + if (vmIsTrailingZeros) { + for (; ; ) { + long vmDiv10 = vm / 10; + if (vm - 10 * vmDiv10 != 0) { + break; + } + long vpDiv10 = vp / 10; + long vrDiv10 = vr / 10; + vrIsTrailingZeros &= lastRemovedDigit == 0; + lastRemovedDigit = (int) (vr - 10 * vrDiv10); + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + } + + if (vrIsTrailingZeros && lastRemovedDigit == 5 && (vr & 1) == 0) { + // Round even + lastRemovedDigit = 4; + } + + long output = vr; + if ((vr == vm && (!acceptBounds || !vmIsTrailingZeros)) || lastRemovedDigit >= 5) { + output++; + } + e10Out[0] = e10 + removed; + return output; + } else { + // Optimized loop when no trailing zeros + boolean roundUp = false; + long vpDiv100 = vp / 100; + long vmDiv100 = vm / 100; + if (vpDiv100 > vmDiv100) { + long vrDiv100 = vr / 100; + long vrMod100 = vr - 100 * vrDiv100; + roundUp = vrMod100 >= 50; + vr = vrDiv100; + vp = vpDiv100; + vm = vmDiv100; + removed += 2; + } + + for (; ; ) { + long vpDiv10 = vp / 10; + long vmDiv10 = vm / 10; + if (vpDiv10 <= vmDiv10) { + break; + } + long vrDiv10 = vr / 10; + roundUp = (vr - 10 * vrDiv10) >= 5; + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + + long output = vr + (vr == vm || roundUp ? 1 : 0); + e10Out[0] = e10 + removed; + return output; + } + } + + /** + * Returns the number of decimal digits in a value known to fit in 17 digits. + */ + static int decimalLength17(long v) { + if (v >= 10_000_000_000_000_000L) return 17; + if (v >= 1_000_000_000_000_000L) return 16; + if (v >= 100_000_000_000_000L) return 15; + if (v >= 10_000_000_000_000L) return 14; + if (v >= 1_000_000_000_000L) return 13; + if (v >= 100_000_000_000L) return 12; + if (v >= 10_000_000_000L) return 11; + if (v >= 1_000_000_000L) return 10; + if (v >= 100_000_000L) return 9; + if (v >= 10_000_000L) return 8; + if (v >= 1_000_000L) return 7; + if (v >= 100_000L) return 6; + if (v >= 10_000L) return 5; + if (v >= 1_000L) return 4; + if (v >= 100L) return 3; + if (v >= 10L) return 2; + return 1; + } +} diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java index 0f0f9a219..ddaf1736d 100644 --- a/core/src/main/java/io/questdb/client/std/Unsafe.java +++ b/core/src/main/java/io/questdb/client/std/Unsafe.java @@ -207,8 +207,8 @@ private static long AccessibleObject_override_fieldOffset() { // object headers (JEP 450, enabled by default in JDK 27) shrink the // header from 12 to 8 bytes, and a hard-coded 12/16 would then point // inside the header -- the Unsafe write to `override` would silently - // miss and setAccessible() would have no effect (surfacing as - // IllegalAccessError from the FdBig double-formatting bridge on JDK 27). + // miss and setAccessible() would have no effect, so makeAccessible() callers + // would fail with IllegalAccessError on JDK 27. // The probe tracks compact (8), compressed (12), uncompressed (16) and // 32-bit (8) layouts automatically. return firstFieldBoundaryOffset(); diff --git a/core/src/main/java11/io/questdb/client/std/Compat.java b/core/src/main/java11/io/questdb/client/std/Compat.java index 86635036e..f6d39aba4 100644 --- a/core/src/main/java11/io/questdb/client/std/Compat.java +++ b/core/src/main/java11/io/questdb/client/std/Compat.java @@ -45,6 +45,15 @@ public static long currentPid() { } } + /** + * Returns the most significant 64 bits of the 128-bit product of two longs. + * Delegates to {@code Math.multiplyHigh()} on Java 9+, which HotSpot + * intrinsifies; the Java 8 variant computes the same result by hand. + */ + public static long multiplyHigh(long x, long y) { + return Math.multiplyHigh(x, y); + } + /** * Spin-loop pause hint. Delegates to {@code Thread.onSpinWait()} on Java 9+; * the Java 8 variant is a no-op (no such hint exists there). diff --git a/core/src/main/java11/io/questdb/client/std/FdBig.java b/core/src/main/java11/io/questdb/client/std/FdBig.java deleted file mode 100644 index e6d69490a..000000000 --- a/core/src/main/java11/io/questdb/client/std/FdBig.java +++ /dev/null @@ -1,207 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.reflect.Method; - -/** - * Thin bridge over the JDK's internal {@code jdk.internal.math.FDBigInteger}, - * used only by the {@link Numbers} double-to-string slow path. This is the - * Java 9+ variant; the parallel copy under {@code src/main/java8} targets - * {@code sun.misc.FDBigInteger} directly. Keeping the type behind this wrapper - * lets {@code Numbers} carry a single, JDK-agnostic copy of the algorithm. - *

      - * The bignum class is never named in source. Up to JDK 25 it was a - * {@code public} class in a non-exported package, so a compile-time - * {@code --add-exports} plus a reflective runtime export was enough to reach - * it. JDK 26 (JDK-8366017) made the class itself package-private, and no - * module export can make a non-public type accessible. Instead, every method - * is resolved once into a {@code static final} {@link MethodHandle}: - *

        - *
      1. {@link Class#forName(String)} loads the class; loading performs no - * accessibility check.
        2. - *
      2. {@link Class#getDeclaredMethod} finds the member; lookup performs no - * accessibility check either.
        3. - *
      3. {@link Unsafe#makeAccessible} flips {@code AccessibleObject.override} - * directly, bypassing {@code setAccessible}'s module check.
        4. - *
      4. {@link MethodHandles.Lookup#unreflect} sees the override flag and - * resolves through the JDK's trusted {@code IMPL_LOOKUP}, which skips access - * checks entirely and yields a plain direct method handle.
        5. - *
      - * The handles are {@code static final} and invoked via {@code invokeExact} - * with erased ({@code Object}) signatures, so the JIT treats them as - * constants and inlines the calls: the slow path costs the same as a direct - * call did before. The technique works on JDK 9 through 27. - *

      - * JDK 27 needs no change here, but it exposed a latent bug in - * {@link Unsafe#makeAccessible}: compact object headers (JEP 450, enabled by - * default in JDK 27) shrink the object header to 8 bytes, so the - * {@code override} field that step 3 writes moved from offset 12/16 to 8. - * {@code Unsafe} now derives that offset by measuring the first-field boundary - * instead of hard-coding it, which tracks compact, compressed, uncompressed - * and 32-bit layouts alike. The remaining long-term liability is - * {@code sun.misc.Unsafe} itself being removed from a future JDK; the durable - * answer then is a self-contained bignum that needs no JDK-internal access at - * all. The {@code mrjar-smoke} 27-ea CI job guards the current behaviour. - */ -final class FdBig { - private static final MethodHandle ADD_AND_CMP; // (Object, Object, Object) int - private static final MethodHandle CMP; // (Object, Object) int - private static final MethodHandle GET_NORMALIZATION_BIAS; // (Object) int - private static final MethodHandle LEFT_SHIFT; // (Object, int) Object - private static final MethodHandle MULT_BY_10; // (Object) Object - private static final MethodHandle QUO_REM_ITERATION; // (Object, Object) int - private static final MethodHandle VALUE_OF_MUL_POW52; // (long, int, int) Object - private static final MethodHandle VALUE_OF_POW52; // (int, int) Object - - private final Object value; - - private FdBig(Object value) { - this.value = value; - } - - static FdBig valueOfMulPow52(long value, int p5, int p2) { - try { - return new FdBig((Object) VALUE_OF_MUL_POW52.invokeExact(value, p5, p2)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - static FdBig valueOfPow52(int p5, int p2) { - try { - return new FdBig((Object) VALUE_OF_POW52.invokeExact(p5, p2)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int addAndCmp(FdBig x, FdBig y) { - try { - return (int) ADD_AND_CMP.invokeExact(value, x.value, y.value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int cmp(FdBig other) { - try { - return (int) CMP.invokeExact(value, other.value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int getNormalizationBias() { - try { - return (int) GET_NORMALIZATION_BIAS.invokeExact(value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - FdBig leftShift(int shift) { - try { - return new FdBig((Object) LEFT_SHIFT.invokeExact(value, shift)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - FdBig multBy10() { - try { - return new FdBig((Object) MULT_BY_10.invokeExact(value)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int quoRemIteration(FdBig s) { - try { - return (int) QUO_REM_ITERATION.invokeExact(value, s.value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - /** - * Resolves {@code owner.name(params)} to a method handle of type - * {@code erased} (the bignum type replaced by {@code Object}), without the - * caller needing access to {@code owner}. {@code unreflect} yields a direct - * handle; {@link MethodHandle#asType} then conforms it to the erased type. - * For the handles that narrow an {@code Object} parameter back to the bignum - * type that wraps the direct handle in a cast-inserting adapter (a - * {@code BoundMethodHandle}); only the two {@code valueOf*} handles, which - * merely widen the return to {@code Object}, stay direct. Either way C2 - * inlines the chain and folds the casts, so every call costs the same as a - * direct call and allocates nothing (verified by JMH {@code gc.alloc.rate.norm}). - */ - private static MethodHandle handle( - MethodHandles.Lookup lookup, - Class owner, - String name, - MethodType erased, - Class... params - ) throws ReflectiveOperationException { - Method m = owner.getDeclaredMethod(name, params); - // Sets AccessibleObject.override without setAccessible()'s module - // check. Lookup.unreflect() then treats the member as pre-authorised - // and resolves it via IMPL_LOOKUP, i.e. with no access check at all. - Unsafe.makeAccessible(m); - return lookup.unreflect(m).asType(erased); - } - - private static RuntimeException rethrow(Throwable t) { - if (t instanceof RuntimeException) { - return (RuntimeException) t; - } - if (t instanceof Error) { - throw (Error) t; - } - return new IllegalStateException(t); - } - - static { - try { - final Class big = Class.forName("jdk.internal.math.FDBigInteger"); - final MethodHandles.Lookup lookup = MethodHandles.lookup(); - final Class obj = Object.class; - final Class i32 = int.class; - VALUE_OF_MUL_POW52 = handle(lookup, big, "valueOfMulPow52", MethodType.methodType(obj, long.class, i32, i32), long.class, i32, i32); - VALUE_OF_POW52 = handle(lookup, big, "valueOfPow52", MethodType.methodType(obj, i32, i32), i32, i32); - ADD_AND_CMP = handle(lookup, big, "addAndCmp", MethodType.methodType(i32, obj, obj, obj), big, big); - CMP = handle(lookup, big, "cmp", MethodType.methodType(i32, obj, obj), big); - GET_NORMALIZATION_BIAS = handle(lookup, big, "getNormalizationBias", MethodType.methodType(i32, obj)); - LEFT_SHIFT = handle(lookup, big, "leftShift", MethodType.methodType(obj, obj, i32), i32); - MULT_BY_10 = handle(lookup, big, "multBy10", MethodType.methodType(obj, obj)); - QUO_REM_ITERATION = handle(lookup, big, "quoRemIteration", MethodType.methodType(i32, obj, obj), big); - } catch (ReflectiveOperationException e) { - throw new ExceptionInInitializerError(e); - } - } -} diff --git a/core/src/main/java8/io/questdb/client/std/Compat.java b/core/src/main/java8/io/questdb/client/std/Compat.java index 843b4aa8a..1a9459e28 100644 --- a/core/src/main/java8/io/questdb/client/std/Compat.java +++ b/core/src/main/java8/io/questdb/client/std/Compat.java @@ -59,6 +59,25 @@ public static long currentPid() { * Spin-loop pause hint. No-op on Java 8 — {@code Thread.onSpinWait()} does * not exist there. The parallel Java 9+ variant delegates to it. */ + /** + * Returns the most significant 64 bits of the 128-bit product of two longs. + * Java 8 has no {@code Math.multiplyHigh()}, so this splits both operands + * into 32-bit halves and reassembles the high word from the partial + * products. The Java 9+ variant delegates to the intrinsic instead. + */ + public static long multiplyHigh(long x, long y) { + long x1 = x >> 32; + long x2 = x & 0xFFFFFFFFL; + long y1 = y >> 32; + long y2 = y & 0xFFFFFFFFL; + long z2 = x2 * y2; + long t = x1 * y2 + (z2 >>> 32); + long z1 = t & 0xFFFFFFFFL; + long z0 = t >> 32; + z1 += x2 * y1; + return x1 * y1 + z0 + (z1 >> 32); + } + public static void onSpinWait() { } } diff --git a/core/src/main/java8/io/questdb/client/std/FdBig.java b/core/src/main/java8/io/questdb/client/std/FdBig.java deleted file mode 100644 index a3c2e0822..000000000 --- a/core/src/main/java8/io/questdb/client/std/FdBig.java +++ /dev/null @@ -1,75 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import sun.misc.FDBigInteger; - -/** - * Thin bridge over the JDK's internal {@code FDBigInteger}, used only by the - * {@link Numbers} double-to-string slow path. This is the Java 8 variant: the - * bignum class lives in {@code sun.misc}. The parallel copy under - * {@code src/main/java11} targets {@code jdk.internal.math.FDBigInteger}. The - * public surface is identical across both JDKs, so {@code Numbers} carries a - * single, JDK-agnostic copy of the algorithm. - */ -final class FdBig { - private final FDBigInteger value; - - private FdBig(FDBigInteger value) { - this.value = value; - } - - static FdBig valueOfMulPow52(long value, int p5, int p2) { - return new FdBig(FDBigInteger.valueOfMulPow52(value, p5, p2)); - } - - static FdBig valueOfPow52(int p5, int p2) { - return new FdBig(FDBigInteger.valueOfPow52(p5, p2)); - } - - int addAndCmp(FdBig x, FdBig y) { - return value.addAndCmp(x.value, y.value); - } - - int cmp(FdBig other) { - return value.cmp(other.value); - } - - int getNormalizationBias() { - return value.getNormalizationBias(); - } - - FdBig leftShift(int shift) { - return new FdBig(value.leftShift(shift)); - } - - FdBig multBy10() { - return new FdBig(value.multBy10()); - } - - int quoRemIteration(FdBig s) { - return value.quoRemIteration(s.value); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java b/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java index f38ebf4d2..e1e209afc 100644 --- a/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java +++ b/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java @@ -82,8 +82,13 @@ public void testIsValidTableName() { Assert.assertFalse(TableUtils.isValidTableName("\\..", 127)); Assert.assertFalse(TableUtils.isValidTableName("/..", 127)); Assert.assertFalse(TableUtils.isValidTableName("../", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uFEFFt", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uD800t", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uDB80t", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uDFFFt", 127)); Assert.assertTrue(TableUtils.isValidTableName("table name", 127)); + Assert.assertTrue(TableUtils.isValidTableName("t\uD83D\uDE00t", 127)); Assert.assertTrue(TableUtils.isValidTableName("table name.csv", 127)); Assert.assertTrue(TableUtils.isValidTableName("table-name", 127)); Assert.assertTrue(TableUtils.isValidTableName("table_name", 127)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java new file mode 100644 index 000000000..25a9df4b8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java @@ -0,0 +1,228 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; + +public class WebSocketClientSchemaNegotiationTest { + @Test + public void testDefaultDoesNotRequestAndIgnoresUnsolicitedConfirmation() throws Exception { + ScriptedSocket socket = new ScriptedSocket("X-QWP-Schema: enabled\r\n"); + try (TestClient client = new TestClient(socket)) { + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertFalse(socket.request().contains("X-QWP-Request-Schema:")); + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + + @Test + public void testRequestedConfirmationAndAbsentLegacyResponse() throws Exception { + ScriptedSocket socket = new ScriptedSocket("X-QWP-Schema: EnAbLeD\r\n", ""); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertTrue(socket.request().contains("X-QWP-Request-Schema: true\r\n")); + Assert.assertTrue(client.isQwpSchemaEnabled()); + + client.disconnect(); + Assert.assertFalse("disconnect must clear per-connection confirmation", client.isQwpSchemaEnabled()); + socket.nextResponse(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertTrue("the one-way request survives reuse", socket.request().contains("X-QWP-Request-Schema: true\r\n")); + Assert.assertFalse("an absent response must not retain stale state", client.isQwpSchemaEnabled()); + } + } + + @Test + public void testRejectsInvalidEmptyAndDuplicateConfirmations() throws Exception { + assertRejected("X-QWP-Schema: disabled\r\n", "Invalid X-QWP-Schema"); + assertRejected("X-QWP-Schema: \r\n", "Invalid X-QWP-Schema"); + assertRejected("X-QWP-Schema: enabled\r\nX-QWP-Schema: enabled\r\n", "Duplicate X-QWP-Schema"); + } + + @Test + public void testDoesNotAcceptPrefixedOrValueSpoofHeaders() throws Exception { + String[] headers = { + "Not-X-QWP-Schema: enabled\r\n", + "X-Comment: X-QWP-Schema: enabled\r\n" + }; + for (String header : headers) { + ScriptedSocket socket = new ScriptedSocket(header); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + } + + private static void assertRejected(String headers, String message) throws Exception { + ScriptedSocket socket = new ScriptedSocket(headers); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + try { + client.upgrade("/write/v4", 1000, null); + Assert.fail("expected invalid schema confirmation"); + } catch (HttpClientException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains(message)); + } + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + + private static final class ScriptedSocket implements Socket { + private final String[] extraHeaders; + private int responseIndex; + private byte[] response; + private int responsePosition; + private String request; + + private ScriptedSocket(String... extraHeaders) { + this.extraHeaders = extraHeaders; + } + + @Override + public void close() { + } + + @Override + public int getFd() { + return 1; + } + + @Override + public boolean isClosed() { + return false; + } + + void nextResponse() { + responseIndex++; + response = null; + responsePosition = 0; + } + + @Override + public void of(int fd) { + } + + String request() { + return request; + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + if (response == null || responsePosition == response.length) { + return 0; + } + int count = Math.min(bufferLen, response.length - responsePosition); + for (int i = 0; i < count; i++) { + Unsafe.getUnsafe().putByte(bufferPtr + i, response[responsePosition++]); + } + return count; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + byte[] bytes = new byte[bufferLen]; + for (int i = 0; i < bufferLen; i++) bytes[i] = Unsafe.getUnsafe().getByte(bufferPtr + i); + request = new String(bytes, StandardCharsets.US_ASCII); + String key = header(request, "Sec-WebSocket-Key:"); + response = response(key, extraHeaders[responseIndex]); + responsePosition = 0; + return bufferLen; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + + private static String header(String request, String name) { + int start = request.indexOf(name) + name.length(); + int end = request.indexOf("\r\n", start); + return request.substring(start, end).trim(); + } + + private static byte[] response(String key, String extraHeaders) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + String accept = Base64.getEncoder().encodeToString(sha1.digest( + (key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").getBytes(StandardCharsets.US_ASCII))); + return ("HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + accept + "\r\n" + + extraHeaders + + "\r\n").getBytes(StandardCharsets.US_ASCII); + } catch (Exception e) { + throw new AssertionError(e); + } + } + } + + private static final class TestClient extends WebSocketClient { + private TestClient(ScriptedSocket socket) { + super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 1d83dff2a..fc303ba85 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -256,6 +256,55 @@ public void testCatchUpCapGapMinEscalationWindowConfigStringNotSupportedForTcp() + "catch_up_cap_gap_min_escalation_window_millis=300000;")); } + @Test + public void testSchemaModeBuilderNotSupportedForTcp() { + assertThrows("schema_mode is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .schemaMode(Sender.SchemaMode.STRICT)); + } + + @Test + public void testSchemaModeConfigStringInvalidRejected() { + assertThrows("invalid schema_mode [value=maybe, allowed-values=[auto, strict, off]]", + () -> Sender.builder("ws::addr=localhost:9000;schema_mode=maybe;")); + } + + @Test + public void testSchemaModeConfigStringNotSupportedForTcp() { + assertThrows("schema_mode is only supported for WebSocket transport", + () -> Sender.builder("tcp::addr=localhost:9009;schema_mode=auto;")); + } + + @Test + public void testSchemaModeConfigStringValuesAreCaseInsensitive() { + Assert.assertEquals("AUTO", + Sender.builder("ws::addr=localhost:9000;schema_mode=Auto;").wsConfigSnapshotForTest().get("schema_mode")); + Assert.assertEquals("OFF", + Sender.builder("ws::addr=localhost:9000;schema_mode=OFF;").wsConfigSnapshotForTest().get("schema_mode")); + Assert.assertNull(Sender.builder("ws::addr=localhost:9000;").wsConfigSnapshotForTest().get("schema_mode")); + } + + @Test + public void testSchemaWaitMillisBuilderNotSupportedForTcp() { + assertThrows("schema_wait_millis is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .schemaWaitMillis(1000)); + } + + @Test + public void testSchemaWaitMillisConfigStringZeroRejected() { + assertThrows("schema_wait_millis must be > 0: 0", + () -> Sender.builder("ws::addr=localhost:9000;schema_wait_millis=0;")); + } + + @Test + public void testSchemaWaitMillisUnsetInSnapshot() { + Assert.assertEquals(-1L, + Sender.builder("ws::addr=localhost:9000;").wsConfigSnapshotForTest().get("schema_wait_millis")); + } + @Test public void testCatchUpCapGapMinEscalationWindowUnsetInSnapshot() { // wsConfigSnapshotForTest puts the raw field, not the resolved default (see diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java new file mode 100644 index 000000000..1217a2d13 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java @@ -0,0 +1,261 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +public class QwpSchemaFeedbackResponseTest { + + @Test + public void testUpdatesAreAtomicAndEveryPrefixIsRejected() { + byte[] valid = updates("orders", "trades"); + WebSocketResponse response = new WebSocketResponse(); + withNative(valid, (ptr, length) -> { + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertEquals(2, response.getSchemaUpdateCount()); + Assert.assertEquals("orders", response.getSchemaUpdateTableName(0)); + Assert.assertEquals(0, response.getSchemaUpdate(0).getRequestId()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, response.getSchemaUpdate(0).getResult()); + for (int n = 0; n < length; n++) { + Assert.assertFalse("prefix " + n, response.readFrom(ptr, n, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.isSchemaInvalidation()); + } + }); + } + + @Test + public void testInvalidationAndFailedReadClearVisibleFeedback() { + WebSocketResponse response = new WebSocketResponse(); + byte[] invalidation = base((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)); + withNative(invalidation, (ptr, length) -> { + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertTrue(response.isSchemaInvalidation()); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.readFrom(ptr, length - 1, true)); + Assert.assertFalse(response.isSchemaInvalidation()); + Assert.assertFalse(response.hasSchemaUpdates()); + }); + } + + @Test + public void testNegotiationModesAndDurableAckRules() { + byte[] updates = updates("t"); + withNative(updates, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, false))); + byte[] reserved = base((byte) (WebSocketResponse.STATUS_OK | 0x40)); + withNative(reserved, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + byte[] durable = new byte[]{(byte) (WebSocketResponse.STATUS_DURABLE_ACK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL), 0, 0}; + withNative(durable, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testDuplicateNamesAreCaseInsensitive() { + byte[] duplicate = updates("foo", "FOO"); + withNative(duplicate, (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertFalse(response.readFrom(ptr, length, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + }); + } + + @Test + public void testNestedEnvelopeAndNonZeroFeedbackRequestIdRejected() { + byte[] nested = updatesWithPayload("t", new byte[]{'Q', 'W', 'P', '1', 1, 0, 0, 0, 0, 0, 0, 0}); + withNative(nested, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + byte[] payload = missingPayload(); + payload[1] = 1; + byte[] nonZero = updatesWithPayload("t", payload); + withNative(nonZero, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testInvalidSchemaColumnNamesRejectWholeAckAndNackWithoutPublishingPrefix() { + byte[] valid = missingPayload(); + byte[][] invalidSchemas = { + knownPayload("x", "x"), + knownPayload("x", "X"), + knownPayload("Ä", "ä"), + knownPayload("a/b"), + knownPayload("a\nb") + }; + for (byte[] invalid : invalidSchemas) { + for (byte status : new byte[]{WebSocketResponse.STATUS_OK, WebSocketResponse.STATUS_SCHEMA_MISMATCH}) { + byte[] feedback = updatesWithPayloads(status, new String[]{"first", "second"}, new byte[][]{valid, invalid}); + withNative(feedback, (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertFalse(response.readFrom(ptr, length, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.isSchemaInvalidation()); + }); + } + } + } + + @Test + public void testNackUpdatesPreserveBaseError() { + byte[] payload = missingPayload(); + byte[] name = "bad_table".getBytes(StandardCharsets.UTF_8); + byte[] message = "bad row".getBytes(StandardCharsets.UTF_8); + ByteBuffer b = ByteBuffer.allocate(11 + message.length + 2 + 2 + name.length + 4 + payload.length) + .order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_SCHEMA_MISMATCH | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + b.putLong(19).putShort((short) message.length).put(message).putShort((short) 1); + b.putShort((short) name.length).put(name).putInt(payload.length).put(payload); + withNative(b.array(), (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertEquals(WebSocketResponse.STATUS_SCHEMA_MISMATCH, response.getStatus()); + Assert.assertEquals(19, response.getSequence()); + Assert.assertEquals("bad row", response.getErrorMessage()); + Assert.assertEquals("bad_table", response.getSchemaUpdateTableName(0)); + }); + } + + @Test + public void testHostileCountLengthAndUtf8RejectedWithoutPublication() { + byte[] count = base((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + count = java.util.Arrays.copyOf(count, count.length + 2); + count[count.length - 2] = (byte) 0xff; + count[count.length - 1] = (byte) 0xff; + withNative(count, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + + byte[] invalidUtf8 = updates("t"); + invalidUtf8[15] = (byte) 0xc0; + withNative(invalidUtf8, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + + byte[] oversized = updates("t"); + ByteBuffer.wrap(oversized).order(ByteOrder.LITTLE_ENDIAN).putInt(16, -1); + withNative(oversized, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testOneMiBBoundAppliesOnlyToUpdates() { + int count = 3000; + String prefix = new String(new char[123]).replace('\0', '\u4e00'); + int nameLength = 373; + int length = 11 + count * (2 + nameLength + 8); + Assert.assertTrue(length > QwpSchemaProtocol.MAX_FRAME_SIZE); + byte[] base = new byte[length]; + ByteBuffer b = ByteBuffer.wrap(base).order(ByteOrder.LITTLE_ENDIAN); + b.put(WebSocketResponse.STATUS_OK).putLong(1).putShort((short) count); + for (int i = 0; i < count; i++) { + byte[] name = (prefix + String.format(java.util.Locale.ROOT, "%04d", i)).getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(nameLength, name.length); + b.putShort((short) nameLength).put(name).putLong(i); + } + withNative(base, (ptr, n) -> Assert.assertTrue(new WebSocketResponse().readFrom(ptr, n, false))); + base[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL); + withNative(base, (ptr, n) -> Assert.assertTrue(new WebSocketResponse().readFrom(ptr, n, true))); + byte[] payload = missingPayload(); + byte[] update = java.util.Arrays.copyOf(base, base.length + 2 + 2 + 1 + 4 + payload.length); + ByteBuffer trailer = ByteBuffer.wrap(update).order(ByteOrder.LITTLE_ENDIAN); + // Not chained off position(): that returns Buffer, not ByteBuffer, on Java 8. + trailer.position(base.length); + trailer.putShort((short) 1).putShort((short) 1).put((byte) 't').putInt(payload.length).put(payload); + update[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES); + withNative(update, (ptr, n) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, n, true))); + } + + private static byte[] base(byte status) { + ByteBuffer b = ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN); + b.put(status).putLong(7).putShort((short) 0); + return b.array(); + } + + private static byte[] missingPayload() { + return ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) QwpSchemaProtocol.RESULT_MISSING).array(); + } + + private static byte[] knownPayload(String... columnNames) { + int size = 26; + for (String name : columnNames) { + size += 2 + name.getBytes(StandardCharsets.UTF_8).length + 6; + } + ByteBuffer b = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) -1).putShort((short) columnNames.length); + for (String name : columnNames) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + b.putShort((short) bytes.length).put(bytes).putInt(5).putShort((short) 0); + } + return b.array(); + } + + private static byte[] updates(String... names) { + return updatesWithPayloads(names, missingPayload()); + } + + private static byte[] updatesWithPayload(String name, byte[] payload) { + return updatesWithPayloads(new String[]{name}, payload); + } + + private static byte[] updatesWithPayloads(String[] names, byte[] payload) { + byte[][] payloads = new byte[names.length][]; + java.util.Arrays.fill(payloads, payload); + return updatesWithPayloads(WebSocketResponse.STATUS_OK, names, payloads); + } + + private static byte[] updatesWithPayloads(byte status, String[] names, byte[][] payloads) { + int size = 13; + for (int i = 0; i < names.length; i++) { + size += 2 + names[i].getBytes(StandardCharsets.UTF_8).length + 4 + payloads[i].length; + } + ByteBuffer b = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (status | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + b.putLong(7).putShort((short) 0).putShort((short) names.length); + for (int i = 0; i < names.length; i++) { + String name = names[i]; + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + b.putShort((short) bytes.length).put(bytes).putInt(payloads[i].length).put(payloads[i]); + } + return b.array(); + } + + private static void withNative(byte[] bytes, NativeAssertion assertion) { + long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, bytes[i]); + } + assertion.run(ptr, bytes.length); + } finally { + Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private interface NativeAssertion { + void run(long ptr, int length); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFreshnessTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFreshnessTest.java new file mode 100644 index 000000000..8399b953d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFreshnessTest.java @@ -0,0 +1,296 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpSchemaFreshness; +import io.questdb.client.std.Rnd; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Pins when a cached table schema counts as confirmed by the server. The + * property test checks the guarantee the sender relies on against a log of + * every confirmation the test handed out: a snapshot is never reported fresh + * without a confirmation, or a failed re-check, within the age limit. + */ +public class QwpSchemaFreshnessTest { + private static final long MAX_AGE = 1_000; + + @Test + public void testAcceptedRowEndsRunOfRejectedRows() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + for (int i = 1; i < QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK; i++) { + freshness.onRowRejected(); + } + freshness.onRowAccepted(); + for (int i = 1; i < QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK; i++) { + freshness.onRowRejected(); + } + Assert.assertFalse(freshness.isRecheckDue()); + Assert.assertFalse(freshness.takeRecheck(0, MAX_AGE)); + freshness.onRowRejected(); + Assert.assertTrue(freshness.isRecheckDue()); + } + + @Test + public void testConfirmationAtAgeLimitIsFreshAndOneNanoLaterExpired() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.confirm(100); + Assert.assertFalse(freshness.isExpired(100, MAX_AGE, -1)); + Assert.assertFalse(freshness.isExpired(100 + MAX_AGE, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(101 + MAX_AGE, MAX_AGE, -1)); + } + + @Test + public void testConfirmationsSurviveClockWrap() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.confirm(Long.MAX_VALUE - 10); + // System.nanoTime() may wrap; only differences carry meaning. + Assert.assertFalse(freshness.isExpired(Long.MIN_VALUE + 10, MAX_AGE, -1)); + freshness.confirm(Long.MIN_VALUE + 5); + Assert.assertFalse(freshness.isExpired(Long.MIN_VALUE + 5 + MAX_AGE, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(Long.MIN_VALUE + 6 + MAX_AGE, MAX_AGE, -1)); + } + + @Test + public void testCoveredFrameConfirmsAsOfItsFlush() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.markConsulted(); + freshness.onFramePublished(5, 200, 4); + Assert.assertTrue("an unacknowledged frame confirms nothing", freshness.isExpired(200, MAX_AGE, 4)); + Assert.assertFalse(freshness.isExpired(200 + MAX_AGE, MAX_AGE, 5)); + Assert.assertTrue(freshness.isExpired(201 + MAX_AGE, MAX_AGE, 5)); + } + + @Test + public void testEarlierConfirmationNeverReplacesLaterOne() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.confirm(500); + freshness.confirm(100); + Assert.assertFalse(freshness.isExpired(500 + MAX_AGE, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(501 + MAX_AGE, MAX_AGE, -1)); + + // A covered frame flushed before the latest confirmation does not age it either. + freshness.onFramePublished(7, 50, 6); + Assert.assertFalse(freshness.isExpired(500 + MAX_AGE, MAX_AGE, 7)); + Assert.assertTrue(freshness.isExpired(501 + MAX_AGE, MAX_AGE, 7)); + } + + @Test + public void testFailedCheckDefersNextCheckByAgeLimit() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.deferCheck(300); + Assert.assertFalse(freshness.isExpired(300 + MAX_AGE, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(301 + MAX_AGE, MAX_AGE, -1)); + + freshness.confirm(0); + freshness.deferCheck(5_000); + Assert.assertFalse(freshness.isExpired(5_000 + MAX_AGE, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(5_001 + MAX_AGE, MAX_AGE, -1)); + // A confirmation after the failed check counts from its own time. + freshness.confirm(7_000); + Assert.assertFalse(freshness.isExpired(7_000 + MAX_AGE, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(7_001 + MAX_AGE, MAX_AGE, -1)); + } + + @Test + public void testFrameAfterCoveredFrameWaitsInTurn() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.markConsulted(); + freshness.onFramePublished(5, 200, 4); + freshness.markConsulted(); + // Publishing the next frame first confirms the covered one, then waits itself. + freshness.onFramePublished(6, 300, 5); + Assert.assertFalse(freshness.isExpired(200 + MAX_AGE, MAX_AGE, 5)); + Assert.assertTrue(freshness.isExpired(201 + MAX_AGE, MAX_AGE, 5)); + Assert.assertFalse(freshness.isExpired(300 + MAX_AGE, MAX_AGE, 6)); + Assert.assertTrue(freshness.isExpired(301 + MAX_AGE, MAX_AGE, 6)); + } + + @Test + public void testNeverConfirmedIsExpired() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + Assert.assertTrue(freshness.isExpired(0, MAX_AGE, -1)); + Assert.assertTrue(freshness.isExpired(0, Long.MAX_VALUE, Long.MAX_VALUE)); + Assert.assertFalse(freshness.isConsulted()); + } + + @Test + public void testOldestUnacknowledgedFrameWaits() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.markConsulted(); + freshness.onFramePublished(5, 200, 4); + freshness.markConsulted(); + freshness.onFramePublished(6, 300, 4); + // Frame 6 was published while frame 5 waited, so only frame 5 confirms. + Assert.assertFalse(freshness.isExpired(200 + MAX_AGE, MAX_AGE, 6)); + Assert.assertTrue(freshness.isExpired(201 + MAX_AGE, MAX_AGE, 6)); + } + + @Test + public void testPublishClearsConsultedMark() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + freshness.markConsulted(); + Assert.assertTrue(freshness.isConsulted()); + freshness.onFramePublished(0, 0, -1); + Assert.assertFalse(freshness.isConsulted()); + } + + @Test + public void testRecheckFollowsRunOfRejectedRows() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + for (int i = 1; i < QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK; i++) { + freshness.onRowRejected(); + Assert.assertFalse(freshness.isRecheckDue()); + Assert.assertFalse(freshness.takeRecheck(0, MAX_AGE)); + } + freshness.onRowRejected(); + Assert.assertTrue(freshness.isRecheckDue()); + Assert.assertTrue(freshness.takeRecheck(0, MAX_AGE)); + Assert.assertFalse("taking the re-check ends the run", freshness.isRecheckDue()); + Assert.assertFalse(freshness.takeRecheck(0, MAX_AGE)); + } + + @Test + public void testRecheckRunsAtMostOncePerInterval() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + rejectRun(freshness); + Assert.assertTrue(freshness.takeRecheck(Long.MAX_VALUE - 10, MAX_AGE)); + rejectRun(freshness); + // Differences only, so the interval holds across a clock wrap. + Assert.assertFalse(freshness.takeRecheck(Long.MAX_VALUE - 10 + MAX_AGE, MAX_AGE)); + Assert.assertTrue("a run deferred by the interval stays due", freshness.isRecheckDue()); + Assert.assertTrue(freshness.takeRecheck(Long.MAX_VALUE - 9 + MAX_AGE, MAX_AGE)); + Assert.assertFalse(freshness.isRecheckDue()); + } + + @Test + public void testRecheckWaitsIntervalAfterUnansweredCheck() { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + rejectRun(freshness); + Assert.assertTrue(freshness.takeRecheck(0, MAX_AGE)); + // The lookup waited past the interval and got no answer. + freshness.deferCheck(MAX_AGE + 1); + rejectRun(freshness); + Assert.assertFalse(freshness.takeRecheck(2 * MAX_AGE + 1, MAX_AGE)); + Assert.assertTrue(freshness.isRecheckDue()); + Assert.assertTrue(freshness.takeRecheck(2 * MAX_AGE + 2, MAX_AGE)); + } + + @Test + public void testRunOfRejectedRowsOutlastsIntRange() { + // A sender that keeps rejecting rows while the wire is down takes no re-check, + // so the run grows until a connection comes back. It must still be due then. + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + for (long i = 0; i <= Integer.MAX_VALUE; i++) { + freshness.onRowRejected(); + } + Assert.assertTrue(freshness.isRecheckDue()); + Assert.assertTrue(freshness.takeRecheck(0, MAX_AGE)); + Assert.assertFalse(freshness.isRecheckDue()); + } + + @Test + public void testRandomSequencesNeverReportUnconfirmedSnapshotFresh() { + Rnd rnd = TestUtils.generateRandom(null); + for (int run = 0; run < 2_000; run++) { + QwpSchemaFreshness freshness = new QwpSchemaFreshness(); + List frames = new ArrayList<>(); + List confirmations = new ArrayList<>(); + List failedChecks = new ArrayList<>(); + long now = rnd.nextLong(); + long okFsn = -1; + long nextFsn = 0; + int steps = 1 + rnd.nextInt(40); + for (int step = 0; step < steps; step++) { + now += rnd.nextInt(700); + switch (rnd.nextInt(5)) { + case 0: + // A DESCRIBE confirms as of a moment no later than now. + long describedAt = now - rnd.nextInt(50); + freshness.confirm(describedAt); + confirmations.add(describedAt); + break; + case 1: + long flushedAt = now - rnd.nextInt(50); + nextFsn += 1 + rnd.nextInt(3); + freshness.markConsulted(); + freshness.onFramePublished(nextFsn, flushedAt, okFsn); + frames.add(new long[]{nextFsn, flushedAt}); + break; + case 2: + okFsn = Math.min(nextFsn, okFsn + rnd.nextInt(4)); + break; + case 3: + freshness.deferCheck(now); + failedChecks.add(now); + break; + default: + break; + } + if (!freshness.isExpired(now, MAX_AGE, okFsn)) { + Assert.assertTrue("run " + run + " step " + step + ": reported fresh without a confirmation", + isJustified(now, okFsn, frames, confirmations, failedChecks)); + } + } + } + } + + private static boolean isJustified( + long now, + long okFsn, + List frames, + List confirmations, + List failedChecks + ) { + for (int i = 0, n = confirmations.size(); i < n; i++) { + if (now - confirmations.get(i) <= MAX_AGE) { + return true; + } + } + for (int i = 0, n = failedChecks.size(); i < n; i++) { + if (now - failedChecks.get(i) <= MAX_AGE) { + return true; + } + } + for (int i = 0, n = frames.size(); i < n; i++) { + long[] frame = frames.get(i); + if (frame[0] <= okFsn && now - frame[1] <= MAX_AGE) { + return true; + } + } + return false; + } + + private static void rejectRun(QwpSchemaFreshness freshness) { + for (int i = 0; i < QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK; i++) { + freshness.onRowRejected(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java new file mode 100644 index 000000000..acbb97816 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -0,0 +1,5208 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpSchemaFreshness; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Compat; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaSenderIntegrationTest { + private static final String DECIMAL_TEXT_CORPUS = + "/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv"; + private static final long SCHEMA_MAX_AGE_NANOS = + TimeUnit.MILLISECONDS.toNanos(QwpWebSocketSender.SCHEMA_MAX_AGE_MILLIS); + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 921, 931, ColumnType.getDecimalType(5, 2)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .decimalColumn("value", "1.20m") + .decimalColumn("value", "not-a-decimal") + .atNow(); + + sender.uuidColumn("failed_b", 11, 12); + try { + sender.decimalColumn("value", "not-a-decimal"); + Assert.fail("expected invalid textual decimal"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.decimalColumn("value", "2.5").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimal64Table( + "events", 2, 120, 250); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDecimalTextOverloadFormatsTextAndPreservesNullNoOps() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(941 + targetType, 951, targetType); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", "00123.4500").atNow(); + sender.decimalColumn("value", "1.2300m").atNow(); + sender.decimalColumn("value", "Infinity") + .decimalColumn("value", "not-a-decimal") + .atNow(); + sender.decimalColumn("value", (CharSequence) null) + .decimalColumn("value", "") + .atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).parsedDecimalTextTable( + "events"); + } + } + }); + } + + @Test + public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() throws Exception { + assertMemoryLeak(() -> { + int[] targets = { + ColumnType.BYTE, ColumnType.SHORT, ColumnType.INT, + ColumnType.LONG, ColumnType.FLOAT, ColumnType.DOUBLE + }; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(901 + i, 911 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -1).atNow(); + + sender.byteColumn("value", (byte) 99); + try { + switch (i % 3) { + case 0: + sender.byteColumn("failed_b", (byte) 1); + break; + case 1: + sender.shortColumn("failed_b", (short) 1); + break; + default: + sender.intColumn("failed_b", 1); + break; + } + Assert.fail("expected unsupported small-integer-to-UUID conversion"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) 2).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerNumericTable( + "events", numericWireType(target)); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIntegerTemporalGenerationRemainsDateBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(981, 991, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 11).atNow(); + + handler.targetType = ColumnType.TIMESTAMP_NANO; + handler.version = 992; + sender.table("events"); + try { + sender.stringColumn("value", "1970-01-01T00:00:00.000001Z"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").intColumn("value", 12).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)) + .twoDateAndTimestampBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.DATE, ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(961 + i, 971 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -1).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) 2).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).integerTemporalTable( + "events", temporalWireType(target)); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testSmallIntegerTextGenerationRemainsSymbolBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1021, 1031, ColumnType.SYMBOL); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 7).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 1032; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 2); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").shortColumn("value", (short) 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)) + .twoSymbolAndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(1041 + i, 1051 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -128).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", Short.MAX_VALUE).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerTextTable( + "events", target); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testSmallIntegerDecimalGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1061, 1071, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 1).atNow(); + + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 1072; + sender.table("events"); + try { + sender.shortColumn("value", (short) 1_000); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").shortColumn("value", (short) 1_000).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoSmallIntegerDecimalBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1081, 1091, ColumnType.getDecimalType(18, 2)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 42).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) -7).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerDecimalTable( + "events"); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntToIpv4GenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1101, 1111, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").intColumn("value", 1).atNow(); + + handler.targetType = ColumnType.IPv4; + handler.version = 1112; + sender.table("events"); + try { + sender.intColumn("value", 1_000); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").intColumn("value", 1_000).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoIntDecimalAndIpv4Blocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1121, 1131, ColumnType.IPv4); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 0).atNow(); + + sender.intColumn("value", 0x0a000001); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.intColumn("value", 1).atNow(); + sender.intColumn("value", -1).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).intToIpv4Table("events"); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaSenderIntegrationTest.class.getResourceAsStream(DECIMAL_TEXT_CORPUS); + Assert.assertNotNull(DECIMAL_TEXT_CORPUS, stream); + boolean[][] sourceTargets = new boolean[3][2]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + DecimalTextVector vector = new DecimalTextVector(line.split("\t", -1)); + sourceTargets[vector.sourceIndex()][vector.targetIndex()] = true; + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 721 + count, 731 + count, vector.targetType()); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events"); + vector.append(sender, "value"); + if (vector.isNull()) { + sender.uuidColumn("failed_b", 11, 12); + } + sender.atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + if (vector.isNull()) { + reader.uuidOnlyTable("events", 11, 12); + } else { + reader.decimalTextTable( + "events", vector.expectedText); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(24, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target pair " + source + '/' + target, + sourceTargets[source][target]); + } + } + }); + } + + @Test + public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversRows() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(781, 791, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .decimalColumn("value", Decimal64.NULL_VALUE) + .decimalColumn("value", new Decimal64(12_340, 2)) + .binaryColumn("value", new byte[]{1}) + .atNow(); + + sender.decimalColumn("value", Decimal256.MAX_VALUE); + try { + sender.decimalColumn("failed_b", new Decimal64(1, 0)); + Assert.fail("expected unsupported DECIMAL64-to-UUID conversion"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.decimalColumn("value", new Decimal64(-125, 3)).atNow(); + sender.table("events").atNow(); + sender.decimalColumn("value", new Decimal256( + 0x161bcca7119915b5L, + 0x0764b4abe8652979L, + 0x7775a5f171950fffL, + 0xffffffffffffffffL, + 76)).atNow(); + sender.decimalColumn("value", new Decimal64(0, 0)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimalTextLifecycleTable("events"); + } + }); + } + + @Test + public void testNativeDecimalTextMatchesServerScalarFormattingForRawPublicValues() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(801, 811, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + Decimal64 wrappedScale = new Decimal64(123, 1); + wrappedScale.setScale(257); + sender.table("events") + .decimalColumn("value", new Decimal128(Long.MAX_VALUE, -1L)) + .atNow(); + sender.decimalColumn("value", wrappedScale).atNow(); + sender.decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimalTextTable( + "events", + "170141183460469231731687303715884105727", + "12.3", + "4.20" + ); + } + }); + } + + @Test + public void testNativeDecimalRollbackAndTargetScaleWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(16, 4); + LongTextSchemaHandler handler = new LongTextSchemaHandler(601, 611, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(150, 2)).atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.decimalColumn("value", Decimal128.fromLong(100_000_000_000_000_000L, 0)); + Assert.fail("expected target precision overflow"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.decimalColumn("value", new Decimal64(25, 2)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).decimal64Table("events", 4, 15_000, 2_500); + } + }); + } + + @Test + public void testNarrowDecimalGenerationRemainsPinnedBeforeWideRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 621, 631, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", Decimal128.fromLong(1, 0)).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 0); + handler.version = 632; + sender.table("events"); + try { + sender.decimalColumn("value", Decimal128.fromLong(1_000, 0)); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", Decimal128.fromLong(1_000, 0)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDecimalScaleGenerationRemainsPinnedBeforeScaleRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 641, 651, ColumnType.getDecimalType(18, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(1, 0)).atNow(); + handler.targetType = ColumnType.getDecimalType(18, 4); + handler.version = 652; + sender.table("events"); + try { + sender.decimalColumn("value", new Decimal64(12_345, 4)); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", new Decimal64(12_345, 4)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimal64ScaleBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 661, 671, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 1).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 672; + sender.table("events"); + try { + sender.longColumn("value", 100_000_000_000_000L); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 100_000_000_000_000L).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 681, 691, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "1").atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 692; + sender.table("events"); + try { + sender.stringColumn("value", "100000000000000"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "100000000000000").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testFloatingDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 701, 711, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").floatColumn("value", 1.0f).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 712; + sender.table("events"); + try { + sender.doubleColumn("value", 100_000_000_000_000.0); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleColumn("value", 100_000_000_000_000.0).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testNativeDecimalTextGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 741, 751, ColumnType.getDecimalType(5, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(12_340, 4)).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 752; + sender.table("events"); + try { + sender.decimalColumn("value", new Decimal64(250_000, 4)); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", new Decimal64(250_000, 4)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimal64AndVarcharBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToGeoHashNullWidthsRollbackAndRecovery() throws Exception { + assertMemoryLeak(() -> { + for (int bits : new int[]{1, 8, 60}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 471 + bits, 481, ColumnType.getGeoHashTypeWithBits(bits)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).nullGeoHashTable("events", bits); + } + } + + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 551, 561, ColumnType.getGeoHashTypeWithBits(8)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "zz").atNow(); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "a"); + Assert.fail("expected invalid GeoHash text"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "04").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).geoHashTable("events", 8, 0xff, 1); + } + }); + } + + @Test + public void testGeoHashPrecisionGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 571, 581, ColumnType.getGeoHashTypeWithBits(60)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "0123456789bc").atNow(); + handler.targetType = ColumnType.getGeoHashTypeWithBits(5); + handler.version = 582; + sender.table("events"); + try { + sender.stringColumn("value", "z"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoGeoHashBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToLong256RollbackAndRecoveryWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(431, 441, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "0x0"); + Assert.fail("expected odd-length LONG256 rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "0x0123456789abcdeffedcba9876543210").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).long256Table("events"); + } + }); + } + + @Test + public void testLong256GenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(451, 461, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .stringColumn("value", "0x0123456789abcdeffedcba9876543210") + .atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 462; + sender.table("events"); + try { + sender.stringColumn("value", "not-long256"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "not-long256").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoLong256AndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToCharRollbackAndRecoveryWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(391, 401, ColumnType.CHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "A").atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.binaryColumn("value", new byte[]{1}); + Assert.fail("expected BINARY to CHAR rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.stringColumn("value", "\ud83d").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).charTable("events", 'A', '?'); + } + }); + } + + @Test + public void testVarcharGenerationRemainsPinnedBeforeCharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(411, 421, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "native-a").atNow(); + handler.targetType = ColumnType.CHAR; + handler.version = 422; + sender.table("events"); + try { + sender.binaryColumn("value", new byte[]{1}); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").charColumn("value", '\uffff').atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoVarcharAndCharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testTimestampToTextTargetsRollbackAndRecoverWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(351 + target, 361, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.MICROS).atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); + Assert.fail("expected timestamp range error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.timestampColumn("value", 1, ChronoUnit.MILLIS).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).timestampTextTable( + "events", "1970-01-01T00:00:00.001Z"); + } + } + }); + } + + @Test + public void testNativeTimestampGenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(371, 381, ColumnType.TIMESTAMP_NANO); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.NANOS).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 382; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 0); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").timestampColumn("value", Long.MAX_VALUE, ChronoUnit.NANOS).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoTimestampAndVarcharBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToTimestampTargetsUsePinnedWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(301 + target, 311, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "not-a-timestamp"); + Assert.fail("expected timestamp parse error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.stringColumn("value", "1969-12-31T23:59:59.999999Z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).longTimestampTable( + "events", + target, + target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000 + ); + } + } + }); + } + + @Test + public void testStringToDateUsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1141, 1151, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "not-a-date"); + Assert.fail("expected DATE parse error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.stringColumn("value", "1969-12-31").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .stringDateTable("events", -86_400_000L); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDateGenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1161, 1171, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "1970-01-01").atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 1172; + sender.table("events"); + try { + sender.stringColumn("value", "native-a"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "native-a").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)) + .twoDateAndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleArrayUsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1181, 1191, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.doubleArray("value", new double[]{99.0}); + Assert.fail("expected array rank mismatch"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events"); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleArrayGenerationRemainsPinnedAcrossRankChange() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1201, 1211, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + handler.targetType = array1d; + handler.version = 1212; + sender.table("events"); + try { + sender.doubleArray("value", new double[]{3.0, 4.0}); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleArray("value", new double[]{3.0, 4.0}).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)) + .twoDoubleArrayRankBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongArrayRejectionRollsBackWithoutRefresh() throws Exception { + assertMemoryLeak(() -> { + for (int overload = 0; overload < 4; overload++) { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1221, 1231, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + switch (overload) { + case 0: + sender.longArray("value", new long[]{99L}); + break; + case 1: + sender.longArray("value", new long[][]{{99L}}); + break; + case 2: + sender.longArray("value", new long[][][]{{{99L}}}); + break; + default: + try (LongArray array = new LongArray(1)) { + sender.longArray("value", array.append(99L)); + } + break; + } + Assert.fail("expected unsupported LONG_ARRAY setter [overload=" + overload + "]"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + // Only the two completed DOUBLE[] rows reach the wire; failed_b is rolled back. + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events"); + Assert.assertEquals("unsupported input must not refresh [overload=" + overload + "]", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongArrayRejectionKeepsPinnedSnapshotAfterSchemaChange() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1241, 1251, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + handler.targetType = array1d; + handler.version = 1252; + for (int attempt = 0; attempt < 2; attempt++) { + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.longArray("value", new long[]{3L, 4L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("schema changes cannot implement LONG_ARRAY", 1, + handler.describeRequests.get()); + } + // A supported input of the new rank is still rejected against the + // pinned snapshot; the batch does not discover the rank change. + try { + sender.doubleArray("value", new double[]{3.0, 4.0}); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleArray("value", new double[]{3.0, 4.0}).atNow(); + sender.flush(); + // The rebound block holds the recovered row alone: rollback dropped + // failed_b with the rejected row. + new FrameReader(handler.awaitDataFrames(2)) + .twoDoubleArrayRankBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test(timeout = 10_000) + public void testLongArrayRejectionDoesNotDependOnDescribeAvailability() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_wait_millis=100;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.respondToDescribe = false; + sender.table("events").stringColumn("partial", "discard"); + try { + sender.longArray("id", new long[]{99L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(321, 331, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "native-a").atNow(); + handler.targetType = ColumnType.TIMESTAMP_MICRO; + handler.version = 332; + sender.table("events"); + try { + sender.stringColumn("failed_b", "not-a-uuid"); + Assert.fail("expected invalid UUID"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "1970-01-01T00:00:00.000001Z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoVarcharAndTimestampBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongToTimestampTargetsUseRawUnitsAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", Long.MIN_VALUE); + sender.binaryColumn("value", new byte[]{1}).atNow(); + sender.table("events").longColumn("value", -1); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .longTimestampTable("events", target, Long.MAX_VALUE); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongGenerationRemainsNativeBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(271, 281, ColumnType.LONG); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 11).atNow(); + handler.targetType = ColumnType.TIMESTAMP_NANO; + handler.version = 282; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-long"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 12).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoLongAndTimestampBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(231, 241, ColumnType.DOUBLE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleColumn("value", -0.0).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 242; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-double"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleColumn("value", 1e23).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDoubleAndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testRunOfRejectedRowsFlushesPendingRowsAndAdoptsChangedSchema() throws Exception { + assertMemoryLeak(() -> { + for (String mode : new String[]{"auto", "strict"}) { + for (String transaction : new String[]{"off", "on"}) { + for (boolean isRecreated : new boolean[]{false, true}) { + String run = mode + ", transaction=" + transaction + ", recreated=" + isRecreated; + LongTextSchemaHandler handler = new LongTextSchemaHandler(251, 261, ColumnType.INT); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = frozenClock(Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + mode + ";transaction=" + transaction + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;"))) { + sender.table("events").longColumn("value", 7).atNow(); + // ALTER COLUMN value TYPE DOUBLE, or DROP TABLE and CREATE TABLE with + // a DOUBLE column, which gives the table a new id. No frame of the + // open batch reaches the server, so no reply reports the change. + handler.targetType = ColumnType.DOUBLE; + if (isRecreated) { + handler.tableId = 252; + } else { + handler.version = 262; + } + rejectRun(sender); + Assert.assertEquals(run, 1, handler.describeRequests.get()); + // The run of rejected rows makes the next row look the table up. The + // server reports the change, so the sender flushes the pending INT row + // and converts this row to DOUBLE. + sender.table("events").doubleColumn("value", 2.5).atNow(); + Assert.assertEquals(run, 2, handler.describeRequests.get()); + Assert.assertEquals(run, 1, handler.awaitDataFrames(1).size()); + sender.flush(); + List frames = handler.awaitDataFrames(2); + // Like an auto-flush, the sender's flush leaves a transaction open: + // only the application's flush() commits it. + Assert.assertEquals(run, transaction.equals("on") ? QwpConstants.FLAG_DEFER_COMMIT : 0, + frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_DEFER_COMMIT); + Assert.assertEquals(run, 0, + frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_DEFER_COMMIT); + new FrameReader(frames).intThenDoubleBlocks("events", 7, 2.5); + } + } + } + } + }); + } + + @Test + public void testRunOfRejectedRowsAdoptsChangedSchemaWithEmptyBatch() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(253, 263, ColumnType.INT); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 7).atNow(); + sender.flush(); + handler.awaitDataFrames(1); + // ALTER COLUMN value TYPE DOUBLE. The frozen clock keeps the cached + // schema within the age limit, so each new batch takes it unchanged. + handler.targetType = ColumnType.DOUBLE; + handler.version = 264; + rejectRun(sender); + Assert.assertEquals(1, handler.describeRequests.get()); + // Nothing is pending, so the sender adopts the new schema without a flush. + sender.table("events").doubleColumn("value", 2.5).atNow(); + Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals(1, handler.dataFrameCount()); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).intThenDoubleBlocks("events", 7, 2.5); + } + }); + } + + @Test(timeout = 20_000) + public void testRunOfRejectedRowsKeepsSnapshotWhenLookupFails() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(255, 265, ColumnType.INT); + // The schema wait exceeds the test timeout, which proves that a re-check + // waits only for SCHEMA_RECHECK_WAIT_MILLIS. + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_wait_millis=60000;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").longColumn("value", 7).atNow(); + handler.targetType = ColumnType.DOUBLE; + handler.version = 266; + handler.isDescribeSilent = true; + // The wait outlasts the interval between re-checks. + handler.onDescribeIgnored = () -> clock.set(SCHEMA_MAX_AGE_NANOS + 1); + rejectRun(sender); + // The re-check gets no answer, so the row meets the cached schema. + assertDoubleRejected(sender, 2.5); + Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals(0, handler.dataFrameCount()); + + // The row rejected after the failed re-check starts the next run. The + // interval runs from the end of the failed wait, so a server that never + // answers cannot hold the producer all the time. + rejectRows(sender, QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK - 1); + assertDoubleRejected(sender, 2.5); + Assert.assertEquals(2, handler.describeRequests.get()); + + // The lookup that got no answer leaves the next one free to run. + handler.isDescribeSilent = false; + clock.set(2 * (SCHEMA_MAX_AGE_NANOS + 1)); + sender.table("events").doubleColumn("value", 2.5).atNow(); + Assert.assertEquals(3, handler.describeRequests.get()); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).intThenDoubleBlocks("events", 7, 2.5); + } + }); + } + + @Test + public void testRejectionsNoTypeChangeExplainsLookNothingUp() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(257, 267, ColumnType.INT); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 7).atNow(); + handler.targetType = ColumnType.DOUBLE; + handler.version = 268; + for (int i = 0; i < QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK; i++) { + // A column name no table can have. + try { + sender.table("events").longColumn("bad\nname", 1); + Assert.fail("expected invalid column name rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + // A designated timestamp outside the range of its column type. + try { + sender.table("events").longColumn("value", 8).at(Long.MAX_VALUE, ChronoUnit.MICROS); + Assert.fail("expected designated timestamp rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + } + sender.table("events").longColumn("value", 9).atNow(); + Assert.assertEquals(1, handler.describeRequests.get()); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).intTable("events", 7, 9); + } + }); + } + + @Test + public void testRunOfRejectedRowsLooksUpUnchangedSchemaOncePerSecond() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(271, 281, ColumnType.INT); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").longColumn("value", 7).atNow(); + rejectRun(sender); + sender.table("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + + rejectRun(sender); + sender.table("events"); + Assert.assertEquals("a second run within the age limit looks nothing up", 2, + handler.describeRequests.get()); + clock.set(SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(3, handler.describeRequests.get()); + + // An accepted row ends a run, so rejections on both sides of it look + // nothing up even where together they would make a run. + clock.set(3 * SCHEMA_MAX_AGE_NANOS); + rejectRows(sender, QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK - 1); + sender.table("events").longColumn("value", 8).atNow(); + rejectRows(sender, QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK - 1); + sender.table("events"); + Assert.assertEquals(3, handler.describeRequests.get()); + + // The schema never changed, so no lookup flushed the open batch: both + // rows travel in the one frame of the explicit flush. + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + List frames = handler.awaitDataFrames(1); + Assert.assertEquals(1, frames.size()); + new FrameReader(frames).intTable("events", 7, 8); + } + }); + } + + @Test(timeout = 10_000) + public void testRunOfRejectedRowsWaitsForTheWireBeforeLookingUp() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + LongTextSchemaHandler first = new LongTextSchemaHandler(291, 301, ColumnType.INT); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").longColumn("value", 7).atNow(); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + rejectRun(sender); + // The wire is down, so the row neither looks up nor waits; the test + // timeout proves the latter. The run stays due for the next connection. + sender.table("events"); + Assert.assertEquals(1, first.describeRequests.get()); + + // The column changed type during the outage. + LongTextSchemaHandler second = new LongTextSchemaHandler(291, 302, ColumnType.DOUBLE); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.table("events").doubleColumn("value", 2.5).atNow(); + Assert.assertEquals(1, second.describeRequests.get()); + sender.flush(); + new FrameReader(second.awaitDataFrames(2)).intThenDoubleBlocks("events", 7, 2.5); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test + public void testFloatingToTextTargetsUsePinnedWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (String input : new String[]{"FLOAT", "DOUBLE"}) { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(211 + target, 221, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events"); + if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(0x7fc00002)); + else sender.doubleColumn("value", Double.longBitsToDouble(0x7ff8000000000002L)); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events"); + if ("FLOAT".equals(input)) sender.floatColumn("value", 0.1f); + else sender.doubleColumn("value", 1e23); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected floating row rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(1)); + else sender.doubleColumn("value", Double.longBitsToDouble(1)); + sender.atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).floatingTextTable( + "events", target, + "FLOAT".equals(input) ? "0.10000000149011612" : "1.0E23", + "FLOAT".equals(input) ? "1.401298464324817E-45" : "5.0E-324" + ); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + } + }); + } + + @Test + public void testUuidGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(191, 201, ColumnType.UUID); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L) + .atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 202; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-uuid"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("value", 0x2122232425262728L, 0x3132333435363738L).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoUuidAndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(171 + target, 181, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").uuidColumn("value", 7, 0); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L).atNow(); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.longTextTable( + "events", + target, + "11121314-1516-1718-0102-030405060708" + ); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.IPv4, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(211 + target, 221, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .ipv4Column("value", 0) + .ipv4Column("value", "not-an-ip") + .atNow(); + + sender.table("events").ipv4Column("value", 0x05060708); + try { + sender.ipv4Column("failed_b", 0x090a0b0c); + Assert.fail("expected IPv4 to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.uuidColumn("failed_b", 1, 2); + try { + sender.ipv4Column("value", "not-an-ip"); + Assert.fail("expected invalid IPv4 text"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + // Two consecutive rows were rejected, so this row re-checks the + // schema once. It has not changed: the row keeps the pinned wire. + sender.ipv4Column("value", ".....255.1.2.3......").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).ipv4Table( + "events", target, 0xff010203); + Assert.assertEquals("a rejected setter must not look up metadata, but the" + + " row after a run of rejected rows re-checks once", 2, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIpv4GenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(231, 241, ColumnType.IPv4); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").ipv4Column("value", 0x01020304).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 242; + sender.table("events"); + try { + sender.ipv4Column("value", "not-an-ip"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").ipv4Column("value", 0x05060708).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoIpv4AndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testGeoHashTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + int[] targets = {geo20, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .geoHashColumn("value", "u33d") + .atNow(); + + sender.table("events").geoHashColumn("value", 0x11111, 20); + try { + sender.geoHashColumn("failed_b", 1, 20); + Assert.fail("expected GEOHASH to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.geoHashColumn("value", 0x12345, 20).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).geoHashTargetTable( + "events", target, 0xd0c6c, 0x12345, 20); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testGeoHashGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + LongTextSchemaHandler handler = new LongTextSchemaHandler(281, 291, geo20); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").geoHashColumn("value", 0xabcde, 20).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 292; + sender.table("events"); + try { + sender.stringColumn("value", "not-geohash"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").geoHashColumn("value", 0x12345, 20).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoNativeGeoHashAndVarcharBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.LONG256, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE) + .long256Column("value", 9, 10, 11, 12) + .atNow(); + + sender.table("events").long256Column("value", 5, 6, 7, 8); + try { + sender.long256Column("failed_b", 9, 10, 11, 12); + Assert.fail("expected LONG256 to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.long256Column("value", 1, 2, 3, 4).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).long256TargetTable( + "events", target, 1, 2, 3, 4); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLong256GenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(281, 291, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").long256Column("value", 1, 2, 3, 4).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 292; + sender.table("events"); + try { + sender.stringColumn("value", "not-long256"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").long256Column("value", 5, 6, 7, 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoNativeLong256AndVarcharBlocks( + "events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(121 + target, 131, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("value", Long.MIN_VALUE); + // MIN is an effective null write, so this unsupported duplicate must be ignored. + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").longColumn("value", 7); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + // The failed middle row is auto-cancelled; continue without selecting the table again. + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.longTextTable("events", target, Long.toString(Long.MAX_VALUE)); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongSymbolGenerationRemainsPinnedAcrossTextRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(151, 161, ColumnType.SYMBOL); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("value", 7).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 162; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 2); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)) + .twoSymbolAndVarcharBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testAsymmetricUuidRowsArePinnedAndInvalidMiddleRowRollsBack() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 41, 73); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("id", "11121314-1516-1718-0102-030405060708").atNow(); + sender.table("events"); + try { + sender.stringColumn("failed_b", "must-roll-back"); + sender.stringColumn("id", "not-a-uuid"); + Assert.fail("expected invalid UUID"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.uuidColumn("id", 0x2122232425262728L, 0x3132333435363738L).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + FrameReader reader = new FrameReader(frame); + reader.schemaTable("events", 2, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + Assert.assertEquals(0x2122232425262728L, reader.i64()); + Assert.assertEquals(0x3132333435363738L, reader.i64()); + reader.eof(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDuplicateSetterSkipsInvalidConversionAndRefresh() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 7, 9); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2); + sender.stringColumn("id", "invalid and ignored").atNow(); + sender.flush(); + Assert.assertNotNull(handler.awaitDataFrame()); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testExplicitAndServerAssignedDesignatedTimestampUseSchemaFrames() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 11, 12); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).at(Instant.ofEpochSecond(3, 4)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + } + }); + } + + @Test + public void testMissingTableInfersColumnsAndUsesCachedAbsence() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("missing").longColumn("x", 7).atNow(); + sender.table("missing").longColumn("x", 8).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("missing", 2, "x", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(8, reader.i64()); + reader.eof(); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testKnownAdoptionWaitsForBatchBoundary() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, 31, 41); + handler.isSchemaFeedbackEnabled = true; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 11).atNow(); + handler.result = QwpSchemaProtocol.RESULT_KNOWN; + // The pending batch keeps its inferred snapshot: a conflicting input + // fails against the inferred LONG and the failed row rolls back whole. + try { + sender.stringColumn("failed_b", "partial"); + sender.stringColumn("id", "11121314-1516-1718-0102-030405060708"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals(1, handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("id", "21222324-2526-2728-3132-333435363738").atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrames(2)); + reader.twoInferredAdoptionBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 1).atNow(); + try { + sender.stringColumn("id", "different inferred type"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("id", 3).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", 2, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + reader.eof(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testAutoUnavailableSchemaUsesLegacyFrame() throws Exception { + assertMemoryLeak(() -> { + for (int result : new int[]{QwpSchemaProtocol.RESULT_UNAVAILABLE, QwpSchemaProtocol.RESULT_TOO_LARGE}) { + SchemaHandler handler = new SchemaHandler(result, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "value", "legacy"); + } + } + }); + } + + @Test + public void testTooLargeSchemaIsDescribedOnce() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_TOO_LARGE, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + // AUTO once re-sent a blocking DESCRIBE for every batch of a table + // too wide to describe; the cached negative answer now serves them. + for (int i = 0; i < 5; i++) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + } + for (byte[] frame : handler.awaitDataFrames(5)) { + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + SchemaHandler strictHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_TOO_LARGE, -1, -1); + try (TestWebSocketServer server = schemaServer(strictHandler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + for (int i = 0; i < 2; i++) { + try { + sender.table("events").stringColumn("value", "strict"); + Assert.fail("STRICT must reject a table too wide to describe"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + } + // STRICT has no legacy rows to earn feedback, so it asks again each time. + Assert.assertEquals(2, strictHandler.describeRequests.get()); + } + }); + } + + @Test + public void testAgedSchemaIsLookedUpAgainAtNextBatch() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + Assert.assertEquals(1, handler.describeRequests.get()); + + clock.set(SCHEMA_MAX_AGE_NANOS); + sender.table("events"); + Assert.assertEquals("a schema confirmed exactly the age limit ago is fresh", + 1, handler.describeRequests.get()); + + clock.set(SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + // The lookup confirmed the schema, so the batch's first setter does not ask again. + sender.uuidColumn("id", 3, 4).atNow(); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testAcknowledgedFrameExtendsSchemaLifetime() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + + long flushNanos = SCHEMA_MAX_AGE_NANOS * 3 / 5; + clock.set(flushNanos); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The OK reply confirms the schema as of the second flush. + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS); + sender.table("events"); + Assert.assertEquals(1, handler.describeRequests.get()); + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testUnacknowledgedFrameDoesNotExtendSchemaLifetime() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1, false); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + clock.set(SCHEMA_MAX_AGE_NANOS * 3 / 5); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + handler.awaitDataFrames(2); + Assert.assertEquals(1, handler.describeRequests.get()); + + // Neither frame was acknowledged, so only the DESCRIBE at 0 confirms. + clock.set(SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testDurableAckModeConfirmsSchemaAtOkLevel() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + handler.isOkCarryingSeqTxn = true; + TestWebSocketServer durableServer = new TestWebSocketServer(handler, true); + durableServer.setAdvertiseSchema(true); + try (TestWebSocketServer server = start(durableServer); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";request_durable_ack=on;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + AtomicLong clock = new AtomicLong(); + ws.setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + awaitOkAcks(ws, 1); + + long flushNanos = SCHEMA_MAX_AGE_NANOS * 3 / 5; + clock.set(flushNanos); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + awaitOkAcks(ws, 2); + // The server never sends a durable ACK, so the trim watermark stays put. + Assert.assertEquals(-1, sender.getAckedFsn()); + + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS); + sender.table("events"); + Assert.assertEquals("an OK reply confirms the schema without a durable ACK", + 1, handler.describeRequests.get()); + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testUnansweredRecheckKeepsCachedSchemaAndDefersNextCheck() throws Exception { + assertMemoryLeak(() -> { + for (String mode : new String[]{"auto", "strict"}) { + for (boolean isSilent : new boolean[]{true, false}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + mode + ";schema_wait_millis=200;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + String label = "mode=" + mode + ", silent=" + isSilent; + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + + if (isSilent) { + handler.respondToDescribe = false; + } else { + handler.result = QwpSchemaProtocol.RESULT_UNAVAILABLE; + } + clock.set(SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(label, 2, handler.describeRequests.get()); + // The failed re-check defers the next one, so the first setter does not + // wait again, and the row converts against the cached UUID target. + sender.stringColumn("id", "11121314-1516-1718-0102-030405060708").atNow(); + Assert.assertEquals(label, 2, handler.describeRequests.get()); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrames(2).get(1)); + reader.schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + reader.eof(); + } + } + } + }); + } + + @Test + public void testDeniedRecheckFailsRow() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + + handler.result = QwpSchemaProtocol.RESULT_DENIED; + clock.set(SCHEMA_MAX_AGE_NANOS + 1); + for (int attempt = 0; attempt < 2; attempt++) { + try { + sender.table("events"); + Assert.fail("expected access denial"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.ACCESS_DENIED, e.getReason()); + } + } + // Unlike an unanswered re-check, a denial never falls back to the cached + // schema: the retry asks again and fails too. + Assert.assertEquals(3, handler.describeRequests.get()); + } + }); + } + + @Test(timeout = 10_000) + public void testFrameWrittenWhileWireIsDownConfirmsSchemaAfterReconnect() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // The wire is down: the batch converts against the cached schema. + long offlineFlushNanos = SCHEMA_MAX_AGE_NANOS * 3 / 5; + clock.set(offlineFlushNanos); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + long fsn = sender.flushAndGetSequence(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + + // The reply to the replayed frame confirms the schema as of its flush. + clock.set(offlineFlushNanos + SCHEMA_MAX_AGE_NANOS); + sender.table("events"); + Assert.assertEquals(0, second.describeRequests.get()); + clock.set(offlineFlushNanos + SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test + public void testSplitFlushFramesConfirmSchema() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 81, 82); + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + // Each single-table frame fits the cap; the combined two-table frame does not. + server.setAdvertisedMaxBatchSize(48); + try (TestWebSocketServer ignored = start(server); + Sender sender = sender(server)) { + AtomicLong clock = new AtomicLong(); + ((QwpWebSocketSender) sender).setNanoClockForTesting(clock::get); + sender.table("a").uuidColumn("id", 1, 2).atNow(); + sender.table("b").uuidColumn("id", 3, 4).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + long flushNanos = SCHEMA_MAX_AGE_NANOS * 3 / 5; + clock.set(flushNanos); + sender.table("a").uuidColumn("id", 5, 6).atNow(); + sender.table("b").uuidColumn("id", 7, 8).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + Assert.assertEquals("each batch split into one frame per table", 4, handler.awaitDataFrames(4).size()); + Assert.assertEquals(2, handler.describeRequests.get()); + + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS); + sender.table("a"); + sender.table("b"); + Assert.assertEquals(2, handler.describeRequests.get()); + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS + 1); + sender.table("a"); + sender.table("b"); + Assert.assertEquals(4, handler.describeRequests.get()); + } + }); + } + + @Test(timeout = 20_000) + public void testSymbolDictRecycleKeepsSchemaConfirmation() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 41, 73); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + AtomicLong clock = new AtomicLong(); + ws.setNanoClockForTesting(clock::get); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + long flushNanos = SCHEMA_MAX_AGE_NANOS * 3 / 5; + clock.set(flushNanos); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + + // The recycle restarts the engine's FSNs. The frame acknowledged before + // it must still confirm the schema, and the unacknowledged frame after + // it must not, though its engine FSN is below the pre-recycle ones. + handler.acknowledgeData = false; + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS / 5); + sender.resetSymbolDictionary(); + sender.table("events"); + Assert.assertEquals(1, ws.getSymbolDictEpoch()); + sender.uuidColumn("id", 5, 6).atNow(); + sender.flush(); + // The frame reaching the server proves the rebuilt loop's wire is up. + handler.awaitDataFrames(3); + Assert.assertEquals(1, handler.describeRequests.get()); + + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS); + sender.table("events"); + Assert.assertEquals(1, handler.describeRequests.get()); + clock.set(flushNanos + SCHEMA_MAX_AGE_NANOS + 1); + sender.table("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testAutoTooLargeSchemaLegacyRowJoinsPendingSchemaBatch() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.result = QwpSchemaProtocol.RESULT_TOO_LARGE; + + // Schema-validated and legacy tables share one frame format, so the + // legacy row joins the pending batch. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(1); + Assert.assertEquals(2, ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN) + .getShort(6) & 0xffff); + } + }); + } + + @Test + public void testStrictInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() throws Exception { + assertMemoryLeak(() -> { + int[] results = { + QwpSchemaProtocol.RESULT_DENIED, + QwpSchemaProtocol.RESULT_UNAVAILABLE, + QwpSchemaProtocol.RESULT_TOO_LARGE + }; + LineSenderSchemaException.Reason[] reasons = { + LineSenderSchemaException.Reason.ACCESS_DENIED, + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE + }; + for (int i = 0; i < results.length; i++) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + handler.isSchemaFeedbackEnabled = true; + // STRICT surfaces every lookup failure; AUTO would write the + // RESULT_UNAVAILABLE and RESULT_TOO_LARGE cases with the legacy contract instead. + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").longColumn("id", 1).atNow(); + handler.result = results[i]; + // The pending batch never consults the server: the conflict is + // local and only the partial row rolls back. + try { + sender.stringColumn("failed_only", "partial"); + sender.stringColumn("id", "different inferred type"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("id", 3).atNow(); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", 2, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + reader.eof(); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The server could not describe the table for the stale frame, so + // its ACK invalidated the cache. The next explicit table selection + // looks the table up and reports the failure there. + try { + sender.table("events"); + Assert.fail("expected lookup failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reasons[i], e.getReason()); + } + Assert.assertEquals(2, handler.describeRequests.get()); + handler.result = QwpSchemaProtocol.RESULT_MISSING; + sender.table("events").longColumn("id", 5).atNow(); + sender.flush(); + reader = new FrameReader(handler.awaitDataFrames(2).get(1)); + reader.schemaTable("events", 1, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(5, reader.i64()); + reader.eof(); + Assert.assertEquals(3, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testTableNegotiatesBeforeNullNoops() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").doubleArray("values", (double[]) null); + sender.longArray("values", (long[]) null); + sender.longArray("values", (long[][]) null); + sender.longArray("values", (long[][][]) null); + sender.longArray("values", (LongArray) null); + sender.ipv4Column("address", (CharSequence) null); + Assert.assertEquals(1, handler.describeRequests.get()); + sender.cancelRow(); + } + }); + } + + @Test + public void testUnsupportedSetterEvictsSnapshotAndInvalidNameDoesNotRefresh() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 2, 3); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events"); + try { + sender.stringColumn("bad\nname", "x"); + Assert.fail("expected invalid column name"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("column name")); + } + Assert.assertEquals(1, handler.describeRequests.get()); + sender.table("events"); + try { + sender.longColumn("id", 1); + Assert.fail("expected UUID target rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + // The snapshot type of a known column rejected the value, which a type + // change could explain, so the rejection evicted the snapshot and the + // next row looks the table up again. + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyFrameAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + for (boolean isStrict : new boolean[]{false, true}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + (isStrict ? "strict" : "auto") + ";close_flush_timeout_millis=0;")) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + FrameReader reader = new FrameReader(frame); + reader.legacyVarcharTable("events", "value", "legacy"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testOldPeerUsesLegacySmallIntegerWireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .byteColumn("b", (byte) -7) + .shortColumn("s", (short) 42) + .intColumn("i", Integer.MIN_VALUE) + .atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + new FrameReader(frame).legacySmallIntegerTable("events"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyIpv4WireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").ipv4Column("value", 0xc0a80101).atNow(); + sender.table("events").ipv4Column("value", "10.20.30.40").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + new FrameReader(frame).legacyIpv4Table( + "events", "value", 0xc0a80101, 0x0a141e28); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyLong256WireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").long256Column("value", 1, 2, 3, 4).atNow(); + sender.table("events").long256Column( + "value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + new FrameReader(frame).legacyLong256Table("events", "value"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyGeoHashWireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").geoHashColumn("value", 3, 5).atNow(); + sender.table("events").geoHashColumn("value", "U").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + new FrameReader(frame).legacyGeoHashTable("events", "value"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 51, 52); + handler.isSchemaFeedbackEnabled = true; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.version = 53; + sender.table("events"); + try { + sender.stringColumn("id", "still-invalid"); + Assert.fail("expected UUID validation error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + // The rejection neither looks up metadata nor splits the batch: the + // next row joins the block pinned to version 52. + sender.uuidColumn("id", 3, 4).atNow(); + long fsn = sender.flushAndGetSequence(); + new FrameReader(handler.awaitDataFrame()).uuidTable("events", 1, 2, 3, 4); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The stale frame's ACK carries version 53; the next batch adopts it. + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2).get(1)).uuidTable("events", 5, 6); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testPendingLegacyBatchStaysLegacyAcrossUpgradeAndNextBatchAdoptsSchema() throws Exception { + assertMemoryLeak(() -> { + for (boolean isStrict : new boolean[]{false, true}) { + int port = TestPorts.findUnusedPort(); + SchemaHandler legacyHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + TestWebSocketServer legacy = new TestWebSocketServer(legacyHandler, false, null, port); + legacy.start(); + Assert.assertTrue(legacy.awaitStart(5, TimeUnit.SECONDS)); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = Sender.builder("ws::addr=localhost:" + port + + ";schema_mode=" + (isStrict ? "strict" : "auto") + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;") + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build(); + TestWebSocketServer upgraded = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + legacy.close(); + Assert.assertTrue("sender did not observe the legacy connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + sender.table("events").stringColumn("legacy_value", "B"); + SchemaHandler schemaHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 61, 62); + upgraded = new TestWebSocketServer(schemaHandler, false, null, port); + upgraded.setAdvertiseSchema(true); + upgraded.start(); + Assert.assertTrue(upgraded.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed and servicing frames", + schemaHandler.awaitPong()); + sender.atNow(); + // A batch has one wire contract: the pending batch already holds + // legacy rows, so this row stays legacy although the supporting + // connection is up. The first row after the flush adopts schema mode. + sender.table("events").stringColumn("legacy_value", "C").atNow(); + sender.flush(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = schemaHandler.awaitDataFrames(2); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B", "C"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, schemaHandler.describeRequests.get()); + } finally { + sender.close(); + legacy.close(); + if (upgraded != null) { + upgraded.close(); + } + } + } + }); + } + + @Test(timeout = 10_000) + public void testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); + TestWebSocketServer server = null; + try { + Instant value = Instant.ofEpochSecond(3, 456_789_123); + // No server is listening: this effective write must nevertheless + // complete on the caller thread using the existing legacy conversion. + sender.table("events").timestampColumn("ts", value).atNow(); + + SchemaHandler handler = new SchemaHandler( + QwpSchemaProtocol.RESULT_KNOWN, 71, 72, true, -1); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + + // The pending batch is legacy, so this row stays legacy too. The + // first row after the flush adopts schema mode. + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyTimestampTable( + "events", "ts", 3_456_789L, 3_456_789L); + new FrameReader(frames.get(1)).schemaTimestampTable( + "events", "ts", 3_456_789_123L); + Assert.assertEquals(1, handler.describeRequests.get()); + } finally { + sender.close(); + if (server != null) { + server.close(); + } + } + }); + } + + @Test + public void testSchemaBoundBatchCapSplitsPerTable() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 81, 82); + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + // Each single-table frame fits the cap; the combined two-table frame does not. + server.setAdvertisedMaxBatchSize(48); + try (TestWebSocketServer ignored = start(server); + Sender sender = sender(server)) { + sender.table("a").uuidColumn("id", 1, 2).atNow(); + sender.table("b").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = handler.awaitDataFrames(2); + for (int i = 0; i < 2; i++) { + byte[] frame = frames.get(i); + Assert.assertTrue(frame.length <= 48); + Assert.assertEquals(1, + ByteBuffer.wrap(frame).order(ByteOrder.LITTLE_ENDIAN).getShort(6) & 0xffff); + } + Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT) != 0); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + FrameReader first = new FrameReader(frames.get(0)); + first.schemaTable("a", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, first.u8()); + Assert.assertEquals(1, first.i64()); + Assert.assertEquals(2, first.i64()); + first.eof(); + FrameReader second = new FrameReader(frames.get(1)); + second.schemaTable("b", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, second.u8()); + Assert.assertEquals(3, second.i64()); + Assert.assertEquals(4, second.i64()); + second.eof(); + } + }); + } + + @Test(timeout = 10_000) + public void testUnboundedSchemaWaitStillResolves() throws Exception { + assertMemoryLeak(() -> { + for (String mode : new String[]{"strict", "auto"}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + // Long.MAX_VALUE once overflowed the absolute deadline into a zero + // wait: STRICT failed the row and AUTO fell back to legacy framing. + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + mode + ";schema_wait_millis=" + Long.MAX_VALUE + ";" + + "auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertEquals(mode, 1, handler.describeRequests.get()); + handler.awaitDataFrame(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;schema_wait_millis=1000;" + + "initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + QwpTableBuffer b = ((QwpWebSocketSender) sender).getTableBuffer("b"); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + QwpTableBuffer events = ((QwpWebSocketSender) sender).getTableBuffer("events"); + handler.respondToDescribe = false; + try { + sender.table("b"); + Assert.fail("expected schema negotiation deadline"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + Assert.assertEquals(1, events.getRowCount()); + Assert.assertEquals(0, b.getRowCount()); + handler.respondToDescribe = true; + sender.uuidColumn("id", 3, 4).atNow(); + Assert.assertEquals(1, events.getRowCount()); + Assert.assertEquals(1, b.getRowCount()); + sender.flush(); + handler.awaitDataFrame(); + } + }); + } + + @Test + public void testStrictInterruptedSchemaLookupFailsTypedAndPreservesInterrupt() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 93, 94); + handler.respondToDescribe = false; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + AtomicReference writer = new AtomicReference<>(); + AtomicBoolean interruptPreserved = new AtomicBoolean(); + Future result = executor.submit(() -> { + writer.set(Thread.currentThread()); + entered.countDown(); + try { + sender.table("events").uuidColumn("id", 1, 2); + return null; + } catch (LineSenderSchemaException e) { + interruptPreserved.set(Thread.currentThread().isInterrupted()); + return e; + } + }); + try { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + writer.get().interrupt(); + LineSenderSchemaException error = result.get(5, TimeUnit.SECONDS); + Assert.assertNotNull(error); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, error.getReason()); + Assert.assertTrue("writer interrupt status was cleared", interruptPreserved.get()); + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictAsyncWithRecoveredBacklogWaitsForFirstConnection() throws Exception { + assertMemoryLeak(() -> { + File sfRoot = temporaryFolder.newFolder("strict-async-recovery"); + int unavailablePort = publishUnackedSchemaBacklog(sfRoot, 95, 96); + + // Under an asynchronous initial connect, STRICT waits for the first + // connection before it picks a contract, recovered backlog or not. With + // no server, the wait runs out its budget and fails typed. + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + unavailablePort + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;initial_connect_retry=async;schema_mode=strict;" + + "schema_wait_millis=250;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;")) { + try { + sender.table("events"); + Assert.fail("STRICT took the legacy contract before the first connection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoAsyncRecoveredSchemaBacklogWritesLegacyOfflineAndReplaysBacklog() throws Exception { + assertMemoryLeak(() -> { + File sfRoot = temporaryFolder.newFolder("auto-async-recovery"); + int port = publishUnackedSchemaBacklog(sfRoot, 97, 98); + + TestWebSocketServer server = null; + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;initial_connect_retry=async;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + // AUTO (the default) never waits on a wire that has not come up, so + // this row takes the legacy contract right away. + sender.table("events").stringColumn("legacy_value", "A").atNow(); + + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 97, 98); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + sender.flush(); + + // The recovered frame replays first, then the legacy batch. + List frames = handler.awaitDataFrames(2); + FrameReader recovered = new FrameReader(frames.get(0)); + recovered.schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, recovered.u8()); + Assert.assertEquals(1, recovered.i64()); + Assert.assertEquals(2, recovered.i64()); + recovered.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("events", "legacy_value", "A"); + Assert.assertEquals(0, handler.describeRequests.get()); + } finally { + if (server != null) { + server.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoOutageWritesUnseenTableLegacyAndNextBatchAdoptsSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 111, 112); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + first.awaitDataFrame(); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // The wire is down and "other" is not cached. AUTO must not wait out + // the 30 s default schema budget (the test timeout proves it); the row + // takes the legacy contract and queues in store-and-forward. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 113, 114); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + // The first row after the flush consults the schema again: "other" is + // looked up on the fresh connection and adopts the strict contract. + sender.table("other").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + + List frames = second.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyVarcharTable("other", "legacy_value", "A"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("other", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testReconnectKeepsCachedSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + new FrameReader(first.awaitDataFrame()).schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(1, first.describeRequests.get()); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + // The next batch binds from the snapshot cached before the outage, so + // the reconnect costs no DESCRIBE round trip. + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + FrameReader frame = new FrameReader(second.awaitDataFrame()); + frame.schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, frame.u8()); + Assert.assertEquals(3, frame.i64()); + Assert.assertEquals(4, frame.i64()); + frame.eof(); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoOutageLegacyRowJoinsPendingSchemaBatch() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + // A schema row is pending and unflushed when the wire drops. + sender.table("events").uuidColumn("id", 1, 2).atNow(); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // Schema-validated and legacy tables share one frame format, so the + // legacy row joins the pending batch instead of forcing a flush. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + Assert.assertEquals(1, ((QwpWebSocketSender) sender).getTableBuffer("events").getRowCount()); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + + List frames = second.awaitDataFrames(1); + Assert.assertEquals(2, ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN) + .getShort(6) & 0xffff); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictOutageFailsUnseenTableWithinSchemaWaitAndRecovers() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 131, 132); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=strict;schema_wait_millis=250;", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + first.awaitDataFrame(); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // STRICT keeps waiting for the wire and fails typed at the budget. + try { + sender.table("other"); + Assert.fail("STRICT fell back to legacy encoding during an outage"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + // A cached table is still writable offline: "events" was pinned by the + // flushed batch's snapshot and the cache survives until reconnect. + sender.table("events").uuidColumn("id", 3, 4).atNow(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 131, 132); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.table("other").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + + List frames = second.awaitDataFrames(1); + Assert.assertEquals(2, ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN) + .getShort(6) & 0xffff); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testOffRetainedLegacyFramesFailOverFromCapableToLegacyPeer() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142, false); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=off;", disconnected); + TestWebSocketServer replacement = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + byte[] retained = first.awaitDataFrame(); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertArrayEquals(retained, second.awaitDataFrame()); + Assert.assertTrue(sender.drain(5_000)); + Assert.assertFalse(server.hasRequestedSchema()); + Assert.assertFalse(replacement.hasRequestedSchema()); + Assert.assertEquals(0, first.describeRequests.get()); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testStrictRemembersSchemaHandshakeWithoutPublishedFrames() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=strict;schema_wait_millis=100;", disconnected); + TestWebSocketServer replacement = null; + try { + Assert.assertTrue(first.awaitPong()); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + LineSenderSchemaException outage = Assert.assertThrows(LineSenderSchemaException.class, + () -> sender.table("events")); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, outage.getReason()); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertTrue("legacy replacement was not installed", second.awaitPong()); + LineSenderSchemaException downgrade = Assert.assertThrows(LineSenderSchemaException.class, + () -> sender.table("other")); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, downgrade.getReason()); + Assert.assertEquals(0, first.describeRequests.get()); + Assert.assertEquals(0, second.describeRequests.get()); + Assert.assertEquals(0, second.dataFrames.size()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testAutoDowngradesBeforePublishingSchemaFrames() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer replacement = null; + try { + // Lookup alone must not turn a schema-capable peer into a replay requirement. + sender.table("events"); + Assert.assertEquals(1, first.describeRequests.get()); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertTrue("legacy replacement was not installed", second.awaitPong()); + sender.table("other").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + new FrameReader(second.awaitDataFrame()).legacyVarcharTable("other", "legacy_value", "A"); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testSchemaModeOffNeverDescribesOnSchemaCapableServer() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); + Assert.assertEquals(0, handler.describeRequests.get()); + Assert.assertFalse(server.hasRequestedSchema()); + } + }); + } + + @Test(timeout = 10_000) + public void testAutoAbandonsLookupWhenWireDropsUnderIt() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 151, 152); + first.respondToDescribe = false; + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + // A budget far beyond the test timeout: only the wire drop can end the wait. + Sender sender = outageSender(port, "schema_wait_millis=60000;", disconnected); + ExecutorService executor = Executors.newSingleThreadExecutor(); + TestWebSocketServer restarted = null; + try { + Future row = executor.submit(() -> { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + return null; + }); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (first.describeRequests.get() == 0) { + Assert.assertTrue("lookup was never sent", System.nanoTime() < deadline); + Thread.sleep(5); + } + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + // The connection loss fails the pending lookup and AUTO falls back to + // the legacy contract instead of sitting out the schema budget. + row.get(5, TimeUnit.SECONDS); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 151, 152); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + new FrameReader(second.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); + } finally { + executor.shutdownNow(); + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test + public void testResetDiscardsRowsAndKeepsCachedSnapshot() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 101, 102); + handler.isSchemaFeedbackEnabled = true; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.version = 103; + sender.table("events"); + try { + sender.stringColumn("id", "invalid"); + Assert.fail("expected UUID validation error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + // reset() discards the rows without sending anything, so no ACK can + // deliver version 103: the next batch still pins the cached 102. + sender.reset(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + long fsn = sender.flushAndGetSequence(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(4, reader.i64()); + reader.eof(); + // That frame's ACK carries 103 for the batch after it. + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2).get(1)).uuidTable("events", 5, 6); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testNormalSenderAutomaticallyNegotiatesBeforeFirstEffectiveValue() throws Exception { + assertMemoryLeak(() -> { + DescribeUuidHandler handler = new DescribeUuidHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + sender.table("events"); + try { + sender.stringColumn("id", "not-a-uuid"); + sender.cancelRow(); + Assert.fail("expected schema-directed UUID validation"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + private static final class DescribeUuidHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger describeRequests = new AtomicInteger(); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + ByteBuffer input = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + if (input.getInt(0) != QwpConstants.MAGIC_MESSAGE + || data[5] != QwpSchemaProtocol.FLAG_CONTROL) { + return; + } + describeRequests.incrementAndGet(); + long requestId = input.getLong(QwpConstants.HEADER_SIZE + 1); + byte[] column = "id".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + ByteBuffer response = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + response.putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(17) + .putLong(23) + .putShort((short) -1) + .putShort((short) 1) + .putShort((short) column.length) + .put(column) + .putInt(ColumnType.UUID) + .putShort((short) 0); + try { + client.sendBinary(response.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + @Test(timeout = 20_000) + public void testSymbolDictRecycleKeepsSchemaContractAndCache() throws Exception { + assertMemoryLeak(() -> { + for (String mode : new String[]{"strict", "auto"}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 41, 73); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = frozenClock(Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + mode + ";auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;"))) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + sender.table("events").stringColumn("id", "11121314-1516-1718-0102-030405060708").atNow(); + Assert.assertTrue(mode, sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + Assert.assertEquals(mode, 1, handler.describeRequests.get()); + + // The backlog is acknowledged, so this table() call runs the recycle, + // which rebuilds the I/O loop and connects it in the background. The + // rebuilt loop must keep the schema contract and the cached schema. + sender.resetSymbolDictionary(); + sender.table("events"); + Assert.assertEquals(mode, 1, ws.getSymbolDictEpoch()); + sender.stringColumn("id", "21222324-2526-2728-3132-333435363738").atNow(); + Assert.assertTrue(mode, sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + Assert.assertEquals(mode, 2, server.handshakeCount()); + + // A legacy row would ship the text as a VARCHAR column instead. + FrameReader reader = new FrameReader(handler.awaitDataFrames(2).get(1)); + reader.schemaTable("events", 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(mode, 0, reader.u8()); + Assert.assertEquals(mode, 0x3132333435363738L, reader.i64()); + Assert.assertEquals(mode, 0x2122232425262728L, reader.i64()); + reader.eof(); + Assert.assertEquals(mode, 1, handler.describeRequests.get()); + } + } + }); + } + + private static TestWebSocketServer legacyServer(SchemaHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + private static void assertDoubleRejected(Sender sender, double value) { + try { + sender.table("events").doubleColumn("value", value); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + } + + private static void rejectRows(Sender sender, int count) { + for (int i = 0; i < count; i++) { + assertDoubleRejected(sender, 1.5); + } + } + + // Rejects as many consecutive rows as make the next row re-check the schema. + private static void rejectRun(Sender sender) { + rejectRows(sender, QwpSchemaFreshness.REJECTIONS_BEFORE_RECHECK); + } + + /** + * A sender for the outage tests: no auto-flush, fast reconnect backoff, and a + * connection listener that trips {@code disconnected} when the wire drops. + * {@code extraConfig} is appended verbatim (e.g. {@code schema_mode=strict;}). + */ + private static Sender outageSender(int port, String extraConfig, CountDownLatch disconnected) { + return frozenClock(Sender.builder("ws::addr=localhost:" + port + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;" + extraConfig) + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build()); + } + + /** + * Writes one schema-validated UUID row for "events" through a sender rooted at + * {@code sfRoot} against a server that never acknowledges data, so the frame + * stays in store-and-forward for a later sender to recover. Returns the port + * the server listened on, which is free again when this method returns. + */ + private static int publishUnackedSchemaBacklog(File sfRoot, int tableId, long version) throws Exception { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, tableId, version, false); + try (TestWebSocketServer server = schemaServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;close_flush_timeout_millis=0;"; + try (Sender sender = Sender.fromConfig(cfg)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + handler.awaitDataFrame(); + } + return server.getPort(); + } + } + + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + return start(server); + } + + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler, int port) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + return start(server); + } + + private static TestWebSocketServer start(TestWebSocketServer server) throws Exception { + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + private static byte numericWireType(int targetType) { + switch (targetType) { + case ColumnType.BYTE: + return QwpConstants.TYPE_BYTE; + case ColumnType.SHORT: + return QwpConstants.TYPE_SHORT; + case ColumnType.INT: + return QwpConstants.TYPE_INT; + case ColumnType.LONG: + return QwpConstants.TYPE_LONG; + case ColumnType.FLOAT: + return QwpConstants.TYPE_FLOAT; + case ColumnType.DOUBLE: + return QwpConstants.TYPE_DOUBLE; + default: + throw new AssertionError(ColumnType.nameOf(targetType)); + } + } + + private static byte temporalWireType(int targetType) { + switch (targetType) { + case ColumnType.DATE: + return QwpConstants.TYPE_DATE; + case ColumnType.TIMESTAMP_MICRO: + return QwpConstants.TYPE_TIMESTAMP; + case ColumnType.TIMESTAMP_NANO: + return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: + throw new AssertionError(ColumnType.nameOf(targetType)); + } + } + + private static void awaitOkAcks(QwpWebSocketSender sender, long count) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (sender.getTotalAcks() < count) { + if (System.nanoTime() - deadline > 0) { + Assert.fail("timed out waiting for " + count + " OK replies; received " + sender.getTotalAcks()); + } + Compat.onSpinWait(); + } + } + + /** + * Freezes the sender's clock, so a cached schema never outlives the schema age + * limit however long a test waits; the tests that pin the limit move their own + * clock. + */ + private static Sender frozenClock(Sender sender) { + ((QwpWebSocketSender) sender).setNanoClockForTesting(() -> 0L); + return sender; + } + + private static Sender sender(TestWebSocketServer server) { + return frozenClock(Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")); + } + + private static final class DecimalTextVector { + private final String caseId; + private final String expectedText; + private final String inputForm; + private final String sourceType; + private final int sourceScale; + private final String[] limbs; + private final String targetType; + + private DecimalTextVector(String[] fields) { + Assert.assertEquals(String.join("\t", fields), 10, fields.length); + caseId = fields[0]; + sourceType = fields[1]; + sourceScale = Integer.parseInt(fields[2]); + limbs = new String[]{fields[3], fields[4], fields[5], fields[6]}; + targetType = fields[7]; + inputForm = fields[8]; + expectedText = fields[9]; + } + + private void append(Sender sender, String column) { + if ("JAVA_NULL".equals(inputForm)) { + appendJavaNull(sender, column); + return; + } + if ("NULL_VALUE".equals(inputForm)) { + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, Decimal64.NULL_VALUE); + return; + case "DECIMAL128": + sender.decimalColumn(column, Decimal128.NULL_VALUE); + return; + case "DECIMAL256": + sender.decimalColumn(column, Decimal256.NULL_VALUE); + return; + default: + throw new AssertionError(sourceType); + } + } + Assert.assertEquals("VALUE", inputForm); + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, new Decimal64(limb(0), sourceScale)); + return; + case "DECIMAL128": + sender.decimalColumn(column, new Decimal128(limb(1), limb(0), sourceScale)); + return; + case "DECIMAL256": + sender.decimalColumn(column, + new Decimal256(limb(3), limb(2), limb(1), limb(0), sourceScale)); + return; + default: + throw new AssertionError(sourceType); + } + } + + private void appendJavaNull(Sender sender, String column) { + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, (Decimal64) null); + return; + case "DECIMAL128": + sender.decimalColumn(column, (Decimal128) null); + return; + case "DECIMAL256": + sender.decimalColumn(column, (Decimal256) null); + return; + default: + throw new AssertionError(sourceType); + } + } + + private boolean isNull() { + return !"VALUE".equals(inputForm); + } + + private long limb(int index) { + Assert.assertNotEquals("-", limbs[index]); + return Long.parseUnsignedLong(limbs[index], 16); + } + + private int sourceIndex() { + switch (sourceType) { + case "DECIMAL64": + return 0; + case "DECIMAL128": + return 1; + case "DECIMAL256": + return 2; + default: + throw new AssertionError(sourceType); + } + } + + private int targetIndex() { + if ("STRING".equals(targetType)) { + return 0; + } + if ("VARCHAR".equals(targetType)) { + return 1; + } + throw new AssertionError(targetType); + } + + private int targetType() { + if ("STRING".equals(targetType)) { + return ColumnType.STRING; + } + if ("VARCHAR".equals(targetType)) { + return ColumnType.VARCHAR; + } + throw new AssertionError(targetType); + } + } + + /** + * Builds the cumulative ACK the fake servers send for a data frame. Like the real + * server, a handler attaches the current schema of {@code tableName} when the + * snapshot it last delivered to the client is stale, so the client learns about + * a change on the batch boundary without a DESCRIBE of its own. + */ + private static byte[] ackWithSchemaUpdate(long sequence, String tableName, byte[] schemaPayload) { + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + ByteBuffer ack = ByteBuffer.allocate(1 + 8 + 2 + 2 + 2 + name.length + 4 + schemaPayload.length) + .order(ByteOrder.LITTLE_ENDIAN); + ack.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(sequence).putShort((short) 0) + .putShort((short) 1).putShort((short) name.length).put(name) + .putInt(schemaPayload.length).put(schemaPayload); + return ack.array(); + } + + /** The ACK a server sends when it cannot produce the schema snapshot a stale frame calls for. */ + private static byte[] ackInvalidateAll(long sequence) { + ByteBuffer ack = ByteBuffer.allocate(1 + 8 + 2).order(ByteOrder.LITTLE_ENDIAN); + ack.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)) + .putLong(sequence).putShort((short) 0); + return ack.array(); + } + + private static byte[] schemaControlFrame(byte[] schemaPayload) { + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + schemaPayload.length) + .order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(schemaPayload.length) + .put(schemaPayload); + return out.array(); + } + + /** + * Walks one QWP data frame, or a sequence of single-table frames. A table + * pins one schema snapshot per batch, so two schema generations of the same + * table always arrive in two frames; the two-block helpers read the second + * block from the second frame, skipping its message header and symbol + * dictionary prelude. + */ + private static final class FrameReader { + private final List frames; + private int frameIndex; + private ByteBuffer in; + + private FrameReader(byte[] data) { + this(Collections.singletonList(data)); + } + + private FrameReader(List frames) { + this.frames = frames; + this.in = ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN); + } + + private void nextFrameIfExhausted() { + if (in.hasRemaining() || frameIndex + 1 >= frames.size()) { + return; + } + frameIndex++; + in = ByteBuffer.wrap(frames.get(frameIndex)).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + varint(); + for (int i = 0, n = varint(); i < n; i++) { + string(); + } + } + + private long i64() { + return in.getLong(); + } + + private void legacyVarcharTable(String table, String column, String... values) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(values.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + int total = 0; + StringBuilder joined = new StringBuilder(); + for (String value : values) { + joined.append(value); + total += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(total, in.getInt()); + } + byte[] bytes = joined.toString().getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[total]; + in.get(actual); + Assert.assertArrayEquals(bytes, actual); + eof(); + } + + private void legacyTimestampTable(String table, String column, long... expectedMicros) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(expectedMicros.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + for (long micros : expectedMicros) { + Assert.assertEquals(micros, i64()); + } + eof(); + } + + private void schemaTimestampTable( + String table, + String column, + long expectedNanos + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, column, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedNanos, i64()); + eof(); + } + + private void legacyIpv4Table(String table, String column, int... values) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(values.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_IPv4, u8()); + Assert.assertEquals(0, u8()); + for (int value : values) { + Assert.assertEquals(value, in.getInt()); + } + eof(); + } + + private void legacyLong256Table(String table, String column) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, u8()); + Assert.assertEquals(0, u8()); + for (long value : new long[]{1, 2, 3, 4}) { + Assert.assertEquals(value, i64()); + } + for (int i = 0; i < 4; i++) { + Assert.assertEquals(Long.MIN_VALUE, i64()); + } + eof(); + } + + private void legacySmallIntegerTable(String table) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(3, varint()); + Assert.assertEquals("b", string()); + Assert.assertEquals(QwpConstants.TYPE_BYTE, u8()); + Assert.assertEquals("s", string()); + Assert.assertEquals(QwpConstants.TYPE_SHORT, u8()); + Assert.assertEquals("i", string()); + Assert.assertEquals(QwpConstants.TYPE_INT, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals((byte) -7 & 0xff, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(42, in.getShort()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Integer.MIN_VALUE, in.getInt()); + eof(); + } + + private void longTextTable( + String table, + int targetType, + String expectedValue + ) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + // Global dictionary registration is append-only: the rolled-back B value + // remains an unused dictionary entry, while no B row survives. + Assert.assertEquals("7", string()); + Assert.assertEquals(expectedValue, string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, 2, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(1, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + } + eof(); + } + + private void ipv4Table(String table, int targetType, int expectedValue) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = targetType == ColumnType.IPv4 + ? QwpConstants.TYPE_IPv4 + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, 2, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (wireType == QwpConstants.TYPE_IPv4) { + Assert.assertEquals(expectedValue, in.getInt()); + } else { + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(9, in.getInt()); + Assert.assertEquals("255.1.2.3", stringBytes(9)); + } + eof(); + } + + private void intToIpv4Table(String table) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 4, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, u8()); + Assert.assertEquals(3, u8()); + Assert.assertEquals(1, in.getInt()); + Assert.assertEquals(-1, in.getInt()); + eof(); + } + + private void long256TargetTable( + String table, + int targetType, + long l0, + long l1, + long l2, + long l3 + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = targetType == ColumnType.LONG256 + ? QwpConstants.TYPE_LONG256 + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, 2, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (wireType == QwpConstants.TYPE_LONG256) { + Assert.assertEquals(l0, i64()); + Assert.assertEquals(l1, i64()); + Assert.assertEquals(l2, i64()); + Assert.assertEquals(l3, i64()); + } else { + String expected = "0x04000000000000000300000000000000020000000000000001"; + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + } + eof(); + } + + private void long256Table(String table) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0xfedcba9876543210L, i64()); + Assert.assertEquals(0x0123456789abcdefL, i64()); + Assert.assertEquals(0, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void geoHashTable( + String table, int bits, int first, int second + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(bits, varint()); + Assert.assertEquals(first, u8()); + Assert.assertEquals(second, u8()); + eof(); + } + + private void geoHashTargetTable( + String table, + int targetType, + long first, + long second, + int precision + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = ColumnType.isGeoHash(targetType) + ? QwpConstants.TYPE_GEOHASH + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, 2, "value", wireType); + if (wireType == QwpConstants.TYPE_GEOHASH) { + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(precision, varint()); + for (long value : new long[]{first, second}) { + for (int i = 0; i < (precision + 7) / 8; i++) { + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), u8()); + } + } + } else { + String firstText = "11010000110001101100"; + String secondText = "00010010001101000101"; + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(firstText.length(), in.getInt()); + Assert.assertEquals(firstText.length() + secondText.length(), in.getInt()); + Assert.assertEquals(firstText, stringBytes(firstText.length())); + Assert.assertEquals(secondText, stringBytes(secondText.length())); + } + eof(); + } + + private void legacyGeoHashTable(String table, String column) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, varint()); + Assert.assertEquals(3, u8()); + Assert.assertEquals(26, u8()); + eof(); + } + + private void decimal64Table( + String table, int scale, long first, long second + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(scale, u8()); + Assert.assertEquals(first, i64()); + Assert.assertEquals(second, i64()); + eof(); + } + + private void decimalTextLifecycleTable(String table) { + StringBuilder maxScale = new StringBuilder(78).append("0."); + for (int i = 0; i < 76; i++) { + maxScale.append('9'); + } + String expected = "123.40-0.125" + maxScale + '0'; + + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 5, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0x04, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals(12, in.getInt()); + Assert.assertEquals(90, in.getInt()); + Assert.assertEquals(91, in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void decimalTextTable(String table, String... expectedValues) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, expectedValues.length, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + int totalLength = 0; + StringBuilder expected = new StringBuilder(); + for (String value : expectedValues) { + expected.append(value); + totalLength += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(totalLength, in.getInt()); + } + byte[] expectedBytes = expected.toString().getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[totalLength]; + in.get(actual); + Assert.assertArrayEquals(expectedBytes, actual); + eof(); + } + + private void parsedDecimalTextTable(String table) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 4, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0x0c, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals(12, in.getInt()); + Assert.assertEquals("123.451.2300", stringBytes(12)); + eof(); + } + + private void twoDecimalBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1_000, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void twoDecimal64ScaleBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(12_345, i64()); + eof(); + } + + private void twoDecimal64AndVarcharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(12_340, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(7, in.getInt()); + Assert.assertEquals("25.0000", stringBytes(7)); + eof(); + } + + private void twoDecimalWidthAndScaleBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(10_000, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, u8()); + Assert.assertEquals(-8_446_744_073_709_551_616L, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void nullGeoHashTable(String table, int bits) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(bits, varint()); + eof(); + } + + private void smallIntegerNumericTable(String table, byte wireType) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 4, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + assertSmallIntegerValue(wireType, -1); + assertSmallIntegerValue(wireType, 2); + assertSmallIntegerValue(wireType, 3); + eof(); + } + + private void smallIntegerDecimalTable(String table) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 4, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(2, u8()); + Assert.assertEquals(4_200, i64()); + Assert.assertEquals(-700, i64()); + Assert.assertEquals(214_748_364_700L, i64()); + eof(); + } + + private void smallIntegerTextTable(String table, int targetType) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(4, varint()); + Assert.assertEquals("-128", string()); + Assert.assertEquals("99", string()); + Assert.assertEquals("32767", string()); + Assert.assertEquals("2147483647", string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, 4, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(3, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(4, in.getInt()); + Assert.assertEquals(9, in.getInt()); + Assert.assertEquals(19, in.getInt()); + Assert.assertEquals("-128327672147483647", stringBytes(19)); + } + eof(); + } + + private void integerTemporalTable(String table, byte wireType) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 5, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + if (wireType != QwpConstants.TYPE_DATE) { + Assert.assertEquals(0, u8()); + } + Assert.assertEquals(-1, i64()); + Assert.assertEquals(2, i64()); + Assert.assertEquals(3, i64()); + Assert.assertEquals(Long.MAX_VALUE, i64()); + eof(); + } + + private void assertSmallIntegerValue(byte wireType, long value) { + switch (wireType) { + case QwpConstants.TYPE_BYTE: + Assert.assertEquals((byte) value & 0xff, u8()); + break; + case QwpConstants.TYPE_SHORT: + Assert.assertEquals((short) value, in.getShort()); + break; + case QwpConstants.TYPE_INT: + Assert.assertEquals((int) value, in.getInt()); + break; + case QwpConstants.TYPE_LONG: + Assert.assertEquals(value, in.getLong()); + break; + case QwpConstants.TYPE_FLOAT: + Assert.assertEquals(Float.floatToRawIntBits((float) value), in.getInt()); + break; + case QwpConstants.TYPE_DOUBLE: + Assert.assertEquals(Double.doubleToRawLongBits((double) value), in.getLong()); + break; + default: + throw new AssertionError(wireType); + } + } + + private void twoGeoHashBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(60, varint()); + Assert.assertEquals(0x443214c74254bL, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, varint()); + Assert.assertEquals(31, u8()); + eof(); + } + + private void twoDateAndTimestampBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(12, i64()); + eof(); + } + + private void twoLong256AndVarcharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0xfedcba9876543210L, i64()); + Assert.assertEquals(0x0123456789abcdefL, i64()); + Assert.assertEquals(0, i64()); + Assert.assertEquals(0, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(11, in.getInt()); + Assert.assertEquals("not-long256", stringBytes(11)); + eof(); + } + + private void twoNativeLong256AndVarcharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(0, u8()); + for (long value : new long[]{1, 2, 3, 4}) { + Assert.assertEquals(value, i64()); + } + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + String expected = "0x08000000000000000700000000000000060000000000000005"; + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void twoNativeGeoHashAndVarcharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(20, varint()); + Assert.assertEquals(0xde, u8()); + Assert.assertEquals(0xbc, u8()); + Assert.assertEquals(0x0a, u8()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + String expected = "00010010001101000101"; + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void longTimestampTable( + String table, int targetType, long expectedValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", + targetType == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedValue, i64()); + eof(); + } + + private void twoLongAndTimestampBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(12, i64()); + eof(); + } + + private void twoVarcharAndTimestampBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_TIMESTAMP); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + eof(); + } + + private void timestampTextTable( + String table, String expectedLastValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedLastValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + eof(); + } + + private void stringDateTable( + String table, long expectedValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(expectedValue, i64()); + eof(); + } + + private void doubleArrayTable(String table) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{1, 2}, 1.0, 2.0); + assertDoubleArray(new int[]{2, 1}, 3.0, 4.0); + eof(); + } + + private void twoDoubleArrayRankBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{1, 2}, 1.0, 2.0); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{2}, 3.0, 4.0); + eof(); + } + + private void assertDoubleArray(int[] shape, double... values) { + Assert.assertEquals(shape.length, u8()); + for (int dimension : shape) { + Assert.assertEquals(dimension, in.getInt()); + } + for (double value : values) { + Assert.assertEquals(Double.doubleToRawLongBits(value), in.getLong()); + } + } + + private void twoDateAndVarcharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + eof(); + } + + private void charTable(String table, char first, char second) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 2, "value", QwpConstants.TYPE_CHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(first, in.getChar()); + Assert.assertEquals(second, in.getChar()); + eof(); + } + + private void twoVarcharAndCharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_CHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0xffff, in.getChar()); + eof(); + } + + private void twoTimestampAndVarcharBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Long.MIN_VALUE, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + String expected = "2262-04-11T23:47:16.854Z"; + byte[] bytes = expected.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(bytes.length, in.getInt()); + byte[] actual = new byte[bytes.length]; + in.get(actual); + Assert.assertArrayEquals(bytes, actual); + eof(); + } + + private void floatingTextTable( + String table, + int targetType, + String rolledBackValue, + String expectedValue + ) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(rolledBackValue, string()); + Assert.assertEquals(expectedValue, string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, 2, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(1, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + } + eof(); + } + + private void messageHeader(int tableCount) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + if (frames.size() == 1) { + Assert.assertEquals(tableCount, in.getShort() & 0xffff); + } else { + // One generation per frame: the blocks span the frame sequence. + Assert.assertEquals(tableCount, frames.size()); + Assert.assertEquals(1, in.getShort() & 0xffff); + } + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + } + + private void schemaBlockHeader( + String table, + int rows, + String column, + byte wireType + ) { + nextFrameIfExhausted(); + Assert.assertEquals(table, string()); + Assert.assertEquals(rows, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(wireType & 0xff, u8()); + } + + private void twoSymbolAndVarcharBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("7", string()); + + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_SYMBOL); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(1, in.getInt()); + Assert.assertEquals('8', u8()); + eof(); + } + + private void twoSmallIntegerDecimalBlocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, u8()); + Assert.assertEquals(100_000_000, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void twoIntDecimalAndIpv4Blocks( + String table + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1_000, in.getInt()); + eof(); + } + + private void twoUuidAndVarcharBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x0102030405060708L, i64()); + Assert.assertEquals(0x1112131415161718L, i64()); + + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(36, in.getInt()); + Assert.assertEquals("31323334-3536-3738-2122-232425262728", stringBytes(36)); + eof(); + } + + private void twoIpv4AndVarcharBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x01020304, in.getInt()); + + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(7, in.getInt()); + Assert.assertEquals("5.6.7.8", stringBytes(7)); + eof(); + } + + private void intTable(String table, int... values) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, values.length, "value", QwpConstants.TYPE_INT); + Assert.assertEquals(0, u8()); + for (int value : values) { + Assert.assertEquals(value, in.getInt()); + } + eof(); + } + + private void intThenDoubleBlocks(String table, int intValue, double doubleValue) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_INT); + Assert.assertEquals(0, u8()); + Assert.assertEquals(intValue, in.getInt()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Double.doubleToRawLongBits(doubleValue), i64()); + eof(); + } + + private void twoDoubleAndVarcharBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Double.doubleToRawLongBits(-0.0), i64()); + schemaBlockHeader(table, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals("1.0E23", stringBytes(6)); + eof(); + } + + private String stringBytes(int length) { + byte[] bytes = new byte[length]; + in.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private void eof() { + Assert.assertEquals("unread frames", frames.size() - 1, frameIndex); + Assert.assertFalse("unexpected trailing column or value bytes", in.hasRemaining()); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + in.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private void schemaTable(String table, int rows, String column, byte type) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + u8(); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(rows, varint()); + int columns = varint(); + Assert.assertEquals(1, columns); + Assert.assertEquals(column, string()); + Assert.assertEquals(type & 0xff, u8()); + } + + private void uuidTable(String table, long... loHiPairs) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, loHiPairs.length / 2, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + for (long word : loHiPairs) { + Assert.assertEquals(word, i64()); + } + eof(); + } + + private void twoInferredAdoptionBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + Assert.assertEquals(table, string()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("id", string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + + schemaBlockHeader(table, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x3132333435363738L, i64()); + Assert.assertEquals(0x2122232425262728L, i64()); + eof(); + } + + private void uuidOnlyTable(String table, long lo, long hi) { + schemaTable(table, 1, "failed_b", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(lo, i64()); + Assert.assertEquals(hi, i64()); + eof(); + } + + private int u8() { + return in.get() & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int b; + do { + b = u8(); + result |= (b & 0x7f) << shift; + shift += 7; + } while ((b & 0x80) != 0); + return result; + } + } + + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private volatile boolean acknowledgeData; + private final List dataFrames = new ArrayList<>(); + private final CountDownLatch pong = new CountDownLatch(1); + private final AtomicInteger describeRequests = new AtomicInteger(); + private final int designatedIndex; + private volatile int result; + private volatile boolean respondToDescribe = true; + private final int tableId; + private volatile long version; + private long ackSequence; + // The snapshot the client last received for "events", through DESCRIBE or ACK + // feedback. With feedback enabled, a data frame acknowledged while that + // snapshot is stale carries the update, as the real server does. + private int deliveredResult = -1; + private long deliveredVersion = -1; + private volatile boolean isSchemaFeedbackEnabled; + // OK replies name a table seqTxn, so in durable-ack mode they wait for a + // durable ACK, which this handler never sends. + private volatile boolean isOkCarryingSeqTxn; + + private SchemaHandler(int result, int tableId, long version) { + this(result, tableId, version, true); + } + + private SchemaHandler(int result, int tableId, long version, boolean acknowledgeData) { + this(result, tableId, version, acknowledgeData, 2); + } + + private SchemaHandler( + int result, + int tableId, + long version, + boolean acknowledgeData, + int designatedIndex + ) { + this.result = result; + this.tableId = tableId; + this.version = version; + this.acknowledgeData = acknowledgeData; + this.designatedIndex = designatedIndex; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + describeRequests.incrementAndGet(); + if (!respondToDescribe) { + return; + } + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); + int nameLength = request.getShort(QwpConstants.HEADER_SIZE + 1 + Long.BYTES) & 0xffff; + byte[] name = new byte[nameLength]; + request.position(QwpConstants.HEADER_SIZE + 1 + Long.BYTES + Short.BYTES); + request.get(name); + String tableName = new String(name, StandardCharsets.UTF_8); + int responseTableId = tableId + ("b".equals(tableName) ? 1 : 0); + if ("events".equals(tableName)) { + deliveredResult = result; + deliveredVersion = version; + } + send(client, schemaControlFrame(schemaPayload(requestId, responseTableId))); + return; + } + dataFrames.add(data); + notifyAll(); + if (!acknowledgeData) { + return; + } + send(client, ack()); + } + + private byte[] ack() { + long sequence = ackSequence++; + if (isOkCarryingSeqTxn) { + return okWithSeqTxn(sequence); + } + if (!isSchemaFeedbackEnabled || (result == deliveredResult && version == deliveredVersion)) { + return QwpWireTestUtils.buildAck(sequence); + } + if (result != QwpSchemaProtocol.RESULT_KNOWN && result != QwpSchemaProtocol.RESULT_MISSING) { + return ackInvalidateAll(sequence); + } + deliveredResult = result; + deliveredVersion = version; + return ackWithSchemaUpdate(sequence, "events", schemaPayload(0, tableId)); + } + + private static byte[] okWithSeqTxn(long sequence) { + byte[] name = "events".getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(1 + 8 + 2 + 2 + name.length + 8).order(ByteOrder.LITTLE_ENDIAN) + .put(WebSocketResponse.STATUS_OK) + .putLong(sequence) + .putShort((short) 1) + .putShort((short) name.length) + .put(name) + .putLong(sequence + 1) + .array(); + } + + private static void send(TestWebSocketServer.ClientHandler client, byte[] frame) { + try { + client.sendBinary(frame); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onOpen(TestWebSocketServer.ClientHandler client) { + try { + client.sendPing(new byte[]{0x51, 0x57, 0x50}); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onPong(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length == 3 && data[0] == 0x51 && data[1] == 0x57 && data[2] == 0x50) { + pong.countDown(); + } + } + + private boolean awaitPong() throws InterruptedException { + return pong.await(5, TimeUnit.SECONDS); + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private synchronized List awaitDataFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.size() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for " + count + " data frames; received " + dataFrames.size()); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return new ArrayList<>(dataFrames); + } + + private byte[] schemaPayload(long requestId, int responseTableId) { + byte[] id = "id".getBytes(StandardCharsets.UTF_8); + byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); + byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1; + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + payloadLength += 4 + 8 + 2 + 2 + + 2 + id.length + 4 + 2 + + 2 + failed.length + 4 + 2 + + 2 + ts.length + 4 + 2; + } + ByteBuffer out = ByteBuffer.allocate(payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + out.putInt(responseTableId).putLong(version).putShort((short) designatedIndex).putShort((short) 3) + .putShort((short) id.length).put(id).putInt(ColumnType.UUID).putShort((short) 0) + .putShort((short) failed.length).put(failed).putInt(ColumnType.STRING).putShort((short) 0) + .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); + } + return out.array(); + } + } + + private static final class LongTextSchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final List dataFrames = new ArrayList<>(); + private final AtomicInteger describeRequests = new AtomicInteger(); + private final CountDownLatch pong = new CountDownLatch(1); + private long ackSequence; + // Counts DESCRIBE requests but answers none of them. + private volatile boolean isDescribeSilent; + private volatile Runnable onDescribeIgnored = () -> { + }; + private volatile int tableId; + private volatile int targetType; + private volatile long version; + // The version the client last received, through DESCRIBE or ACK feedback. + private long deliveredVersion = -1; + + private LongTextSchemaHandler(int tableId, long version, int targetType) { + this.tableId = tableId; + this.version = version; + this.targetType = targetType; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + describeRequests.incrementAndGet(); + if (isDescribeSilent) { + onDescribeIgnored.run(); + return; + } + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); + deliveredVersion = version; + send(client, schemaControlFrame(schemaPayload(requestId))); + return; + } + dataFrames.add(data); + notifyAll(); + long sequence = ackSequence++; + if (version == deliveredVersion) { + send(client, QwpWireTestUtils.buildAck(sequence)); + } else { + deliveredVersion = version; + send(client, ackWithSchemaUpdate(sequence, "events", schemaPayload(0))); + } + } + + private static void send(TestWebSocketServer.ClientHandler client, byte[] frame) { + try { + client.sendBinary(frame); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + // The client answers this ping once its I/O thread installed the connection. + @Override + public void onOpen(TestWebSocketServer.ClientHandler client) { + try { + client.sendPing(new byte[]{0x51, 0x57, 0x50}); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onPong(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length == 3 && data[0] == 0x51 && data[1] == 0x57 && data[2] == 0x50) { + pong.countDown(); + } + } + + private boolean awaitPong() throws InterruptedException { + return pong.await(5, TimeUnit.SECONDS); + } + + private synchronized int dataFrameCount() { + return dataFrames.size(); + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for LONG-to-text data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private synchronized List awaitDataFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.size() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for " + count + " data frames; received " + dataFrames.size()); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return new ArrayList<>(dataFrames); + } + + private byte[] schemaPayload(long requestId) { + byte[] value = "value".getBytes(StandardCharsets.UTF_8); + byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); + byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + + 2 + value.length + 4 + 2 + + 2 + failed.length + 4 + 2 + + 2 + ts.length + 4 + 2; + ByteBuffer out = ByteBuffer.allocate(payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(tableId).putLong(version).putShort((short) 2).putShort((short) 3) + .putShort((short) value.length).put(value).putInt(targetType).putShort((short) 0) + .putShort((short) failed.length).put(failed).putInt(ColumnType.UUID).putShort((short) 0) + .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); + return out.array(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 9767eaf9c..9d0c9437f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -43,6 +43,27 @@ */ public class QwpWebSocketEncoderTest { + @Test + public void testHeaderRejectsTableCountOutsideUnsignedShortRange() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.writeHeader(0, 0); + encoder.writeHeader(0xffff, 0); + assertIllegalArgument(() -> encoder.writeHeader(-1, 0)); + assertIllegalArgument(() -> encoder.writeHeader(0x1_0000, 0)); + } + }); + } + + private static void assertIllegalArgument(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + // expected + } + } + @Test public void testBufferResetAndReuse() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 0df432cc7..f8ae12ca3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -51,6 +51,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -192,7 +193,8 @@ public void testBinaryColumnAfterCloseThrows() throws Exception { @Test public void testBinaryColumnRejectsNullArray() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.binaryColumn("x", (byte[]) null); @@ -207,7 +209,8 @@ public void testBinaryColumnRejectsNullArray() throws Exception { @Test public void testBinaryColumnRejectsNullDirectByteSlice() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.binaryColumn("x", (io.questdb.client.std.bytes.DirectByteSlice) null); @@ -264,7 +267,8 @@ public void testCancelRowAfterCloseThrows() throws Exception { @Test public void testCancelRowDiscardsPartialRow() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("test"); sender.longColumn("x", 1); sender.boolColumn("y", true); @@ -289,6 +293,28 @@ public void testCancelRowNoOpWithoutTable() throws Exception { }); } + @Test + public void testSelectedTableSupportsEmptyFlushAndImplicitRows() throws Exception { + assertMemoryLeak(() -> { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { + sender.table("t"); + sender.flush(); + + sender.longColumn("v", 1).atNow(); + sender.longColumn("discarded", 2); + sender.cancelRow(); + sender.longColumn("v", 3).atNow(); + + sender.longColumn("v", 4); + sender.table("t"); // same-table selection does not disturb a partial row + sender.boolColumn("flag", true).atNow(); + + Assert.assertEquals(3, sender.getTableBuffer("t").getRowCount()); + } + }); + } + @Test public void testCancelledRowDoesNotShipItsTable() throws Exception { assertMemoryLeak(() -> { @@ -582,7 +608,6 @@ public void testIpv4ColumnStringNullReturnsThis() throws Exception { // return the sender so chained builders keep working. assertMemoryLeak(() -> { try (QwpWebSocketSender sender = createUnconnectedSender()) { - sender.table("t"); Assert.assertSame(sender, sender.ipv4Column("addr", null)); } }); @@ -596,7 +621,8 @@ public void testIpv4ColumnStringRejectsNullLiteral() throws Exception { // real IPv4". Reject it as an invalid IPv4 address; the user can // pass a null reference (or omit the setter) to mark the row null. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); for (String input : new String[]{"null", "NULL", "Null", "nUlL"}) { try { @@ -622,7 +648,8 @@ public void testIpv4ColumnStringRejectsZeroAddress() throws Exception { // valid dotted-quad. Reject it on the client so the user gets a // clear error instead of a value that disappears on read. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.ipv4Column("addr", "0.0.0.0"); @@ -726,15 +753,8 @@ public void testOperationsAfterCloseThrow() throws Exception { @Test public void testPendingBytesMatchesGroundTruthAcrossTableSwitches() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - // Bypass ensureConnected: sendRow only short-circuits on - // connected==true and we don't drive any path that touches - // the cursor engine in this test. - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { // Round-robin three tables to exercise the per-row delta // logic across switches. The running pendingBytes must @@ -756,12 +776,8 @@ public void testPendingBytesMatchesGroundTruthAcrossTableSwitches() throws Excep @Test public void testPendingBytesUnchangedByCancelRow() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t0").longColumn("a", 1).longColumn("b", 2).atNow(); long committed = sender.getPendingBytes(); @@ -802,12 +818,8 @@ public void testRowAcceptedWhenServerAdvertisesNoCap() throws Exception { // the volatile serverMaxBatchSize to 0 mid-row, and a torn read that observed // the drop between the guard and the throw must not spuriously reject the row. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(0); Assert.assertEquals(0, sender.getServerMaxBatchSize()); @@ -831,12 +843,8 @@ public void testRowExceedingServerBatchCapThrows() throws Exception { // already exceed the server's advertised cap, before nextRow() commits it, so // the flush cannot build an oversize WS frame the server closes with 1009. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(64); Assert.assertEquals(64, sender.getServerMaxBatchSize()); @@ -872,12 +880,8 @@ public void testCumulativeBufferedBytesDoNotTripPerRowCap() throws Exception { // row once the table's CUMULATIVE buffered bytes crossed the cap; this // pins the wiring the single-row tests cannot distinguish. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(64); // Three rows of 22-26 bytes each: every row fits the 64-byte cap, @@ -887,6 +891,9 @@ public void testCumulativeBufferedBytesDoNotTripPerRowCap() throws Exception { } QwpTableBuffer buf = sender.getTableBuffer("t"); Assert.assertEquals(3, buf.getRowCount()); + // The accumulated frame is intentionally larger than the cap; + // this test covers the per-row guard, not close-time flushing. + sender.reset(); } }); } @@ -1023,12 +1030,71 @@ public void testIllegalTableNameIsEscapedInTheMessage() throws Exception { }); } + @Test + public void testUnpairedSurrogateTableNameIsRejectedBeforeConnect() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + for (char invalidChar : new char[]{'\uD800', '\uDB80', '\uDFFF'}) { + try { + sender.table("bad" + invalidChar + "name"); + Assert.fail("an unpaired surrogate must be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table name contains illegal characters")); + } + } + } + }); + } + private static MicrobatchBuffer getMicrobatchBuffer(QwpWebSocketSender sender, String fieldName) throws Exception { Field field = QwpWebSocketSender.class.getDeclaredField(fieldName); field.setAccessible(true); return (MicrobatchBuffer) field.get(sender); } + private static QwpWebSocketSender connectLegacySender( + TestWebSocketServer server, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos + ) { + CursorSendEngine engine = new CursorSendEngine(null, 4L * 1024 * 1024); + try { + return QwpWebSocketSender.connect( + "localhost", server.getPort(), null, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + null, DurableAckTiers.NONE, engine, 0L); + } catch (Throwable t) { + engine.close(); + throw t; + } + } + + private static TestWebSocketServer startLegacyServer() throws Exception { + AtomicLong nextSequence = new AtomicLong(); + TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(QwpWireTestUtils.buildAck(nextSequence.getAndIncrement())); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + boolean started = false; + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + started = true; + return server; + } finally { + if (!started) { + server.close(); + } + } + } + private static int indexOf(byte[] frame, String ascii) { int n = ascii.length(); for (int i = 0, last = frame.length - n; i <= last; i++) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java new file mode 100644 index 000000000..ae10a518f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java @@ -0,0 +1,146 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * {@code Long.MAX_VALUE} as a wait budget means "no limit". It once overflowed an + * absolute nanoTime deadline, so the waits below returned at once instead. + */ +public class QwpWebSocketSenderUnboundedWaitTest { + + @Test(timeout = 20_000) + public void testCloseFlushWaitsWithoutLimit() throws Exception { + assertMemoryLeak(() -> { + HeldAckHandler handler = new HeldAckHandler(); + try (TestWebSocketServer server = start(handler)) { + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;close_flush_timeout_millis=" + Long.MAX_VALUE + ";"); + sender.table("t").longColumn("v", 1).atNow(); + sender.flush(); + Assert.assertTrue(handler.received.await(5, TimeUnit.SECONDS)); + AtomicReference error = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + error.set(t); + } + }, "unbounded-close"); + closer.start(); + try { + closer.join(200); + Assert.assertTrue("close() must wait for the ACK", closer.isAlive()); + } finally { + handler.release.countDown(); + closer.join(10_000); + } + Assert.assertFalse(closer.isAlive()); + Assert.assertNull(error.get()); + Assert.assertTrue(handler.acked.get()); + } + }); + } + + @Test(timeout = 20_000) + public void testDrainWaitsWithoutLimit() throws Exception { + assertMemoryLeak(() -> { + HeldAckHandler handler = new HeldAckHandler(); + try (TestWebSocketServer server = start(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;close_flush_timeout_millis=0;")) { + sender.table("t").longColumn("v", 1).atNow(); + AtomicReference drained = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + Thread drainer = new Thread(() -> { + try { + drained.set(sender.drain(Long.MAX_VALUE)); + } catch (Throwable t) { + error.set(t); + } + }, "unbounded-drain"); + drainer.start(); + try { + Assert.assertTrue(handler.received.await(5, TimeUnit.SECONDS)); + drainer.join(200); + Assert.assertTrue("drain() must wait for the ACK", drainer.isAlive()); + } finally { + handler.release.countDown(); + drainer.join(10_000); + } + Assert.assertFalse(drainer.isAlive()); + Assert.assertNull(error.get()); + Assert.assertEquals(Boolean.TRUE, drained.get()); + } + }); + } + + private static TestWebSocketServer start(HeldAckHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + // Holds the ACK for the first data frame until the test releases it. + private static final class HeldAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicBoolean acked = new AtomicBoolean(); + private final CountDownLatch received = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private long sequence; + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + received.countDown(); + try { + if (!release.await(10, TimeUnit.SECONDS)) { + return; + } + client.sendBinary(QwpWireTestUtils.buildAck(sequence++)); + acked.set(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java index 870e0ceed..b40ecef9a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java @@ -749,16 +749,16 @@ public void testCarriedInterruptCommitsRecycleAndKeepsFlag() throws Exception { } /** - * A pending REBUILD resume must refuse to complete under an in-progress - * row. The row's symbol ids belong to the outgoing dictionary; committing - * the swap underneath it would ship them against the fresh one and the - * server would decode a different symbol. at()/atNow() always arrive with - * a row in progress, so only table(), flush() and drain() can complete a - * pending rebuild: the refused row is rolled back and the next table() - * completes the swap. + * A pending REBUILD resume must never complete under a row that holds + * values. The row's symbol ids would belong to the outgoing dictionary; + * committing the swap underneath it would ship them against the fresh one + * and the server would decode a different symbol. The first setter of a + * row resolves the table's schema before the row takes its first value, + * and that resolution completes the pending rebuild, so the row's symbol + * id comes from the fresh dictionary. */ @Test(timeout = 60_000L) - public void testRebuildResumeRefusesAnInProgressRow() throws Exception { + public void testRebuildResumeCompletesBeforeTheRowTakesAValue() throws Exception { assertMemoryLeak(() -> { try (TestWebSocketServer server = ackingServer()) { try (Sender sender = Sender.fromConfig(cfg(server))) { @@ -784,20 +784,11 @@ public void testRebuildResumeRefusesAnInProgressRow() throws Exception { } Assert.assertEquals(0, ws.getSymbolDictEpoch()); - // The caller ignores the failure and finishes a row on the - // still-selected table: its symbol id is an OLD-dictionary id. - try { - sender.symbol("s", "x").longColumn("v", 2L).atNow(); - Assert.fail("atNow() must not complete the pending rebuild under an in-progress row"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), - "a symbol dictionary recycle is completing; finish or cancel the in-progress row and retry"); - } - Assert.assertEquals("the swap must not commit underneath the row", - 0, ws.getSymbolDictEpoch()); - - // The refused row was rolled back; the next table() completes the swap. - sender.table("t").symbol("s", "x").longColumn("v", 3L).atNow(); + // The caller ignores the failure and starts a row on the still-selected table. + sender.symbol("s", "x"); + Assert.assertEquals("the swap must complete before the row takes a value", + 1, ws.getSymbolDictEpoch()); + sender.longColumn("v", 2L).atNow(); Assert.assertEquals(1, ws.getSymbolDictEpoch()); long f2 = sender.flushAndGetSequence(); Assert.assertTrue(sender.awaitAckedFsn(f2, 5_000)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java index b70bdc4f7..299bba248 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java @@ -400,7 +400,8 @@ public void testUnknownStatusByte() throws Exception { int size = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE; long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); try { - Unsafe.getUnsafe().putByte(ptr, (byte) 0xFF); + // Unknown low-six-bit status remains a legacy error response. + Unsafe.getUnsafe().putByte(ptr, (byte) 0x3F); Unsafe.getUnsafe().putLong(ptr + 1, 1L); Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); @@ -409,8 +410,8 @@ public void testUnknownStatusByte() throws Exception { WebSocketResponse parsed = new WebSocketResponse(); Assert.assertTrue(parsed.readFrom(ptr, size)); - Assert.assertEquals((byte) 0xFF, parsed.getStatus()); - Assert.assertEquals("UNKNOWN(255)", parsed.getStatusName()); + Assert.assertEquals((byte) 0x3F, parsed.getStatus()); + Assert.assertEquals("UNKNOWN(63)", parsed.getStatusName()); } finally { Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java new file mode 100644 index 000000000..826c17bc0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java @@ -0,0 +1,593 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.DurableAckTiers; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class CursorWebSocketSchemaCoordinatorTest { + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testColdHitCaseAliasAndForcedRefreshUseProductionTransport() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("cold").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + try { + loop.resolveSchema(new String(new char[QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + 1]), 5_000); + Assert.fail("expected invalid table name"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, handler.requests.get()); + } + QwpSchemaResponse first = loop.resolveSchema("Trades", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, first.getResult()); + Assert.assertEquals(1, handler.requests.get()); + Assert.assertEquals(1, loop.resolveSchema("trades", 5_000).getTableId()); + Assert.assertEquals(1, handler.requests.get()); + Assert.assertEquals(2, loop.refreshSchema("TRADES", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testMissingIsCachedButRemoteFailuresAreTyped() throws Exception { + DescribeHandler missing = new DescribeHandler(true, QwpSchemaProtocol.RESULT_MISSING); + try (TestWebSocketServer server = server(missing); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("missing").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("Absent", 5_000).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("ABSENT", 0).getResult()); + Assert.assertEquals(1, missing.requests.get()); + } finally { + loop.close(); + } + } + assertRemoteReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "denied"); + assertRemoteReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "unavailable"); + } + + @Test + public void testTooLargeIsCachedButUnavailableIsNot() throws Exception { + // A table too wide to describe stays too wide until its version changes, + // which ACK feedback reports; re-asking every batch would cost a round trip. + DescribeHandler tooLarge = new DescribeHandler(true, QwpSchemaProtocol.RESULT_TOO_LARGE); + try (TestWebSocketServer server = server(tooLarge); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("too-large").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("Wide", 5_000).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("WIDE", 0).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.peekSchema("wide").getResult()); + Assert.assertEquals(1, tooLarge.requests.get()); + } finally { + loop.close(); + } + } + DescribeHandler unavailable = new DescribeHandler(true, QwpSchemaProtocol.RESULT_UNAVAILABLE); + try (TestWebSocketServer server = server(unavailable); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("unavailable").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.resolveSchema("t", 5_000)); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(2, unavailable.requests.get()); + Assert.assertNull(loop.peekSchema("t")); + } finally { + loop.close(); + } + } + } + + @Test + public void testTooLargeFeedbackIsCached() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("too-large-feedback").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + loop.resolveSchema("seed", 5_000); + handler.sendFeedback("Wide", QwpSchemaProtocol.RESULT_TOO_LARGE); + // The marker reply follows the feedback on the wire, so the I/O thread + // has applied the feedback once the marker lookup returns. + loop.refreshSchema("marker", 5_000); + int afterMarker = handler.requests.get(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("wide", 0).getResult()); + Assert.assertEquals(afterMarker, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testOrderedFeedbackInvalidationAndDurableDemultiplexing() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler, true); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("feedback").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort(), true, true)) { + Assert.assertTrue(client.isServerDurableAckEnabled()); + CursorWebSocketSendLoop loop = loop(client, engine, true); + try { + loop.start(); + loop.resolveSchema("seed", 5_000); + + handler.sendFeedback("Feedback", QwpSchemaProtocol.RESULT_MISSING); + handler.durableBeforeReply = true; + loop.refreshSchema("marker", 5_000); + int afterMarker = handler.requests.get(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("feedback", 0).getResult()); + Assert.assertEquals(afterMarker, handler.requests.get()); + + handler.sendInvalidation(); + loop.refreshSchema("marker", 5_000); + loop.resolveSchema("feedback", 5_000); + Assert.assertEquals(afterMarker + 2, handler.requests.get()); + Assert.assertEquals(-1, engine.ackedFsn()); + Assert.assertEquals(-1, engine.publishedFsn()); + } finally { + loop.close(); + } + } + } + + @Test + public void testFailedRefreshEvictsPriorCachedSchema() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("refresh-eviction").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertEquals(1, loop.resolveSchema("cached", 5_000).getTableId()); + handler.reply = false; + Future refresh = executor.submit(() -> loop.refreshSchema("CACHED", 200)); + Assert.assertTrue(handler.awaitRequests(2)); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, refresh); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("cached", 0)); + handler.reply = true; + Assert.assertEquals(3, loop.resolveSchema("cached", 5_000).getTableId()); + Assert.assertEquals(3, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testRevalidateAsksEvenWhenCachedAndReplacesEntry() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("revalidate").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(1, loop.resolveSchema("Trades", 5_000).getTableId()); + Assert.assertEquals(2, loop.revalidateSchema("TRADES", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + Assert.assertEquals(2, loop.resolveSchema("trades", 0).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testRevalidateTimeoutKeepsCachedEntry() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("revalidate-timeout").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertEquals(1, loop.resolveSchema("cached", 5_000).getTableId()); + handler.reply = false; + Future revalidate = executor.submit(() -> loop.revalidateSchema("CACHED", 200)); + Assert.assertTrue(handler.awaitRequests(2)); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, revalidate); + // Unlike a refresh, the failed revalidation leaves the entry in place. + Assert.assertEquals(1, loop.resolveSchema("cached", 0).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testRevalidateKeepsEntryWhenUnavailableAndEvictsItWhenDenied() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("revalidate-results").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(1, loop.resolveSchema("t", 5_000).getTableId()); + + handler.result = QwpSchemaProtocol.RESULT_UNAVAILABLE; + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.revalidateSchema("t", 5_000)); + Assert.assertEquals(1, loop.peekSchema("t").getTableId()); + + handler.result = QwpSchemaProtocol.RESULT_DENIED; + assertReason(LineSenderSchemaException.Reason.ACCESS_DENIED, () -> loop.revalidateSchema("t", 5_000)); + Assert.assertNull(loop.peekSchema("t")); + Assert.assertEquals(3, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testTimeoutIgnoresLateReplyAndNextRequestSucceeds() throws Exception { + DescribeHandler handler = new DescribeHandler(false); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("late").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Future timedOut = executor.submit(() -> loop.resolveSchema("late", 200)); + Assert.assertTrue(handler.awaitFirstRequest()); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, timedOut); + handler.replyToFirst(); + handler.reply = true; + Assert.assertEquals(2, loop.resolveSchema("late", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testAbandonOnDisconnectFailsPendingLookupBeforeItsDeadline() throws Exception { + DescribeHandler handler = new DescribeHandler(false); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("abandon").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + loop.setAbandonSchemaLookupOnDisconnect(true); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertTrue(loop.isWireUp()); + // A budget far beyond the assertion window: only the drop can end it. + Future pending = executor.submit(() -> loop.resolveSchema("t", 60_000)); + Assert.assertTrue(handler.awaitFirstRequest()); + server.close(); + try { + pending.get(5, TimeUnit.SECONDS); + Assert.fail("lookup survived the connection loss"); + } catch (ExecutionException e) { + LineSenderSchemaException cause = (LineSenderSchemaException) e.getCause(); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, cause.getReason()); + Assert.assertTrue(cause.getMessage(), cause.getMessage().contains("connection lost")); + } + Assert.assertFalse(loop.isWireUp()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testPeekReturnsOnlyCachedSchema() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("peek").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertNull(loop.peekSchema("Trades")); + Assert.assertEquals(0, handler.requests.get()); + QwpSchemaResponse resolved = loop.resolveSchema("Trades", 5_000); + Assert.assertSame(resolved, loop.peekSchema("trades")); + Assert.assertEquals(1, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testUnnegotiatedConnectionIsUnavailable() throws Exception { + io.questdb.client.test.tools.TestUtils.assertMemoryLeak(() -> { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = new TestWebSocketServer(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("legacy").getAbsolutePath(), 1 << 20)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (WebSocketClient client = connect(server.getPort(), false)) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(0, handler.requests.get()); + } finally { + loop.close(); + } + } + } + }); + } + + @Test + public void testMalformedControlFailsLookupWithoutAckFallback() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("malformed-control").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(1, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + private static CursorWebSocketSendLoop loop(WebSocketClient client, CursorSendEngine engine) { + return loop(client, engine, false); + } + + private static CursorWebSocketSendLoop loop(WebSocketClient client, CursorSendEngine engine, boolean durable) { + return new CursorWebSocketSendLoop(client, engine, 0, null, 1, 4, + durable ? DurableAckTiers.REPLICATED | DurableAckTiers.LEGACY_TRUE : DurableAckTiers.NONE); + } + + private void assertRemoteReason(int result, LineSenderSchemaException.Reason reason, String directory) throws Exception { + DescribeHandler handler = new DescribeHandler(true, result); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder(directory).getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(reason, () -> loop.resolveSchema("t", 5_000)); + } finally { + loop.close(); + } + } + } + + private static TestWebSocketServer server(DescribeHandler handler) throws Exception { + return server(handler, false); + } + + private static TestWebSocketServer server(DescribeHandler handler, boolean durable) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler, durable); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static WebSocketClient connect(int port) { + return connect(port, true); + } + + private static WebSocketClient connect(int port, boolean schema) { + return connect(port, schema, false); + } + + private static WebSocketClient connect(int port, boolean schema, boolean durable) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + client.connect("localhost", port); + if (schema) { + client.requestQwpSchema(); + } + client.setQwpDurableAckTiers(durable + ? DurableAckTiers.REPLICATED | DurableAckTiers.LEGACY_TRUE + : DurableAckTiers.NONE); + client.upgrade("/write/v4", 5_000, null); + return client; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + try { + action.run(); + Assert.fail("expected schema failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + } + } + + private static void assertFutureReason( + LineSenderSchemaException.Reason reason, + Future future + ) throws Exception { + try { + future.get(5, TimeUnit.SECONDS); + Assert.fail("expected schema failure"); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof LineSenderSchemaException); + Assert.assertEquals(reason, ((LineSenderSchemaException) e.getCause()).getReason()); + } + } + + private static File taskTempRoot() { + File root = new File("target/schema-coordination-2.7/tmp").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class DescribeHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger requests = new AtomicInteger(); + private final CountDownLatch firstRequest = new CountDownLatch(1); + private volatile boolean reply; + private volatile boolean durableBeforeReply; + private volatile boolean malformedReply; + private TestWebSocketServer.ClientHandler client; + private long firstRequestId; + private volatile int result; + + private DescribeHandler(boolean reply) { + this(reply, QwpSchemaProtocol.RESULT_KNOWN); + } + + private DescribeHandler(boolean reply, int result) { + this.reply = reply; + this.result = result; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, + ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).getInt()); + Assert.assertEquals(QwpSchemaProtocol.FLAG_CONTROL, data[5]); + long requestId = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).getLong(QwpConstants.HEADER_SIZE + 1); + this.client = client; + int requestCount = requests.incrementAndGet(); + notifyAll(); + if (requestCount == 1) { + firstRequestId = requestId; + firstRequest.countDown(); + } + if (reply) { + if (malformedReply) { + ByteBuffer malformed = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN); + sendRaw(client, malformed.putInt(QwpConstants.MAGIC_MESSAGE).array()); + return; + } + if (durableBeforeReply) { + durableBeforeReply = false; + sendRaw(client, new byte[]{WebSocketResponse.STATUS_DURABLE_ACK, 0, 0}); + } + send(client, requestId, requests.get(), result); + } + } + + synchronized void replyToFirst() throws IOException { + send(client, firstRequestId, 1, result); + } + + boolean awaitFirstRequest() throws InterruptedException { + return firstRequest.await(5, TimeUnit.SECONDS); + } + + synchronized boolean awaitRequests(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (requests.get() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return true; + } + + synchronized void sendFeedback(String tableName, int result) { + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + byte[] schema = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) result).array(); + ByteBuffer b = ByteBuffer.allocate(11 + 2 + 2 + name.length + 4 + schema.length) + .order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(0).putShort((short) 0).putShort((short) 1) + .putShort((short) name.length).put(name).putInt(schema.length).put(schema); + sendRaw(client, b.array()); + } + + synchronized void sendInvalidation() { + ByteBuffer b = ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)) + .putLong(0).putShort((short) 0); + sendRaw(client, b.array()); + } + + private static void send(TestWebSocketServer.ClientHandler client, long requestId, int tableId, int result) { + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1; + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + payloadLength += 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + } + ByteBuffer b = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + b.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payloadLength).put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId).put((byte) result); + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + b.putInt(tableId).putLong(1).putShort((short) -1).putShort((short) 1) + .putShort((short) column.length).put(column).putInt(ColumnType.LONG).putShort((short) 0); + } + sendRaw(client, b.array()); + } + + private static void sendRaw(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(data); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java new file mode 100644 index 000000000..12e48b188 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -0,0 +1,742 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.runners.model.Statement; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSchemaLifecycleTest { + @Rule + public final TestRule memoryLeak = (base, description) -> new Statement() { + @Override + public void evaluate() throws Throwable { + AtomicReference failure = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + try { + base.evaluate(); + } catch (Throwable e) { + failure.set(e); + } + }); + if (failure.get() != null) { + throw failure.get(); + } + } + }; + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testCloseReleasesLongDeadlineWaiterBeforeLoopStarts() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("close-before-start"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + Lookup lookup = lookup(loop, "never-started", Long.MAX_VALUE, null); + Assert.assertTrue(lookup.entered.await(5, TimeUnit.SECONDS)); + loop.close(); + lookup.join(); + assertUnavailable(lookup.error.get()); + } + } + + @Test + public void testCloseReleasesLookupAlreadySentByActiveLoop() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("close-active"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup lookup = lookup(loop, "active", Long.MAX_VALUE, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + loop.close(); + lookup.join(); + assertUnavailable(lookup.error.get()); + } + } + + @Test + public void testInterruptionRestoresFlagAndReleasesSlotForNextLookup() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("interrupt"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup interrupted = lookup(loop, "first", Long.MAX_VALUE, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + interrupted.thread.interrupt(); + interrupted.join(); + assertUnavailable(interrupted.error.get()); + Assert.assertTrue(interrupted.interrupted.get()); + + handler.reply = true; + QwpSchemaResponse next = loop.resolveSchema("second", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, next.getResult()); + Assert.assertEquals("second", handler.lastName.get()); + } + } + + @Test + public void testDisconnectRetriesCurrentLookupRetainsCacheAndRejectsStaleReply() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + QwpSchemaResponse cached = loop.resolveSchema("cached", 5_000); + handler.disconnectName = "lost"; + handler.disconnectsRemaining = 2; + Lookup lost = lookup(loop, "lost", 5_000, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + Assert.assertSame(cached, loop.resolveSchema("CACHED", 5_000)); + Lookup busy = lookup(loop, "fresh", 5_000, null); + busy.join(); + assertUnavailable(busy.error.get()); + Assert.assertTrue(busy.error.get().getMessage().contains("another schema lookup")); + handler.sendStaleBeforeNextReply = true; + reconnect.allow.countDown(); + lost.join(); + Assert.assertNull(String.valueOf(lost.error.get()), lost.error.get()); + Assert.assertNotNull(lost.response.get()); + Assert.assertEquals(4, lost.response.get().getTableId()); + Assert.assertSame(lost.response.get(), loop.resolveSchema("lost", 0)); + Assert.assertEquals(3, server.handshakeCount()); + + // The cache survives the reconnect. ACK feedback on the new + // connection refreshes any entry that went stale meanwhile. + int requestsBeforeCacheProbe = handler.requests; + Assert.assertSame(cached, loop.resolveSchema("cached", 5_000)); + Assert.assertEquals(requestsBeforeCacheProbe, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testQueuedUnsentLookupSurvivesDisconnect() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("queued-disconnect"); + DisconnectingClient client = new DisconnectingClient()) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + reconnect.allow.countDown(); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Assert.assertTrue(client.receiving.await(5, TimeUnit.SECONDS)); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + client.disconnect.countDown(); + queued.join(); + Assert.assertNull(String.valueOf(queued.error.get()), queued.error.get()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, queued.response.get().getResult()); + Assert.assertEquals(1, handler.requests); + } finally { + client.disconnect.countDown(); + } + } + } + } + + @Test + public void testQueuedLookupBecomesUnavailableAtLegacyReplacement() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("queued-legacy-replacement"); + DisconnectingClient client = new DisconnectingClient()) { + server.setAdvertiseSchema(false); + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + reconnect.allow.countDown(); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Assert.assertTrue(client.receiving.await(5, TimeUnit.SECONDS)); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + client.disconnect.countDown(); + queued.join(); + assertUnavailable(queued.error.get()); + Assert.assertTrue(loop.isWireUp()); + Assert.assertFalse(loop.isSchemaEnabled()); + Assert.assertEquals(0, handler.requests); + } finally { + client.disconnect.countDown(); + } + } + } + } + + @Test + public void testReconnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + handler.disconnectName = "deadline"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-deadline"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup lookup = lookup(loop, "deadline", 1_000, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + Assert.assertTrue(lookup.error.get().getMessage().contains("timed out")); + reconnect.allow.countDown(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, + loop.resolveSchema("probe", 5_000).getResult()); + Assert.assertEquals(2, handler.requests); + Assert.assertEquals("probe", handler.lastName.get()); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testInterruptDuringReconnectReleasesSlotForQueuedLookup() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + handler.disconnectName = "interrupted"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-interrupt"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup interrupted = lookup(loop, "interrupted", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + interrupted.thread.interrupt(); + interrupted.join(); + assertUnavailable(interrupted.error.get()); + Assert.assertTrue(interrupted.interrupted.get()); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + reconnect.allow.countDown(); + queued.join(); + Assert.assertNull(String.valueOf(queued.error.get()), queued.error.get()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, queued.response.get().getResult()); + Assert.assertEquals(2, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testCloseDuringReconnectReleasesLookupBeforeFactoryReturns() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.disconnectName = "closed"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-close"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + AtomicReference closeError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable e) { + closeError.set(e); + } + }); + try { + loop.start(); + Lookup lookup = lookup(loop, "closed", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + closer.start(); + lookup.join(); + assertUnavailable(lookup.error.get()); + Assert.assertTrue(lookup.error.get().getMessage().contains("closed")); + } finally { + reconnect.allow.countDown(); + closer.join(5_000); + Assert.assertFalse(closer.isAlive()); + } + Assert.assertNull(String.valueOf(closeError.get()), closeError.get()); + Assert.assertEquals(1, handler.requests); + } + } + } + + @Test + public void testLookupsDuringOutageDoNotShortenReconnectBackoff() throws Exception { + SchemaHandler handler = new SchemaHandler(); + FailingReconnectFactory reconnect = new FailingReconnectFactory(); + try (CursorSendEngine engine = engine("outage-backoff")) { + TestWebSocketServer server = server(handler); + CursorWebSocketSendLoop loop = null; + try { + loop = new CursorWebSocketSendLoop(connect(server.getPort()), engine, 0, + reconnect, 200, 200); + loop.start(); + server.close(); + Assert.assertTrue(reconnect.firstAttempt.await(5, TimeUnit.SECONDS)); + int attemptsBefore = reconnect.attempts.get(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + int lookups = 0; + while (System.nanoTime() < deadline) { + try { + loop.resolveSchema("outage", 1); + Assert.fail("the wire is down"); + } catch (LineSenderSchemaException expected) { + lookups++; + } + } + int attempts = reconnect.attempts.get() - attemptsBefore; + // A 200 ms backoff plus up to 200 ms jitter allows at most 5 attempts per second. + Assert.assertTrue("lookups=" + lookups + ", attempts=" + attempts, attempts <= 6); + Assert.assertTrue("lookups=" + lookups, lookups > 50); + } finally { + if (loop != null) { + loop.close(); + } + server.close(); + } + } + } + + @Test + public void testMalformedCurrentReplyCancelsLookupEvenWithReconnectAvailable() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-malformed"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup lookup = lookup(loop, "malformed", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + handler.malformedReply = false; + handler.reply = true; + reconnect.allow.countDown(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, + loop.resolveSchema("probe", 5_000).getResult()); + Assert.assertEquals(2, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testAsyncInitialConnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("async-initial")) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(null, engine, reconnect)) { + loop.start(); + Lookup lookup = lookup(loop, "deadline", 25, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + reconnect.allow.countDown(); + Assert.assertTrue(reconnect.connected.await(5, TimeUnit.SECONDS)); + QwpSchemaResponse probe = loop.resolveSchema("probe", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, probe.getResult()); + Assert.assertEquals(1, handler.requests); + Assert.assertEquals("probe", handler.lastName.get()); + } finally { + reconnect.allow.countDown(); + } + } + } + + @Test + public void testMalformedTimedOutReplyDoesNotPoisonIdleConnection() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("stale-idle"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup stale = lookup(loop, "stale", 1_000, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + stale.join(); + assertUnavailable(stale.error.get()); + handler.sendMalformedLastRequestAndFeedback("idle_barrier"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("idle_barrier", 0).getResult()); + break; + } catch (LineSenderSchemaException e) { + assertUnavailable(e); + if (System.nanoTime() >= deadline) { + Assert.fail("schema feedback barrier was not observed"); + } + Thread.yield(); + } + } + Assert.assertEquals(1, handler.requests); + handler.reply = true; + QwpSchemaResponse response = loop.resolveSchema("usable", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, response.getResult()); + Assert.assertEquals("usable", handler.lastName.get()); + Assert.assertEquals(1, server.handshakeCount()); + } + } + + @Test + public void testMalformedOldReplyDoesNotFailNewLookupOrPopulateOldCache() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("stale-pending"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup stale = lookup(loop, "stale", 1_000, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + stale.join(); + assertUnavailable(stale.error.get()); + handler.sendStaleBeforeNextReply = true; + QwpSchemaResponse current = loop.resolveSchema("current", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, current.getResult()); + Assert.assertEquals(2, current.getTableId()); + Assert.assertEquals(1, server.handshakeCount()); + + handler.reply = true; + QwpSchemaResponse staleProbe = loop.resolveSchema("stale", 5_000); + Assert.assertEquals(3, staleProbe.getTableId()); + Assert.assertEquals(3, handler.requests); + } + } + + @Test + public void testMalformedMatchingReplyFailsTransportAndReleasesWaiter() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("malformed-current"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup lookup = lookup(loop, "current", Long.MAX_VALUE, null); + lookup.join(); + assertUnavailable(lookup.error.get()); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + loop.checkError(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Invalid schema control response:")); + break; + } + if (System.nanoTime() >= deadline) { + Assert.fail("malformed schema response did not publish the transport error"); + } + Thread.yield(); + } + } + } + + private CursorSendEngine engine(String name) throws IOException { + return new CursorSendEngine(temp.newFolder(name).getAbsolutePath(), 1 << 20); + } + + private static CursorWebSocketSendLoop loop( + WebSocketClient client, + CursorSendEngine engine, + CursorWebSocketSendLoop.ReconnectFactory reconnect + ) { + return new CursorWebSocketSendLoop( + client, engine, 0, reconnect, 1, 4); + } + + private static Lookup lookup( + CursorWebSocketSendLoop loop, + String table, + long timeoutMillis, + CountDownLatch completed + ) { + Lookup lookup = new Lookup(); + lookup.thread = new Thread(() -> { + lookup.entered.countDown(); + try { + lookup.response.set(loop.resolveSchema(table, timeoutMillis)); + } catch (Throwable e) { + lookup.error.set(e); + } finally { + lookup.interrupted.set(Thread.currentThread().isInterrupted()); + if (completed != null) { + completed.countDown(); + } + } + }, "schema-lookup-" + table); + lookup.thread.start(); + return lookup; + } + + private static void assertUnavailable(Throwable error) { + Assert.assertTrue(String.valueOf(error), error instanceof LineSenderSchemaException); + Assert.assertEquals( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + ((LineSenderSchemaException) error).getReason()); + } + + private static TestWebSocketServer server(SchemaHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static WebSocketClient connect(int port) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + client.requestQwpSchema(); + client.upgrade("/write/v4", 5_000, null); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + private static File taskTempRoot() { + File root = new File("target/schema-coordination-2.7/tmp").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class Lookup { + private final AtomicReference error = new AtomicReference<>(); + private final CountDownLatch entered = new CountDownLatch(1); + private final AtomicBoolean interrupted = new AtomicBoolean(); + private final AtomicReference response = new AtomicReference<>(); + private Thread thread; + + private void awaitWaiting() throws InterruptedException { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != Thread.State.TIMED_WAITING) { + Assert.assertTrue("lookup did not wait", thread.isAlive() && System.nanoTime() < deadline); + Thread.yield(); + } + } + + private void join() throws InterruptedException { + thread.join(5_000); + Assert.assertFalse("lookup thread did not finish", thread.isAlive()); + } + } + + private static final class FailingReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final CountDownLatch firstAttempt = new CountDownLatch(1); + + @Override + public WebSocketClient reconnect() { + attempts.incrementAndGet(); + firstAttempt.countDown(); + throw new RuntimeException("server is down"); + } + } + + private static final class GatedReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final CountDownLatch allow = new CountDownLatch(1); + private final CountDownLatch attempted = new CountDownLatch(1); + private final CountDownLatch connected = new CountDownLatch(1); + private final int port; + + private GatedReconnectFactory(int port) { + this.port = port; + } + + @Override + public WebSocketClient reconnect() { + attempted.countDown(); + try { + if (!allow.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("replacement was not released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + WebSocketClient client = connect(port); + connected.countDown(); + return client; + } + } + + // Fault injection only: hold the receive poll until the producer has queued + // a lookup, then fail before the I/O thread can send that lookup. + private static final class DisconnectingClient extends WebSocketClient { + private final CountDownLatch disconnect = new CountDownLatch(1); + private final CountDownLatch receiving = new CountDownLatch(1); + + private DisconnectingClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + receiving.countDown(); + try { + Assert.assertTrue(disconnect.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + throw new LineSenderException("injected transport disconnect"); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicReference lastName = new AtomicReference<>(); + private final CountDownLatch requestSeen = new CountDownLatch(1); + private volatile String disconnectName; + private int disconnectsRemaining = 1; + private volatile TestWebSocketServer.ClientHandler lastClient; + private volatile long lastRequestId; + private volatile boolean malformedReply; + private volatile boolean reply; + private volatile boolean sendStaleBeforeNextReply; + private volatile int requests; + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + ByteBuffer input = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = input.getLong(QwpConstants.HEADER_SIZE + 1); + int nameLength = input.getShort(QwpConstants.HEADER_SIZE + 9) & 0xffff; + String name = new String(data, QwpConstants.HEADER_SIZE + 11, nameLength, StandardCharsets.UTF_8); + requests++; + lastClient = client; + long staleRequestId = lastRequestId; + lastRequestId = requestId; + lastName.set(name); + requestSeen.countDown(); + if (name.equals(disconnectName) && disconnectsRemaining-- > 0) { + try { + client.sendClose(1001, "test disconnect"); + } catch (IOException e) { + throw new AssertionError(e); + } + } else if (sendStaleBeforeNextReply) { + sendStaleBeforeNextReply = false; + sendMalformed(client, staleRequestId); + send(client, staleRequestId, 999); + send(client, requestId, requests); + } else if (malformedReply) { + sendMalformed(client, requestId); + } else if (reply) { + send(client, requestId, requests); + } + } + + private synchronized void sendMalformedLastRequestAndFeedback(String tableName) { + sendMalformed(lastClient, lastRequestId); + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + byte[] schema = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0) + .put((byte) QwpSchemaProtocol.RESULT_MISSING).array(); + ByteBuffer feedback = ByteBuffer.allocate(11 + 2 + 2 + name.length + 4 + schema.length) + .order(ByteOrder.LITTLE_ENDIAN); + feedback.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(0).putShort((short) 0).putShort((short) 1) + .putShort((short) name.length).put(name).putInt(schema.length).put(schema); + try { + lastClient.sendBinary(feedback.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static void sendMalformed(TestWebSocketServer.ClientHandler client, long requestId) { + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + 9).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(9).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static void send(TestWebSocketServer.ClientHandler client, long requestId, int tableId) { + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payloadLength).put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId).put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(tableId) + .putLong(1).putShort((short) -1).putShort((short) 1) + .putShort((short) column.length).put(column).putInt(ColumnType.LONG).putShort((short) 0); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java deleted file mode 100644 index f2452dde0..000000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java +++ /dev/null @@ -1,251 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client.sf.cursor; - -import io.questdb.client.LineSenderServerException; -import io.questdb.client.SenderError; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.std.Unsafe; -import org.junit.Assert; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; - -/** - * Pinpointed tests for the latched-error contract on {@link CursorWebSocketSendLoop}: - * {@code recordFatal} → {@link CursorWebSocketSendLoop#getTerminalError} + - * {@link CursorWebSocketSendLoop#getLastTerminalServerError} + - * {@link CursorWebSocketSendLoop#checkError}. Bypasses the constructor entirely - * via {@code Unsafe.allocateInstance} to avoid the live wire/engine dependencies - * — the latch is a self-contained piece of state. - */ -public class CursorWebSocketSendLoopErrorLatchTest { - - @Test - public void testCheckErrorRethrowsLineSenderException() throws Exception { - // checkError must rethrow the SAME LineSenderException instance, not - // a wrapper. Producers depend on this so getServerError() works on - // typed throws. - CursorWebSocketSendLoop loop = newBareLoop(); - SenderError err = newSenderError(); - LineSenderServerException original = new LineSenderServerException(err); - setField(loop, "terminalError", original); - Assert.assertNull(loop.getSynchronouslySurfacedError()); - - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrown) { - Assert.assertSame("checkError must rethrow LineSenderException unchanged", - original, thrown); - Assert.assertSame(err, - ((LineSenderServerException) thrown).getServerError()); - } - Assert.assertSame("checkError must mark the exact latched throwable as user-owned", - original, loop.getSynchronouslySurfacedError()); - // a synchronously surfaced latch is owned -- the close() safety net - // must stay silent now - loop.checkUnsurfacedError(); - } - - @Test - public void testRecordFatalWrapsNonLineSenderThrowableOnce() throws Exception { - // Non-LineSenderException causes (NPE, IOException, etc.) are wrapped - // in a LineSenderException ONCE, at latch time -- so producers always - // see one exception type AND every rethrow delivers the same instance, - // which close() relies on to suppress double-signals by identity. - CursorWebSocketSendLoop loop = newBareLoop(); - Throwable raw = new RuntimeException("oh no"); - invokeRecordFatal(loop, raw); - Assert.assertNull(loop.getSynchronouslySurfacedError()); - - LineSenderException first = null; - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrown) { - Assert.assertNotSame(raw, thrown); - Assert.assertSame(raw, thrown.getCause()); - Assert.assertTrue(thrown.getMessage().contains("oh no")); - first = thrown; - } - Assert.assertSame("the latch must hold the wrapper, not the raw cause", - first, loop.getTerminalError()); - Assert.assertSame("ownership tracks the latched wrapper", - first, loop.getSynchronouslySurfacedError()); - loop.checkUnsurfacedError(); // owned -> silent - - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrownAgain) { - Assert.assertSame("every rethrow must deliver the same instance", - first, thrownAgain); - } - } - - @Test - public void testCheckErrorIsNoopWhenNoLatch() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Assert.assertNull(loop.getTerminalError()); - loop.checkError(); // must not throw - } - - @Test - public void testCheckUnsurfacedErrorThrowsOnceThenStaysSilent() throws Exception { - // The close() safety net: an unowned latch must rethrow exactly like - // checkError; once any synchronous caller has owned the error, it - // must stay silent -- unlike checkError, which keeps throwing. - CursorWebSocketSendLoop loop = newBareLoop(); - loop.checkUnsurfacedError(); // no latch -> silent - - LineSenderException e = new LineSenderException("wire fail"); - setField(loop, "terminalError", e); - try { - loop.checkUnsurfacedError(); - Assert.fail("an unowned latch must rethrow from the safety net"); - } catch (LineSenderException thrown) { - Assert.assertSame(e, thrown); - } - - loop.checkUnsurfacedError(); // the throw above made the caller owner -> silent - - try { - loop.checkError(); - Assert.fail("checkError must keep rethrowing a latched error"); - } catch (LineSenderException thrown) { - Assert.assertSame(e, thrown); - } - } - - @Test - public void testGetTerminalErrorReturnsLatchedThrowable() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Throwable e = new LineSenderException("boom"); - setField(loop, "terminalError", e); - Assert.assertSame(e, loop.getTerminalError()); - } - - @Test - public void testGetTerminalErrorIsNullBeforeAnyFailure() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Assert.assertNull("loops with no latched error must report null", - loop.getTerminalError()); - } - - @Test - public void testRecordFatalLatchesThrowableOnly() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - // running must be true initially so we can verify recordFatal flips it. - setField(loop, "running", true); - Throwable e = new LineSenderException("wire fail"); - - invokeRecordFatal(loop, e); - - Assert.assertSame(e, loop.getTerminalError()); - Assert.assertNull("typed payload must be null for a wire-level fatal", - loop.getLastTerminalServerError()); - Assert.assertFalse("recordFatal must stop the loop", - (Boolean) getField(loop, "running")); - } - - @Test - public void testRecordFatalLatchesBothThrowableAndSenderError() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - setField(loop, "running", true); - SenderError err = newSenderError(); - LineSenderServerException ex = new LineSenderServerException(err); - - invokeRecordFatal(loop, ex); - - Assert.assertSame(ex, loop.getTerminalError()); - Assert.assertSame("typed payload is derived from the latched LineSenderServerException", - err, loop.getLastTerminalServerError()); - Assert.assertFalse((Boolean) getField(loop, "running")); - } - - @Test - public void testRecordFatalIsIdempotent() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - setField(loop, "running", true); - SenderError firstErr = newSenderError(); - SenderError secondErr = newSenderError(); - LineSenderServerException first = new LineSenderServerException(firstErr); - LineSenderServerException second = new LineSenderServerException(secondErr); - - invokeRecordFatal(loop, first); - invokeRecordFatal(loop, second); - - // Only the first failure latches — subsequent calls must not - // overwrite, otherwise a follow-on cascade would mask the original - // root cause. - Assert.assertSame("first throwable must remain latched", - first, loop.getTerminalError()); - Assert.assertSame("first SenderError must remain latched", - firstErr, loop.getLastTerminalServerError()); - } - - private static SenderError newSenderError() { - return new SenderError( - SenderError.Category.SCHEMA_MISMATCH, - SenderError.Policy.TERMINAL, - 0x03, - "test-msg", - 7L, - 100L, 100L, - "tbl", - System.nanoTime() - ); - } - - private static CursorWebSocketSendLoop newBareLoop() throws Exception { - // Bypass the real constructor — we don't need a wire client or engine - // to test the latched-error contract. - return (CursorWebSocketSendLoop) Unsafe.getUnsafe() - .allocateInstance(CursorWebSocketSendLoop.class); - } - - private static void setField(Object target, String name, Object value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.set(target, value); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - private static void invokeRecordFatal(CursorWebSocketSendLoop loop, Throwable t) - throws Exception { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( - "recordFatal", Throwable.class); - m.setAccessible(true); - m.invoke(loop, t); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java index bbc92f3ec..e79fb8074 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java @@ -25,16 +25,12 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendCounters; +import io.questdb.client.cutlass.qwp.client.DurableAckTiers; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; /** @@ -42,16 +38,55 @@ * machinery used to swallow {@link java.lang.Error} (OOM, LinkageError, * StackOverflowError) into an indefinite "transport outage" retry with only * a throttled, possibly-null-message WARN as a trace. A JVM/programming - * failure is not a transport outage -- retrying cannot clear it -- so every - * retry loop must rethrow {@code Error}, after latching it as terminal where - * a producer could otherwise hang in {@code checkError()}. - *

      - * Uses the same {@code Unsafe.allocateInstance} bare-loop pattern as - * {@link CursorWebSocketSendLoopErrorLatchTest}: the retry loops only touch - * the fields wired below, so no live wire client or engine is needed. + * failure is not a transport outage -- retrying cannot clear it -- so the + * blocking connection helper must rethrow {@code Error} immediately and the + * background loop must latch it for the producer before stopping. */ public class CursorWebSocketSendLoopJvmErrorTest { + @Test(timeout = 30_000) + public void testIoLoopLatchesReconnectJvmErrorAndShutsDown() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + LinkageError injected = new LinkageError("simulated JVM failure"); + try (CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024)) { + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + () -> { + attempts.incrementAndGet(); + throw injected; + }, + 1L, + 4L, + DurableAckTiers.NONE + ); + try { + loop.start(); + long deadlineNanos = System.nanoTime() + 5_000_000_000L; + while ((loop.getTerminalError() == null || loop.isRunning()) + && System.nanoTime() < deadlineNanos) { + Thread.yield(); + } + + Throwable terminal = loop.getTerminalError(); + Assert.assertNotNull("I/O loop did not latch the JVM Error", terminal); + Assert.assertEquals("a JVM Error must not be retried", 1, attempts.get()); + Assert.assertFalse("a terminal JVM Error must stop the loop", loop.isRunning()); + try { + loop.checkError(); + Assert.fail("producer-facing checkError() must surface the terminal failure"); + } catch (LineSenderException thrown) { + Assert.assertSame("checkError() must rethrow the latched wrapper", terminal, thrown); + Assert.assertSame("the wrapper must retain the original JVM Error", injected, thrown.getCause()); + } + } finally { + // Also proves that the real I/O-thread shutdown latch is released. + loop.close(); + } + } + } + @Test public void testConnectWithRetryPropagatesJvmError() { // The budgeted blocking initial-connect helper must not burn the @@ -75,109 +110,4 @@ public void testConnectWithRetryPropagatesJvmError() { Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); } - @Test - public void testConnectLoopPropagatesJvmErrorAndLatchesTerminal() throws Exception { - // The background per-outage reconnect loop must (1) latch the Error - // as terminal FIRST -- a producer parked in checkError() would - // otherwise never observe the failure -- and (2) rethrow so the I/O - // thread dies loudly instead of reconnect-looping forever. - CursorWebSocketSendLoop loop = newBareLoop(); - AtomicInteger attempts = new AtomicInteger(); - wireReconnectPlumbing(loop, attempts); - - Method connectLoop = CursorWebSocketSendLoop.class.getDeclaredMethod( - "connectLoop", Throwable.class, String.class, long.class); - connectLoop.setAccessible(true); - try { - connectLoop.invoke(loop, new LineSenderException("initial wire failure"), "reconnect", 0L); - Assert.fail("a JVM Error must escape connectLoop, not be retried"); - } catch (InvocationTargetException ite) { - Assert.assertTrue("expected LinkageError, got " + ite.getCause(), - ite.getCause() instanceof LinkageError); - } - Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); - assertErrorLatchedAndStopped(loop); - } - - @Test - public void testIoLoopDoesNotFunnelJvmErrorIntoReconnect() throws Exception { - // ioLoop's catch (Throwable) used to funnel EVERYTHING into - // fail(t) -> connectLoop(t, "reconnect"). An Error must instead be - // latched as terminal and rethrown; the finally still counts down - // the shutdown latch so close() cannot hang on the dead thread. - CursorWebSocketSendLoop loop = newBareLoop(); - AtomicInteger attempts = new AtomicInteger(); - wireReconnectPlumbing(loop, attempts); - CountDownLatch shutdownLatch = new CountDownLatch(1); - setField(loop, "shutdownLatch", shutdownLatch); - // client == null + running routes ioLoop into attemptInitialConnect - // -> connectLoop -> the throwing factory, exercising the full funnel. - - Method ioLoop = CursorWebSocketSendLoop.class.getDeclaredMethod("ioLoop"); - ioLoop.setAccessible(true); - try { - ioLoop.invoke(loop); - Assert.fail("a JVM Error must escape ioLoop, not re-enter the reconnect loop"); - } catch (InvocationTargetException ite) { - Assert.assertTrue("expected LinkageError, got " + ite.getCause(), - ite.getCause() instanceof LinkageError); - } - Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); - Assert.assertEquals("shutdown latch must count down so close() cannot hang", - 0L, shutdownLatch.getCount()); - assertErrorLatchedAndStopped(loop); - } - - private static void assertErrorLatchedAndStopped(CursorWebSocketSendLoop loop) - throws Exception { - Throwable terminal = loop.getTerminalError(); - Assert.assertNotNull("Error must be latched as terminal for checkError()", terminal); - Assert.assertTrue("latch wraps the raw Error once", - terminal instanceof LineSenderException); - Assert.assertTrue("latched cause must be the original Error", - terminal.getCause() instanceof LinkageError); - Assert.assertFalse("recordFatal must stop the loop", - (Boolean) getField(loop, "running")); - try { - loop.checkError(); - Assert.fail("producer-facing checkError must surface the latched terminal"); - } catch (LineSenderException thrown) { - Assert.assertSame(terminal, thrown); - } - } - - /** - * Wires the minimal state the reconnect paths dereference: a factory - * throwing {@link LinkageError} on every attempt, live {@code running}, - * and the attempt counters (field initializers do not run under - * {@code Unsafe.allocateInstance}). - */ - private static void wireReconnectPlumbing(CursorWebSocketSendLoop loop, - AtomicInteger attempts) throws Exception { - CursorWebSocketSendLoop.ReconnectFactory factory = () -> { - attempts.incrementAndGet(); - throw new LinkageError("simulated JVM failure"); - }; - setField(loop, "reconnectFactory", factory); - setField(loop, "running", true); - setField(loop, "counters", new CursorSendCounters()); - } - - private static CursorWebSocketSendLoop newBareLoop() throws Exception { - // Bypass the real constructor -- no wire client or engine needed. - return (CursorWebSocketSendLoop) Unsafe.getUnsafe() - .allocateInstance(CursorWebSocketSendLoop.class); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - private static void setField(Object target, String name, Object value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.set(target, value); - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java new file mode 100644 index 000000000..153bd4160 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -0,0 +1,369 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import io.questdb.client.test.tools.TestUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +public class QwpSchemaReplayNetworkTest { + private static final int SEGMENT_SIZE = 1 << 20; + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testPublicSenderSchemaFrameSurvivesRestartByteForByte() throws Exception { + File root = temp.newFolder("public-sender-restart"); + String slot = new File(root, "default").getAbsolutePath(); + SchemaRecordingHandler firstHandler = new SchemaRecordingHandler(false); + byte[] expected; + try (TestWebSocketServer firstPeer = new TestWebSocketServer(firstHandler)) { + firstPeer.setAdvertiseSchema(true); + firstPeer.start(); + Assert.assertTrue(firstPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(firstPeer.getPort(), root, false))) { + sender.table("small_replay").byteColumn("value", (byte) -7).atNow(); + Assert.assertEquals(0, sender.flushAndGetSequence()); + expected = firstHandler.awaitDataFrame(); + } + } + assertRetained(slot, 0); + + SchemaRecordingHandler replayHandler = new SchemaRecordingHandler(true); + try (TestWebSocketServer replayPeer = new TestWebSocketServer(replayHandler)) { + replayPeer.setAdvertiseSchema(true); + replayPeer.start(); + Assert.assertTrue(replayPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(replayPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertArrayEquals(expected, replayHandler.awaitDataFrame()); + assertNoQuarantine(root); + } + + @Test + public void testSchemaFeedbackAckAdvancesRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-ack"); + seedBacklog(new File(root, "default").getAbsolutePath()); + FeedbackHandler handler = new FeedbackHandler(false); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(peer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals(2, handler.frames.get()); + assertNoQuarantine(root); + } + + @Test + public void testMalformedSchemaFeedbackDoesNotAdvanceRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-malformed"); + String slot = new File(root, "default").getAbsolutePath(); + seedBacklog(slot); + FeedbackHandler handler = new FeedbackHandler(true); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + Sender sender = Sender.fromConfig(config(peer.getPort(), root, false)); + try { + Assert.assertTrue("malformed feedback must be parsed, rejected, and trigger replay", + handler.awaitDistinctClients(5, TimeUnit.SECONDS)); + } finally { + sender.close(); + } + } + Assert.assertTrue(handler.frames.get() > 0); + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test + public void testInvalidSchemaColumnsDoNotAdvanceRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-invalid-columns"); + String slot = new File(root, "default").getAbsolutePath(); + seedBacklog(slot); + FeedbackHandler handler = new FeedbackHandler(false, true); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + Sender sender = Sender.fromConfig(config(peer.getPort(), root, false)); + try { + Assert.assertTrue("invalid schema feedback must be rejected and replayed", + handler.awaitDistinctClients(5, TimeUnit.SECONDS)); + } finally { + sender.close(); + } + } + Assert.assertTrue(handler.frames.get() > 0); + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + private static void assertNoQuarantine(File root) { + File[] files = root.listFiles(); + Assert.assertNotNull(files); + for (File file : files) { + Assert.assertFalse(file.getName(), file.getName().contains(".unreplayable-")); + Assert.assertFalse(file.getName(), file.getName().equals(OrphanScanner.FAILED_SENTINEL_NAME)); + File failed = new File(file, OrphanScanner.FAILED_SENTINEL_NAME); + Assert.assertFalse(failed.getAbsolutePath(), failed.exists()); + } + } + + private static void assertRetained(String slot, long publishedFsn) { + try (CursorSendEngine recovered = new CursorSendEngine(slot, SEGMENT_SIZE)) { + Assert.assertEquals(-1, recovered.ackedFsn()); + Assert.assertEquals(publishedFsn, recovered.publishedFsn()); + } + } + + private static String config(int port, File root, boolean durableAck) { + return "ws::addr=localhost:" + port + + ";sf_dir=" + root.getAbsolutePath() + + ";sender_id=default;sf_max_segment_bytes=1m;request_durable_ack=" + + (durableAck ? "on" : "off") + ";close_flush_timeout_millis=0;"; + } + + private static byte[] copy(QwpWebSocketEncoder encoder, int length) { + byte[] bytes = new byte[length]; + long address = encoder.getBuffer().getBufferPtr(); + for (int i = 0; i < length; i++) { + bytes[i] = io.questdb.client.std.Unsafe.getUnsafe().getByte(address + i); + } + return bytes; + } + + private static byte[][] seedBacklog(String slot) { + try (CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer legacy = table("replay", 1); + QwpTableBuffer schema = table("replay", 2)) { + byte[] first = copy(encoder, encoder.encode(legacy)); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), first.length)); + byte[] second = copy(encoder, encoder.encode(schema)); + Assert.assertEquals(1, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), second.length)); + return new byte[][]{first, second}; + } + } + + private static QwpTableBuffer table(String name, long value) { + QwpTableBuffer table = new QwpTableBuffer(name); + table.getOrCreateColumn("n", QwpConstants.TYPE_LONG, true).addLong(value); + table.nextRow(); + return table; + } + + private static File taskTempRoot() { + File root = new File("target/schema-replay-network").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class SchemaRecordingHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledge; + private final List dataFrames = new ArrayList<>(); + + private SchemaRecordingHandler(boolean acknowledge) { + this.acknowledge = acknowledge; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + sendSchema(client, request.getLong(QwpConstants.HEADER_SIZE + 1)); + return; + } + dataFrames.add(data); + notifyAll(); + if (acknowledge) { + try { + client.sendBinary(ok(0)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for schema data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private static byte[] ok(long sequence) { + return ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN) + .put(WebSocketResponse.STATUS_OK).putLong(sequence).putShort((short) 0).array(); + } + + private static void sendSchema(TestWebSocketServer.ClientHandler client, long requestId) { + byte[] value = "value".getBytes(StandardCharsets.UTF_8); + byte[] timestamp = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + Long.BYTES + 1 + Integer.BYTES + Long.BYTES + + Short.BYTES + Short.BYTES + + Short.BYTES + value.length + Integer.BYTES + Short.BYTES + + Short.BYTES + timestamp.length + Integer.BYTES + Short.BYTES; + ByteBuffer response = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + response.putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(301) + .putLong(401) + .putShort((short) 1) + .putShort((short) 2) + .putShort((short) value.length) + .put(value) + .putInt(ColumnType.INT) + .putShort((short) 0) + .putShort((short) timestamp.length) + .put(timestamp) + .putInt(ColumnType.TIMESTAMP_NANO) + .putShort((short) 0); + try { + client.sendBinary(response.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private static final class FeedbackHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger frames = new AtomicInteger(); + private final CountDownLatch distinctClients; + private final boolean malformed; + private final boolean invalidColumns; + private TestWebSocketServer.ClientHandler lastClient; + + private FeedbackHandler(boolean malformed) { + this(malformed, false); + } + + private FeedbackHandler(boolean malformed, boolean invalidColumns) { + this.malformed = malformed; + this.invalidColumns = invalidColumns; + this.distinctClients = new CountDownLatch(malformed || invalidColumns ? 2 : 1); + } + + private boolean awaitDistinctClients(long timeout, TimeUnit unit) throws InterruptedException { + return distinctClients.await(timeout, unit); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + int sequence = frames.getAndIncrement(); + byte[] response = schemaFeedbackOk(sequence); + if (malformed) { + response = java.util.Arrays.copyOf(response, response.length - 1); + } else if (invalidColumns) { + response = schemaFeedbackInvalidColumns(sequence); + } + try { + client.sendBinary(response); + if (client != lastClient) { + lastClient = client; + distinctClients.countDown(); + } + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static byte[] schemaFeedbackOk(long sequence) { + byte[] tableName = "replay".getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.allocate(11 + 2 + 2 + tableName.length + 4 + 10) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES) + .putLong(sequence).putShort((short) 0).putShort((short) 1) + .putShort((short) tableName.length).put(tableName).putInt(10) + .put((byte) 2).putLong(0).put((byte) 1); + return buffer.array(); + } + + private static byte[] schemaFeedbackInvalidColumns(long sequence) { + byte[] tableName = "replay".getBytes(StandardCharsets.UTF_8); + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 26 + 2 * (2 + column.length + 6); + ByteBuffer buffer = ByteBuffer.allocate(11 + 2 + 2 + tableName.length + 4 + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES) + .putLong(sequence).putShort((short) 0).putShort((short) 1) + .putShort((short) tableName.length).put(tableName).putInt(payloadLength) + .put((byte) 2).putLong(0).put((byte) 0).putInt(1).putLong(1) + .putShort((short) -1).putShort((short) 2); + for (byte name : new byte[]{'x', 'X'}) { + buffer.putShort((short) 1).put(name).putInt(5).putShort((short) 0); + } + return buffer.array(); + } + } + +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index ace6d4f15..a4cea0ded 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -46,6 +46,7 @@ public void testGetFixedTypeSize() { Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_SHORT)); Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_CHAR)); Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_INT)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_IPv4)); Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_LONG)); Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_FLOAT)); Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DOUBLE)); @@ -67,6 +68,7 @@ public void testGetFixedTypeSize() { public void testGetTypeName() { Assert.assertEquals("BOOLEAN", QwpConstants.getTypeName(TYPE_BOOLEAN)); Assert.assertEquals("INT", QwpConstants.getTypeName(TYPE_INT)); + Assert.assertEquals("IPv4", QwpConstants.getTypeName(TYPE_IPv4)); Assert.assertEquals("VARCHAR", QwpConstants.getTypeName(TYPE_VARCHAR)); Assert.assertEquals("TIMESTAMP", QwpConstants.getTypeName(TYPE_TIMESTAMP)); Assert.assertEquals("TIMESTAMP_NANOS", QwpConstants.getTypeName(TYPE_TIMESTAMP_NANOS)); @@ -94,6 +96,7 @@ public void testIsFixedWidthType() { Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_SHORT)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_CHAR)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_INT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_IPv4)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_FLOAT)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DOUBLE)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java new file mode 100644 index 000000000..beaf1de4d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java @@ -0,0 +1,127 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +/** + * Per-row cost of writing an exact-type row through {@link QwpSchemaBinding} + * versus the legacy {@link QwpTableBuffer#getOrCreateColumn} path. Both write + * the same 10 columns (5 LONG, 5 DOUBLE) in the same order, so the difference + * is the binding's per-value column resolution. Run with {@link #main}. + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 2) +@Fork(1) +public class QwpSchemaBindingBenchmark { + private static final int COLUMN_COUNT = 10; + private static final int ROWS_PER_BATCH = 1_000; + private final String[] names = new String[COLUMN_COUNT]; + private QwpSchemaBinding binding; + private QwpTableBuffer legacyBuffer; + @Param({"2", "14"}) + private int nameLength; + private QwpTableBuffer schemaBuffer; + private long value; + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(QwpSchemaBindingBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void legacyRow() { + long v = value++; + for (int i = 0; i < COLUMN_COUNT; i += 2) { + legacyBuffer.getOrCreateColumn(names[i], QwpConstants.TYPE_LONG, true).addLong(v); + legacyBuffer.getOrCreateColumn(names[i + 1], QwpConstants.TYPE_DOUBLE, true).addDouble(v); + } + legacyBuffer.nextRow(); + if (legacyBuffer.getRowCount() == ROWS_PER_BATCH) { + legacyBuffer.reset(); + } + } + + @Benchmark + public void schemaRow() { + long v = value++; + for (int i = 0; i < COLUMN_COUNT; i += 2) { + binding.longColumn(names[i], v); + binding.doubleColumn(names[i + 1], v); + } + schemaBuffer.nextRow(); + if (schemaBuffer.getRowCount() == ROWS_PER_BATCH) { + schemaBuffer.reset(); + } + } + + @Setup(Level.Trial) + public void setUp() { + byte[][] columns = new byte[COLUMN_COUNT][]; + for (int i = 0; i < COLUMN_COUNT; i++) { + StringBuilder name = new StringBuilder().append((char) ('a' + i)); + while (name.length() < nameLength) { + name.append('x'); + } + names[i] = name.toString(); + columns[i] = QwpSchemaTestFixtures.column(names[i], i % 2 == 0 ? ColumnType.LONG : ColumnType.DOUBLE); + } + legacyBuffer = new QwpTableBuffer("t"); + schemaBuffer = new QwpTableBuffer("t"); + binding = QwpSchemaTestFixtures.binding(schemaBuffer, columns); + } + + @TearDown(Level.Trial) + public void tearDown() { + legacyBuffer.close(); + schemaBuffer.close(); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java new file mode 100644 index 000000000..7f03eb11c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java @@ -0,0 +1,426 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingBinaryTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/binary-input.tsv"; + + @Test + public void testConformanceCorpusUsesExactBinaryWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBinaryTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertEquals(line, "BINARY", fields[3]); + Assert.assertEquals(line, "VALID", fields[6]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + if ("BINARY".equals(fields[1])) { + binding.binaryColumn("value", bytes(fields[2])); + } else if ("STRING".equals(fields[1])) { + binding.stringColumn("value", "".equals(fields[2]) ? null : utf16(fields[2])); + } else { + throw new AssertionError(fields[1]); + } + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, 1); + if ("".equals(fields[4])) { + Assert.assertEquals(fields[0], 1, reader.u8()); + Assert.assertEquals(fields[0], 1, reader.u8()); + Assert.assertEquals(fields[0], 0, reader.i32()); + } else { + byte[] expected = bytes(fields[4]); + if ("binary_all_256".equals(fields[0])) { + assertAllByteValues(bytes(fields[2])); + assertAllByteValues(expected); + assertAllByteValues(bytes(fields[5])); + } + Assert.assertEquals(fields[0], 0, reader.u8()); + Assert.assertEquals(fields[0], 0, reader.i32()); + Assert.assertEquals(fields[0], expected.length, reader.i32()); + Assert.assertArrayEquals(fields[0], expected, reader.bytes(expected.length)); + } + Assert.assertEquals(fields[0], encoder.getBuffer().getPosition(), reader.position()); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testAllBinaryOverloadsCopyBeforeReturning() throws Exception { + assertMemoryLeak(() -> { + long address = Unsafe.malloc(6, MemoryTag.NATIVE_DEFAULT); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + byte[] array = {1, 2}; + binding.binaryColumn("value", array); + array[0] = 9; + buffer.nextRow(); + + put(address, new byte[]{3, 4}); + DirectByteSlice slice = new DirectByteSlice().of(address, 2); + binding.binaryColumn("value", slice); + put(address, new byte[]{8, 8}); + slice.of(0, 0); + buffer.nextRow(); + + put(address, new byte[]{5, 6}); + binding.binaryColumn("value", address, 2); + put(address, new byte[]{7, 7}); + buffer.nextRow(); + Unsafe.free(address, 6, MemoryTag.NATIVE_DEFAULT); + address = 0; + binding.binaryColumn("value", 0, 0); + buffer.nextRow(); + + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 4, 1); + Assert.assertEquals(0, reader.u8()); + for (int offset : new int[]{0, 2, 4, 6, 6}) { + Assert.assertEquals(offset, reader.i32()); + } + Assert.assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6}, reader.bytes(6)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } finally { + if (address != 0) { + Unsafe.free(address, 6, MemoryTag.NATIVE_DEFAULT); + } + } + }); + } + + @Test + public void testDuplicatePrecedesBinaryArgumentValidation() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.binaryColumn("value", new byte[]{42}) + .binaryColumn("value", (byte[]) null) + .binaryColumn("value", (DirectByteSlice) null) + .binaryColumn("value", 0, 1) + .binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1) + .stringColumn("value", new ThrowingCharSequence(new TestCharSequenceException())); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(42, reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.stringColumn("value", "x").binaryColumn("value", (byte[]) null); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals('x', reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testInvalidBinaryArgumentsAreTypedAndCheckedBeforeMemoryAccess() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + assertInvalid("binary value", () -> binding.binaryColumn("value", (byte[]) null)); + rollback(buffer); + assertInvalid("binary slice", () -> binding.binaryColumn("value", (DirectByteSlice) null)); + rollback(buffer); + assertInvalid("non-negative", () -> binding.binaryColumn("value", new DirectByteSlice().of(1, -1))); + rollback(buffer); + assertInvalid("pointer", () -> binding.binaryColumn("value", new DirectByteSlice().of(0, 1))); + rollback(buffer); + assertInvalid("non-negative", () -> binding.binaryColumn("value", 1, -1)); + rollback(buffer); + assertInvalid("pointer", () -> binding.binaryColumn("value", 0, 1)); + rollback(buffer); + assertInvalid("maximum", () -> binding.binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1)); + } + } + + @Test + public void testBitmapCrossesSixtyFourRowsWithPresentAndOmittedValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + for (int row = 0; row < 130; row++) { + if ((row & 1) == 0) { + binding.binaryColumn("value", new byte[]{(byte) row}); + } + buffer.nextRow(); + } + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 130, 1); + Assert.assertEquals(1, reader.u8()); + for (int i = 0; i < 16; i++) { + Assert.assertEquals(0xaa, reader.u8()); + } + Assert.assertEquals(0x02, reader.u8()); + for (int i = 0; i <= 65; i++) { + Assert.assertEquals(i, reader.i32()); + } + for (int row = 0; row < 130; row += 2) { + Assert.assertEquals(row & 0xff, reader.u8()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.BINARY), column("existing_b", ColumnType.BINARY), + column("only_b", ColumnType.BINARY), column("c", ColumnType.BINARY)); + binding.binaryColumn("a", new byte[]{1}); + binding.binaryColumn("existing_b", new byte[]{2}); + buffer.nextRow(); + TestCharSequenceException existingFailure = new TestCharSequenceException(); + Assert.assertSame(existingFailure, Assert.assertThrows(TestCharSequenceException.class, + () -> binding.stringColumn("existing_b", new ThrowingCharSequence(existingFailure)))); + rollback(buffer); + TestCharSequenceException newColumnFailure = new TestCharSequenceException(); + Assert.assertSame(newColumnFailure, Assert.assertThrows(TestCharSequenceException.class, + () -> binding.stringColumn("only_b", new ThrowingCharSequence(newColumnFailure)))); + rollback(buffer); + binding.binaryColumn("c", new byte[]{3}); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 3); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); + Assert.assertEquals("existing_b", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); + assertSparseBinary(reader, 2, 1); + assertSparseBinary(reader, 2, 2); + assertSparseBinary(reader, 1, 3); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testStringAppendIsExceptionAtomicForVarcharRuntimeAndError() throws Exception { + assertMemoryLeak(() -> { + for (Throwable failure : new Throwable[]{new TestCharSequenceException(), new TestCharSequenceError()}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.VARCHAR)); + binding.stringColumn("value", "a"); + buffer.nextRow(); + ThrowingCharSequence input = failure instanceof Error + ? new ThrowingCharSequence(failure, 4096, 300) + : new ThrowingCharSequence(failure); + Throwable actual = Assert.assertThrows(failure.getClass(), + () -> binding.stringColumn("value", input)); + Assert.assertSame(failure, actual); + rollback(buffer); + binding.stringColumn("value", "c"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 2, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + for (int offset : new int[]{0, 1, 2}) { + Assert.assertEquals(offset, reader.i32()); + } + Assert.assertArrayEquals(new byte[]{'a', 'c'}, reader.bytes(2)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } + }); + } + + @Test + public void testResetClearAndSchemaGuards() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.binaryColumn("value", new byte[]{7}); + buffer.nextRow(); + buffer.reset(); + binding.binaryColumn("value", new byte[]{8}); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(8, reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + Assert.assertThrows(IllegalStateException.class, () -> binding.binaryColumn("value", new byte[]{1})); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.STRING))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.INT))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.BINARY, new byte[]{1}))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, 0, column("value", ColumnType.BINARY))); + } + }); + } + + private static void assertSparseBinary(QwpTestWireReader reader, int bitmap, int value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(value, reader.u8()); + } + + private static void assertAllByteValues(byte[] bytes) { + Assert.assertEquals(256, bytes.length); + for (int i = 0; i < bytes.length; i++) { + Assert.assertEquals(i, bytes[i] & 0xff); + } + } + + private static void assertUnsupported(QwpSchemaBinding binding) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> binding.binaryColumn("value", new byte[]{1})); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, error.getReason()); + } + + private static void assertInvalid(String detail, Runnable runnable) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(error.getMessage(), LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("table=t")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=BINARY")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=BINARY")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains(detail)); + } + + private static void rollback(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static void put(long address, byte[] bytes) { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(address + i, bytes[i]); + } + } + + private static byte[] bytes(String hex) { + Assert.assertEquals(hex, 0, hex.length() & 1); + byte[] result = new byte[hex.length() / 2]; + for (int i = 0; i < result.length; i++) { + result[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16); + } + return result; + } + + private static String utf16(String hex) { + Assert.assertEquals(hex, 0, hex.length() & 3); + char[] chars = new char[hex.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + return tableReader(encoder, size, rows, columns, QwpConstants.TYPE_BINARY); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns, int wireType) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + return reader; + } + + private static final class ThrowingCharSequence implements CharSequence { + private final Throwable failure; + private final int failureIndex; + private final int length; + + private ThrowingCharSequence(Throwable failure) { + this(failure, 2, 1); + } + + private ThrowingCharSequence(Throwable failure, int length, int failureIndex) { + this.failure = failure; + this.length = length; + this.failureIndex = failureIndex; + } + + @Override + public int length() { + return length; + } + + @Override + public char charAt(int index) { + if (index < failureIndex) { + return 'x'; + } + if (failure instanceof Error) { + throw (Error) failure; + } + throw (RuntimeException) failure; + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + } + + private static final class TestCharSequenceException extends RuntimeException { + } + + private static final class TestCharSequenceError extends Error { + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java new file mode 100644 index 000000000..abb1f6c6d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java @@ -0,0 +1,310 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingBooleanTest { + private static final String BOOLEAN_CORPUS = "/io/questdb/client/cutlass/qwp/boolean-to-target.tsv"; + private static final String STRING_CORPUS = "/io/questdb/client/cutlass/qwp/string-to-boolean.tsv"; + + @Test + public void testBooleanCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBooleanTest.class.getResourceAsStream(BOOLEAN_CORPUS); + Assert.assertNotNull(BOOLEAN_CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 6, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(fields[2]))); + Assert.assertTrue(fields[1], "TRUE".equals(fields[1]) || "FALSE".equals(fields[1])); + binding.boolColumn("value", "TRUE".equals(fields[1])); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), wireType(fields[3]), 1); + Assert.assertEquals(0, reader.u8()); + assertValue(reader, fields[3], fields[4]); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testStringCorpusParsesStrictAsciiAndUsesBooleanWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBooleanTest.class.getResourceAsStream(STRING_CORPUS); + Assert.assertNotNull(STRING_CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 4, fields.length); + CharSequence value = "".equals(fields[1]) ? null : utf16(fields[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BOOLEAN)); + if ("".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("value", value)); + } else { + binding.stringColumn("value", value); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_BOOLEAN, 1); + if (value == null) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(fields[2]), reader.u8()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(80, count); + }); + } + + @Test + public void testBooleanPackingPastEightAndSixtyFourWithOmissions() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BOOLEAN)); + for (int row = 0; row < 137; row++) { + if ((row & 1) == 0) { + binding.boolColumn("value", (row & 3) == 0); + } + buffer.nextRow(); + } + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_BOOLEAN, 137); + Assert.assertEquals(1, reader.u8()); + byte[] nulls = new byte[18]; + for (int i = 0; i < 17; i++) nulls[i] = (byte) 0xaa; + Assert.assertArrayEquals(nulls, reader.bytes(nulls.length)); + byte[] expected = new byte[9]; + for (int i = 0; i < 8; i++) expected[i] = 0x55; + expected[8] = 0x15; + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + + buffer.reset(); + for (int row = 0; row < 70; row++) buffer.nextRow(); + QwpTestWireReader allNull = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_BOOLEAN, 70); + Assert.assertEquals(1, allNull.u8()); + byte[] allNulls = new byte[9]; + for (int i = 0; i < 8; i++) allNulls[i] = (byte) 0xff; + allNulls[8] = 0x3f; + Assert.assertArrayEquals(allNulls, allNull.bytes(allNulls.length)); + Assert.assertEquals(encoder.getBuffer().getPosition(), allNull.position()); + } + }); + } + + @Test + public void testDuplicateRollbackFailedRowAndLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.BOOLEAN), column("bad", ColumnType.BOOLEAN), column("c", ColumnType.INT)); + binding.boolColumn("a", true); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "invalid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.boolColumn("c", true); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + StringBuilder mutable = new StringBuilder("TrUe"); + binding.stringColumn("a", mutable).boolColumn("a", false); + mutable.replace(0, mutable.length(), "false"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reset, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + buffer.clear(); + assertIllegalState(() -> binding.boolColumn("a", true)); + assertIllegalState(() -> binding.stringColumn("a", "true")); + } + }); + } + + @Test + public void testDuplicatePrecedesUnsupportedCrossSetterAndInvalidParse() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("flag", ColumnType.BOOLEAN), column("sym", ColumnType.SYMBOL)); + binding.boolColumn("flag", true).stringColumn("flag", "invalid"); + binding.stringColumn("sym", "first").boolColumn("sym", false); + buffer.nextRow(); + Assert.assertTrue(encoder.encode(buffer) > 0); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("sym", ColumnType.SYMBOL), column("future", ColumnType.BOOLEAN, new byte[]{1}), + column("flagged", ColumnType.BOOLEAN | 0x10000), column("plain", ColumnType.BOOLEAN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("sym", true)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.symbol("plain", null)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.longColumn("plain", Long.MIN_VALUE)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("future", true)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("flagged", "true")); + rollback(buffer); + binding.boolColumn("missing", true); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.BOOLEAN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("d", true)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("d", null)); + } + } + + private static void assertValue(QwpTestWireReader reader, String type, String expected) { + switch (type) { + case "BOOLEAN": Assert.assertEquals(Integer.parseInt(expected), reader.u8()); break; + case "BYTE": Assert.assertEquals(Integer.parseInt(expected), reader.u8()); break; + case "SHORT": Assert.assertEquals(Integer.parseInt(expected), reader.i16()); break; + case "INT": Assert.assertEquals(Integer.parseInt(expected), reader.i32()); break; + case "LONG": Assert.assertEquals(Long.parseLong(expected), reader.i64()); break; + case "FLOAT": Assert.assertEquals(Float.floatToRawIntBits(Float.parseFloat(expected)), Float.floatToRawIntBits(reader.f32())); break; + case "DOUBLE": Assert.assertEquals(Double.doubleToRawLongBits(Double.parseDouble(expected)), Double.doubleToRawLongBits(reader.f64())); break; + case "VARCHAR": + byte[] bytes = hex(expected); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(bytes.length, reader.i32()); + Assert.assertArrayEquals(bytes, reader.bytes(bytes.length)); + break; + default: throw new AssertionError("unknown wire type: " + type); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "BOOLEAN": return ColumnType.BOOLEAN; + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + case "STRING": return ColumnType.STRING; + case "VARCHAR": return ColumnType.VARCHAR; + default: throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BOOLEAN": return QwpConstants.TYPE_BOOLEAN; + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + case "VARCHAR": return QwpConstants.TYPE_VARCHAR; + default: throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } + + private static String utf16(String value) { + Assert.assertEquals(0, value.length() & 3); + char[] chars = new char[value.length() / 4]; + for (int i = 0; i < chars.length; i++) chars[i] = (char) Integer.parseInt(value.substring(i * 4, i * 4 + 4), 16); + return new String(chars); + } + + private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } + private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java new file mode 100644 index 000000000..84a260663 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java @@ -0,0 +1,211 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; + +public class QwpSchemaBindingCharTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/char-conversions.tsv"; + private static final String HEADER = "# case_id\tinput_kind\tutf16_hex\texpected_char\texpected_null"; + + @Test + public void testCharToTextUsesTargetNativeUtf8() { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", targetType))); + binding.charColumn("value", 'A'); + buffer.nextRow(); + binding.charColumn("value", '\u03a9'); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(3, reader.i32()); + Assert.assertEquals('A', reader.u8()); + Assert.assertEquals(0xce, reader.u8()); + Assert.assertEquals(0xa9, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + } + } + + @Test + public void testCorpusUsesExactNativeCharWire() throws Exception { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean headerSeen = false; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + Assert.assertTrue("true".equals(fields[4]) || "false".equals(fields[4])); + Assert.assertEquals(Integer.parseInt(fields[3]) == 0, Boolean.parseBoolean(fields[4])); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + if ("CHAR".equals(fields[1])) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", ColumnType.CHAR))); + String input = fromUtf16Hex(fields[2]); + Assert.assertEquals(1, input.length()); + binding.charColumn("value", input.charAt(0)); + } else if ("STRING".equals(fields[1])) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", ColumnType.CHAR))); + binding.stringColumn("value", "".equals(fields[2]) ? null : fromUtf16Hex(fields[2])); + } else { + throw new AssertionError("unknown input kind: " + fields[1]); + } + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_CHAR); + if ("".equals(fields[2])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(fields[3]), reader.u16()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(23, count); + } + + @Test + public void testDuplicateOmissionRollbackResetAndInference() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, + column("a", ColumnType.CHAR), column("bad", ColumnType.UUID), column("c", ColumnType.CHAR))); + binding.stringColumn("a", "A").binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "B"); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("bad", 'C')); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "Z"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 3, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(6, reader.u8()); + Assert.assertEquals('A', reader.u16()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals('Z', reader.u16()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "Q"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals('Q', reset.u16()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "A"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.charColumn("value", 'A'); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_CHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + } + + @Test + public void testUnsupportedAndDesignatedTargetsRemainRejected() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, + column("value", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("value", 'A')); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("ts", 'A')); + } + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", targetType))); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.charColumn("value", '\ud800')); + } + } + } + + private static String fromUtf16Hex(String hex) { + StringBuilder value = new StringBuilder(hex.length() / 4); + for (int i = 0; i < hex.length(); i += 4) { + value.append((char) Integer.parseInt(hex.substring(i, i + 4), 16)); + } + return value.toString(); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java new file mode 100644 index 000000000..4afa3669a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java @@ -0,0 +1,155 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * A binding resolves each column against the schema once, when it creates the + * buffer column, and later writes reuse that resolution. These tests pin that + * the reuse keeps every per-value rule of the first write. + */ +public class QwpSchemaBindingColumnResolutionTest { + + @Test + public void testDuplicateWriteToResolvedColumnKeepsFirstValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 1); + buffer.nextRow(); + binding.longColumn("v", 2); + binding.longColumn("V", 3); + buffer.nextRow(); + assertInts(buffer.getColumn(0), 1, 2); + } + }); + } + + @Test + public void testInferredColumnTypeConflictIsDetectedOnLaterRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("x", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleColumn("x", 2.0), "inferred column type conflict"); + buffer.cancelCurrentRow(); + binding.longColumn("X", 3); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + QwpTableBuffer.ColumnBuffer column = buffer.getColumn(0); + Assert.assertEquals(QwpConstants.TYPE_LONG, column.getType()); + Assert.assertEquals(2, column.getSize()); + Assert.assertEquals(1, Unsafe.getUnsafe().getLong(column.getDataAddress())); + Assert.assertEquals(3, Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES)); + } + }); + } + + @Test + public void testInvalidNameIsRejectedAfterColumnsExist() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("", 2), "invalid column name"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn(null, 2), "invalid column name"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("v?", 2), "invalid column name"); + Assert.assertEquals(1, buffer.getColumnCount()); + } + }); + } + + @Test + public void testRejectedSchemaColumnsStayRejectedAcrossRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, + column("ts", ColumnType.TIMESTAMP_MICRO), + parameterizedColumn("p", ColumnType.LONG), + column("v", ColumnType.INT)); + for (int attempt = 0; attempt < 2; attempt++) { + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("ts", 1), "designated timestamp writes"); + buffer.cancelCurrentRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("p", 1), "parameterized target type"); + buffer.cancelCurrentRow(); + binding.longColumn("v", attempt); + buffer.nextRow(); + } + Assert.assertEquals("rejected targets must not create buffer columns", 1, buffer.getColumnCount()); + assertInts(buffer.getColumn(0), 0, 1); + } + }); + } + + @Test + public void testResolvedColumnKeepsConversionAndRangeChecks() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 7); + buffer.nextRow(); + binding.longColumn("V", 8); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("v", 1L << 40)); + buffer.cancelCurrentRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.binaryColumn("v", new byte[]{1}), "conversion is not implemented"); + buffer.cancelCurrentRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + Assert.assertEquals(QwpConstants.TYPE_INT, buffer.getColumn(0).getType()); + assertInts(buffer.getColumn(0), 7, 8); + } + }); + } + + private static void assertInts(QwpTableBuffer.ColumnBuffer column, int... expected) { + Assert.assertEquals(expected.length, column.getSize()); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals(expected[i], Unsafe.getUnsafe().getInt(column.getDataAddress() + (long) i * Integer.BYTES)); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java new file mode 100644 index 000000000..0f7771e8d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java @@ -0,0 +1,273 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingDoubleArrayTest { + + @Test + public void testAllPublicRepresentationsUseExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + int[] shape32 = new int[ColumnType.ARRAY_NDIMS_LIMIT]; + java.util.Arrays.fill(shape32, 1); + double payloadNaN = Double.longBitsToDouble(0x7ff8000000000042L); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray array = new DoubleArray(shape32)) { + array.append(10.5); + QwpSchemaBinding binding = binding(buffer, + column("a1", arrayType(1)), + column("a2", arrayType(2)), + column("a3", arrayType(3)), + column("a32", arrayType(ColumnType.ARRAY_NDIMS_LIMIT))); + binding.doubleArray("a1", new double[]{payloadNaN, -0.0, Double.POSITIVE_INFINITY}); + binding.doubleArray("a2", new double[][]{{3.0, 4.0}, {5.0, 6.0}}); + binding.doubleArray("a3", new double[][][]{{{7.0}, {8.0}}}); + binding.doubleArray("a32", array); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 4); + columnDef(reader, "a1"); + columnDef(reader, "a2"); + columnDef(reader, "a3"); + columnDef(reader, "a32"); + array(reader, new int[]{3}, payloadNaN, -0.0, Double.POSITIVE_INFINITY); + array(reader, new int[]{2, 2}, 3.0, 4.0, 5.0, 6.0); + array(reader, new int[]{1, 2, 1}, 7.0, 8.0); + array(reader, shape32, 10.5); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNullRepresentationsAreNoopsAndWrapperRankTracksReshape() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray array = new DoubleArray(1, 1)) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.doubleArray("a", (double[]) null) + .doubleArray("b", (double[][]) null) + .doubleArray("c", (double[][][]) null) + .doubleArray("d", (DoubleArray) null); + Assert.assertEquals(0, buffer.getColumnDefs().length); + + Assert.assertEquals(2, array.getDimensionality()); + array.reshape(1); + Assert.assertEquals(1, array.getDimensionality()); + array.append(42.0); + binding.doubleArray("value", array); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + columnDef(reader, "value"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 1, new int[]{1}, new double[]{42.0}); + Assert.assertEquals(size, reader.position()); + } + }); + + try (DoubleArray array = new DoubleArray(1)) { + array.close(); + Assert.assertThrows(LineSenderException.class, array::getDimensionality); + } + } + + @Test + public void testKnownRankDuplicateRollbackAndShapeValidation() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray closedDuplicate = new DoubleArray(1)) { + QwpSchemaBinding binding = binding(buffer, + column("v", arrayType(1)), column("m", arrayType(2))); + + binding.doubleArray("v", new double[]{1.0}); + binding.doubleArray("v", new double[][]{{99.0}}); // duplicate: first value wins + closedDuplicate.close(); + binding.doubleArray("v", closedDuplicate); // duplicate: do not inspect an ignored value + buffer.nextRow(); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[][]{{2.0}}), + "sourceDims=2", "targetDims=1"); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + Assert.assertThrows(LineSenderException.class, + () -> binding.doubleArray("m", new double[][]{{2.0}, {3.0, 4.0}})); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.doubleArray("v", new double[0]); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1); + columnDef(reader, "v"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 1, new int[]{1}, new double[]{1.0}); + rawArray(reader, 1, new int[]{0}, new double[0]); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testMissingSchemaPinsFirstEffectiveRank() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.doubleArray("v", new double[][]{{1.0, 2.0}}); + buffer.nextRow(); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{3.0}), + "sourceDims=1", "inferredDims=2"); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.doubleArray("v", new double[][]{{4.0}, {5.0}}); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1); + columnDef(reader, "v"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 2, new int[]{1, 2}, new double[]{1.0, 2.0}); + rawArray(reader, 2, new int[]{2, 1}, new double[]{4.0, 5.0}); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{6.0}), + "sourceDims=1", "inferredDims=2"); + } + }); + } + + @Test + public void testRejectsWrongElementTypeWeakRankParametersAndDesignatedColumn() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.encodeArrayType(ColumnType.LONG, 1))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "targetType=LONG[]"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + int weak = arrayType(1) | (1 << 19); + QwpSchemaBinding binding = binding(buffer, column("v", weak)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "conversion is not implemented"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("v", arrayType(1), new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "parameterized target type"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("v", arrayType(1))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "designated timestamp writes"); + } + } + + @Test + public void testRejectsStringValuesForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("a", "zz")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("a", "")); + } + } + }); + } + + @Test + public void testRejectsGeoHashValuesForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.geoHashColumn("a", 1023L, 10)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.geoHashColumn("a", "zz")); + } + } + }); + } + + @Test + public void testStringNullUsesArrayNullWireForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + binding.stringColumn("a", null); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + columnDef(reader, "a"); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + private static void array(QwpTestWireReader reader, int[] shape, double... values) { + Assert.assertEquals(0, reader.u8()); + rawArray(reader, shape.length, shape, values); + } + + private static int arrayType(int dimensions) { + return ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions); + } + + private static void columnDef(QwpTestWireReader reader, String name) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE_ARRAY, reader.u8()); + } + + private static void rawArray(QwpTestWireReader reader, int expectedDims, int[] shape, double[] values) { + Assert.assertEquals(expectedDims, reader.u8()); + for (int dimension : shape) { + Assert.assertEquals(dimension, reader.i32()); + } + for (double value : values) { + Assert.assertEquals(Double.doubleToRawLongBits(value), reader.i64()); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java new file mode 100644 index 000000000..33e7abd3f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java @@ -0,0 +1,295 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv"; + private static final String HEADER = "# case_id\tsource_type\tinput_bits_hex\ttarget_type\t" + + "target_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\t" + + "expected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingFloatingDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[][] sourceTargets = new boolean[2][6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 12, fields.length); + Vector vector = new Vector(fields); + sourceTargets[vector.sourceIndex()][targetIndex(vector.targetType)] = true; + assertVector(vector); + count++; + } + } + Assert.assertEquals(60, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target pair " + source + '/' + target, + sourceTargets[source][target]); + } + } + }); + } + + @Test + public void testDuplicateNullRollbackAndOmissionLifecycle() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(3, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + // Duplicate suppression precedes both an invalid conversion and null normalization. + binding.floatColumn("value", 1.0f) + .doubleColumn("value", 1.25) + .doubleColumn("value", Double.NEGATIVE_INFINITY); + buffer.nextRow(); + + // A null first value also wins over a later invalid duplicate. + binding.doubleColumn("value", Double.longBitsToDouble(0x7ff8000000000042L)) + .floatColumn("value", 1.25f); + buffer.nextRow(); + + // Row B must disappear without disturbing completed A or following C. + binding.doubleColumn("only_b", 2.0); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.doubleColumn("value", 1.25)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.floatColumn("value", -2.5f); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_DECIMAL64, 4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0a, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-25, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL64, 1); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void assertInvalid(Vector vector) throws Exception { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> vector.append(binding, "value")); + Assert.assertEquals(vector.caseId, LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(vector.caseId, error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=value")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=" + vector.sourceType)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=DECIMAL")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } + } + + private static void assertVector(Vector vector) throws Exception { + try { + if (vector.invalid()) { + assertInvalid(vector); + return; + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + vector.append(binding, "value"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType), 1); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + return reader; + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + return 5; + } + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final long inputBits; + private final String outcome; + private final String sourceType; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + sourceType = fields[1]; + Assert.assertTrue(sourceType.equals("FLOAT") || sourceType.equals("DOUBLE")); + inputBits = Long.parseUnsignedLong(fields[2], 16); + Assert.assertEquals(sourceType.equals("FLOAT") ? 8 : 16, fields[2].length()); + targetType = fields[3]; + targetPrecision = Integer.parseInt(fields[4]); + targetScale = Integer.parseInt(fields[5]); + outcome = fields[6]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) { + expectedLimbs[i] = parseHex(fields[7 + i]); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[7 + i]); + } + if (outcome.equals("NULL")) { + Assert.assertEquals("NULL", fields[11]); + } else if (outcome.equals("INVALID")) { + Assert.assertEquals("-", fields[11]); + } else { + Assert.assertFalse(fields[11].isEmpty()); + Assert.assertNotEquals("-", fields[11]); + } + } + + private void append(QwpSchemaBinding binding, String column) { + if ("FLOAT".equals(sourceType)) { + binding.floatColumn(column, Float.intBitsToFloat((int) inputBits)); + } else { + binding.doubleColumn(column, Double.longBitsToDouble(inputBits)); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private int sourceIndex() { + return "FLOAT".equals(sourceType) ? 0 : 1; + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java new file mode 100644 index 000000000..c58adaae3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java @@ -0,0 +1,265 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingFloatingTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] f = line.split("\\t", -1); + Assert.assertEquals(line, 7, f.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(f[3]))); + Runnable append = () -> append(binding, f[1], f[2]); + if ("".equals(f[5])) { + Assert.assertEquals(f[0], "", f[6]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + } else { + append.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), wireType(f[4]), 1, 1); + if ("".equals(f[5])) { + Assert.assertEquals(f[0], + "BYTE".equals(f[3]) || "SHORT".equals(f[3]) ? "0" : "NULL", f[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertArrayEquals(f[0], hex(f[5]), reader.bytes(hex(f[5]).length)); + } + Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(350, count); + }); + } + + @Test + public void testNaNIsMissingWithOtherNullAndPresentValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DOUBLE)); + binding.floatColumn("value", Float.intBitsToFloat(0x7fc12345)); + buffer.nextRow(); + buffer.nextRow(); + binding.doubleColumn("value", -0.0d); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_DOUBLE, 3, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals(0x8000000000000000L, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testDuplicatePrecedesInvalidAndUnsupportedConversion() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("n", ColumnType.LONG), column("uuid", ColumnType.UUID)); + binding.floatColumn("n", 1).doubleColumn("n", Double.POSITIVE_INFINITY); + binding.uuidColumn("uuid", 1, 2).floatColumn("uuid", Float.NaN); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 1, 2); + Assert.assertEquals("n", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals("uuid", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.FLOAT), column("only_b", ColumnType.DOUBLE), + column("bad", ColumnType.BYTE), column("c", ColumnType.LONG)); + binding.doubleColumn("a", 1.5); + buffer.nextRow(); + binding.floatColumn("only_b", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.doubleColumn("bad", 128)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.floatColumn("c", 3); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_FLOAT, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(Float.floatToRawIntBits(1.5f), reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testResetClearAndStaleBinding() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.FLOAT)); + binding.doubleColumn("value", 9.25); + buffer.nextRow(); + buffer.reset(); + binding.floatColumn("value", -0.0f); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_FLOAT, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x80000000, reader.i32()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + assertIllegalState(() -> binding.floatColumn("value", 1)); + assertIllegalState(() -> binding.doubleColumn("value", 1)); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("uuid", ColumnType.UUID), column("future", ColumnType.FLOAT, new byte[]{1}), + column("flagged", ColumnType.DOUBLE | 0x10000), column("bool", ColumnType.BOOLEAN), + column("timestamp", ColumnType.TIMESTAMP), column("binary", ColumnType.BINARY), + column("byte", ColumnType.BYTE), column("short", ColumnType.SHORT)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("uuid", Float.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("future", Double.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("flagged", 1)); + rollback(buffer); + binding.doubleColumn("missing", 1); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("bool", Float.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("timestamp", Double.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("binary", Float.NaN)); + rollback(buffer); + assertInvalidContext(binding, buffer, "byte", true); + assertInvalidContext(binding, buffer, "short", false); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.DOUBLE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("d", Float.NaN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("d", Double.NaN)); + } + } + + private static void append(QwpSchemaBinding binding, String inputType, String bits) { + if ("FLOAT".equals(inputType)) { + binding.floatColumn("value", Float.intBitsToFloat((int) Long.parseLong(bits, 16))); + } else if ("DOUBLE".equals(inputType)) { + binding.doubleColumn("value", Double.longBitsToDouble(Long.parseUnsignedLong(bits, 16))); + } else { + throw new AssertionError("unknown input type: " + inputType); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "BYTE": return ColumnType.BYTE; case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BYTE": return QwpConstants.TYPE_BYTE; case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + Assert.assertEquals(value, 0, value.length() & 1); byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring((bytes.length - 1 - i) * 2, (bytes.length - i) * 2), 16); + return bytes; + } + + private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } + private static void assertInvalidContext(QwpSchemaBinding binding, QwpTableBuffer buffer, String column, boolean floatInput) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> { if (floatInput) binding.floatColumn(column, 128); else binding.doubleColumn(column, Double.POSITIVE_INFINITY); }); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=" + column)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=" + (floatInput ? "FLOAT" : "DOUBLE"))); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + column.toUpperCase())); + rollback(buffer); + } + private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java new file mode 100644 index 000000000..2ae401101 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java @@ -0,0 +1,315 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-text.tsv"; + + @Test + public void testCorpusUsesExactTargetTextWire() throws Exception { + assertMemoryLeak( + () -> { + InputStream stream = + QwpSchemaBindingFloatingTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = + new BufferedReader( + new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + boolean symbol = "SYMBOL".equals(fields[3]); + QwpSchemaBinding binding = + binding(buffer, column("value", targetType(fields[3]))); + append(binding, fields[1], fields[2], "value"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = + tableReader( + encoder, + size, + symbol + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR, + 1); + boolean missing = "".equals(fields[5]); + Assert.assertEquals(missing, "".equals(fields[6])); + Assert.assertEquals(missing ? 1 : 0, reader.u8()); + if (missing) { + Assert.assertEquals(1, reader.u8()); + if (symbol) { + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + } + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals( + fields[6], + new String(expected, StandardCharsets.UTF_8)); + if (symbol) { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals( + expected, reader.bytes(expected.length)); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError( + "case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(48, count); + }); + } + + @Test + public void testNaNDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak( + () -> { + for (int target : + new int[] {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = + target == ColumnType.SYMBOL + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR; + QwpSchemaBinding binding = binding(buffer, column("value", target)); + binding.floatColumn("value", Float.intBitsToFloat(0x7fc00002)) + .binaryColumn("value", new byte[] {1}); + buffer.nextRow(); + binding.doubleColumn("value", -0.0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("-0.0", reader.string()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(4, reader.i32()); + Assert.assertEquals("-0.0", reader.ascii(4)); + } + Assert.assertEquals(size, reader.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = + binding( + buffer, + column("a", ColumnType.STRING), + column("bad", ColumnType.UUID), + column("c", ColumnType.VARCHAR)); + binding.doubleColumn("a", 1e23); + buffer.nextRow(); + binding.floatColumn("a", 0.1f); + assertReason( + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleColumn("bad", 2.0)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.floatColumn("c", Float.intBitsToFloat(1)); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarchar(reader, 6, "1.0E23"); + assertVarchar(reader, 5, "1.401298464324817E-45"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.doubleColumn("a", Double.NEGATIVE_INFINITY); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + assertVarchar(reset, 0, "-Infinity"); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testSymbolFormatterScratchIsCopied() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.doubleColumn("value", 42.0); + buffer.nextRow(); + binding.doubleColumn("value", 43.0); + buffer.nextRow(); + binding.doubleColumn("value", 42.0); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("42.0", reader.string()); + Assert.assertEquals("43.0", reader.string()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testNativeInferenceAndUnsupportedTargets() { + assertNative("FLOAT", "value", 0x80000000L); + assertNative("DOUBLE", "value", 0x8000000000000000L); + for (int target : new int[] {ColumnType.TIMESTAMP, ColumnType.UUID, ColumnType.BOOLEAN}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", target)); + assertReason( + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> + binding.doubleColumn( + "value", Double.longBitsToDouble(0x7ff8000000000002L))); + } + } + } + + private static void assertNative(String inputType, String name, long bits) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + append( + binding, + inputType, + inputType.equals("FLOAT") + ? String.format("%08x", bits) + : String.format("%016x", bits), + name); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals( + inputType.equals("FLOAT") ? QwpConstants.TYPE_FLOAT : QwpConstants.TYPE_DOUBLE, + reader.u8()); + Assert.assertEquals(0, reader.u8()); + if (inputType.equals("FLOAT")) { + Assert.assertEquals((int) bits, reader.i32()); + } else { + Assert.assertEquals(bits, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + + private static void append(QwpSchemaBinding binding, String type, String bits, String name) { + if ("FLOAT".equals(type)) + binding.floatColumn(name, Float.intBitsToFloat((int) Long.parseUnsignedLong(bits, 16))); + else binding.doubleColumn(name, Double.longBitsToDouble(Long.parseUnsignedLong(bits, 16))); + } + + private static void assertVarchar(QwpTestWireReader reader, int bitmap, String expected) { + if (bitmap == 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + } + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length(), reader.i32()); + Assert.assertEquals(expected, reader.ascii(expected.length())); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) { + return ColumnType.STRING; + } + if ("VARCHAR".equals(value)) { + return ColumnType.VARCHAR; + } + if ("SYMBOL".equals(value)) { + return ColumnType.SYMBOL; + } + throw new AssertionError(value); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java new file mode 100644 index 000000000..9a20ed183 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java @@ -0,0 +1,243 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-decimal.tsv"; + private static final String HEADER = "# case_id\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[] targets = new boolean[6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 11, fields.length); + Vector vector = new Vector(fields); + targets[targetIndex(vector.targetType)] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + if (vector.invalid()) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("value", vector.input)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } else { + binding.longColumn("value", vector.input); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType)); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(39, count); + for (int i = 0; i < targets.length; i++) { + Assert.assertTrue("missing target " + i, targets[i]); + } + }); + } + + @Test + public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(2, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + binding.longColumn("value", 1).longColumn("value", 10); + buffer.nextRow(); + + binding.longColumn("only_b", 1); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("value", 10)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.longColumn("value", -1); + buffer.nextRow(); + binding.longColumn("value", Long.MIN_VALUE).longColumn("value", 1); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0c, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-10, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 1); + decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + return 5; + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final long input; + private final String outcome; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + input = Long.parseLong(fields[1]); + targetType = fields[2]; + targetPrecision = Integer.parseInt(fields[3]); + targetScale = Integer.parseInt(fields[4]); + outcome = fields[5]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) { + expectedLimbs[i] = parseHex(fields[6 + i]); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[6 + i]); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java new file mode 100644 index 000000000..0b9f67156 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java @@ -0,0 +1,283 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-text.tsv"; + + @Test + public void testCorpusUsesExactTextWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = wireType(fields[3]); + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); + binding.longColumn("value", Long.parseLong(fields[1])); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 1); + if ("".equals(fields[4])) { + Assert.assertEquals("", fields[5]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) Assert.assertEquals(0, reader.i32()); + else Assert.assertEquals(0, reader.varint()); + } else { + byte[] expected = hex(fields[4]); + Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals(0, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + } else { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = targetType == ColumnType.SYMBOL + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR; + QwpSchemaBinding binding = binding(buffer, column("value", targetType)); + binding.longColumn("value", Long.MIN_VALUE).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.bytes(2)); + } else { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } + Assert.assertEquals(size, reader.position()); + } + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.STRING), column("bad", ColumnType.UUID), + column("c", ColumnType.VARCHAR), column("sym", ColumnType.SYMBOL)); + binding.longColumn("a", 10).stringColumn("a", "ignored"); + buffer.nextRow(); + binding.longColumn("sym", 7).longColumn("sym", Long.MIN_VALUE); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("bad", 20)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.longColumn("c", 30); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_VARCHAR, 3, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(6, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); + Assert.assertArrayEquals(new byte[]{'1', '0'}, reader.bytes(2)); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); + Assert.assertArrayEquals(new byte[]{'3', '0'}, reader.bytes(2)); + Assert.assertEquals(size, reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.longColumn("value", 8); + buffer.nextRow(); + buffer.reset(); + binding.longColumn("value", 42); + buffer.nextRow(); + binding.longColumn("value", 43); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(2, reset.varint()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reset.stringBytes()); + Assert.assertArrayEquals(new byte[]{'4', '3'}, reset.stringBytes()); + Assert.assertEquals(0, reset.varint()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(0, reset.varint()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingSchemaInfersNativeLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.longColumn("value", 42); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + Assert.assertEquals(QwpConstants.TYPE_LONG, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(42, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testOwnerBackedSymbolUsesGlobalDictionary() throws Exception { + assertMemoryLeak(() -> { + TestWebSocketServer.WebSocketServerHandler handler = new TestWebSocketServer.WebSocketServerHandler() { }; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender senderApi = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + QwpWebSocketSender sender = (QwpWebSocketSender) senderApi; + Assert.assertEquals(0, sender.getOrAddGlobalSymbol("prefix")); + GlobalSymbolDictionary mirror = new GlobalSymbolDictionary(); + Assert.assertEquals(0, mirror.getOrAddSymbol("prefix")); + try (QwpTableBuffer buffer = new QwpTableBuffer("t", sender); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.longColumn("value", 42); + buffer.nextRow(); + binding.longColumn("value", 43); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("42")); + Assert.assertEquals(2, sender.getOrAddGlobalSymbol("43")); + Assert.assertEquals(0, buffer.getColumn(0).getSymbolDictionarySize()); + mirror.getOrAddSymbol("42"); + mirror.getOrAddSymbol("43"); + encoder.beginMessage(1, mirror, -1, 2); + encoder.addTable(buffer); + int size = encoder.finishMessage(); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(3, reader.varint()); + Assert.assertArrayEquals("prefix".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("42".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("43".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + } + }); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + return tableReader(encoder, size, wireType, rows, 1); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + if (columns == 1) { + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + } + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) return ColumnType.STRING; + if ("VARCHAR".equals(value)) return ColumnType.VARCHAR; + if ("SYMBOL".equals(value)) return ColumnType.SYMBOL; + throw new AssertionError(value); + } + + private static byte wireType(String value) { + return "SYMBOL".equals(value) ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR; + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java new file mode 100644 index 000000000..1347185c8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java @@ -0,0 +1,185 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongTimestampTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv"; + + @Test + public void testCorpusUsesExactTargetUnitWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongTimestampTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + int target = targetType(fields[2]); + byte wireType = wireType(fields[3]); + long value = Long.parseLong(fields[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.longColumn("value", value); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", wireType); + if ("".equals(fields[4])) { + Assert.assertEquals("true", fields[5]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals("false", fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(12, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + byte wireType = target == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.longColumn("value", Long.MIN_VALUE).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.longColumn("value", Long.MIN_VALUE + 1); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, "value", wireType); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.MIN_VALUE + 1, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.longColumn("value", Long.MAX_VALUE); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, 1, "value", wireType); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(Long.MAX_VALUE, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.TIMESTAMP_MICRO), + column("bad", ColumnType.UUID), + column("c", ColumnType.TIMESTAMP_NANO)); + binding.longColumn("a", -1); + buffer.nextRow(); + binding.longColumn("a", 2); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("bad", 3)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.longColumn("c", 4); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNamedDesignatedRejection() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.longColumn("value", Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.longColumn("ts", 1)); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; + } + + private static byte wireType(String value) { + return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java new file mode 100644 index 000000000..f144d097b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java @@ -0,0 +1,571 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingNativeDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv"; + private static final String HEADER = "# case_id\tsource_type\tsource_scale\tsource_ll_hex\tsource_lh_hex\tsource_hl_hex\tsource_hh_hex\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWireAndPreservesCaller() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingNativeDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + Assert.assertFalse(line.isEmpty()); + String[] f = line.split("\t", -1); + Assert.assertEquals(line, 16, f.length); + Assert.assertTrue("VALUE".equals(f[10]) || "INVALID".equals(f[10])); + assertTargetTag(f[7], Integer.parseInt(f[8])); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", ColumnType.getDecimalType(Integer.parseInt(f[8]), Integer.parseInt(f[9])))); + Object value = decimal(f); + long[] before = limbs(value); + int scale = scale(value); + if ("INVALID".equals(f[10])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> append(binding, value)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(0, buffer.getColumnCount()); + } else { + append(binding, value); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType(f[7])); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(f[9]), reader.u8()); + assertExpectedLimbs(reader, f); + Assert.assertEquals(size, reader.position()); + } + Assert.assertEquals(scale, scale(value)); + Assert.assertArrayEquals(before, limbs(value)); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(60, count); + }); + } + + @Test + public void testDecimalTextOverloadUsesEveryExactTargetWidth() throws Exception { + assertMemoryLeak(() -> { + int[] precisions = {2, 4, 9, 18, 38, 76}; + for (int precision : precisions) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", ColumnType.getDecimalType(precision, 1))); + binding.decimalColumn("value", "1.2", new Decimal256()); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireTypeForPrecision(precision)); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(12, reader.i64()); + int limbs = precision <= 18 ? 1 : precision <= 38 ? 2 : 4; + for (int i = 1; i < limbs; i++) { + Assert.assertEquals(0, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testDecimalTextOverloadSuppressesDuplicatesAndRollsBackInvalidRows() throws Exception { + assertMemoryLeak(() -> { + int decimal = ColumnType.getDecimalType(5, 1); + Decimal256 scratch = new Decimal256(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", decimal), column("only_b", decimal)); + + binding.decimalColumn("value", "1.2", scratch) + .decimalColumn("value", "not-a-decimal", scratch); + buffer.nextRow(); + + binding.decimalColumn("only_b", "2", scratch); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.decimalColumn("value", "not-a-decimal", scratch)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.decimalColumn("value", "NaN", scratch) + .decimalColumn("value", "still-not-a-decimal", scratch); + buffer.nextRow(); + binding.decimalColumn("value", (CharSequence) null, scratch) + .decimalColumn("value", "", scratch); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 3, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x06, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(12, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadValidatesTargetAfterDuplicateBeforeParsing() throws Exception { + assertMemoryLeak(() -> { + Decimal256 scratch = new Decimal256(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.UUID)); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", "not-a-decimal", scratch)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.uuidColumn("value", 1, 2) + .decimalColumn("value", "not-a-decimal", scratch); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadMissingInfersDecimal256AtNaturalScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + Decimal256 scratch = new Decimal256(); + binding.decimalColumn("value", "NaN", scratch); + buffer.nextRow(); + binding.decimalColumn("value", "12.3400", scratch); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256, 2); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1234, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadMissingSpecialInfersDecimal256ScaleZeroNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.decimalColumn("value", "-Infinity", new Decimal256()); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.decimalColumn("value", "5.678", new Decimal256()); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = unknownTableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(3, reset.u8()); + Assert.assertEquals(5678, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testDuplicateRollbackResetOmissionAndRetainedTargetScale() throws Exception { + assertMemoryLeak(() -> { + int decimal = ColumnType.getDecimalType(18, 4); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", decimal), column("bad", ColumnType.getDecimalType(2, 0)), column("c", decimal)); + binding.decimalColumn("a", new Decimal64(15, 1)).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.decimalColumn("a", new Decimal64(2, 0)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.decimalColumn("bad", new Decimal128(0, 1000, 0))); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.decimalColumn("c", new Decimal256(0, 0, 0, 25, 2)); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); + decimalDefinition(reader, "a", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reader, "c", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(4, reader.u8()); + Assert.assertEquals(15000, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(4, reader.u8()); + Assert.assertEquals(2500, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.decimalColumn("c", new Decimal64(3, 0)); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + decimalDefinition(reset, "a", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reset, "c", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(4, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(4, reset.u8()); + Assert.assertEquals(30000, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingInfersNativeDecimalWidthAndScale() throws Exception { + assertMemoryLeak(() -> { + Object[] values = { + new Decimal64(12, 1), + new Decimal128(3, 4, 2), + new Decimal256(5, 6, 7, 8, 3) + }; + byte[] types = {QwpConstants.TYPE_DECIMAL64, QwpConstants.TYPE_DECIMAL128, QwpConstants.TYPE_DECIMAL256}; + for (int i = 0; i < values.length; i++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + append(binding, values[i]); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = unknownTableReader(encoder, size, types[i]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(scale(values[i]), reader.u8()); + for (long limb : wireLimbs(values[i])) { + Assert.assertEquals(limb, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testMalformedParameterizedAndDesignatedTargetsRemainRejected() throws Exception { + assertMemoryLeak(() -> { + Decimal64 value = new Decimal64(1, 0); + int[] malformed = { + ColumnType.DECIMAL64, + ColumnType.getDecimalType(18, 4) | 0x10000, + ColumnType.getDecimalType(18, 4) | 0x40000000 + }; + for (int type : malformed) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", type)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, + column("value", ColumnType.getDecimalType(18, 4))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + parameterizedColumn("value", ColumnType.getDecimalType(18, 4))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + }); + } + + @Test + public void testLegacyBufferStillChoosesScaleFromFirstValueInEachBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn( + "value", QwpConstants.TYPE_DECIMAL64, true); + column.addDecimal64(new Decimal64(12, 2)); + buffer.nextRow(); + int firstSize = encoder.encode(buffer); + QwpTestWireReader first = legacyTableReader(encoder, firstSize); + Assert.assertEquals(0, first.u8()); + Assert.assertEquals(2, first.u8()); + Assert.assertEquals(12, first.i64()); + Assert.assertEquals(firstSize, first.position()); + + buffer.reset(); + column.addDecimal64(new Decimal64(34, 3)); + buffer.nextRow(); + int secondSize = encoder.encode(buffer); + QwpTestWireReader second = legacyTableReader(encoder, secondSize); + Assert.assertEquals(0, second.u8()); + Assert.assertEquals(3, second.u8()); + Assert.assertEquals(34, second.i64()); + Assert.assertEquals(secondSize, second.position()); + } + }); + } + + @Test + public void testLegacyNullOnlyDecimalUsesWireScaleZeroWithoutLockingNextBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn( + "value", QwpConstants.TYPE_DECIMAL256, true); + column.addDecimal256(Decimal256.NULL_VALUE); + buffer.nextRow(); + + int nullSize = encoder.encode(buffer); + QwpTestWireReader nullOnly = legacyTableReader(encoder, nullSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(1, nullOnly.u8()); + Assert.assertEquals(1, nullOnly.u8()); + Assert.assertEquals(0, nullOnly.u8()); + Assert.assertEquals(nullSize, nullOnly.position()); + + buffer.reset(); + column.addDecimal256(new Decimal256(0, 0, 0, 1234, 4)); + buffer.nextRow(); + int valueSize = encoder.encode(buffer); + QwpTestWireReader value = legacyTableReader(encoder, valueSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(0, value.u8()); + Assert.assertEquals(4, value.u8()); + Assert.assertEquals(1234, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(valueSize, value.position()); + } + }); + } + + private static void append(QwpSchemaBinding binding, Object value) { + if (value instanceof Decimal64) { + binding.decimalColumn("value", (Decimal64) value); + } else if (value instanceof Decimal128) { + binding.decimalColumn("value", (Decimal128) value); + } else { + binding.decimalColumn("value", (Decimal256) value); + } + } + + private static void assertExpectedLimbs(QwpTestWireReader reader, String[] f) { + int count = wireLongCount(f[7]); + for (int i = 0; i < count; i++) { + Assert.assertEquals(parseHex(f[11 + i]), reader.i64()); + } + for (int i = count; i < 4; i++) { + Assert.assertEquals("-", f[11 + i]); + } + } + + private static void assertTargetTag(String name, int precision) { + String expected; + if (precision <= 2) { + expected = "DECIMAL8"; + } else if (precision <= 4) { + expected = "DECIMAL16"; + } else if (precision <= 9) { + expected = "DECIMAL32"; + } else if (precision <= 18) { + expected = "DECIMAL64"; + } else if (precision <= 38) { + expected = "DECIMAL128"; + } else { + expected = "DECIMAL256"; + } + Assert.assertEquals(expected, name); + } + + private static Object decimal(String[] f) { + int scale = Integer.parseInt(f[2]); + long ll = parseHex(f[3]); + switch (f[1]) { + case "DECIMAL64": + return new Decimal64(ll, scale); + case "DECIMAL128": + Decimal128 decimal = new Decimal128(); + decimal.of(parseHex(f[4]), ll, scale); + return decimal; + case "DECIMAL256": + return new Decimal256(parseHex(f[6]), parseHex(f[5]), parseHex(f[4]), ll, scale); + default: + throw new AssertionError("unknown source type " + f[1]); + } + } + + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static long[] limbs(Object value) { + if (value instanceof Decimal64) { + return new long[]{((Decimal64) value).getValue()}; + } + if (value instanceof Decimal128) { + return new long[]{((Decimal128) value).getHigh(), ((Decimal128) value).getLow()}; + } + Decimal256 decimal = (Decimal256) value; + return new long[]{decimal.getHh(), decimal.getHl(), decimal.getLh(), decimal.getLl()}; + } + + private static QwpTestWireReader legacyTableReader(QwpWebSocketEncoder encoder, int size) { + return legacyTableReader(encoder, size, QwpConstants.TYPE_DECIMAL64); + } + + private static QwpTestWireReader legacyTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + decimalDefinition(reader, "value", type); + return reader; + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int scale(Object value) { + if (value instanceof Decimal64) { + return ((Decimal64) value).getScale(); + } + if (value instanceof Decimal128) { + return ((Decimal128) value).getScale(); + } + return ((Decimal256) value).getScale(); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static QwpTestWireReader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + return unknownTableReader(encoder, size, type, 1); + } + + private static QwpTestWireReader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static byte wireTypeForPrecision(int precision) { + if (precision <= 18) { + return QwpConstants.TYPE_DECIMAL64; + } + if (precision <= 38) { + return QwpConstants.TYPE_DECIMAL128; + } + return QwpConstants.TYPE_DECIMAL256; + } + + private static long[] wireLimbs(Object value) { + if (value instanceof Decimal64) { + return new long[]{((Decimal64) value).getValue()}; + } + if (value instanceof Decimal128) { + Decimal128 decimal = (Decimal128) value; + return new long[]{decimal.getLow(), decimal.getHigh()}; + } + Decimal256 decimal = (Decimal256) value; + return new long[]{decimal.getLl(), decimal.getLh(), decimal.getHl(), decimal.getHh()}; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java new file mode 100644 index 000000000..643f97364 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java @@ -0,0 +1,184 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringDateTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-date.tsv"; + + @Test + public void testCorpusUsesExactDateWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringDateTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("case_id\tinput_kind\tinput\toutcome\texpected_raw\tfeature", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + CharSequence value = input(fields[1], fields[2]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DATE)); + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + Assert.assertEquals(0, buffer.getRowCount()); + } else { + Assert.assertTrue(line, "VALID".equals(fields[3]) || "NULL".equals(fields[3])); + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", QwpConstants.TYPE_DATE); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(53, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.DATE), column("bad", ColumnType.UUID), column("c", ColumnType.DATE)); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-02"); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-03"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "1969-12-31"); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(13, reader.u8()); + Assert.assertEquals(86_400_000L, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(-86_400_000L, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "1970-01-01 00:00:00.001UTC"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testNativeInferenceParameterizedAndDesignatedBehavior() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "1970-01-01"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DATE, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "1970-01-01")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("value", ColumnType.DATE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "1970-01-01")); + } + } + + private static CharSequence input(String kind, String value) { + Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); + if ("".equals(value)) { + return null; + } + if ("".equals(value)) { + return ""; + } + if ("TEXT".equals(kind)) { + return value; + } + String[] units = value.split(","); + char[] chars = new char[units.length]; + for (int i = 0; i < units.length; i++) { + chars[i] = (char) Integer.parseInt(units[i], 16); + } + return new String(chars); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java new file mode 100644 index 000000000..02bf19c24 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java @@ -0,0 +1,223 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-decimal.tsv"; + private static final String HEADER = "# case_id\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[] targets = new boolean[6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 11, fields.length); + Vector vector = new Vector(fields); + targets[targetIndex(vector.targetType)] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + if (vector.invalid()) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", vector.input)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } else { + binding.stringColumn("value", vector.input); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType)); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(55, count); + for (int i = 0; i < targets.length; i++) { + Assert.assertTrue("missing target " + i, targets[i]); + } + }); + } + + @Test + public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(3, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + binding.stringColumn("value", "1.0").stringColumn("value", "invalid"); + buffer.nextRow(); + + binding.stringColumn("only_b", "2.0"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", "100.0")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.stringColumn("value", "-2.5"); + buffer.nextRow(); + binding.stringColumn("value", null).stringColumn("value", "invalid"); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0c, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-25, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 1); + decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": return 0; + case "DECIMAL16": return 1; + case "DECIMAL32": return 2; + case "DECIMAL64": return 3; + case "DECIMAL128": return 4; + default: return 5; + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) return QwpConstants.TYPE_DECIMAL128; + if ("DECIMAL256".equals(target)) return QwpConstants.TYPE_DECIMAL256; + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) return 2; + if ("DECIMAL256".equals(target)) return 4; + return 1; + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final String input; + private final String outcome; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + input = "".equals(fields[1]) ? null : "".equals(fields[1]) ? "" : fields[1]; + targetType = fields[2]; + targetPrecision = Integer.parseInt(fields[3]); + targetScale = Integer.parseInt(fields[4]); + outcome = fields[5]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) expectedLimbs[i] = parseHex(fields[6 + i]); + for (int i = limbs; i < 4; i++) Assert.assertEquals("-", fields[6 + i]); + if (outcome.equals("INVALID")) { + Assert.assertEquals("-", fields[10]); + } else if (outcome.equals("NULL")) { + Assert.assertEquals("NULL", fields[10]); + } else { + Assert.assertNotEquals("-", fields[10]); + Assert.assertNotEquals("NULL", fields[10]); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) return "DECIMAL8"; + if (precision <= 4) return "DECIMAL16"; + if (precision <= 9) return "DECIMAL32"; + if (precision <= 18) return "DECIMAL64"; + if (precision <= 38) return "DECIMAL128"; + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java new file mode 100644 index 000000000..c75fd43ca --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java @@ -0,0 +1,280 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringGeoHashTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv"; + private static final String HEADER = "# case_id\tinput\tbits\toutcome\tvalue_hex"; + + @Test + public void testCorpusUsesExactGeoHashWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean headerSeen = false; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse(headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + int bits = Integer.parseInt(fields[2]); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", ColumnType.getGeoHashTypeWithBits(bits)))); + CharSequence value = "".equals(fields[1]) ? null + : "".equals(fields[1]) ? "" : fields[1]; + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + } else if ("VALUE".equals(fields[3]) || "NULL".equals(fields[3])) { + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bits, reader.varint()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(bits, reader.varint()); + long expected = Long.parseUnsignedLong(fields[4], 16); + for (int i = 0; i < (bits + 7) / 8; i++) { + Assert.assertEquals((int) ((expected >>> (8 * i)) & 0xff), reader.u8()); + } + } + Assert.assertEquals(size, reader.position()); + } else { + throw new AssertionError("unknown outcome: " + fields[3]); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue(headerSeen); + Assert.assertEquals(18, count); + }); + } + + @Test + public void testDuplicateBitmapRollbackResetInferenceAndGuards() throws Exception { + assertMemoryLeak(() -> { + int geo8 = ColumnType.getGeoHashTypeWithBits(8); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("a", geo8), column("bad", ColumnType.LONG), column("c", geo8))); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "zz"); + buffer.nextRow(); + binding.stringColumn("a", ""); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "zz"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "z")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "04"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 5, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + assertGeo(reader, 29, 8, 0xff); + assertGeo(reader, 15, 8, 1); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "04"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); + assertGeo(reset, 0, 8, 1); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(8, reset.varint()); + Assert.assertEquals(resetSize, reset.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "z"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.unsupportedColumn("value", "GEOHASH")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, parameterizedColumn("value", geo8))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "z")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "z")); + } + }); + } + + @Test + public void testMalformedPackedGeoHashTargetsRemainTypedUnsupported() throws Exception { + assertMemoryLeak(() -> { + int geoFlag = 1 << 16; + int[] malformed = { + ColumnType.GEOHASH, + geoFlag | ColumnType.GEOBYTE, + geoFlag | (61 << 8) | ColumnType.GEOLONG, + geoFlag | (8 << 8) | ColumnType.STRING, + ColumnType.getGeoHashTypeWithBits(8) | (1 << 24) + }; + for (int targetType : malformed) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", targetType))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "zz")); + } + } + }); + } + + @Test + public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exception { + assertMemoryLeak(() -> { + for (int bits = 1; bits <= 60; bits++) { + int targetType = ColumnType.getGeoHashTypeWithBits(bits); + StringBuilder input = new StringBuilder((bits + 4) / 5); + for (int i = 0; i < (bits + 4) / 5; i++) { + input.append('z'); + } + long max = (1L << bits) - 1; + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", targetType))); + binding.stringColumn("value", input); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + assertGeo(reader, 0, bits, max); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("value", input); + buffer.nextRow(); + binding.stringColumn("value", null); + buffer.nextRow(); + binding.stringColumn("value", ""); + buffer.nextRow(); + size = encoder.encode(buffer); + reader = tableReader(encoder, size, 3); + assertGeo(reader, 6, bits, max); + Assert.assertEquals(size, reader.position()); + } + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + buffer.getOrCreateColumn("value", QwpConstants.TYPE_GEOHASH, true) + .addGeoHash(0x0fffffffffffffffL, 60); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(60, reader.varint()); + Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + buffer.getOrCreateColumn("value", QwpConstants.TYPE_GEOHASH, false) + .addGeoHash(0x0fffffffffffffffL, 60); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(60, reader.varint()); + Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + private static void assertGeo(QwpTestWireReader reader, int bitmap, int bits, long value) { + Assert.assertEquals(bitmap < 0 ? 0 : 1, reader.u8()); + if (bitmap >= 0) { + Assert.assertEquals(bitmap, reader.u8()); + } + Assert.assertEquals(bits, reader.varint()); + for (int i = 0; i < (bits + 7) / 8; i++) { + Assert.assertEquals((int) ((value >>> (8 * i)) & 0xff), reader.u8()); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java new file mode 100644 index 000000000..02742b4de --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java @@ -0,0 +1,212 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringLong256Test { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv"; + private static final String HEADER = "# case_id\tinput\toutcome\tl0_hex\tl1_hex\tl2_hex\tl3_hex"; + + @Test + public void testCorpusUsesExactLong256Wire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringLong256Test.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + boolean headerSeen = false; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.LONG256)); + if ("VALUE".equals(fields[2])) { + binding.stringColumn("value", fields[1]); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + Assert.assertEquals(0, reader.u8()); + for (int i = 3; i < 7; i++) { + Assert.assertEquals(parseHexLong(fields[i]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } else if ("NULL".equals(fields[2])) { + Assert.assertEquals("", fields[1]); + binding.stringColumn("value", null); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } else if ("INVALID".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", fields[1])); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(0, buffer.getColumnDefs().length); + } else { + throw new AssertionError("unknown outcome: " + fields[2]); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(23, count); + }); + } + + @Test + public void testDuplicateNullOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.LONG256), column("bad", ColumnType.LONG), column("c", ColumnType.LONG256)); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "0x01").stringColumn("a", "invalid duplicate"); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "0x02"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "0x03")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "0x04"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(4, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + assertLong256Values(reader, 13, 1); + assertLong256Values(reader, 7, 4); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "0x05"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(5, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingStringInferenceRemainsVarchar() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "0x00"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + } + + @Test + public void testParameterizedDesignatedAndOtherTargetsRemainUnsupported() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + parameterizedColumn("value", ColumnType.LONG256), column("other", ColumnType.UUID)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "0x00")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("other", "0x00")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "0x00")); + } + } + + private static void assertLong256Values(QwpTestWireReader reader, int bitmap, long value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(value, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + return tableReader(encoder, size, rows, "value"); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(column, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + return reader; + } + + private static long parseHexLong(String value) { + return Long.parseUnsignedLong(value, 16); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java new file mode 100644 index 000000000..8a7bd3a20 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java @@ -0,0 +1,292 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringNumericTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-numeric.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringNumericTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] f = line.split("\\t", -1); + Assert.assertEquals(line, 7, f.length); + Assert.assertTrue(line, "VALID".equals(f[6]) || "INVALID".equals(f[6])); + CharSequence value = "".equals(f[1]) ? null : utf16(f[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(f[2]))); + if ("INVALID".equals(f[6])) { + Assert.assertEquals(f[0], "", f[4]); + Assert.assertEquals(f[0], "", f[5]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("value", value)); + } else { + binding.stringColumn("value", value); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), wireType(f[3]), 1, 1); + if ("".equals(f[4])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + byte[] expected = littleEndianHex(f[4]); + Assert.assertArrayEquals(f[0], expected, reader.bytes(expected.length)); + } + Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError | RuntimeException e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(429, count); + }); + } + + @Test + public void testDuplicatePrecedesInvalidParseAndUnsupportedConversion() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("n", ColumnType.INT), column("uuid", ColumnType.UUID)); + binding.stringColumn("n", "42").stringColumn("n", "invalid"); + binding.uuidColumn("uuid", 1, 2).stringColumn("uuid", "not-a-uuid"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 1, 2); + Assert.assertEquals("n", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals("uuid", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(42, reader.i32()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.TIMESTAMP_MICRO)); + binding.timestampColumn("value", 123, ChronoUnit.MICROS) + .stringColumn("value", "not-a-timestamp-conversion"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_TIMESTAMP, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(123, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.INT), column("only_b", ColumnType.LONG), + column("bad", ColumnType.SHORT), column("c", ColumnType.DOUBLE)); + binding.stringColumn("a", "1"); + buffer.nextRow(); + binding.stringColumn("only_b", "2"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("bad", "+3")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "-0.0"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x8000000000000000L, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testResetClearAndMutableInput() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.LONG)); + StringBuilder mutable = new StringBuilder("7"); + binding.stringColumn("value", mutable); + mutable.replace(0, 1, "9"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + + buffer.reset(); + binding.stringColumn("value", "8"); + buffer.nextRow(); + reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(8, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + assertIllegalState(() -> binding.stringColumn("value", "1")); + } + }); + } + + @Test + public void testNullAndParsedSentinelsHaveDistinctWirePresence() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.INT)); + binding.stringColumn("value", "-2147483648"); + buffer.nextRow(); + binding.stringColumn("value", null); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), QwpConstants.TYPE_INT, 2, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(Integer.MIN_VALUE, reader.i32()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("bool", ColumnType.BOOLEAN), column("date", ColumnType.DATE, new byte[]{1}), + column("future", ColumnType.INT, new byte[]{1}), + column("flagged", ColumnType.LONG | 0x10000)); + assertSchemaError(LineSenderSchemaException.Reason.INVALID_VALUE, "bool", "BOOLEAN", () -> binding.stringColumn("bool", "2")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "date", "DATE", () -> binding.stringColumn("date", "1")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "future", "INT", () -> binding.stringColumn("future", "1")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "flagged", "unknown", () -> binding.stringColumn("flagged", "1")); + rollback(buffer); + binding.stringColumn("missing", "1"); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.DOUBLE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("d", null)); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError(value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError(value); + } + } + + private static String utf16(String hex) { + Assert.assertEquals(0, hex.length() & 3); + char[] chars = new char[hex.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static byte[] littleEndianHex(String hex) { + Assert.assertEquals(0, hex.length() & 1); + byte[] bytes = new byte[hex.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring((bytes.length - 1 - i) * 2, (bytes.length - i) * 2), 16); + } + return bytes; + } + + private static void rollback(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static void assertIllegalState(Runnable runnable) { + Assert.assertThrows(IllegalStateException.class, runnable::run); + } + + private static void assertSchemaError( + LineSenderSchemaException.Reason reason, + String column, + String target, + Runnable runnable + ) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("table=t")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=" + column)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=STRING")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + target)); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java new file mode 100644 index 000000000..305a8d564 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java @@ -0,0 +1,212 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringTimestampTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv"; + + @Test + public void testCorpusUsesExactTargetUnitWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringTimestampTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("case_id\ttarget\tinput_kind\tinput\toutcome\texpected_raw\tfeature", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertTrue(line, "VALID".equals(fields[4]) || "NULL".equals(fields[4]) || "INVALID".equals(fields[4])); + int target = targetType(fields[1]); + byte wireType = wireType(fields[1]); + CharSequence value = input(fields[2], fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + if ("INVALID".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + Assert.assertEquals(0, buffer.getRowCount()); + count++; + continue; + } + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", wireType); + if ("NULL".equals(fields[4])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(110, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + byte wireType = target == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.stringColumn("value", null).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("value", "1970-01-01T00:00:00.000001Z"); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, "value", wireType); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? 1 : 1000, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("value", "1969-12-31T23:59:59.999999Z"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, 1, "value", wireType); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.TIMESTAMP_MICRO), + column("bad", ColumnType.UUID), + column("c", ColumnType.TIMESTAMP_NANO)); + binding.stringColumn("a", "1969-12-31T23:59:59.999999Z"); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-01T00:00:00.000002Z"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "1970-01-01T00:00:00.000004Z"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4000, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNamedDesignatedRejection() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "1970-01-01T00:00:00Z"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(20, reader.i32()); + Assert.assertEquals("1970-01-01T00:00:00Z", reader.ascii(20)); + Assert.assertEquals(size, reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "1970-01-01T00:00:00Z")); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + Assert.assertTrue(value, "TIMESTAMP".equals(value) || "TIMESTAMP_NANOS".equals(value)); + return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; + } + + private static CharSequence input(String kind, String value) { + Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); + if ("".equals(value)) { + return null; + } + if ("TEXT".equals(kind)) { + return value; + } + String[] units = value.split(","); + char[] chars = new char[units.length]; + for (int i = 0; i < units.length; i++) { + chars[i] = (char) Integer.parseInt(units[i], 16); + } + return new String(chars); + } + + private static byte wireType(String value) { + return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java new file mode 100644 index 000000000..eb7345c2d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -0,0 +1,2210 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.schemaResult; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingTest { + @Test + public void testTimestampInputConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + int count = 0; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try { + int targetType = timestampColumnType(fields[4]); + byte wireType = timestampWireType(fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + Runnable write; + if ("LONG".equals(fields[1])) { + long input = Long.parseLong(fields[2]); + ChronoUnit unit = ChronoUnit.valueOf(fields[3]); + write = () -> rows.timestampColumn("ts", input, unit); + } else if ("INSTANT".equals(fields[1])) { + Instant input = Instant.parse(fields[2]); + Assert.assertEquals("-", fields[3]); + write = () -> rows.timestampColumn("ts", input); + } else { + throw new AssertionError("unknown input_kind: " + fields[1]); + } + if ("".equals(fields[5])) { + Assert.assertEquals("", fields[6]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, write); + continue; + } + write.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.u8()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(102, count); + } + } + + @Test + public void testTimestampOverloadValidationAndDuplicateSuppression() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("ts", Instant.ofEpochSecond(0, 123)) + .timestampColumn("ts", (Instant) null) + .timestampColumn("ts", Long.MAX_VALUE, ChronoUnit.WEEKS); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, + QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(123, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", (Instant) null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.WEEKS)); + } + for (int targetType : new int[]{ColumnType.LONG, ColumnType.TIMESTAMP_NANO | 0x10000}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", (Instant) null)); + } + } + }); + } + + @Test + public void testTimestampInstantFailureRollsBackOnlyCurrentRowColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("late", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("micro", Instant.ofEpochSecond(1)); + buffer.nextRow(); + rows.timestampColumn("micro", Instant.ofEpochSecond(2)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("late", Instant.MAX)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.timestampColumn("micro", Instant.ofEpochSecond(3)) + .timestampColumn("late", Instant.ofEpochSecond(0, 4)); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1_000_000, reader.i64()); + Assert.assertEquals(3_000_000, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testTimestampUnitConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/timestamp-units.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + int count = 0; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try { + long input = Long.parseLong(fields[1]); + ChronoUnit unit = ChronoUnit.valueOf(fields[2]); + int targetType = timestampColumnType(fields[3]); + byte wireType = timestampWireType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + if ("".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", input, unit)); + continue; + } + rows.timestampColumn("ts", input, unit); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.u8()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(48, count); + } + } + + @Test + public void testTimestampValidationDuplicateAndRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("nano", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("micro", 7, ChronoUnit.MICROS) + .timestampColumn("micro", Long.MAX_VALUE, null) + .timestampColumn("micro", Long.MAX_VALUE, ChronoUnit.SECONDS); + buffer.nextRow(); + + rows.timestampColumn("micro", 8, ChronoUnit.MICROS); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("nano", Long.MAX_VALUE, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + rows.timestampColumn("micro", 9, ChronoUnit.MICROS) + .timestampColumn("nano", 10, ChronoUnit.NANOS); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(9, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testTimestampNullUnsupportedAndBindingLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("nano", ColumnType.TIMESTAMP_NANO)); + rows.stringColumn("micro", null).stringColumn("nano", null); + buffer.nextRow(); + rows.stringColumn("micro", null); + buffer.nextRow(); + QwpTestWireReader nulls = tableReader(encoder, encoder.encode(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(1, nulls.u8()); + Assert.assertEquals(3, nulls.u8()); + Assert.assertEquals(0, nulls.u8()); + Assert.assertEquals(1, nulls.u8()); + Assert.assertEquals(3, nulls.u8()); + Assert.assertEquals(0, nulls.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), nulls.position()); + + buffer.reset(); + rows.timestampColumn("nano", 11, ChronoUnit.NANOS); + buffer.nextRow(); + buffer.clear(); + assertIllegalState(() -> rows.timestampColumn("nano", 12, ChronoUnit.NANOS)); + assertIllegalState(() -> rows.timestampColumn("nano", Instant.EPOCH)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("x", ColumnType.LONG)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("x", Long.MIN_VALUE, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.timestampColumn("missing", 1, ChronoUnit.MICROS); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", 1, null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.WEEKS)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + for (int targetType : new int[]{ColumnType.TIMESTAMP_MICRO | 0x10000, ColumnType.TIMESTAMP_NANO | 0x10000}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", Instant.EPOCH)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.stringColumn("ts", null)); + } + }); + } + @Test + public void testClassifiesSchemaLessResponses() throws Exception { + assertCtorReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "access denied"); + assertCtorReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "temporarily unavailable"); + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)).longColumn("x", 1); + } + assertCtorReason(QwpSchemaProtocol.RESULT_TOO_LARGE, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "response limit"); + } + + @Test + public void testInfersNativeTargetsForMissingSchemaAndColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.longColumn("l", 1) + .stringColumn("s", "x") + .symbol("sym", "y") + .boolColumn("b", true) + .floatColumn("f", 1) + .doubleColumn("d", 2) + .uuidColumn("u", 3, 4) + .binaryColumn("bin", new byte[]{5}) + .timestampColumn("tm", 6, ChronoUnit.MICROS) + .timestampColumn("tn", 7, ChronoUnit.NANOS) + .timestampColumn("ti", Instant.EPOCH); + buffer.nextRow(); + byte[] expected = { + QwpConstants.TYPE_LONG, QwpConstants.TYPE_VARCHAR, QwpConstants.TYPE_SYMBOL, + QwpConstants.TYPE_BOOLEAN, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE, + QwpConstants.TYPE_UUID, QwpConstants.TYPE_BINARY, QwpConstants.TYPE_TIMESTAMP, + QwpConstants.TYPE_TIMESTAMP_NANOS, QwpConstants.TYPE_TIMESTAMP + }; + QwpColumnDef[] definitions = buffer.getColumnDefs(); + Assert.assertEquals(expected.length, definitions.length); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals("column " + i, expected[i], definitions[i].getTypeCode()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.uuidColumn("u", 0x0102030405060708L, 0x1112131415161718L) + .floatColumn("f", Float.intBitsToFloat(0x3fc00000)) + .stringColumn("s", "quest"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, + QwpConstants.TYPE_UUID, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x3fc00000, reader.i32()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(5, reader.i32()); + Assert.assertEquals("quest", reader.ascii(5)); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("known", ColumnType.LONG)); + rows.longColumn("missing", 1) + .unsupportedColumn("missing", "INT") + .uuidColumn("missing", 2, 3); + buffer.nextRow(); + LineSenderSchemaException conflict = Assert.assertThrows( + LineSenderSchemaException.class, + () -> rows.uuidColumn("missing", 4, 5) + ); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, conflict.getReason()); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.longColumn("a", 1); + buffer.nextRow(); + try { + rows.stringColumn("failed_only", new CharSequence() { + @Override + public char charAt(int index) { + if (index == 1) { + throw new IllegalStateException("fixture failure"); + } + return 'x'; + } + + @Override + public int length() { + return 2; + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + }); + Assert.fail("expected fixture failure"); + } catch (IllegalStateException e) { + Assert.assertEquals("fixture failure", e.getMessage()); + } + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("a", 3); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer micros = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(micros, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.designatedTimestamp(1, ChronoUnit.MICROS); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, micros.getColumnDefs()[0].getTypeCode()); + micros.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.designatedTimestamp(2, ChronoUnit.NANOS)); + micros.cancelCurrentRow(); + micros.rollbackUncommittedColumns(); + } + try (QwpTableBuffer nanos = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(nanos, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.designatedTimestamp(1, ChronoUnit.NANOS); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, nanos.getColumnDefs()[0].getTypeCode()); + } + }); + } + + @Test + public void testDuplicateUsesFirstValueAcrossInputSetters() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.uuidColumn("u", 1, 2).longColumn("u", 3); + rows.longColumn("l", 4).stringColumn("l", "not-a-long"); + buffer.nextRow(); + rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef") + .stringColumn("u", "malformed"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testEmptyAndIncompleteEncodeContracts() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("l", ColumnType.LONG)); + Assert.assertEquals(0, encoder.encode(buffer)); + rows.longColumn("l", 1); + assertIllegalState(() -> encoder.encode(buffer)); + buffer.nextRow(); + int size = encoder.encode(buffer); + assertLongFrame(encoder, size, 1); + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + buffer.reset(); + Assert.assertEquals(0, encoder.encode(buffer)); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + }); + } + + @Test + public void testLifecycleAndNameValidation() throws Exception { + try (QwpTableBuffer invalid = new QwpTableBuffer("bad\nname")) { + assertIllegalArgument(() -> new QwpSchemaBinding(invalid, known(column("x", ColumnType.LONG)))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("x", ColumnType.LONG)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("bad\nname", 1)); + rows.longColumn("missing", 1); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding stale = rows(buffer, column("x", ColumnType.LONG)); + buffer.clear(); + assertIllegalState(() -> stale.longColumn("x", 1)); + assertIllegalState(() -> stale.timestampColumn("x", Instant.EPOCH)); + } + } + + @Test + public void testSmallIntegerNumericConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + try { + long input = Long.parseLong(fields[2]); + int targetType = numericColumnType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", targetType)); + Runnable append = () -> appendSmallInteger(rows, fields[1], input, "n"); + if ("".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + } else { + append.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, + numericWireType(fields[3])); + if ("".equals(fields[4])) { + assertNumericNull(reader, fields[3]); + } else { + Assert.assertEquals(0, reader.u8()); + assertNumericValue(reader, fields[3], fields[4]); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(64, count); + } + + @Test + public void testSmallIntegerTextConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try { + long input = Long.parseLong(fields[2]); + byte wireType = textWireType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", textColumnType(fields[3]))); + appendSmallInteger(rows, fields[1], input, "value"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, wireType); + if ("".equals(fields[4])) { + Assert.assertEquals("", fields[5]); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("row zero must be NULL", 1, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + } + } else { + byte[] expected = hexBytes(fields[4]); + Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals("value must be bitmap-present", 0, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(fields[5], reader.string()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + for (byte value : expected) { + Assert.assertEquals(value & 0xff, reader.u8()); + } + } + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(30, count); + } + + @Test + public void testSmallIntegerDecimalConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv"); + Assert.assertNotNull(stream); + boolean[][] sourceTargets = new boolean[3][6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals( + "# case_id\tinput_type\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome" + + "\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql", + lines.readLine() + ); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 12, fields.length); + try { + long input = Long.parseLong(fields[2]); + int precision = Integer.parseInt(fields[4]); + int scale = Integer.parseInt(fields[5]); + Assert.assertEquals(decimalStorageType(precision), fields[3]); + sourceTargets[smallIntegerSourceIndex(fields[1])][decimalStorageIndex(fields[3])] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows( + buffer, + column("value", ColumnType.getDecimalType(precision, scale)) + ); + Runnable append = () -> appendSmallInteger(rows, fields[1], input, "value"); + if ("INVALID".equals(fields[6])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + rollbackCurrentRow(buffer); + } else { + append.run(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader( + encoder, + size, + 1, + decimalWireType(fields[3]) + ); + if ("NULL".equals(fields[6])) { + Assert.assertEquals("NULL", fields[11]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals("VALUE", fields[6]); + Assert.assertEquals(0, reader.u8()); + } + Assert.assertEquals(scale, reader.u8()); + if ("VALUE".equals(fields[6])) { + int limbs = decimalWireLongCount(fields[3]); + for (int i = 0; i < limbs; i++) { + Assert.assertEquals(parseHexLong(fields[7 + i]), reader.i64()); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[7 + i]); + } + } + Assert.assertEquals(size, reader.position()); + } + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(30, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target coverage " + source + '/' + target, + sourceTargets[source][target]); + } + } + } + + @Test + public void testIntegerTemporalConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + try { + long input = Long.parseLong(fields[2]); + int targetType = temporalColumnType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + appendIntegerTemporal(rows, fields[1], input, "value"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader( + encoder, + encoder.encode(buffer), + 1, + temporalWireType(fields[3]) + ); + if ("".equals(fields[4])) { + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("row zero must be NULL", 1, reader.u8()); + } else { + Assert.assertEquals("value must be bitmap-present", 0, reader.u8()); + } + if (!"DATE".equals(fields[3])) { + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + } + if (!"".equals(fields[4])) { + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(24, count); + } + + @Test + public void testSmallIntegerDuplicateFirstAndFailureRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("a", ColumnType.LONG), + column("only_b", ColumnType.SHORT), + column("bad", ColumnType.BYTE), + column("c", ColumnType.DOUBLE)); + rows.byteColumn("a", (byte) -1).intColumn("a", Integer.MIN_VALUE); + buffer.nextRow(); + + rows.shortColumn("only_b", (short) 5); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.shortColumn("bad", (short) 128)); + rollbackCurrentRow(buffer); + + rows.intColumn("c", 3); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, + QwpConstants.TYPE_LONG, QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(Double.doubleToRawLongBits(3), reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testIntMinIsNullWithCompanionMissingRowsAcrossNumericTargets() throws Exception { + assertMemoryLeak(() -> { + for (String target : new String[]{"BYTE", "SHORT", "INT", "LONG", "FLOAT", "DOUBLE"}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", numericColumnType(target))); + rows.intColumn("n", Integer.MIN_VALUE); + buffer.nextRow(); + buffer.nextRow(); + rows.intColumn("n", 7); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, numericWireType(target)); + Assert.assertEquals(target, 1, reader.u8()); + Assert.assertEquals(target, 3, reader.u8()); + if ("FLOAT".equals(target)) { + assertNumericValue(reader, target, "0x40e00000"); + } else if ("DOUBLE".equals(target)) { + assertNumericValue(reader, target, "0x401c000000000000"); + } else { + assertNumericValue(reader, target, "7"); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testIntToIpv4UsesTargetNativeWireAndNulls() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ip", ColumnType.IPv4)); + rows.intColumn("ip", Integer.MIN_VALUE); + buffer.nextRow(); + rows.intColumn("ip", 0); + buffer.nextRow(); + rows.intColumn("ip", 1); + buffer.nextRow(); + rows.intColumn("ip", Integer.MIN_VALUE + 1); + buffer.nextRow(); + rows.intColumn("ip", Integer.MAX_VALUE); + buffer.nextRow(); + rows.intColumn("ip", -1); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 7, QwpConstants.TYPE_IPv4); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("INT null, IPv4 zero and omission must be NULL", 0x43, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(Integer.MIN_VALUE + 1, reader.i32()); + Assert.assertEquals(Integer.MAX_VALUE, reader.i32()); + Assert.assertEquals(-1, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testSmallIntegerMissingSchemaUsesNativeWireTypes() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.byteColumn("b", (byte) -1).shortColumn("s", (short) 2).intColumn("i", 3); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, + QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0xff, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(2, reader.u16()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(3, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("uuid", ColumnType.UUID), + column("ip", ColumnType.IPv4), + column("future", ColumnType.INT, new byte[]{1}), + column("future_date", ColumnType.DATE, new byte[]{1}), + column("future_text", ColumnType.VARCHAR, new byte[]{1}), + column("future_decimal", ColumnType.getDecimalType(18, 2), new byte[]{1}), + column("future_ipv4", ColumnType.IPv4, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("uuid", (byte) 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("ip", (byte) 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("ip", (short) 2)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("future", (short) 2)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("future_date", (byte) 3)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("future_text", 4)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("future_decimal", (short) 5)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("future_ipv4", 6)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("uuid", Integer.MIN_VALUE)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("designated", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("designated", 1)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding stale = rows(buffer, column("n", ColumnType.LONG)); + buffer.clear(); + assertIllegalState(() -> stale.byteColumn("n", (byte) 1)); + assertIllegalState(() -> stale.shortColumn("n", (short) 2)); + assertIllegalState(() -> stale.intColumn("n", 3)); + } + } + + @Test + public void testLongNumericConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/long-to-numeric.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + int count = 0; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 4, fields.length); + try { + long input = Long.parseLong(fields[1]); + int targetType = numericColumnType(fields[2]); + byte wireType = numericWireType(fields[2]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", targetType)); + if ("".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.longColumn("n", input)); + continue; + } + rows.longColumn("n", input); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); + if ("".equals(fields[3])) { + if (input == Long.MIN_VALUE) { + assertNumericNull(reader, fields[2]); + } else { + Assert.assertEquals("INT sentinel is carried as a non-null wire value", 0, reader.u8()); + Assert.assertEquals(Integer.MIN_VALUE, reader.i32()); + } + } else { + Assert.assertEquals(0, reader.u8()); + assertNumericValue(reader, fields[2], fields[3]); + } + Assert.assertEquals(size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(47, count); + } + } + + @Test + public void testLongNumericDuplicateFirstAndFailureRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("b", ColumnType.BYTE), column("i", ColumnType.INT)); + rows.longColumn("b", 7).longColumn("b", 128).stringColumn("b", "unsupported"); + rows.longColumn("i", 1); + buffer.nextRow(); + rows.longColumn("i", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("b", 128)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("b", 3).longColumn("i", 4); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(4, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLongRangeDiagnosticIsReadableAndDoesNotEchoValue() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("b", ColumnType.BYTE)); + try { + rows.longColumn("b", 9_876_543_210L); + Assert.fail("expected range failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + Assert.assertFalse(e.isRetryable()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("table=t")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("column=b")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("inputType=LONG")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("targetType=BYTE(" + ColumnType.BYTE + ")")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("outside target type range")); + Assert.assertFalse(e.getMessage(), e.getMessage().contains("9876543210")); + } + } + } + + @Test + public void testLongNullRejectsUuidUnlessDuplicate() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.longColumn("u", Long.MIN_VALUE)); + rows.uuidColumn("u", 1, 2).longColumn("u", Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLongNumericDuplicateFirstAcrossEveryTarget() throws Exception { + assertMemoryLeak(() -> { + byte[][] schema = { + column("b", ColumnType.BYTE), column("s", ColumnType.SHORT), + column("i", ColumnType.INT), column("l", ColumnType.LONG), + column("f", ColumnType.FLOAT), column("d", ColumnType.DOUBLE)}; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, schema); + String[] names = {"b", "s", "i", "l", "f", "d"}; + for (String name : names) { + rows.longColumn(name, 7).stringColumn(name, "not-numeric").uuidColumn(name, 1, 2); + } + rows.longColumn("b", 128).longColumn("s", 32768).longColumn("i", 2147483648L); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.u16()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.i32()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Float.floatToRawIntBits(7), reader.i32()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Double.doubleToRawLongBits(7), reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNumericFailureRemovesColumnsIntroducedOnlyInFailedRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("a", ColumnType.LONG), column("only_b", ColumnType.BYTE), + column("bad", ColumnType.SHORT)); + rows.longColumn("a", 1); + buffer.nextRow(); + rows.longColumn("only_b", 5); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("bad", 32768)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("a", 3); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNumericExtensionRejectedOnlyWhenUsed() throws Exception { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("future_float", ColumnType.FLOAT, new byte[]{1}), column("plain", ColumnType.BYTE)); + rows.longColumn("plain", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.longColumn("future_float", 2)); + } + } + + @Test + public void testNullStringSupportsAllNumericTargets() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("b", ColumnType.BYTE), column("s", ColumnType.SHORT), + column("i", ColumnType.INT), column("l", ColumnType.LONG), + column("f", ColumnType.FLOAT), column("d", ColumnType.DOUBLE)); + rows.stringColumn("b", null).stringColumn("s", null).stringColumn("i", null) + .stringColumn("l", null).stringColumn("f", null).stringColumn("d", null); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); + assertNumericNull(reader, "BYTE"); + assertNumericNull(reader, "SHORT"); + assertNumericNull(reader, "INT"); + assertNumericNull(reader, "LONG"); + assertNumericNull(reader, "FLOAT"); + assertNumericNull(reader, "DOUBLE"); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNullStringSupportsLongAndUuidTargets() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.stringColumn("u", null).stringColumn("l", null); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testParameterExtensionRejectedOnlyWhenColumnUsed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("future", ColumnType.LONG, new byte[]{1}), column("plain", ColumnType.LONG)); + rows.longColumn("plain", 1); + buffer.nextRow(); + int size = encoder.encode(buffer); + assertLongFrame(encoder, size, 1); + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.longColumn("future", 2)); + } + }); + } + + @Test + public void testUnknownFullTypeFlagsRejectedOnlyWhenUsed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("future", ColumnType.LONG | 0x10000), column("plain", ColumnType.LONG)); + rows.longColumn("plain", 7); + buffer.nextRow(); + int size = encoder.encode(buffer); + assertLongFrame(encoder, size, 7); + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.longColumn("future", 2)); + } + }); + } + + @Test + public void testCancelAndResetRemoveOnlyIntendedRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("l", ColumnType.LONG)); + rows.longColumn("l", 1); + buffer.nextRow(); + rows.longColumn("l", 2); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("l", 3); + buffer.nextRow(); + int size = encoder.encode(buffer); + assertLongFrame(encoder, size, 1, 3); + buffer.reset(); + rows.longColumn("l", 4); + buffer.nextRow(); + size = encoder.encode(buffer); + assertLongFrame(encoder, size, 4); + } + }); + } + + @Test + public void testOwnerRollbackAfterSetterFailurePreservesCompletedRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.longColumn("l", 1); + buffer.nextRow(); + rows.longColumn("l", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.stringColumn("u", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG, QwpConstants.TYPE_UUID); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testAttachmentLifecycleAndStaleBindingGuards() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaResponse schema = known(column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + QwpSchemaBinding old = new QwpSchemaBinding(buffer, schema); + assertIllegalState(() -> new QwpSchemaBinding(buffer, schema)); + buffer.reset(); + old.longColumn("l", 1); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + buffer.clear(); + + QwpSchemaBinding current = new QwpSchemaBinding(buffer, schema); + current.longColumn("l", 7); + assertIllegalState(() -> old.longColumn("l", 8)); + assertIllegalState(() -> old.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef")); + assertIllegalState(() -> old.uuidColumn("u", 1, 2)); + buffer.nextRow(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testBindingRequiresEmptyLayout() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + buffer.getOrCreateColumn("l", QwpConstants.TYPE_LONG, true).addLong(1); + buffer.nextRow(); + buffer.reset(); + Assert.assertEquals(1, buffer.getColumnCount()); + assertIllegalState(() -> new QwpSchemaBinding(buffer, known(column("l", ColumnType.LONG)))); + } + } + + @Test + public void testBoundEncodingGuardsDoNotMutateEncoder() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer sentinel = new QwpTableBuffer("s"); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + sentinel.getOrCreateColumn("x", QwpConstants.TYPE_LONG, true).addLong(9); + sentinel.nextRow(); + encoder.encode(sentinel); + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + // An empty bound table cannot join a multi-table message. + assertEncoderUnchanged(encoder, () -> encoder.encodeWithDeltaDict( + buffer, new GlobalSymbolDictionary(), -1, -1)); + binding.longColumn("l", 1); + // A bound table with an incomplete row cannot be encoded. + assertEncoderUnchanged(encoder, () -> encoder.encode(buffer)); + assertEncoderUnchanged(encoder, () -> encoder.encodeWithDeltaDict( + buffer, new GlobalSymbolDictionary(), -1, -1)); + } + }); + } + + @Test + public void testEmptyBoundEncodingContracts() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + Assert.assertEquals(0, encoder.encode(buffer)); + Assert.assertEquals(0, encoder.getBuffer().getPosition()); + encoder.beginMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertEncoderUnchanged(encoder, () -> encoder.addTable(buffer)); + } + }); + } + + @Test + public void testUuidConformanceCorpus() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 3, fields.length); + CharSequence input = "".equals(fields[1]) ? null : fields[1]; + String expected = fields[2]; + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + if ("".equals(expected)) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.stringColumn("u", input)); + continue; + } + rows.stringColumn("u", input); + buffer.nextRow(); + QwpTableBuffer.ColumnBuffer column = buffer.getColumnCount() == 0 ? null : buffer.getColumn(0); + if ("".equals(expected)) { + // NULL is either a null-bitmap entry or the UUID null sentinel + // (both limbs Long.MIN_VALUE), which the server reads as NULL. + Assert.assertTrue(fields[0], column == null || column.isNull(0) + || (Unsafe.getUnsafe().getLong(column.getDataAddress()) == Long.MIN_VALUE + && Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES) == Long.MIN_VALUE)); + } else { + UUID uuid = UUID.fromString(expected); + Assert.assertNotNull(fields[0], column); + Assert.assertFalse(fields[0], column.isNull(0)); + // The buffer stores the low limb first, as on the wire. + Assert.assertEquals(fields[0], uuid.getLeastSignificantBits(), + Unsafe.getUnsafe().getLong(column.getDataAddress())); + Assert.assertEquals(fields[0], uuid.getMostSignificantBits(), + Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES)); + } + } + } + } + Assert.assertEquals(20, count); + }); + } + + @Test + public void testStringAndTypedUuidEncodeSameLimbs() throws Exception { + assertMemoryLeak(() -> { + long hi = 0x0123456789abcdefL; + long lo = 0xfedcba9876543210L; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + rows.stringColumn("u", "01234567-89ab-cdef-fedc-ba9876543210"); + buffer.nextRow(); + rows.uuidColumn("u", lo, hi); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(lo, reader.i64()); + Assert.assertEquals(hi, reader.i64()); + Assert.assertEquals(lo, reader.i64()); + Assert.assertEquals(hi, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testIpv4ConformanceCorpusUsesExactTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_kind\tinput\toutcome\texpected_hex\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + for (int targetType : new int[]{ColumnType.IPv4, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ip", targetType)); + Runnable append = "INT".equals(fields[1]) + ? () -> rows.ipv4Column("ip", Integer.parseInt(fields[2])) + : () -> rows.ipv4Column("ip", fields[2]); + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + continue; + } + append.run(); + buffer.nextRow(); + byte wireType = targetType == ColumnType.IPv4 + ? QwpConstants.TYPE_IPv4 + : QwpConstants.TYPE_VARCHAR; + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), 1, wireType); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + } + } else { + Assert.assertEquals("VALUE", fields[3]); + Assert.assertEquals(0, reader.u8()); + if (wireType == QwpConstants.TYPE_IPv4) { + Assert.assertEquals((int) Long.parseUnsignedLong(fields[4], 16), reader.i32()); + } else { + byte[] expected = fields[5].getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertEquals(fields[5], reader.ascii(expected.length)); + } + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(16, count); + } + + @Test + public void testIpv4DuplicateRollbackNullOmissionAndInference() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("ip", ColumnType.IPv4), column("bad", ColumnType.LONG)); + rows.ipv4Column("ip", 0x01020304).ipv4Column("ip", "not-an-ip"); + buffer.nextRow(); + + rows.ipv4Column("ip", 0x05060708); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("bad", 0x090a0b0c)); + rollbackCurrentRow(buffer); + + rows.ipv4Column("ip", 0); + buffer.nextRow(); + buffer.nextRow(); + rows.ipv4Column("ip", ".0.0.0.0."); + buffer.nextRow(); + rows.ipv4Column("ip", 0x0d0e0f10); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 5, QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0e, reader.u8()); + Assert.assertEquals(0x01020304, reader.i32()); + Assert.assertEquals(0x0d0e0f10, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.ipv4Column("ignored", (CharSequence) null) + .ipv4Column("ip", "192.168.1.1"); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), -1, -1, 1, + QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0xc0a80101, reader.i32()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.ipv4Column("ip", 0); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), -1, -1, 1, + QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testIpv4RejectsUnsupportedParameterizedAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("long_value", ColumnType.LONG), + column("parameterized", ColumnType.IPv4, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("long_value", 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("long_value", "not-an-ip")); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("parameterized", "1.2.3.4")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("ts", 1)); + } + } + + @Test + public void testLong256ConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/long256-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_l0\tinput_l1\tinput_l2\tinput_l3\toutcome\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + long l0 = Long.parseLong(fields[1]); + long l1 = Long.parseLong(fields[2]); + long l2 = Long.parseLong(fields[3]); + long l3 = Long.parseLong(fields[4]); + for (int targetType : new int[]{ColumnType.LONG256, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + rows.long256Column("value", l0, l1, l2, l3); + buffer.nextRow(); + byte wireType = targetType == ColumnType.LONG256 + ? QwpConstants.TYPE_LONG256 + : QwpConstants.TYPE_VARCHAR; + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); + if ("NULL".equals(fields[5])) { + Assert.assertEquals("-", fields[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + } + } else { + Assert.assertEquals("VALUE", fields[5]); + Assert.assertEquals(0, reader.u8()); + if (wireType == QwpConstants.TYPE_LONG256) { + Assert.assertEquals(l0, reader.i64()); + Assert.assertEquals(l1, reader.i64()); + Assert.assertEquals(l2, reader.i64()); + Assert.assertEquals(l3, reader.i64()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(fields[6].length(), reader.i32()); + Assert.assertEquals(fields[6], reader.ascii(fields[6].length())); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(8, count); + }); + } + + @Test + public void testLong256DuplicateNullOmissionRollbackAndMissingInference() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", ColumnType.LONG256), column("bad", ColumnType.UUID)); + rows.long256Column("value", 1, 2, 3, 4) + .long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + buffer.nextRow(); + rows.long256Column("value", 5, 6, 7, 8); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("bad", 9, 10, 11, 12)); + rollbackCurrentRow(buffer); + rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x06, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_LONG256, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLong256DuplicatePrecedesUnsupportedAndRejectsParameterizedDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("bad", ColumnType.UUID), + column("parameterized", ColumnType.LONG256, new byte[]{1})); + rows.uuidColumn("bad", 1, 2) + .long256Column("bad", 3, 4, 5, 6); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("bad", 3, 4, 5, 6)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("parameterized", 1, 2, 3, 4)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("ts", 1, 2, 3, 4)); + } + } + + @Test + public void testNativeGeoHashConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_bits\tprecision_bits\texpected_value_hex\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + long input = Long.parseLong(fields[1]); + int precision = Integer.parseInt(fields[2]); + long expected = Long.parseUnsignedLong(fields[3], 16); + int geoType = ColumnType.getGeoHashTypeWithBits(precision); + for (int targetType : new int[]{geoType, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + rows.geoHashColumn("value", input, precision); + buffer.nextRow(); + byte wireType = ColumnType.isGeoHash(targetType) + ? QwpConstants.TYPE_GEOHASH + : QwpConstants.TYPE_VARCHAR; + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); + if (wireType == QwpConstants.TYPE_GEOHASH) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(precision, reader.varint()); + for (int i = 0; i < (precision + 7) / 8; i++) { + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.u8()); + } + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(fields[4].length(), reader.i32()); + Assert.assertEquals(fields[4], reader.ascii(fields[4].length())); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(15, count); + }); + } + + @Test + public void testNativeGeoHashEveryPrecisionUsesFixedWidthText() throws Exception { + assertMemoryLeak(() -> { + for (int precision = 1; precision <= 60; precision++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + rows(buffer, column("value", ColumnType.STRING)) + .geoHashColumn("value", -1, precision); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(precision, reader.i32()); + for (int i = 0; i < precision; i++) { + Assert.assertEquals('1', reader.u8()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testNativeGeoHashOmissionRollbackInferenceAndGuards() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", geo20), column("bad", ColumnType.UUID)); + rows.geoHashColumn("value", 0xabcde, 20) + .geoHashColumn("value", 0x12345, 20); + buffer.nextRow(); + buffer.nextRow(); + rows.geoHashColumn("value", 0x11111, 20); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("bad", 1, 20)); + rollbackCurrentRow(buffer); + rows.geoHashColumn("value", 0x12345, 20); + buffer.nextRow(); + + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x02, reader.u8()); + Assert.assertEquals(20, reader.varint()); + for (long value : new long[]{0xabcde, 0x12345}) { + for (int i = 0; i < 3; i++) { + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), reader.u8()); + } + } + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.geoHashColumn("value", 0xabcde, 20); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(20, reader.varint()); + Assert.assertEquals(0xde, reader.u8()); + Assert.assertEquals(0xbc, reader.u8()); + Assert.assertEquals(0x0a, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("mismatch", ColumnType.getGeoHashTypeWithBits(15)), + column("unsupported", ColumnType.UUID), + column("parameterized", geo20, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("mismatch", 1, 0)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("mismatch", 1, 61)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("mismatch", 1, 20)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("unsupported", 1, 20)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("parameterized", 1, 20)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("ts", 1, 20)); + } + }); + } + + @Test + public void testNativeGeoHashTextOverloadPreservesSourcePrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("geo", ColumnType.getGeoHashTypeWithBits(20)), + column("text", ColumnType.STRING)); + rows.geoHashColumn("geo", "u33d") + .geoHashColumn("text", "U"); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, + QwpConstants.TYPE_GEOHASH, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(20, reader.varint()); + long expected = Numbers.parseGeoHashBase32("u33d"); + for (int i = 0; i < 3; i++) { + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.u8()); + } + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(5, reader.i32()); + Assert.assertEquals("11010", reader.ascii(5)); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", ColumnType.getGeoHashTypeWithBits(15))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("value", "u33d")); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", (CharSequence) null)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "")); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "0123456789bcd")); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "a")); + } + }); + } + + private static void assertCtorReason(int result, LineSenderSchemaException.Reason reason, String message) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + try { + new QwpSchemaBinding(buffer, schemaResult(result)); + Assert.fail("expected schema exception"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + Assert.assertFalse(e.isRetryable()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains(message)); + } + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + binding.longColumn("l", 7); + buffer.nextRow(); + assertLongFrame(encoder, encoder.encode(buffer), 7); + } + } + + private static void assertLongFrame(QwpWebSocketEncoder encoder, int size, long... values) { + QwpTestWireReader reader = tableReader(encoder, size, values.length, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + for (long value : values) { + Assert.assertEquals(value, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + + private static void appendSmallInteger( + QwpSchemaBinding rows, + String inputType, + long value, + String column + ) { + switch (inputType) { + case "BYTE": + rows.byteColumn(column, (byte) value); + break; + case "SHORT": + rows.shortColumn(column, (short) value); + break; + case "INT": + rows.intColumn(column, (int) value); + break; + default: + throw new AssertionError(inputType); + } + } + + private static int decimalStorageIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + Assert.assertEquals("DECIMAL256", target); + return 5; + } + } + + private static String decimalStorageType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + + private static int decimalWireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static byte decimalWireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static void appendIntegerTemporal( + QwpSchemaBinding rows, + String inputType, + long value, + String column + ) { + switch (inputType) { + case "BYTE": + rows.byteColumn(column, (byte) value); + break; + case "SHORT": + rows.shortColumn(column, (short) value); + break; + case "INT": + rows.intColumn(column, (int) value); + break; + case "LONG": + rows.longColumn(column, value); + break; + default: + throw new AssertionError(inputType); + } + } + + private static byte[] hexBytes(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + + private static void assertNumericValue(QwpTestWireReader reader, String target, String expected) { + switch (target) { + case "BYTE": + Assert.assertEquals(Integer.parseInt(expected) & 0xff, reader.u8()); + break; + case "SHORT": + Assert.assertEquals(Integer.parseInt(expected) & 0xffff, reader.u16()); + break; + case "INT": + Assert.assertEquals(Integer.parseInt(expected), reader.i32()); + break; + case "LONG": + Assert.assertEquals(Long.parseLong(expected), reader.i64()); + break; + case "FLOAT": + Assert.assertEquals((int) Long.parseUnsignedLong(expected.substring(2), 16), reader.i32()); + break; + case "DOUBLE": + Assert.assertEquals(Long.parseUnsignedLong(expected.substring(2), 16), reader.i64()); + break; + default: + Assert.fail(target); + } + } + + private static void assertNumericNull(QwpTestWireReader reader, String target) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(target, 1, reader.u8()); + } + + private static void rollbackCurrentRow(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static int numericColumnType(String target) { + switch (target) { + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError(target); + } + } + + private static long parseHexLong(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int smallIntegerSourceIndex(String source) { + switch (source) { + case "BYTE": + return 0; + case "SHORT": + return 1; + default: + Assert.assertEquals("INT", source); + return 2; + } + } + + private static int timestampColumnType(String target) { + switch (target) { + case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; + case "TIMESTAMP_NS": return ColumnType.TIMESTAMP_NANO; + default: throw new AssertionError(target); + } + } + + private static int temporalColumnType(String target) { + switch (target) { + case "DATE": return ColumnType.DATE; + case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; + case "TIMESTAMP_NS": return ColumnType.TIMESTAMP_NANO; + default: throw new AssertionError(target); + } + } + + private static int textColumnType(String target) { + switch (target) { + case "STRING": return ColumnType.STRING; + case "VARCHAR": return ColumnType.VARCHAR; + case "SYMBOL": return ColumnType.SYMBOL; + default: throw new AssertionError(target); + } + } + + private static byte textWireType(String target) { + return "SYMBOL".equals(target) ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR; + } + + private static byte timestampWireType(String target) { + switch (target) { + case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; + case "TIMESTAMP_NS": return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: throw new AssertionError(target); + } + } + + private static byte temporalWireType(String target) { + switch (target) { + case "DATE": return QwpConstants.TYPE_DATE; + case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; + case "TIMESTAMP_NS": return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: throw new AssertionError(target); + } + } + + private static byte numericWireType(String target) { + switch (target) { + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError(target); + } + } + + private static byte[] copyFrame(QwpWebSocketEncoder encoder, int size) { + byte[] copy = new byte[size]; + long address = encoder.getBuffer().getBufferPtr(); + for (int i = 0; i < size; i++) { + copy[i] = Unsafe.getUnsafe().getByte(address + i); + } + return copy; + } + + private static void assertIllegalArgument(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertIllegalState(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + // expected + } + } + + private static void assertEncoderUnchanged(QwpWebSocketEncoder encoder, Runnable action) { + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + assertIllegalState(action); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + + private static QwpSchemaBinding rows(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte... wireTypes) { + return tableReader(encoder, size, 1, 1, rows, wireTypes); + } + + private static QwpTestWireReader tableReader( + QwpWebSocketEncoder encoder, + int size, + int tableId, + long metadataVersion, + int rows, + byte... wireTypes + ) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(wireTypes.length, reader.varint()); + for (byte wireType : wireTypes) { + reader.string(); + Assert.assertEquals(wireType, reader.u8()); + } + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java new file mode 100644 index 000000000..fcf1324c0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java @@ -0,0 +1,351 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/text-to-text.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 7, fields.length); + try { + String value = "".equals(fields[3]) ? null : utf16(fields[3]); + byte wireType = wireType(fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); + append(binding, fields[1], value); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encode(buffer), wireType); + if (value == null) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + } else { + Assert.assertEquals(0, reader.varint()); + } + } else if (wireType == QwpConstants.TYPE_VARCHAR) { + byte[] expected = hex(fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + count++; + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + } + Assert.assertEquals(90, count); + }); + } + + @Test + public void testFirstValueNullOmissionAndMutableInput() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("text", ColumnType.STRING), column("sym", ColumnType.SYMBOL)); + StringBuilder mutable = new StringBuilder("before"); + binding.symbol("text", mutable).stringColumn("text", "ignored"); + binding.stringColumn("sym", "first").symbol("sym", null); + mutable.replace(0, mutable.length(), "after"); + buffer.nextRow(); + binding.stringColumn("text", null).symbol("sym", null); + buffer.nextRow(); + buffer.nextRow(); + Assert.assertEquals("before".length(), buffer.getColumn(0).getStringDataSize()); + Assert.assertArrayEquals("before".getBytes(StandardCharsets.UTF_8), + bytes(buffer.getColumn(0).getStringDataAddress(), "before".length())); + Assert.assertEquals("first", buffer.getColumn(1).getSymbolValue(0).toString()); + int size = encoder.encode(buffer); + Assert.assertTrue(size > QwpConstants.HEADER_SIZE); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); + } + + @Test + public void testFailureRollbackAndLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaResponse schema = known(-1, + column("a", ColumnType.STRING), column("only_b", ColumnType.SYMBOL), + column("bad", ColumnType.UUID), column("c", ColumnType.VARCHAR)); + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, schema); + binding.stringColumn("a", "A"); + buffer.nextRow(); + binding.symbol("only_b", "B"); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("bad", "invalid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.symbol("c", "C"); + buffer.nextRow(); + Assert.assertEquals(2, buffer.getColumnCount()); + Assert.assertEquals("a", buffer.getColumnDefs()[0].getName()); + Assert.assertEquals("c", buffer.getColumnDefs()[1].getName()); + Assert.assertTrue(encoder.encode(buffer) > 0); + + buffer.reset(); + binding.symbol("a", "reset"); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(5, reset.i32()); + Assert.assertArrayEquals("reset".getBytes(StandardCharsets.UTF_8), reset.bytes(5)); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + buffer.clear(); + assertIllegalState(() -> binding.symbol("a", "stale")); + assertIllegalState(() -> binding.stringColumn("a", "stale")); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("long", ColumnType.LONG), column("future", ColumnType.SYMBOL, new byte[]{1}), + column("flagged", ColumnType.STRING | 0x10000)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("long", null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("future", "x")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("flagged", "x")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.symbol("missing", "x"); + Assert.assertEquals(QwpConstants.TYPE_SYMBOL, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(0, column("d", ColumnType.SYMBOL))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("d", null)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("d", null)); + } + } + + @Test + public void testUnsupportedDuplicateIsIgnoredAcrossTextSetters() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("uuid", ColumnType.UUID), column("long", ColumnType.LONG)); + binding.stringColumn("uuid", "01234567-89ab-cdef-0123-456789abcdef") + .symbol("uuid", null); + binding.longColumn("long", 7).symbol("long", null); + buffer.nextRow(); + Assert.assertTrue(encoder.encode(buffer) > 0); + } + }); + } + + @Test + public void testOwnerBackedGlobalSymbolIdsAndEncoding() throws Exception { + assertMemoryLeak(() -> { + TestWebSocketServer.WebSocketServerHandler handler = new TestWebSocketServer.WebSocketServerHandler() { }; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender senderApi = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + QwpWebSocketSender sender = (QwpWebSocketSender) senderApi; + Assert.assertEquals(0, sender.getOrAddGlobalSymbol("prefix")); + GlobalSymbolDictionary mirror = new GlobalSymbolDictionary(); + Assert.assertEquals(0, mirror.getOrAddSymbol("prefix")); + try (QwpTableBuffer buffer = new QwpTableBuffer("t", sender); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + StringBuilder mutable = new StringBuilder("mutable"); + binding.symbol("value", mutable); + mutable.replace(0, mutable.length(), "changed"); + buffer.nextRow(); + binding.stringColumn("value", "?"); + buffer.nextRow(); + binding.symbol("value", "\ud800"); + buffer.nextRow(); + binding.symbol("value", "mutable"); + buffer.nextRow(); + + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("mutable")); + Assert.assertEquals(2, sender.getOrAddGlobalSymbol("?")); + Assert.assertEquals(3, sender.getOrAddGlobalSymbol("\ud800")); + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("mutable")); + mirror.getOrAddSymbol("mutable"); + mirror.getOrAddSymbol("?"); + mirror.getOrAddSymbol("\ud800"); + Assert.assertEquals(0, buffer.getColumn(0).getSymbolDictionarySize()); + + encoder.beginMessage(1, mirror, -1, 3); + encoder.addTable(buffer); + int size = encoder.finishMessage(); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(4, reader.varint()); + Assert.assertArrayEquals("prefix".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("mutable".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); + Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); + skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 4); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + } + }); + } + + private static void append(QwpSchemaBinding binding, String inputType, CharSequence value) { + if ("STRING".equals(inputType)) { + binding.stringColumn("value", value); + } else if ("SYMBOL".equals(inputType)) { + binding.symbol("value", value); + } else { + throw new AssertionError("unknown input type: " + inputType); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTableHeader(reader, wireType, 1); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "STRING": + return ColumnType.STRING; + case "VARCHAR": + return ColumnType.VARCHAR; + case "SYMBOL": + return ColumnType.SYMBOL; + default: + throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "VARCHAR": + return QwpConstants.TYPE_VARCHAR; + case "SYMBOL": + return QwpConstants.TYPE_SYMBOL; + default: + throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + Assert.assertEquals(0, value.length() & 1); + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + + private static byte[] bytes(long address, int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = Unsafe.getUnsafe().getByte(address + i); + } + return value; + } + + private static String utf16(String value) { + Assert.assertEquals(0, value.length() & 3); + char[] chars = new char[value.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(value.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static void assertIllegalState(Runnable action) { + Assert.assertThrows(IllegalStateException.class, action::run); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java new file mode 100644 index 000000000..ed58ea452 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java @@ -0,0 +1,210 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; + +public class QwpSchemaBindingTimestampTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv"; + + @Test + public void testCorpusUsesExactVarcharBytes() throws Exception { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + boolean headerSeen = false; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals( + "# case_id\tinput_kind\tunit\tvalue\tnanos\toutcome\texpected_text", line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertTrue("LONG".equals(fields[1]) || "INSTANT".equals(fields[1])); + Assert.assertTrue("VALUE".equals(fields[5]) || "INVALID".equals(fields[5])); + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + Runnable append = () -> append(binding, fields); + if ("INVALID".equals(fields[5])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + continue; + } + append.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader( + encoder, encoder.encode(buffer), 1, QwpConstants.TYPE_VARCHAR); + String expectedText = "".equals(fields[6]) ? "" : fields[6]; + byte[] expected = expectedText.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(reader.limit(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError( + "case_id=" + fields[0] + ", target=" + target + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(63, count); + } + + private static void append(QwpSchemaBinding binding, String[] fields) { + if ("INSTANT".equals(fields[1])) { + binding.timestampColumn("value", Instant.ofEpochSecond( + Long.parseLong(fields[3]), Integer.parseInt(fields[4]))); + } else if ("LONG".equals(fields[1])) { + binding.timestampColumn("value", Long.parseLong(fields[3]), ChronoUnit.valueOf(fields[2])); + } else { + throw new AssertionError("unknown input kind: " + fields[1]); + } + } + + @Test + public void testUnitsInstantNullDuplicateOmissionRollbackAndReset() { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", target), column("bad", ColumnType.UUID), column("c", target)); + binding.timestampColumn("a", Long.MIN_VALUE, ChronoUnit.MICROS) + .binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.timestampColumn("a", -1, ChronoUnit.NANOS); + buffer.nextRow(); + buffer.nextRow(); + binding.timestampColumn("a", Instant.ofEpochSecond(-1, 999_999_999)); + buffer.nextRow(); + binding.timestampColumn("a", 1, ChronoUnit.DAYS); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("bad", 1, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.timestampColumn("c", 1000, ChronoUnit.NANOS); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 5, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarchar(reader, 0x14, "", "1970-01-01T00:00:00.000Z", "1969-12-31T23:59:59.999Z"); + assertVarchar(reader, 0x0f, "1970-01-01T00:00:00.000Z"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.timestampColumn("a", 0, ChronoUnit.SECONDS); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + assertVarchar(reset, -1, "1970-01-01T00:00:00.000Z"); + assertVarchar(reset, 0x01); + Assert.assertEquals(resetSize, reset.position()); + } + } + } + + @Test + public void testInferenceDesignatedAndRangeErrors() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.timestampColumn("value", Long.MIN_VALUE, ChronoUnit.NANOS); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.VARCHAR)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", (Instant) null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", 1, null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("value", 1, ChronoUnit.MONTHS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.timestampColumn("value", 1, ChronoUnit.MICROS); + binding.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); + buffer.nextRow(); + } + } + + private static void assertVarchar(QwpTestWireReader reader, int bitmap, String... values) { + if (bitmap < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + } + int total = 0; + StringBuilder joined = new StringBuilder(); + Assert.assertEquals(0, reader.i32()); + for (String value : values) { + total += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(total, reader.i32()); + joined.append(value); + } + Assert.assertArrayEquals(joined.toString().getBytes(StandardCharsets.UTF_8), reader.bytes(total)); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java new file mode 100644 index 000000000..dbb071ee2 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java @@ -0,0 +1,235 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingUuidTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/uuid-to-text.tsv"; + + @Test + public void testCorpusUsesCanonicalVarcharWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingUuidTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertEquals("VARCHAR", fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[3]))); + binding.uuidColumn("value", Long.parseLong(fields[1]), Long.parseLong(fields[2])); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + if ("".equals(fields[5])) { + Assert.assertEquals("", fields[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.i32()); + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals(fields[6], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(16, count); + }); + } + + @Test + public void testDuplicateNullOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType)); + binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE) + .binaryColumn("value", new byte[]{1}) + .longColumn("value", 42); + buffer.nextRow(); + binding.uuidColumn("value", 1, 0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(36, reader.i32()); + Assert.assertEquals("00000000-0000-0000-0000-000000000001", reader.ascii(36)); + Assert.assertEquals(size, reader.position()); + } + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.STRING), column("bad", ColumnType.LONG), column("c", ColumnType.VARCHAR)); + binding.uuidColumn("a", 1, 0); + buffer.nextRow(); + binding.uuidColumn("a", 2, 0); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.uuidColumn("bad", 3, 0)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.uuidColumn("c", 4, 0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarcharValues(reader, 6, "00000000-0000-0000-0000-000000000001"); + assertVarcharValues(reader, 5, "00000000-0000-0000-0000-000000000004"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.uuidColumn("a", 5, 0); + buffer.nextRow(); + int resetSize = encoder.encode(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(36, reset.i32()); + Assert.assertEquals("00000000-0000-0000-0000-000000000005", reset.ascii(36)); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNativeBothMinRemainUuidBytes() { + assertNativeBothMin(known(column("value", ColumnType.UUID)), true); + assertNativeBothMin(missing(), false); + } + + private static void assertNativeBothMin(QwpSchemaResponse schema, boolean known) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, schema); + binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.MIN_VALUE, reader.i64()); + Assert.assertEquals(Long.MIN_VALUE, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testSymbolParameterizedAndDesignatedTargetsRemainUnsupported() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding ordinary = binding(buffer, + column("sym", ColumnType.SYMBOL), parameterizedColumn("text", ColumnType.STRING), + column("ts", ColumnType.TIMESTAMP)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> ordinary.uuidColumn("sym", Long.MIN_VALUE, Long.MIN_VALUE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> ordinary.uuidColumn("text", 1, 0)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding designated = new QwpSchemaBinding(buffer, known(0, column("ts", ColumnType.TIMESTAMP))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> designated.uuidColumn("ts", 1, 0)); + } + } + + private static void assertVarcharValues(QwpTestWireReader reader, int bitmap, String value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(36, reader.i32()); + Assert.assertEquals(value, reader.ascii(36)); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + return tableReader(encoder, size, rows, "value"); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(column, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) return ColumnType.STRING; + if ("VARCHAR".equals(value)) return ColumnType.VARCHAR; + throw new AssertionError(value); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java new file mode 100644 index 000000000..ef1aad876 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java @@ -0,0 +1,329 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.frame; + +public class QwpSchemaProtocolTest { + @Test + public void testAcceptsMaximumResponseColumnNameBoundaries() { + String name = repeat('\u0800', QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH); + QwpSchemaResponse response = decode(knownResponse(1, 1, 1, 0, column(name, 5, new byte[0]))); + Assert.assertEquals(name, response.getColumnName(0)); + Assert.assertEquals(QwpSchemaProtocol.MAX_NAME_UTF8_LENGTH, + name.getBytes(StandardCharsets.UTF_8).length); + } + + @Test + public void testDecodeKnownSchemaAndSkipsUnknownParams() { + byte[] frame = knownResponse(7, 42, 19, 1, + column("id", 32, new byte[]{1, 2, 3}), + column("métric🚀", 0x71234567, new byte[0])); + QwpSchemaResponse response = decode(frame); + Assert.assertEquals(7, response.getRequestId()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, response.getResult()); + Assert.assertTrue(response.hasSchema()); + Assert.assertEquals(42, response.getTableId()); + Assert.assertEquals(19, response.getMetadataVersion()); + Assert.assertEquals(1, response.getDesignatedIndex()); + Assert.assertEquals(2, response.getColumnCount()); + Assert.assertTrue(response.hasColumnExtensionParameters(0)); + Assert.assertFalse(response.hasColumnExtensionParameters(1)); + Assert.assertEquals("métric🚀", response.getColumnName(1)); + Assert.assertEquals(0x71234567, response.getColumnType(1)); + } + + @Test + public void testRejectsInvalidAndDuplicateColumnNames() { + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[0]), column("x", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[0]), column("X", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("Ä", 5, new byte[0]), column("ä", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("a/b", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("a\nb", 5, new byte[0]))); + } + + @Test + public void testAcceptsValidColumnNameCompatibilityBoundaries() { + String max = repeat('\u0800', QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH); + QwpSchemaResponse response = decode(knownResponse(1, 1, 0, -1, + column("metric name", 5, new byte[0]), + column("метрика", 5, new byte[0]), + column("!^[$", 5, new byte[0]), + column(max, 5, new byte[0]))); + Assert.assertEquals(4, response.getColumnCount()); + Assert.assertEquals(max, response.getColumnName(3)); + } + + @Test + public void testDecodeEverySchemaLessResult() { + for (int result = QwpSchemaProtocol.RESULT_MISSING; result <= QwpSchemaProtocol.RESULT_TOO_LARGE; result++) { + QwpSchemaResponse response = decode(resultResponse(11, result)); + Assert.assertEquals(result, response.getResult()); + Assert.assertFalse(response.hasSchema()); + Assert.assertEquals(0, response.getColumnCount()); + Assert.assertEquals(-1, response.getTableId()); + } + } + + @Test + public void testEncodeDescribeExactAsciiWire() { + byte[] encoded = QwpSchemaProtocol.encodeDescribe(0x0102030405060708L, "trades"); + ByteBuffer b = ByteBuffer.wrap(encoded).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, b.getInt()); + Assert.assertEquals(1, b.get() & 0xff); + Assert.assertEquals(QwpSchemaProtocol.FLAG_CONTROL, b.get()); + Assert.assertEquals(0, b.getShort()); + Assert.assertEquals(encoded.length - QwpConstants.HEADER_SIZE, b.getInt()); + Assert.assertEquals(QwpSchemaProtocol.KIND_DESCRIBE, b.get()); + Assert.assertEquals(0x0102030405060708L, b.getLong()); + Assert.assertEquals(6, b.getShort()); + byte[] name = new byte[6]; + b.get(name); + Assert.assertArrayEquals("trades".getBytes(StandardCharsets.UTF_8), name); + Assert.assertFalse(b.hasRemaining()); + } + + @Test + public void testEncodeDescribeUnicodeBoundariesAndRejectsInvalidNames() { + String maxUtf16AndUtf8 = repeat('\u0800', 127); + byte[] encoded = QwpSchemaProtocol.encodeDescribe(1, maxUtf16AndUtf8); + Assert.assertEquals(381, Short.toUnsignedInt(ByteBuffer.wrap(encoded, 21, 2).order(ByteOrder.LITTLE_ENDIAN).getShort())); + + assertEncodeRejected(0, "table"); + assertEncodeRejected(1, ""); + assertEncodeRejected(1, repeat('a', 128)); + assertEncodeRejected(1, repeat('\u0800', 126) + "🚀"); // 128 UTF-16 units + assertEncodeRejected(1, "bad\uD800name"); + assertEncodeRejected(1, "bad\uDC00name"); + assertEncodeRejected(1, "bad/name"); + } + + @Test + public void testGetColumnIndexIsCaseInsensitive() { + QwpSchemaResponse known = decode(knownResponse(1, 1, 1, 1, + column("Id", 5, new byte[0]), + column("métric🚀", 6, new byte[0]))); + Assert.assertEquals(0, known.getColumnIndex("id")); + Assert.assertEquals(0, known.getColumnIndex("ID")); + Assert.assertEquals(1, known.getColumnIndex("MÉTRIC🚀")); + Assert.assertEquals(-1, known.getColumnIndex("absent")); + + QwpSchemaResponse missing = decode(resultResponse(1, QwpSchemaProtocol.RESULT_MISSING)); + Assert.assertEquals(-1, missing.getColumnIndex("id")); + } + + @Test + public void testRejectsMalformedEnvelopesAndTrailingData() { + byte[] valid = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); + assertMalformed(mutate(valid, 0, (byte) 'X')); + assertMalformed(mutate(valid, 4, (byte) 2)); + assertMalformed(mutate(valid, 5, (byte) (QwpSchemaProtocol.FLAG_CONTROL | 1))); + assertMalformed(mutate(valid, 6, (byte) 1)); + assertMalformed(mutate(valid, 12, QwpSchemaProtocol.KIND_DESCRIBE)); + assertMalformed(Arrays.copyOf(valid, valid.length - 1)); + + byte[] trailing = Arrays.copyOf(valid, valid.length + 1); + ByteBuffer.wrap(trailing).order(ByteOrder.LITTLE_ENDIAN).putInt(8, trailing.length - 12); + assertMalformed(trailing); + byte[] oversized = new byte[QwpSchemaProtocol.MAX_FRAME_SIZE + 1]; + assertMalformed(oversized); + } + + @Test + public void testPeekResponseRequestIdValidatesOnlyFixedEnvelopeAndIdentity() { + byte[] valid = resultResponse(7, QwpSchemaProtocol.RESULT_MISSING); + byte[] noResult = responseIdentity(7); + Assert.assertEquals(7, peek(noResult)); + Assert.assertEquals(7, peek(valid)); + assertMalformed(Arrays.copyOf(noResult, 20), true); + assertMalformed(mutate(valid, 0, (byte) 'X'), true); + assertMalformed(mutate(valid, 4, (byte) 2), true); + assertMalformed(mutate(valid, 5, (byte) (QwpSchemaProtocol.FLAG_CONTROL | 1)), true); + assertMalformed(mutate(valid, 6, (byte) 1), true); + assertMalformed(mutate(valid, 12, QwpSchemaProtocol.KIND_DESCRIBE), true); + assertMalformed(resultResponse(0, QwpSchemaProtocol.RESULT_MISSING), true); + + byte[] badLength = valid.clone(); + ByteBuffer.wrap(badLength).order(ByteOrder.LITTLE_ENDIAN).putInt(8, 9); + assertMalformed(badLength, true); + + // Identification deliberately stops before the result/body, while full decoding stays strict. + assertMalformed(noResult); + } + + @Test + public void testRejectsEveryTruncatedKnownResponsePrefix() { + byte[] valid = knownResponse(9, 7, 6, 1, + column("first", 5, new byte[]{1, 2, 3}), + column("секунд", 0x71234567, new byte[0])); + Assert.assertEquals(2, decode(valid).getColumnCount()); + for (int length = 0; length < valid.length; length++) { + byte[] prefix = Arrays.copyOf(valid, length); + if (length >= QwpConstants.HEADER_SIZE) { + ByteBuffer.wrap(prefix).order(ByteOrder.LITTLE_ENDIAN) + .putInt(8, length - QwpConstants.HEADER_SIZE); + } + assertMalformed(prefix); + } + } + + @Test + public void testRejectsInvalidResultsIdentitiesCountsAndIndexes() { + assertMalformed(resultResponse(0, QwpSchemaProtocol.RESULT_MISSING)); + assertMalformed(resultResponse(1, 5)); + assertMalformed(knownResponse(1, -1, 0, -1)); + assertMalformed(knownResponse(1, 1, -1, -1)); + assertMalformed(knownResponse(1, 1, 0, 0)); + assertMalformed(knownResponse(1, 1, 0, -2)); + + byte[] count = knownResponse(1, 1, 0, -1); + ByteBuffer.wrap(count).order(ByteOrder.LITTLE_ENDIAN).putShort(36, (short) 2049); + assertMalformed(count); + } + + @Test + public void testRejectsInvalidUtf8AndFieldBoundsBeforeAllocation() { + byte[] invalidUtf8 = knownResponse(1, 1, 0, -1, columnBytes(new byte[]{(byte) 0xc3, 0x28}, 5, new byte[0])); + assertMalformed(invalidUtf8); + assertMalformed(knownResponse(1, 1, 0, -1, column("", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column(repeat('a', 128), 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[1025]))); + + byte[] truncatedName = knownResponse(1, 1, 0, -1, column("abc", 5, new byte[0])); + ByteBuffer.wrap(truncatedName).order(ByteOrder.LITTLE_ENDIAN).putShort(38, (short) 381); + assertMalformed(truncatedName); + byte[] truncatedParams = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); + ByteBuffer.wrap(truncatedParams).order(ByteOrder.LITTLE_ENDIAN).putShort(45, (short) 1024); + assertMalformed(truncatedParams); + } + + private static void assertEncodeRejected(long requestId, String name) { + try { + QwpSchemaProtocol.encodeDescribe(requestId, name); + Assert.fail("expected invalid describe request"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertMalformed(byte[] frame) { + try { + decode(frame); + Assert.fail("expected malformed frame"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Malformed QWP schema response:")); + } + } + + private static void assertMalformed(byte[] frame, boolean peek) { + try { + if (peek) { + peek(frame); + } else { + decode(frame); + } + Assert.fail("expected malformed frame"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Malformed QWP schema response:")); + } + } + + private static byte[] columnBytes(byte[] name, int type, byte[] params) { + ByteBuffer b = ByteBuffer.allocate(2 + name.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN); + b.putShort((short) name.length).put(name).putInt(type).putShort((short) params.length).put(params); + return b.array(); + } + + private static QwpSchemaResponse decode(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static long peek(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.peekResponseRequestId(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static byte[] knownResponse(long requestId, int tableId, long version, int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) length += column.length; + ByteBuffer b = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(tableId).putLong(version).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) b.put(column); + return frame(b.array()); + } + + private static byte[] mutate(byte[] source, int offset, byte value) { + byte[] copy = source.clone(); + copy[offset] = value; + return copy; + } + + private static String repeat(char c, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + private static byte[] resultResponse(long requestId, int result) { + ByteBuffer b = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + return frame(b.array()); + } + + private static byte[] responseIdentity(long requestId) { + ByteBuffer b = ByteBuffer.allocate(9).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId); + return frame(b.array()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java new file mode 100644 index 000000000..e14c7d14a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java @@ -0,0 +1,161 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +/** + * Shared mechanics for the {@code QwpSchemaBinding*Test} family: schema + * response construction and encoded-table header checks. The fixtures write + * the schema envelope by hand, independently of any production encoder, and + * leave every conversion expectation to the individual tests. + * + *

      All fixtures use table {@code "t"}, table id 1 and metadata version 1 + * unless a parameter says otherwise.

      + */ +final class QwpSchemaTestFixtures { + private static final byte[] NO_PARAMS = new byte[0]; + + private QwpSchemaTestFixtures() { + } + + /** Asserts a non-retryable schema rejection whose message contains every part. */ + static void assertReason(LineSenderSchemaException.Reason reason, Runnable action, String... messageParts) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + for (String part : messageParts) { + Assert.assertTrue(error.getMessage(), error.getMessage().contains(part)); + } + } + + static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return binding(buffer, -1, columns); + } + + static QwpSchemaBinding binding(QwpTableBuffer buffer, int designatedIndex, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designatedIndex, columns)); + } + + static byte[] column(String name, int type) { + return column(name, type, NO_PARAMS); + } + + static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type) + .putShort((short) params.length).put(params).array(); + } + + /** Wraps a control payload in a message frame and decodes it. */ + static QwpSchemaResponse decode(byte[] payload) { + byte[] frame = frame(payload); + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + static byte[] frame(byte[] payload) { + return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put(QwpConstants.VERSION).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload).array(); + } + + static QwpSchemaResponse known(byte[]... columns) { + return known(-1, columns); + } + + static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + static QwpSchemaResponse missing() { + return schemaResult(QwpSchemaProtocol.RESULT_MISSING); + } + + /** A column whose type carries a non-empty extension parameter block. */ + static byte[] parameterizedColumn(String name, int type) { + return column(name, type, new byte[]{1}); + } + + /** Decodes a schema-less response carrying only {@code result}. */ + static QwpSchemaResponse schemaResult(int result) { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) result).array()); + } + + /** Consumes the table prefix and the header of a single column named "value". */ + static void skipTableHeader(QwpTestWireReader reader, byte wireType, int rows) { + skipTablePrefix(reader, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + } + + static void skipTablePrefix(QwpTestWireReader reader, int rows, int columns) { + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + } + + /** Asserts the message header and table prefix, leaving the reader at the first column. */ + static QwpTestWireReader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + skipTablePrefix(reader, rows, columns); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 9c3f5861e..554f7e124 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1246,6 +1246,20 @@ public void testGetOrCreateColumnThrowsWhenExceedingMaxColumnCount() throws Exce }); } + @Test + public void testGetSchemaKeyLowerCasesOnlyWhenNeeded() throws Exception { + assertMemoryLeak(() -> { + String lowerCaseName = "trades"; + try (QwpTableBuffer table = new QwpTableBuffer(lowerCaseName)) { + assertSame(lowerCaseName, table.getSchemaKey()); + } + try (QwpTableBuffer table = new QwpTableBuffer("Trades_ÄB")) { + assertEquals("trades_äb", table.getSchemaKey()); + assertEquals("Trades_ÄB", table.getTableName()); + } + }); + } + @Test public void testLongArrayMultipleRows() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java new file mode 100644 index 000000000..94f2da358 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java @@ -0,0 +1,140 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.std.Unsafe; +import org.junit.Assert; + +import java.nio.charset.StandardCharsets; + +/** + * Bounds-checked cursor over native memory holding an encoded QWP message. + * Tests decode the wire bytes with this reader rather than with production + * decoders, so an encoder bug cannot hide behind a matching decoder bug. + * Multi-byte reads use the native byte order, like the production encoder. + */ +final class QwpTestWireReader { + private final long address; + private final int limit; + private int position; + + QwpTestWireReader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + /** Reads {@code length} raw bytes as a string, without a length prefix. */ + String ascii(int length) { + return new String(bytes(length), StandardCharsets.UTF_8); + } + + byte[] bytes(int length) { + require(length); + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = Unsafe.getUnsafe().getByte(address + position + i); + } + position += length; + return value; + } + + float f32() { + return Float.intBitsToFloat(i32()); + } + + double f64() { + return Double.longBitsToDouble(i64()); + } + + short i16() { + require(Short.BYTES); + short value = Unsafe.getUnsafe().getShort(address + position); + position += Short.BYTES; + return value; + } + + int i32() { + require(Integer.BYTES); + int value = Unsafe.getUnsafe().getInt(address + position); + position += Integer.BYTES; + return value; + } + + long i64() { + require(Long.BYTES); + long value = Unsafe.getUnsafe().getLong(address + position); + position += Long.BYTES; + return value; + } + + int limit() { + return limit; + } + + int position() { + return position; + } + + void skip(int length) { + require(length); + position += length; + } + + /** Reads a varint-prefixed UTF-8 string. */ + String string() { + return new String(stringBytes(), StandardCharsets.UTF_8); + } + + /** Reads the raw bytes of a varint-prefixed string. */ + byte[] stringBytes() { + return bytes(varint()); + } + + int u16() { + return i16() & 0xffff; + } + + int u8() { + require(1); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + int varint() { + int result = 0; + for (int shift = 0; shift < 32; shift += 7) { + int value = u8(); + result |= (value & 0x7f) << shift; + if ((value & 0x80) == 0) { + return result; + } + } + throw new AssertionError("varint exceeds 32 bits at position " + position); + } + + private void require(int length) { + Assert.assertTrue( + "read of " + length + " bytes at " + position + " exceeds limit " + limit, + length >= 0 && position <= limit - length + ); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 102922d6b..8333fcb16 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -146,6 +146,8 @@ public class TestWebSocketServer implements Closeable { // builds. Lets a test force the delta-dictionary catch-up to split across // several frames. Live-updatable via setAdvertisedMaxBatchSize(). private volatile int advertisedMaxBatchSize; + private volatile boolean advertiseSchema; + private volatile boolean hasRequestedSchema; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -227,6 +229,10 @@ public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedExcepti return startLatch.await(timeout, unit); } + public void setAdvertiseSchema(boolean advertiseSchema) { + this.advertiseSchema = advertiseSchema; + } + @Override public void close() { running.set(false); @@ -263,6 +269,10 @@ public int getPort() { return port; } + public boolean hasRequestedSchema() { + return hasRequestedSchema; + } + /** * Number of handshakes the server has completed over its lifetime * (monotonic; never decreases when clients disconnect). @@ -535,6 +545,12 @@ public interface WebSocketServerHandler { default void onBinaryMessage(ClientHandler client, byte[] data) { } + default void onOpen(ClientHandler client) { + } + + default void onPong(ClientHandler client, byte[] data) { + } + default void onPing(ClientHandler client) { } } @@ -606,6 +622,10 @@ public synchronized void sendBinary(byte[] data) throws IOException { writeFrame(WebSocketOpcode.BINARY, data, data.length); } + public synchronized void sendPing(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.PING, data, data.length); + } + /** * Writes a binary frame in socket writes of at most {@code chunkBytes}. * Over TLS every write becomes its own record (records cap at 16 KiB), @@ -721,6 +741,9 @@ private void handleRead() { LOG.error("Failed to send pong", e); } break; + case WebSocketOpcode.PONG: + handler.onPong(this, payload); + break; case WebSocketOpcode.CLOSE: { int code = WebSocketCloseCode.NORMAL_CLOSURE; if (payload.length >= 2) { @@ -787,6 +810,8 @@ private boolean performHandshake() throws IOException { upgradeRequestCount.incrementAndGet(); capturedAuthHeaders.add(authorization); capturedDurableAckRequests.add(durableAckRequest); + hasRequestedSchema |= request.toString().toLowerCase(java.util.Locale.ROOT) + .contains("x-qwp-request-schema: true\r\n"); // Read-path reject: drop the egress upgrade before the 101 so the // query pool's connect fails fast, while ingest write-path upgrades @@ -858,6 +883,9 @@ private boolean performHandshake() throws IOException { } sb.append("X-QWP-Durable-Ack: ").append(granted).append("\r\n"); } + if (advertiseSchema && request.toString().toLowerCase(java.util.Locale.ROOT).contains("x-qwp-request-schema: true\r\n")) { + sb.append("X-QWP-Schema: enabled\r\n"); + } String role = advertisedRole; if (role != null) { sb.append("X-QuestDB-Role: ").append(role).append("\r\n"); @@ -936,6 +964,7 @@ void start() { liveConnections.incrementAndGet(); try { + handler.onOpen(this); // SERVER_INFO is an egress-only frame: send it only on a // read-path (query) connection. An ingest write-path // connection parses every inbound frame as an ACK and diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index 39695ec22..10c63583e 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -73,6 +73,8 @@ public void testEveryIngressKeyIsHonored() { assertHonored("close_flush_timeout_millis=2500", "close_flush_timeout_millis", 2500L); assertHonored("durable_ack_keepalive_interval_millis=900", "durable_ack_keepalive_interval_millis", 900L); assertHonored("initial_connect_retry=async", "initial_connect_retry", "ASYNC"); + assertHonored("schema_mode=strict", "schema_mode", "STRICT"); + assertHonored("schema_wait_millis=1234", "schema_wait_millis", 1234L); assertHonored("keepalive_interval_millis=900", "keepalive_interval_millis", 900L); assertHonored("reconnect_max_duration_millis=12345", "reconnect_max_duration_millis", 12345L); assertHonored("reconnect_initial_backoff_millis=111", "reconnect_initial_backoff_millis", 111L); diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java index 4a481e193..39b89680f 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java @@ -891,7 +891,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal128 d = new Decimal128(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -928,7 +928,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -945,7 +945,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java index 1c461525d..a27d0e7ef 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java @@ -1329,7 +1329,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal256 d = new Decimal256(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -1440,7 +1440,7 @@ public void testOfStringExponentComplexNumbers() throws NumericException { // Complex decimal with exponent int precision = Numbers.decodeLowInt(d.ofString("123.456789e-3")); - Assert.assertEquals(10, precision); + Assert.assertEquals(9, precision); Assert.assertEquals(9, d.getScale()); Assert.assertEquals("0.123456789", d.toString()); @@ -1463,7 +1463,7 @@ public void testOfStringExponentLargeButValid() throws NumericException { // Large negative exponent within bounds precision = Numbers.decodeLowInt(d.ofString("1e-20")); - Assert.assertEquals(21, precision); + Assert.assertEquals(20, precision); Assert.assertEquals(20, d.getScale()); Assert.assertEquals("0.00000000000000000001", d.toString()); } @@ -1530,7 +1530,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -1564,7 +1564,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } @@ -1581,7 +1581,7 @@ public void testOfStringExponentWithPrecisionLimit() throws NumericException { // With scale limit that matches the result scale precision = Numbers.decodeLowInt(d.ofString("1e-3", -1, 3)); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.001", d.toString()); } @@ -1612,7 +1612,7 @@ public void testOfStringExponentWithSuffix() throws NumericException { // With m suffix precision = Numbers.decodeLowInt(d.ofString("2.5e-3m")); - Assert.assertEquals(5, precision); + Assert.assertEquals(4, precision); Assert.assertEquals(4, d.getScale()); Assert.assertEquals("0.0025", d.toString()); } @@ -1832,7 +1832,7 @@ public void testOfStringPrecisionConsistency() throws NumericException { Assert.assertEquals(6, p3); int p4 = Numbers.decodeLowInt(d.ofString("00.001")); - Assert.assertEquals(4, p4); // Leading zeros don't count toward precision + Assert.assertEquals(3, p4); // Leading zeros don't count toward precision; 0.001 needs 3 fractional digits } @Test(expected = NumericException.class) @@ -1946,25 +1946,25 @@ public void testOfStringWithNegativeExponent() throws NumericException { // Basic negative exponent int precision = Numbers.decodeLowInt(d.ofString("1.234e-2")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.01234", d.toString()); // Larger negative exponent precision = Numbers.decodeLowInt(d.ofString("5e-5")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.00005", d.toString()); // Negative exponent with E (uppercase) precision = Numbers.decodeLowInt(d.ofString("7.89E-3")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.00789", d.toString()); // Very small number precision = Numbers.decodeLowInt(d.ofString("1e-10")); - Assert.assertEquals(11, precision); + Assert.assertEquals(10, precision); Assert.assertEquals(10, d.getScale()); Assert.assertEquals("0.0000000001", d.toString()); } @@ -2066,7 +2066,7 @@ public void testOfStringZeroWithDecimal() throws NumericException { Assert.assertEquals("0", d.toString()); precision = Numbers.decodeLowInt(d.ofString("0.000m")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.000", d.toString()); diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java index 769d90120..008989f2a 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java @@ -663,7 +663,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal64 d = new Decimal64(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -700,7 +700,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -717,7 +717,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java new file mode 100644 index 000000000..43ab94d9c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java @@ -0,0 +1,119 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.std; + +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.DecimalParser; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.NumericException; +import org.junit.Assert; +import org.junit.Test; + +public class DecimalParserServerCompatibilityTest { + @Test + public void testFullScaleAndTargetScalePaddingUseServerPrecisionRules() throws NumericException { + assertParsed("0.5", 1, 1, ".5", 1, 1); + assertParsed("0.500", 3, 3, "0.5", 3, 3); + assertParsed("0.100", 3, 3, ".1", 3, 3); + + String fraction = "0." + repeat('9', 76); + assertParsed(fraction, 76, 76, fraction, 76, 76); + } + + @Test + public void testLeadingZerosAndZeroExponentUseServerPrecisionRules() throws NumericException { + assertParsed("0", 1, 0, "0000.e80", 1, 0); + assertParsed("1", 1, 0, "0.01e2", -1, -1); + assertParsed("123", 3, 0, "00.123e3", -1, -1); + assertParsed("0", 1, 0, "0e5", -1, -1); + assertParsed("0.000", 3, 3, "0e-3", 3, 3); + } + + @Test + public void testPrecisionIsAtLeastOneAndZeroFitsFullScale() throws NumericException { + assertParsed("0", 1, 0, "0.000", -1, -1); + assertParsed("0", 1, 0, "0.", -1, -1); + assertParsed("0." + repeat('0', 76), 76, 76, "0", 76, 76); + } + + @Test + public void testSpecialValuesMatchByPrefixLikeServer() throws NumericException { + assertNullParsed("NaN", 38, 4); + assertNullParsed("-Infinity", 76, 8); + // The server's DecimalParser treats NaN and Infinity as prefixes. + assertNullParsed("NaNjunk", 38, 4); + assertNullParsed("NaN ", 38, 4); + assertNullParsed("Infinitygarbage", 76, 8); + assertRejected("Na", 38, 4); + assertRejected("Infinit", 76, 8); + } + + @Test + public void testSuffixOrderAndExtremeExponentsAreRejectedNormally() throws NumericException { + assertParsed("1.23", 3, 2, "1.2300", 38, 2); + assertRejected("1.2300M", 38, 2); + assertRejected("1.2300fm", 38, 2); + assertRejected("1.2300mf", 38, 2); + assertRejected("1e2147483647", 76, 0); + assertRejected("1e-2147483648", 76, 0); + } + + private static void assertParsed( + String expected, + int expectedPrecision, + int expectedScale, + String input, + int precision, + int scale + ) throws NumericException { + Decimal256 viaParser = new Decimal256(); + long parserMeta = DecimalParser.parse( + viaParser, input, 0, input.length(), precision, scale, false, false); + Assert.assertEquals(expected, viaParser.toString()); + Assert.assertEquals(expectedPrecision, Numbers.decodeLowInt(parserMeta)); + Assert.assertEquals(expectedScale, Numbers.decodeHighInt(parserMeta)); + + Decimal256 viaPublicDecimal = new Decimal256(); + long decimalMeta = viaPublicDecimal.ofString( + input, 0, input.length(), precision, scale, false, false); + Assert.assertEquals(expected, viaPublicDecimal.toString()); + Assert.assertEquals(expectedPrecision, Numbers.decodeLowInt(decimalMeta)); + Assert.assertEquals(expectedScale, Numbers.decodeHighInt(decimalMeta)); + } + + private static void assertNullParsed(String input, int precision, int scale) throws NumericException { + Decimal256 viaParser = new Decimal256(); + Assert.assertEquals(0, DecimalParser.parse( + viaParser, input, 0, input.length(), precision, scale, false, false)); + Assert.assertTrue(viaParser.isNull()); + + Decimal256 viaPublicDecimal = new Decimal256(); + Assert.assertEquals(0, viaPublicDecimal.ofString( + input, 0, input.length(), precision, scale, false, false)); + Assert.assertTrue(viaPublicDecimal.isNull()); + } + + private static void assertRejected(String input, int precision, int scale) { + try { + DecimalParser.parse(new Decimal256(), input, 0, input.length(), precision, scale, false, false); + Assert.fail("expected DecimalParser to reject: " + input); + } catch (NumericException expected) { + // expected + } + try { + new Decimal256().ofString(input, 0, input.length(), precision, scale, false, false); + Assert.fail("expected Decimal256 to reject: " + input); + } catch (NumericException expected) { + // expected + } + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + for (int i = 0; i < count; i++) chars[i] = value; + return new String(chars); + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java b/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java index 9076319d3..2d2462a8a 100644 --- a/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java +++ b/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java @@ -49,18 +49,15 @@ * JMH benchmark for {@link Numbers#append(io.questdb.client.std.str.CharSink, double)}, * the double-to-text routine behind every double column the client sends. *

      - * Its bignum slow path goes through the {@code FdBig} bridge over the JDK's - * internal {@code FDBigInteger}. On Java 9+ that bridge binds the bignum - * methods via {@code static final} method handles resolved once at class-init - * (JDK 26 made the class package-private, so it can no longer be named in - * source). This benchmark exists to show the handle-based bridge costs the - * same as the direct calls it replaced: run it once against a client jar - * built from the previous {@code main} and once against the current tree, - * same JDK, and compare {@code shape=extreme} (100% slow path). + * {@code Numbers.append()} formats through Ryu (the same algorithm the server + * uses), which has no bignum slow path. The {@code shape} parameter still + * spans the input distributions that exercised the slow path of the pre-Ryu + * formatter, so the benchmark can compare the two across client versions: + * run it against a jar built from an older client and once against the + * current tree, same JDK. The slow-path shares below refer to that pre-Ryu + * formatter. *

      - * Input shapes, each a fixed pool of 1024 values cycled per invocation. The - * slow-path share was measured by running the pool against a pre-fix jar on - * JDK 26, where the bignum path throws and the fast path does not: + * Input shapes, each a fixed pool of 1024 values cycled per invocation: *

        *
      • {@code simple} -- short literals like {@code 1.0}, {@code 123456.789}; * 0/1024 slow path.
        • diff --git a/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java b/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java index 7bdb23a0a..64b7c2587 100644 --- a/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java +++ b/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java @@ -29,7 +29,7 @@ /** * Standalone (no JUnit) smoke check that double formatting works against the * packaged jar on the running JDK. The extreme-exponent values exercise the - * FdBig fallback of {@code Numbers.appendDouble0}, which resolves + * Ryu double formatter, which resolves * {@code FDBigInteger} from a different JDK-internal package on Java 8 vs 9+; * a jar packaged with the wrong bridge dies here with * {@code NoClassDefFoundError: sun/misc/FDBigInteger} (the 1.3.5-1.3.7 @@ -43,7 +43,7 @@ public static void main(String[] args) { double[] values = { 0.0d, 123.456d, - // FdBig slow-path values + // formerly FdBig slow-path values 1.0E-300, Double.MIN_VALUE, Double.MAX_VALUE, diff --git a/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java b/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java index 8ddaf8a83..44af7bd08 100644 --- a/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java +++ b/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java @@ -43,14 +43,15 @@ *
      • the module name silently changing from {@code io.questdb.client} to the * filename-derived {@code questdb.client} (no module-info.class on JDK 8 * builds and no Automatic-Module-Name),
        • - *
      • JDK 8 builds shipping only the {@code sun.misc.FDBigInteger} FdBig - * bridge, which throws {@code NoClassDefFoundError} on Java 9+ without the - * {@code META-INF/versions/11} counterpart.
        • + *
      • JDK 8 builds shipping only the Java 8 {@code Compat} shim, whose + * {@code multiplyHigh} fallback the Ryu double formatter calls on every + * value, without the {@code META-INF/versions/11} counterpart that lets + * Java 9+ use the {@code Math.multiplyHigh} intrinsic instead.
        • *
      */ public class JarPackagingIT { - private static final String FD_BIG_ENTRY = "io/questdb/client/std/FdBig.class"; - private static final String VERSIONED_FD_BIG_ENTRY = "META-INF/versions/11/" + FD_BIG_ENTRY; + private static final String COMPAT_ENTRY = "io/questdb/client/std/Compat.class"; + private static final String VERSIONED_COMPAT_ENTRY = "META-INF/versions/11/" + COMPAT_ENTRY; @Test public void testDoubleFormattingAgainstPackagedJar() throws Exception { @@ -106,23 +107,20 @@ public void testJarLayout() throws Exception { // JDK 8 build: the shipping layout. Root classes target Java 8, // the java11 bridge rides in META-INF/versions/11. Assert.assertTrue( - "root FdBig of a JDK 8 build must use sun.misc.FDBigInteger", - classReferences(jar, FD_BIG_ENTRY, "sun/misc/FDBigInteger") + "root Compat of a JDK 8 build must use the Java 8 ManagementFactory variant", + classReferences(jar, COMPAT_ENTRY, "java/lang/management/ManagementFactory") ); Assert.assertNull("JDK 8 must not load a root module descriptor", jar.getEntry("module-info.class")); Assert.assertTrue("the released descriptor must declare the token-provider service use", classReferences(jar, "META-INF/versions/11/module-info.class", "io/questdb/client/cutlass/auth/TokenProviderFactory")); - // the java11 bridge never names the class (JDK 26 made it package-private); - // it loads it by dotted name and binds method handles, so the - // constant-pool witness is the dotted string literal - Assert.assertTrue( - "META-INF/versions/11 FdBig must bind jdk.internal.math.FDBigInteger", - classReferences(jar, VERSIONED_FD_BIG_ENTRY, "jdk.internal.math.FDBigInteger") - ); Assert.assertNotNull( "META-INF/versions/11 must carry the java11 Compat shim", - jar.getEntry("META-INF/versions/11/io/questdb/client/std/Compat.class") + jar.getEntry(VERSIONED_COMPAT_ENTRY) + ); + Assert.assertTrue( + "META-INF/versions/11 Compat must use the Math.multiplyHigh intrinsic", + classReferences(jar, VERSIONED_COMPAT_ENTRY, "multiplyHigh") ); // every source in src/main/java11 must ship in versions/11 -- a file added // to the source root but missed by the packaging step would recreate the @@ -137,8 +135,8 @@ public void testJarLayout() throws Exception { // JDK 11+ build: dev/smoke only, never shipped. Root classes are the // java11 variants and the real module descriptor is present. Assert.assertTrue( - "root FdBig of a JDK 11+ build must bind jdk.internal.math.FDBigInteger", - classReferences(jar, FD_BIG_ENTRY, "jdk.internal.math.FDBigInteger") + "root Compat of a JDK 11+ build must use the Math.multiplyHigh intrinsic", + classReferences(jar, COMPAT_ENTRY, "multiplyHigh") ); Assert.assertNotNull("module-info.class missing", jar.getEntry("module-info.class")); } diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index 0467235da..ac61e96f1 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -29,16 +29,23 @@ import io.questdb.client.std.Rnd; import io.questdb.client.std.str.StringSink; import io.questdb.client.std.str.Utf8Sequence; +import io.questdb.client.std.str.Utf8StringSink; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import static io.questdb.client.std.Numbers.ceilPow2; import static org.junit.Assert.assertEquals; public class NumbersTest { private final StringSink sink = new StringSink(); + private final Utf8StringSink utf8Sink = new Utf8StringSink(); private Rnd rnd; @Test(expected = NumericException.class) @@ -140,7 +147,10 @@ public void testFormatDoubleExpNeg() { public void testFormatDoubleFast() { sink.clear(); Numbers.append(sink, -5.9522650387500933e18); - TestUtils.assertEquals("-5.9522650387500933E18", sink); + // Shortest form that still reads back to the same double. The previous + // formatter emitted a 17th digit ("...0933E18"); both parse to + // -0x1.4a6ac610aeb44p62, and Double.toString agrees with this one. + TestUtils.assertEquals("-5.952265038750093E18", sink); } @Test @@ -199,6 +209,141 @@ public void testFormatDoubleRound() { TestUtils.assertEquals("4.4556303333333335E27", sink); } + @Test + public void testFormatDoubleRoundTrip() { + for (int i = 0; i < 100_000; i++) { + long bits = rnd.nextLong(); + double value = Double.longBitsToDouble(bits); + if (Double.isNaN(value) || Double.isInfinite(value)) { + continue; + } + sink.clear(); + utf8Sink.clear(); + Numbers.append(sink, value); + Numbers.append(utf8Sink, value); + TestUtils.assertEquals(sink, utf8Sink); + assertEquals(bits, Double.doubleToRawLongBits(Double.parseDouble(sink.toString()))); + } + } + + @Test + public void testFormatDoubleScale() { + assertFormatDouble("450.31", 450.31, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("120.25", 120.25, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("123.4", 123.456, 0); + assertFormatDouble("123.4", 123.456, -1); + assertFormatDouble("123.45", 123.456, 2); + assertFormatDouble("123.456", 123.456, 3); + assertFormatDouble("0.0", 0.001234, 1); + assertFormatDouble("0.00", 0.001234, 2); + assertFormatDouble("0.0", 0.001234, Integer.MIN_VALUE); + assertFormatDouble("0.001", 0.001234, 3); + assertFormatDouble("-0.0012", -0.001234, 4); + assertFormatDouble("1234000.0", 1234000.0, 1); + // Scientific notation is not truncated by scale. + assertFormatDouble("1.23456789E7", 12345678.9, 1); + assertFormatDouble("1.234E-4", 0.0001234, 1); + } + + @Test + public void testFormatDoubleScratchReuse() { + for (int i = 0; i < 3; i++) { + assertFormatDouble("1.7976931348623157E308", Double.MAX_VALUE, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("1.0", 1.0, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("1.2345678901234567", 1.2345678901234567, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("0.001", 0.001, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("-0.0", -0.0, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("NaN", Double.NaN, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("Infinity", Double.POSITIVE_INFINITY, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("-Infinity", Double.NEGATIVE_INFINITY, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("450.31", 450.31, Numbers.MAX_DOUBLE_SCALE); + } + } + + @Test + public void testFormatDoubleShortDecimalBoundaries() { + // Short decimals below 10^7 take an exact path, the rest Ryu; values on + // either side of each limit of that path. + assertFormatDouble("3.25", 3.25, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("-42.5", -42.5, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("1200000.0", 1_200_000.0, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("9999999.99999999", 9999999.99999999, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("1.0E7", 1e7, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("1.00000005E7", 10000000.5, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("0.12345678", 0.12345678, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("0.123456789", 0.123456789, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("0.001", 0.001, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("9.99E-4", 0.000999, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("1.0E-8", 1e-8, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("5.0E-9", 5e-9, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("0.30000000000000004", 0.1 + 0.2, Numbers.MAX_DOUBLE_SCALE); + assertFormatDouble("450.3", 450.31, 1); + double[] values = { + Math.nextDown(1e7), Math.nextUp(1e7), Math.nextDown(9999999.99999999), Math.nextUp(9999999.99999999), + 1e-8, Math.nextDown(1e-8), Math.nextUp(1e-8), 5e-9, Math.nextDown(5e-9), Math.nextUp(5e-9), + Double.MIN_VALUE, Double.MIN_NORMAL, Double.MAX_VALUE, 1.0 / 3, 2.0 / 3, 0.1 + 0.2 + }; + for (double value : values) { + assertShortestNearest(value); + assertShortestNearest(-value); + } + // Powers of ten and the doubles next to them, where the rounding + // interval can hold the power of ten itself. + for (int exp = -12; exp <= 12; exp++) { + double up = Double.parseDouble("1E" + exp); + double down = up; + for (int i = 0; i < 16; i++) { + assertShortestNearest(up); + assertShortestNearest(down); + up = Math.nextUp(up); + down = Math.nextDown(down); + } + } + } + + @Test + public void testFormatDoubleShortDecimalsMatchJdk() { + // From JDK 19 on, Double.toString prints the shortest decimal that rounds + // to the value, nearest to it (JDK-4511638), in the notation Numbers uses. + Assume.assumeTrue(javaFeatureVersion() >= 19); + for (int scale = 0; scale <= 9; scale++) { + for (int k = 1; k <= 200_000; k++) { + double value = Double.parseDouble(k + "E-" + scale); + assertFormatDoubleAsJdk(value); + assertFormatDoubleAsJdk(-value); + } + } + } + + @Test + public void testFormatDoubleShortestNearestDecimal() { + // Decimals k * 10^-scale, the doubles next to them, and random doubles, + // against the shortest decimal that rounds to the value, computed from + // its definition. This holds on every JDK, unlike Double.toString. + for (int scale = 0; scale <= 9; scale++) { + for (int k = 1; k <= 5_000; k++) { + double value = Double.parseDouble(k + "E-" + scale); + assertShortestNearest(value); + assertShortestNearest(-value); + assertShortestNearest(Math.nextUp(value)); + assertShortestNearest(Math.nextDown(value)); + } + } + Rnd rnd = TestUtils.generateRandom(null); + for (int i = 0; i < 20_000; i++) { + // up to 10 significant digits with up to 9 of them fractional, as + // prices and readings have + double value = Double.parseDouble((rnd.nextLong() >>> 1) % 10_000_000_000L + "E-" + rnd.nextInt(10)); + assertShortestNearest(value); + assertShortestNearest(Math.nextUp(value)); + double random = Double.longBitsToDouble(rnd.nextLong()); + if (!Double.isNaN(random) && !Double.isInfinite(random)) { + assertShortestNearest(random); + } + assertShortestNearest((rnd.nextLong() >>> 11) * 0x1.0p-53 * 1e7); + } + } + @Test public void testFormatDoubleSlowInteractive() { sink.clear(); @@ -536,6 +681,16 @@ public void testParseIPv4LeadingDots3() { Numbers.parseIPv4("...a.1.2.3.4"); } + @Test(expected = NumericException.class) + public void testParseIPv4OnlyDots() { + Numbers.parseIPv4("...."); + } + + @Test(expected = NumericException.class) + public void testParseIPv4SingleDot() { + Numbers.parseIPv4("."); + } + @Test(expected = NumericException.class) public void testParseIPv4MiddleDots() { Numbers.parseIPv4("1..2..3..4"); @@ -730,6 +885,36 @@ public void testParseWrongNan() { Numbers.parseDouble("NaN1"); } + private void assertFormatDouble(String expected, double value, int scale) { + sink.clear(); + utf8Sink.clear(); + Numbers.append(sink, value, scale); + Numbers.append(utf8Sink, value, scale); + TestUtils.assertEquals(expected, sink); + TestUtils.assertEquals(expected, utf8Sink); + } + + private void assertFormatDoubleAsJdk(double value) { + sink.clear(); + Numbers.append(sink, value); + String expected = Double.toString(value); + if (!expected.contentEquals(sink)) { + Assert.fail("value bits " + Double.doubleToRawLongBits(value) + ": expected " + expected + ", got " + sink); + } + } + + private void assertShortestNearest(double value) { + sink.clear(); + Numbers.append(sink, value); + BigDecimal expected = shortestNearest(Math.abs(value)); + BigDecimal actual = new BigDecimal(sink.toString()).abs(); + if (expected.compareTo(actual) != 0) { + Assert.fail("value bits " + Double.doubleToRawLongBits(value) + ": expected digits of " + + expected.stripTrailingZeros().toPlainString() + ", got " + sink); + } + Assert.assertEquals(value, Double.parseDouble(sink.toString()), 0.0); + } + private static void assertParseLongException(String input) { try { Numbers.parseLong(input); @@ -745,4 +930,51 @@ private static void assertHexLongRejected(String hex) { } catch (NumericException expected) { } } + + private static boolean isCloser(BigDecimal candidate, BigDecimal best, BigDecimal exact) { + int cmp = candidate.subtract(exact).abs().compareTo(best.subtract(exact).abs()); + return cmp < 0 || (cmp == 0 && !candidate.unscaledValue().testBit(0)); + } + + private static int javaFeatureVersion() { + String version = System.getProperty("java.specification.version"); + return version.startsWith("1.") ? Integer.parseInt(version.substring(2)) : Integer.parseInt(version); + } + + /** + * The shortest decimal that rounds to {@code value}, nearest to it among + * those, ties to an even last digit, as Ryu defines its result. The interval + * of decimals that round to {@code value} keeps its bounds when the + * significand is even, as round-half-even does. + */ + private static BigDecimal shortestNearest(double value) { + BigDecimal exact = new BigDecimal(value); + BigDecimal half = BigDecimal.valueOf(5, 1); + BigDecimal lo = exact.add(new BigDecimal(Math.nextDown(value))).multiply(half); + // Above Double.MAX_VALUE the next double is infinite; the interval ends + // where the gap to it would. + BigDecimal hi = Double.isInfinite(Math.nextUp(value)) + ? exact.add(exact.subtract(new BigDecimal(Math.nextDown(value))).multiply(half)) + : exact.add(new BigDecimal(Math.nextUp(value))).multiply(half); + boolean isBoundIncluded = (Double.doubleToRawLongBits(value) & 1) == 0; + for (int digits = 1; digits <= 17; digits++) { + BigDecimal nearest = exact.round(new MathContext(digits, RoundingMode.HALF_EVEN)); + BigDecimal unit = nearest.ulp(); + BigDecimal best = null; + // At a power of two the interval is narrower below the value, so the + // nearest candidate can fall outside while its neighbor is inside. + for (BigDecimal candidate : new BigDecimal[]{nearest.subtract(unit), nearest, nearest.add(unit)}) { + int cmpLo = candidate.compareTo(lo); + int cmpHi = candidate.compareTo(hi); + boolean isInside = isBoundIncluded ? cmpLo >= 0 && cmpHi <= 0 : cmpLo > 0 && cmpHi < 0; + if (isInside && (best == null || isCloser(candidate, best, exact))) { + best = candidate; + } + } + if (best != null) { + return best; + } + } + throw new AssertionError("no decimal of up to 17 digits rounds to " + value); + } } diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv new file mode 100644 index 000000000..21a5afe70 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv @@ -0,0 +1,21 @@ +# case_id input_kind input_data_hex expected_wire_type expected_wire_hex expected_sql_hex outcome +# BINARY input is raw bytes; STRING input is UTF-16BE code units. Empty is a present zero-length value. +# is an explicit bitmap null. Hex is lowercase and separator-free. +binary_empty BINARY BINARY VALID +binary_zero BINARY 00 BINARY 00 00 VALID +binary_opaque BINARY 00ff807f010203 BINARY 00ff807f010203 00ff807f010203 VALID +binary_all_256 BINARY 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff BINARY 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff VALID +string_null STRING BINARY VALID +string_empty STRING BINARY VALID +string_ascii STRING 00510075006500730074 BINARY 5175657374 5175657374 VALID +string_embedded_nul STRING 006100000062 BINARY 610062 610062 VALID +string_bmp_two_byte STRING 00e9 BINARY c3a9 c3a9 VALID +string_bmp_three_byte STRING 6c49 BINARY e6b189 e6b189 VALID +string_supplementary STRING d83dde00 BINARY f09f9880 f09f9880 VALID +string_lone_high STRING d800 BINARY 3f 3f VALID +string_high_ascii STRING d8000061 BINARY 3f61 3f61 VALID +string_high_high_low STRING d800d83dde00 BINARY 3ff09f9880 3ff09f9880 VALID +string_lone_low STRING dc00 BINARY 3f 3f VALID +string_low_high_ascii STRING dc00d8000061 BINARY 3f3f61 3f3f61 VALID +string_literal_question STRING 003f BINARY 3f 3f VALID +string_controls STRING 00610009000d000a0062 BINARY 61090d0a62 61090d0a62 VALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv new file mode 100644 index 000000000..8bcc6a231 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv @@ -0,0 +1,19 @@ +# case_id input_boolean target_type expected_wire_type expected_wire_value expected_sql_value +bool_false_boolean FALSE BOOLEAN BOOLEAN 0 false +bool_true_boolean TRUE BOOLEAN BOOLEAN 1 true +bool_false_byte FALSE BYTE BYTE 0 0 +bool_true_byte TRUE BYTE BYTE 1 1 +bool_false_short FALSE SHORT SHORT 0 0 +bool_true_short TRUE SHORT SHORT 1 1 +bool_false_int FALSE INT INT 0 0 +bool_true_int TRUE INT INT 1 1 +bool_false_long FALSE LONG LONG 0 0 +bool_true_long TRUE LONG LONG 1 1 +bool_false_float FALSE FLOAT FLOAT 0 0 +bool_true_float TRUE FLOAT FLOAT 1 1 +bool_false_double FALSE DOUBLE DOUBLE 0 0 +bool_true_double TRUE DOUBLE DOUBLE 1 1 +bool_false_string FALSE STRING VARCHAR 66616c7365 00660061006c00730065 +bool_true_string TRUE STRING VARCHAR 74727565 0074007200750065 +bool_false_varchar FALSE VARCHAR VARCHAR 66616c7365 00660061006c00730065 +bool_true_varchar TRUE VARCHAR VARCHAR 74727565 0074007200750065 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv new file mode 100644 index 000000000..cfb464ac3 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv @@ -0,0 +1,24 @@ +# case_id input_kind utf16_hex expected_char expected_null +char_nul CHAR 0000 0 true +char_ascii CHAR 0041 65 false +char_two_byte CHAR 00e9 233 false +char_three_byte CHAR 20ac 8364 false +char_before_surrogate CHAR d7ff 55295 false +char_high_surrogate CHAR d83d 55357 false +char_low_surrogate CHAR de00 56832 false +char_after_surrogate CHAR e000 57344 false +char_max CHAR ffff 65535 false +string_null STRING 0 true +string_empty STRING 0 true +string_nul STRING 0000 0 true +string_ascii STRING 0041 65 false +string_two_byte STRING 00e9 233 false +string_three_byte STRING 20ac 8364 false +string_before_surrogate STRING d7ff 55295 false +string_after_surrogate STRING e000 57344 false +string_max STRING ffff 65535 false +string_supplementary_first STRING d83dde00 0 true +string_lone_high STRING d83d 63 false +string_lone_low STRING de00 63 false +string_high_then_ascii STRING d83d0041 63 false +string_ascii_then_trailing STRING 004200e9d83dde00 66 false diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv new file mode 100644 index 000000000..f6468b645 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv @@ -0,0 +1,61 @@ +# case_id source_type input_bits_hex target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +float_to_decimal8 FLOAT 41180000 DECIMAL8 2 1 VALUE 000000000000005f - - - 9.5 +double_to_decimal8 DOUBLE 3fb999999999999a DECIMAL8 2 1 VALUE 0000000000000001 - - - 0.1 +float_to_decimal16 FLOAT c1480000 DECIMAL16 4 2 VALUE fffffffffffffb1e - - - -12.50 +double_to_decimal16 DOUBLE 4058ff5c28f5c28f DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +float_to_decimal32 FLOAT 4640e480 DECIMAL32 9 3 VALUE 0000000000bc5f25 - - - 12345.125 +double_to_decimal32 DOUBLE c08f3ffdf3b645a2 DECIMAL32 9 3 VALUE fffffffffff0bdc1 - - - -999.999 +float_to_decimal64 FLOAT 3dcccccd DECIMAL64 18 17 VALUE 002386f278a2bc9c - - - 0.10000000149011612 +double_to_decimal64 DOUBLE 3fb999999999999a DECIMAL64 18 17 VALUE 002386f26fc10000 - - - 0.10000000000000000 +float_to_decimal128 FLOAT 4b800000 DECIMAL128 24 5 VALUE 00000186a0000000 0000000000000000 - - 16777216.00000 +double_to_decimal128 DOUBLE 44b52d02c7e14af6 DECIMAL128 24 0 VALUE 02c7e14af6800000 000000000000152d - - 100000000000000000000000 +float_to_decimal256 FLOAT 7f7fffff DECIMAL256 76 0 VALUE 35211f7ca1800000 ffffff000000000a 0000000000000000 0000000000000000 340282346638528860000000000000000000000 +double_to_decimal256 DOUBLE 3027288e1271f513 DECIMAL256 76 76 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +decimal8_positive_limit DOUBLE 4023cccccccccccd DECIMAL8 2 1 VALUE 0000000000000063 - - - 9.9 +decimal8_negative_limit DOUBLE c023cccccccccccd DECIMAL8 2 1 VALUE ffffffffffffff9d - - - -9.9 +decimal8_positive_over FLOAT 41200000 DECIMAL8 2 1 INVALID - - - - - +decimal8_negative_over DOUBLE c024000000000000 DECIMAL8 2 1 INVALID - - - - - +decimal16_positive_limit DOUBLE 4058ff5c28f5c28f DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +decimal16_negative_fit FLOAT c2c70000 DECIMAL16 4 2 VALUE ffffffffffffd922 - - - -99.50 +decimal16_positive_over DOUBLE 4059000000000000 DECIMAL16 4 2 INVALID - - - - - +decimal16_negative_over FLOAT c2c80000 DECIMAL16 4 2 INVALID - - - - - +decimal32_positive_limit DOUBLE 412e847fff7ced91 DECIMAL32 9 3 VALUE 000000003b9ac9ff - - - 999999.999 +decimal32_positive_fit FLOAT 497423f8 DECIMAL32 9 3 VALUE 000000003b9ac80c - - - 999999.500 +decimal32_positive_over DOUBLE 412e848000000000 DECIMAL32 9 3 INVALID - - - - - +decimal32_negative_over FLOAT c9742400 DECIMAL32 9 3 INVALID - - - - - +decimal64_positive_fit DOUBLE 4376345785d89fff DECIMAL64 18 1 VALUE 0de0b6b3a763ff38 - - - 99999999999999980.0 +decimal64_positive_over DOUBLE 4376345785d8a000 DECIMAL64 18 1 INVALID - - - - - +decimal128_positive_fit DOUBLE 43abc16d674ec7ff DECIMAL128 38 20 VALUE efa958854dc00000 4b3b4ca85a86c25b - - 999999999999999900.00000000000000000000 +decimal128_positive_over DOUBLE 43abc16d674ec800 DECIMAL128 38 20 INVALID - - - - - +decimal256_positive_fit DOUBLE 43e158e460913cff DECIMAL256 76 57 VALUE e000000000000000 c8b1219848c9ee4d 6916687cb4209f75 161bcca711991476 9999999999999998000.000000000000000000000000000000000000000000000000000000000 +decimal256_positive_over DOUBLE 43e158e460913d00 DECIMAL256 76 57 INVALID - - - - - +double_no_rounding DOUBLE 3ff4000000000000 DECIMAL8 2 1 INVALID - - - - - +float_no_rounding FLOAT 3fa00000 DECIMAL8 2 1 INVALID - - - - - +widened_float_point_one_scale_reject FLOAT 3dcccccd DECIMAL8 2 1 INVALID - - - - - +double_point_one_scale_accept DOUBLE 3fb999999999999a DECIMAL8 2 1 VALUE 0000000000000001 - - - 0.1 +float_positive_zero FLOAT 00000000 DECIMAL16 4 2 VALUE 0000000000000000 - - - 0.00 +float_negative_zero FLOAT 80000000 DECIMAL32 9 3 VALUE 0000000000000000 - - - 0.000 +double_positive_zero DOUBLE 0000000000000000 DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +double_negative_zero DOUBLE 8000000000000000 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +float_nan_payload FLOAT 7fc00001 DECIMAL8 2 1 NULL - - - - NULL +float_positive_infinity FLOAT 7f800000 DECIMAL16 4 2 NULL - - - - NULL +float_negative_infinity FLOAT ff800000 DECIMAL32 9 3 NULL - - - - NULL +double_nan_payload DOUBLE 7ff8000000000042 DECIMAL64 18 8 NULL - - - - NULL +double_positive_infinity DOUBLE 7ff0000000000000 DECIMAL128 38 20 NULL - - - - NULL +double_negative_infinity DOUBLE fff0000000000000 DECIMAL256 76 57 NULL - - - - NULL +float_min_value_widened_scale_60 FLOAT 00000001 DECIMAL256 60 60 VALUE 0004fa793930bcd1 0000000000000000 0000000000000000 0000000000000000 0.000000000000000000000000000000000000000000001401298464324817 +float_min_value_widened_scale_59_reject FLOAT 00000001 DECIMAL256 59 59 INVALID - - - - - +float_max_value FLOAT 7f7fffff DECIMAL256 76 0 VALUE 35211f7ca1800000 ffffff000000000a 0000000000000000 0000000000000000 340282346638528860000000000000000000000 +double_one_e_minus_76 DOUBLE 3027288e1271f513 DECIMAL256 76 76 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +double_one_e_minus_77_reject DOUBLE 2ff286d80ec190dc DECIMAL256 76 76 INVALID - - - - - +double_min_normal_reject DOUBLE 0010000000000000 DECIMAL256 76 76 INVALID - - - - - +double_min_value_reject DOUBLE 0000000000000001 DECIMAL256 76 76 INVALID - - - - - +double_max_value_reject DOUBLE 7fefffffffffffff DECIMAL256 76 0 INVALID - - - - - +float_below_two_pow_24 FLOAT 4b7fffff DECIMAL64 18 0 VALUE 0000000000ffffff - - - 16777215 +float_two_pow_24 FLOAT 4b800000 DECIMAL64 18 0 VALUE 0000000001000000 - - - 16777216 +float_above_two_pow_24 FLOAT 4b800001 DECIMAL64 18 0 VALUE 0000000001000002 - - - 16777218 +double_below_two_pow_53 DOUBLE 433fffffffffffff DECIMAL64 18 0 VALUE 001fffffffffffff - - - 9007199254740991 +double_two_pow_53 DOUBLE 4340000000000000 DECIMAL64 18 0 VALUE 0020000000000000 - - - 9007199254740992 +double_above_two_pow_53 DOUBLE 4340000000000001 DECIMAL64 18 0 VALUE 0020000000000002 - - - 9007199254740994 +double_repeating_shortest DOUBLE 3fd5555555555555 DECIMAL64 18 17 VALUE 00766c7d74835552 - - - 0.33333333333333330 +double_large_ryu_boundary DOUBLE 44b52d02c7e14af6 DECIMAL128 24 0 VALUE 02c7e14af6800000 000000000000152d - - 100000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv new file mode 100644 index 000000000..6c5d59e36 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv @@ -0,0 +1,353 @@ +# Seven fields: case_id, input_type, input_raw_bits, target_type, expected_wire_type, expected_wire_bits, expected_sql_value +# Raw bits are lowercase fixed-width hexadecimal. is bitmap null/no value bytes; paired fields are local rejection. +# BITS: means stored floating raw bits. Present INT/LONG minimum uses numeric wire bits and SQL NULL. +float_byte_pos_zero FLOAT 00000000 BYTE BYTE 00 0 +float_byte_neg_zero FLOAT 80000000 BYTE BYTE 00 0 +float_byte_pos_one FLOAT 3f800000 BYTE BYTE 01 1 +float_byte_neg_one FLOAT bf800000 BYTE BYTE ff -1 +float_byte_fraction FLOAT 3fc00000 BYTE BYTE +float_byte_pos_inf FLOAT 7f800000 BYTE BYTE +float_byte_neg_inf FLOAT ff800000 BYTE BYTE +float_byte_nan_payload FLOAT 7fc00001 BYTE BYTE 0 +float_byte_nan_negative FLOAT ffc12345 BYTE BYTE 0 +float_byte_min_subnormal FLOAT 00000001 BYTE BYTE +float_byte_max_finite FLOAT 7f7fffff BYTE BYTE +float_byte_byte_neg_128 FLOAT c3000000 BYTE BYTE 80 -128 +float_byte_byte_pos_127 FLOAT 42fe0000 BYTE BYTE 7f 127 +float_byte_byte_neg_129 FLOAT c3010000 BYTE BYTE +float_byte_byte_pos_128 FLOAT 43000000 BYTE BYTE +float_byte_short_neg_32768 FLOAT c7000000 BYTE BYTE +float_byte_short_pos_32767 FLOAT 46fffe00 BYTE BYTE +float_byte_short_neg_32769 FLOAT c7000100 BYTE BYTE +float_byte_short_pos_32768 FLOAT 47000000 BYTE BYTE +float_byte_int_neg_2147483648 FLOAT cf000000 BYTE BYTE +float_byte_int_pos_2147483647 FLOAT 4f000000 BYTE BYTE +float_byte_int_neg_2147483649 FLOAT cf000000 BYTE BYTE +float_byte_int_pos_2147483648 FLOAT 4f000000 BYTE BYTE +float_byte_long_neg_9223372036854776000 FLOAT df000000 BYTE BYTE +float_byte_long_pos_9223372036854776000 FLOAT 5f000000 BYTE BYTE +float_short_pos_zero FLOAT 00000000 SHORT SHORT 0000 0 +float_short_neg_zero FLOAT 80000000 SHORT SHORT 0000 0 +float_short_pos_one FLOAT 3f800000 SHORT SHORT 0001 1 +float_short_neg_one FLOAT bf800000 SHORT SHORT ffff -1 +float_short_fraction FLOAT 3fc00000 SHORT SHORT +float_short_pos_inf FLOAT 7f800000 SHORT SHORT +float_short_neg_inf FLOAT ff800000 SHORT SHORT +float_short_nan_payload FLOAT 7fc00001 SHORT SHORT 0 +float_short_nan_negative FLOAT ffc12345 SHORT SHORT 0 +float_short_min_subnormal FLOAT 00000001 SHORT SHORT +float_short_max_finite FLOAT 7f7fffff SHORT SHORT +float_short_byte_neg_128 FLOAT c3000000 SHORT SHORT ff80 -128 +float_short_byte_pos_127 FLOAT 42fe0000 SHORT SHORT 007f 127 +float_short_byte_neg_129 FLOAT c3010000 SHORT SHORT ff7f -129 +float_short_byte_pos_128 FLOAT 43000000 SHORT SHORT 0080 128 +float_short_short_neg_32768 FLOAT c7000000 SHORT SHORT 8000 -32768 +float_short_short_pos_32767 FLOAT 46fffe00 SHORT SHORT 7fff 32767 +float_short_short_neg_32769 FLOAT c7000100 SHORT SHORT +float_short_short_pos_32768 FLOAT 47000000 SHORT SHORT +float_short_int_neg_2147483648 FLOAT cf000000 SHORT SHORT +float_short_int_pos_2147483647 FLOAT 4f000000 SHORT SHORT +float_short_int_neg_2147483649 FLOAT cf000000 SHORT SHORT +float_short_int_pos_2147483648 FLOAT 4f000000 SHORT SHORT +float_short_long_neg_9223372036854776000 FLOAT df000000 SHORT SHORT +float_short_long_pos_9223372036854776000 FLOAT 5f000000 SHORT SHORT +float_int_pos_zero FLOAT 00000000 INT INT 00000000 0 +float_int_neg_zero FLOAT 80000000 INT INT 00000000 0 +float_int_pos_one FLOAT 3f800000 INT INT 00000001 1 +float_int_neg_one FLOAT bf800000 INT INT ffffffff -1 +float_int_fraction FLOAT 3fc00000 INT INT +float_int_pos_inf FLOAT 7f800000 INT INT +float_int_neg_inf FLOAT ff800000 INT INT +float_int_nan_payload FLOAT 7fc00001 INT INT NULL +float_int_nan_negative FLOAT ffc12345 INT INT NULL +float_int_min_subnormal FLOAT 00000001 INT INT +float_int_max_finite FLOAT 7f7fffff INT INT +float_int_byte_neg_128 FLOAT c3000000 INT INT ffffff80 -128 +float_int_byte_pos_127 FLOAT 42fe0000 INT INT 0000007f 127 +float_int_byte_neg_129 FLOAT c3010000 INT INT ffffff7f -129 +float_int_byte_pos_128 FLOAT 43000000 INT INT 00000080 128 +float_int_short_neg_32768 FLOAT c7000000 INT INT ffff8000 -32768 +float_int_short_pos_32767 FLOAT 46fffe00 INT INT 00007fff 32767 +float_int_short_neg_32769 FLOAT c7000100 INT INT ffff7fff -32769 +float_int_short_pos_32768 FLOAT 47000000 INT INT 00008000 32768 +float_int_int_neg_2147483648 FLOAT cf000000 INT INT 80000000 NULL +float_int_int_pos_2147483647 FLOAT 4f000000 INT INT +float_int_int_neg_2147483649 FLOAT cf000000 INT INT 80000000 NULL +float_int_int_pos_2147483648 FLOAT 4f000000 INT INT +float_int_long_neg_9223372036854776000 FLOAT df000000 INT INT +float_int_long_pos_9223372036854776000 FLOAT 5f000000 INT INT +float_long_pos_zero FLOAT 00000000 LONG LONG 0000000000000000 0 +float_long_neg_zero FLOAT 80000000 LONG LONG 0000000000000000 0 +float_long_pos_one FLOAT 3f800000 LONG LONG 0000000000000001 1 +float_long_neg_one FLOAT bf800000 LONG LONG ffffffffffffffff -1 +float_long_fraction FLOAT 3fc00000 LONG LONG +float_long_pos_inf FLOAT 7f800000 LONG LONG +float_long_neg_inf FLOAT ff800000 LONG LONG +float_long_nan_payload FLOAT 7fc00001 LONG LONG NULL +float_long_nan_negative FLOAT ffc12345 LONG LONG NULL +float_long_min_subnormal FLOAT 00000001 LONG LONG +float_long_max_finite FLOAT 7f7fffff LONG LONG +float_long_byte_neg_128 FLOAT c3000000 LONG LONG ffffffffffffff80 -128 +float_long_byte_pos_127 FLOAT 42fe0000 LONG LONG 000000000000007f 127 +float_long_byte_neg_129 FLOAT c3010000 LONG LONG ffffffffffffff7f -129 +float_long_byte_pos_128 FLOAT 43000000 LONG LONG 0000000000000080 128 +float_long_short_neg_32768 FLOAT c7000000 LONG LONG ffffffffffff8000 -32768 +float_long_short_pos_32767 FLOAT 46fffe00 LONG LONG 0000000000007fff 32767 +float_long_short_neg_32769 FLOAT c7000100 LONG LONG ffffffffffff7fff -32769 +float_long_short_pos_32768 FLOAT 47000000 LONG LONG 0000000000008000 32768 +float_long_int_neg_2147483648 FLOAT cf000000 LONG LONG ffffffff80000000 -2147483648 +float_long_int_pos_2147483647 FLOAT 4f000000 LONG LONG 0000000080000000 2147483648 +float_long_int_neg_2147483649 FLOAT cf000000 LONG LONG ffffffff80000000 -2147483648 +float_long_int_pos_2147483648 FLOAT 4f000000 LONG LONG 0000000080000000 2147483648 +float_long_long_neg_9223372036854776000 FLOAT df000000 LONG LONG 8000000000000000 NULL +float_long_long_pos_9223372036854776000 FLOAT 5f000000 LONG LONG +float_float_pos_zero FLOAT 00000000 FLOAT FLOAT 00000000 BITS:00000000 +float_float_neg_zero FLOAT 80000000 FLOAT FLOAT 80000000 BITS:80000000 +float_float_pos_one FLOAT 3f800000 FLOAT FLOAT 3f800000 BITS:3f800000 +float_float_neg_one FLOAT bf800000 FLOAT FLOAT bf800000 BITS:bf800000 +float_float_fraction FLOAT 3fc00000 FLOAT FLOAT 3fc00000 BITS:3fc00000 +float_float_pos_inf FLOAT 7f800000 FLOAT FLOAT 7f800000 BITS:7f800000 +float_float_neg_inf FLOAT ff800000 FLOAT FLOAT ff800000 BITS:ff800000 +float_float_nan_payload FLOAT 7fc00001 FLOAT FLOAT NULL +float_float_nan_negative FLOAT ffc12345 FLOAT FLOAT NULL +float_float_min_subnormal FLOAT 00000001 FLOAT FLOAT 00000001 BITS:00000001 +float_float_max_finite FLOAT 7f7fffff FLOAT FLOAT 7f7fffff BITS:7f7fffff +float_float_byte_neg_128 FLOAT c3000000 FLOAT FLOAT c3000000 BITS:c3000000 +float_float_byte_pos_127 FLOAT 42fe0000 FLOAT FLOAT 42fe0000 BITS:42fe0000 +float_float_byte_neg_129 FLOAT c3010000 FLOAT FLOAT c3010000 BITS:c3010000 +float_float_byte_pos_128 FLOAT 43000000 FLOAT FLOAT 43000000 BITS:43000000 +float_float_short_neg_32768 FLOAT c7000000 FLOAT FLOAT c7000000 BITS:c7000000 +float_float_short_pos_32767 FLOAT 46fffe00 FLOAT FLOAT 46fffe00 BITS:46fffe00 +float_float_short_neg_32769 FLOAT c7000100 FLOAT FLOAT c7000100 BITS:c7000100 +float_float_short_pos_32768 FLOAT 47000000 FLOAT FLOAT 47000000 BITS:47000000 +float_float_int_neg_2147483648 FLOAT cf000000 FLOAT FLOAT cf000000 BITS:cf000000 +float_float_int_pos_2147483647 FLOAT 4f000000 FLOAT FLOAT 4f000000 BITS:4f000000 +float_float_int_neg_2147483649 FLOAT cf000000 FLOAT FLOAT cf000000 BITS:cf000000 +float_float_int_pos_2147483648 FLOAT 4f000000 FLOAT FLOAT 4f000000 BITS:4f000000 +float_float_long_neg_9223372036854776000 FLOAT df000000 FLOAT FLOAT df000000 BITS:df000000 +float_float_long_pos_9223372036854776000 FLOAT 5f000000 FLOAT FLOAT 5f000000 BITS:5f000000 +float_double_pos_zero FLOAT 00000000 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 +float_double_neg_zero FLOAT 80000000 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 +float_double_pos_one FLOAT 3f800000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 +float_double_neg_one FLOAT bf800000 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 +float_double_fraction FLOAT 3fc00000 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 +float_double_pos_inf FLOAT 7f800000 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 +float_double_neg_inf FLOAT ff800000 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 +float_double_nan_payload FLOAT 7fc00001 DOUBLE DOUBLE NULL +float_double_nan_negative FLOAT ffc12345 DOUBLE DOUBLE NULL +float_double_min_subnormal FLOAT 00000001 DOUBLE DOUBLE 36a0000000000000 BITS:36a0000000000000 +float_double_max_finite FLOAT 7f7fffff DOUBLE DOUBLE 47efffffe0000000 BITS:47efffffe0000000 +float_double_byte_neg_128 FLOAT c3000000 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 +float_double_byte_pos_127 FLOAT 42fe0000 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 +float_double_byte_neg_129 FLOAT c3010000 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 +float_double_byte_pos_128 FLOAT 43000000 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 +float_double_short_neg_32768 FLOAT c7000000 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 +float_double_short_pos_32767 FLOAT 46fffe00 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 +float_double_short_neg_32769 FLOAT c7000100 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 +float_double_short_pos_32768 FLOAT 47000000 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 +float_double_int_neg_2147483648 FLOAT cf000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +float_double_int_pos_2147483647 FLOAT 4f000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +float_double_int_neg_2147483649 FLOAT cf000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +float_double_int_pos_2147483648 FLOAT 4f000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +float_double_long_neg_9223372036854776000 FLOAT df000000 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 +float_double_long_pos_9223372036854776000 FLOAT 5f000000 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 +double_byte_pos_zero DOUBLE 0000000000000000 BYTE BYTE 00 0 +double_byte_neg_zero DOUBLE 8000000000000000 BYTE BYTE 00 0 +double_byte_pos_one DOUBLE 3ff0000000000000 BYTE BYTE 01 1 +double_byte_neg_one DOUBLE bff0000000000000 BYTE BYTE ff -1 +double_byte_fraction DOUBLE 3ff8000000000000 BYTE BYTE +double_byte_pos_inf DOUBLE 7ff0000000000000 BYTE BYTE +double_byte_neg_inf DOUBLE fff0000000000000 BYTE BYTE +double_byte_nan_payload DOUBLE 7ff8000000000001 BYTE BYTE 0 +double_byte_nan_negative DOUBLE fff8000000000042 BYTE BYTE 0 +double_byte_min_subnormal DOUBLE 0000000000000001 BYTE BYTE +double_byte_max_finite DOUBLE 7fefffffffffffff BYTE BYTE +double_byte_byte_neg_128 DOUBLE c060000000000000 BYTE BYTE 80 -128 +double_byte_byte_pos_127 DOUBLE 405fc00000000000 BYTE BYTE 7f 127 +double_byte_byte_neg_129 DOUBLE c060200000000000 BYTE BYTE +double_byte_byte_pos_128 DOUBLE 4060000000000000 BYTE BYTE +double_byte_short_neg_32768 DOUBLE c0e0000000000000 BYTE BYTE +double_byte_short_pos_32767 DOUBLE 40dfffc000000000 BYTE BYTE +double_byte_short_neg_32769 DOUBLE c0e0002000000000 BYTE BYTE +double_byte_short_pos_32768 DOUBLE 40e0000000000000 BYTE BYTE +double_byte_int_neg_2147483648 DOUBLE c1e0000000000000 BYTE BYTE +double_byte_int_pos_2147483647 DOUBLE 41dfffffffc00000 BYTE BYTE +double_byte_int_neg_2147483649 DOUBLE c1e0000000200000 BYTE BYTE +double_byte_int_pos_2147483648 DOUBLE 41e0000000000000 BYTE BYTE +double_byte_long_neg_9223372036854776000 DOUBLE c3e0000000000000 BYTE BYTE +double_byte_long_pos_9223372036854776000 DOUBLE 43e0000000000000 BYTE BYTE +double_short_pos_zero DOUBLE 0000000000000000 SHORT SHORT 0000 0 +double_short_neg_zero DOUBLE 8000000000000000 SHORT SHORT 0000 0 +double_short_pos_one DOUBLE 3ff0000000000000 SHORT SHORT 0001 1 +double_short_neg_one DOUBLE bff0000000000000 SHORT SHORT ffff -1 +double_short_fraction DOUBLE 3ff8000000000000 SHORT SHORT +double_short_pos_inf DOUBLE 7ff0000000000000 SHORT SHORT +double_short_neg_inf DOUBLE fff0000000000000 SHORT SHORT +double_short_nan_payload DOUBLE 7ff8000000000001 SHORT SHORT 0 +double_short_nan_negative DOUBLE fff8000000000042 SHORT SHORT 0 +double_short_min_subnormal DOUBLE 0000000000000001 SHORT SHORT +double_short_max_finite DOUBLE 7fefffffffffffff SHORT SHORT +double_short_byte_neg_128 DOUBLE c060000000000000 SHORT SHORT ff80 -128 +double_short_byte_pos_127 DOUBLE 405fc00000000000 SHORT SHORT 007f 127 +double_short_byte_neg_129 DOUBLE c060200000000000 SHORT SHORT ff7f -129 +double_short_byte_pos_128 DOUBLE 4060000000000000 SHORT SHORT 0080 128 +double_short_short_neg_32768 DOUBLE c0e0000000000000 SHORT SHORT 8000 -32768 +double_short_short_pos_32767 DOUBLE 40dfffc000000000 SHORT SHORT 7fff 32767 +double_short_short_neg_32769 DOUBLE c0e0002000000000 SHORT SHORT +double_short_short_pos_32768 DOUBLE 40e0000000000000 SHORT SHORT +double_short_int_neg_2147483648 DOUBLE c1e0000000000000 SHORT SHORT +double_short_int_pos_2147483647 DOUBLE 41dfffffffc00000 SHORT SHORT +double_short_int_neg_2147483649 DOUBLE c1e0000000200000 SHORT SHORT +double_short_int_pos_2147483648 DOUBLE 41e0000000000000 SHORT SHORT +double_short_long_neg_9223372036854776000 DOUBLE c3e0000000000000 SHORT SHORT +double_short_long_pos_9223372036854776000 DOUBLE 43e0000000000000 SHORT SHORT +double_int_pos_zero DOUBLE 0000000000000000 INT INT 00000000 0 +double_int_neg_zero DOUBLE 8000000000000000 INT INT 00000000 0 +double_int_pos_one DOUBLE 3ff0000000000000 INT INT 00000001 1 +double_int_neg_one DOUBLE bff0000000000000 INT INT ffffffff -1 +double_int_fraction DOUBLE 3ff8000000000000 INT INT +double_int_pos_inf DOUBLE 7ff0000000000000 INT INT +double_int_neg_inf DOUBLE fff0000000000000 INT INT +double_int_nan_payload DOUBLE 7ff8000000000001 INT INT NULL +double_int_nan_negative DOUBLE fff8000000000042 INT INT NULL +double_int_min_subnormal DOUBLE 0000000000000001 INT INT +double_int_max_finite DOUBLE 7fefffffffffffff INT INT +double_int_byte_neg_128 DOUBLE c060000000000000 INT INT ffffff80 -128 +double_int_byte_pos_127 DOUBLE 405fc00000000000 INT INT 0000007f 127 +double_int_byte_neg_129 DOUBLE c060200000000000 INT INT ffffff7f -129 +double_int_byte_pos_128 DOUBLE 4060000000000000 INT INT 00000080 128 +double_int_short_neg_32768 DOUBLE c0e0000000000000 INT INT ffff8000 -32768 +double_int_short_pos_32767 DOUBLE 40dfffc000000000 INT INT 00007fff 32767 +double_int_short_neg_32769 DOUBLE c0e0002000000000 INT INT ffff7fff -32769 +double_int_short_pos_32768 DOUBLE 40e0000000000000 INT INT 00008000 32768 +double_int_int_neg_2147483648 DOUBLE c1e0000000000000 INT INT 80000000 NULL +double_int_int_pos_2147483647 DOUBLE 41dfffffffc00000 INT INT 7fffffff 2147483647 +double_int_int_neg_2147483649 DOUBLE c1e0000000200000 INT INT +double_int_int_pos_2147483648 DOUBLE 41e0000000000000 INT INT +double_int_long_neg_9223372036854776000 DOUBLE c3e0000000000000 INT INT +double_int_long_pos_9223372036854776000 DOUBLE 43e0000000000000 INT INT +double_long_pos_zero DOUBLE 0000000000000000 LONG LONG 0000000000000000 0 +double_long_neg_zero DOUBLE 8000000000000000 LONG LONG 0000000000000000 0 +double_long_pos_one DOUBLE 3ff0000000000000 LONG LONG 0000000000000001 1 +double_long_neg_one DOUBLE bff0000000000000 LONG LONG ffffffffffffffff -1 +double_long_fraction DOUBLE 3ff8000000000000 LONG LONG +double_long_pos_inf DOUBLE 7ff0000000000000 LONG LONG +double_long_neg_inf DOUBLE fff0000000000000 LONG LONG +double_long_nan_payload DOUBLE 7ff8000000000001 LONG LONG NULL +double_long_nan_negative DOUBLE fff8000000000042 LONG LONG NULL +double_long_min_subnormal DOUBLE 0000000000000001 LONG LONG +double_long_max_finite DOUBLE 7fefffffffffffff LONG LONG +double_long_byte_neg_128 DOUBLE c060000000000000 LONG LONG ffffffffffffff80 -128 +double_long_byte_pos_127 DOUBLE 405fc00000000000 LONG LONG 000000000000007f 127 +double_long_byte_neg_129 DOUBLE c060200000000000 LONG LONG ffffffffffffff7f -129 +double_long_byte_pos_128 DOUBLE 4060000000000000 LONG LONG 0000000000000080 128 +double_long_short_neg_32768 DOUBLE c0e0000000000000 LONG LONG ffffffffffff8000 -32768 +double_long_short_pos_32767 DOUBLE 40dfffc000000000 LONG LONG 0000000000007fff 32767 +double_long_short_neg_32769 DOUBLE c0e0002000000000 LONG LONG ffffffffffff7fff -32769 +double_long_short_pos_32768 DOUBLE 40e0000000000000 LONG LONG 0000000000008000 32768 +double_long_int_neg_2147483648 DOUBLE c1e0000000000000 LONG LONG ffffffff80000000 -2147483648 +double_long_int_pos_2147483647 DOUBLE 41dfffffffc00000 LONG LONG 000000007fffffff 2147483647 +double_long_int_neg_2147483649 DOUBLE c1e0000000200000 LONG LONG ffffffff7fffffff -2147483649 +double_long_int_pos_2147483648 DOUBLE 41e0000000000000 LONG LONG 0000000080000000 2147483648 +double_long_long_neg_9223372036854776000 DOUBLE c3e0000000000000 LONG LONG 8000000000000000 NULL +double_long_long_pos_9223372036854776000 DOUBLE 43e0000000000000 LONG LONG +double_float_pos_zero DOUBLE 0000000000000000 FLOAT FLOAT 00000000 BITS:00000000 +double_float_neg_zero DOUBLE 8000000000000000 FLOAT FLOAT 80000000 BITS:80000000 +double_float_pos_one DOUBLE 3ff0000000000000 FLOAT FLOAT 3f800000 BITS:3f800000 +double_float_neg_one DOUBLE bff0000000000000 FLOAT FLOAT bf800000 BITS:bf800000 +double_float_fraction DOUBLE 3ff8000000000000 FLOAT FLOAT 3fc00000 BITS:3fc00000 +double_float_pos_inf DOUBLE 7ff0000000000000 FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_neg_inf DOUBLE fff0000000000000 FLOAT FLOAT ff800000 BITS:ff800000 +double_float_nan_payload DOUBLE 7ff8000000000001 FLOAT FLOAT NULL +double_float_nan_negative DOUBLE fff8000000000042 FLOAT FLOAT NULL +double_float_min_subnormal DOUBLE 0000000000000001 FLOAT FLOAT 00000000 BITS:00000000 +double_float_max_finite DOUBLE 7fefffffffffffff FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_byte_neg_128 DOUBLE c060000000000000 FLOAT FLOAT c3000000 BITS:c3000000 +double_float_byte_pos_127 DOUBLE 405fc00000000000 FLOAT FLOAT 42fe0000 BITS:42fe0000 +double_float_byte_neg_129 DOUBLE c060200000000000 FLOAT FLOAT c3010000 BITS:c3010000 +double_float_byte_pos_128 DOUBLE 4060000000000000 FLOAT FLOAT 43000000 BITS:43000000 +double_float_short_neg_32768 DOUBLE c0e0000000000000 FLOAT FLOAT c7000000 BITS:c7000000 +double_float_short_pos_32767 DOUBLE 40dfffc000000000 FLOAT FLOAT 46fffe00 BITS:46fffe00 +double_float_short_neg_32769 DOUBLE c0e0002000000000 FLOAT FLOAT c7000100 BITS:c7000100 +double_float_short_pos_32768 DOUBLE 40e0000000000000 FLOAT FLOAT 47000000 BITS:47000000 +double_float_int_neg_2147483648 DOUBLE c1e0000000000000 FLOAT FLOAT cf000000 BITS:cf000000 +double_float_int_pos_2147483647 DOUBLE 41dfffffffc00000 FLOAT FLOAT 4f000000 BITS:4f000000 +double_float_int_neg_2147483649 DOUBLE c1e0000000200000 FLOAT FLOAT cf000000 BITS:cf000000 +double_float_int_pos_2147483648 DOUBLE 41e0000000000000 FLOAT FLOAT 4f000000 BITS:4f000000 +double_float_long_neg_9223372036854776000 DOUBLE c3e0000000000000 FLOAT FLOAT df000000 BITS:df000000 +double_float_long_pos_9223372036854776000 DOUBLE 43e0000000000000 FLOAT FLOAT 5f000000 BITS:5f000000 +double_double_pos_zero DOUBLE 0000000000000000 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 +double_double_neg_zero DOUBLE 8000000000000000 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 +double_double_pos_one DOUBLE 3ff0000000000000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 +double_double_neg_one DOUBLE bff0000000000000 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 +double_double_fraction DOUBLE 3ff8000000000000 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 +double_double_pos_inf DOUBLE 7ff0000000000000 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 +double_double_neg_inf DOUBLE fff0000000000000 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 +double_double_nan_payload DOUBLE 7ff8000000000001 DOUBLE DOUBLE NULL +double_double_nan_negative DOUBLE fff8000000000042 DOUBLE DOUBLE NULL +double_double_min_subnormal DOUBLE 0000000000000001 DOUBLE DOUBLE 0000000000000001 BITS:0000000000000001 +double_double_max_finite DOUBLE 7fefffffffffffff DOUBLE DOUBLE 7fefffffffffffff BITS:7fefffffffffffff +double_double_byte_neg_128 DOUBLE c060000000000000 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 +double_double_byte_pos_127 DOUBLE 405fc00000000000 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 +double_double_byte_neg_129 DOUBLE c060200000000000 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 +double_double_byte_pos_128 DOUBLE 4060000000000000 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 +double_double_short_neg_32768 DOUBLE c0e0000000000000 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 +double_double_short_pos_32767 DOUBLE 40dfffc000000000 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 +double_double_short_neg_32769 DOUBLE c0e0002000000000 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 +double_double_short_pos_32768 DOUBLE 40e0000000000000 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 +double_double_int_neg_2147483648 DOUBLE c1e0000000000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +double_double_int_pos_2147483647 DOUBLE 41dfffffffc00000 DOUBLE DOUBLE 41dfffffffc00000 BITS:41dfffffffc00000 +double_double_int_neg_2147483649 DOUBLE c1e0000000200000 DOUBLE DOUBLE c1e0000000200000 BITS:c1e0000000200000 +double_double_int_pos_2147483648 DOUBLE 41e0000000000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +double_double_long_neg_9223372036854776000 DOUBLE c3e0000000000000 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 +double_double_long_pos_9223372036854776000 DOUBLE 43e0000000000000 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 +float_long_pos_below FLOAT 5effffff LONG LONG 7fffff8000000000 9223371487098961920 +float_long_pos_above FLOAT 5f000001 LONG LONG +float_long_neg_inside FLOAT deffffff LONG LONG 8000008000000000 -9223371487098961920 +float_long_neg_outside FLOAT df000001 LONG LONG +double_long_pos_below DOUBLE 43dfffffffffffff LONG LONG 7ffffffffffffc00 9223372036854774784 +double_long_pos_above DOUBLE 43e0000000000001 LONG LONG +double_long_neg_inside DOUBLE c3dfffffffffffff LONG LONG 8000000000000400 -9223372036854774784 +double_long_neg_outside DOUBLE c3e0000000000001 LONG LONG +float_int_pos_below FLOAT 4effffff INT INT 7fffff80 2147483520 +float_int_pos_at_2p31 FLOAT 4f000000 INT INT +float_int_neg_at_min FLOAT cf000000 INT INT 80000000 NULL +float_int_neg_outside FLOAT cf000001 INT INT +double_int_pos_at_max DOUBLE 41dfffffffc00000 INT INT 7fffffff 2147483647 +double_int_pos_outside DOUBLE 41e0000000000000 INT INT +double_int_neg_at_min DOUBLE c1e0000000000000 INT INT 80000000 NULL +double_int_neg_outside DOUBLE c1e0000000200000 INT INT +float_byte_negative_fraction FLOAT bf000000 BYTE BYTE +float_short_negative_fraction FLOAT bf000000 SHORT SHORT +float_int_negative_fraction FLOAT bf000000 INT INT +float_long_negative_fraction FLOAT bf000000 LONG LONG +float_float_negative_fraction FLOAT bf000000 FLOAT FLOAT bf000000 BITS:bf000000 +float_double_negative_fraction FLOAT bf000000 DOUBLE DOUBLE bfe0000000000000 BITS:bfe0000000000000 +double_byte_negative_fraction DOUBLE bfe0000000000000 BYTE BYTE +double_short_negative_fraction DOUBLE bfe0000000000000 SHORT SHORT +double_int_negative_fraction DOUBLE bfe0000000000000 INT INT +double_long_negative_fraction DOUBLE bfe0000000000000 LONG LONG +double_float_negative_fraction DOUBLE bfe0000000000000 FLOAT FLOAT bf000000 BITS:bf000000 +double_double_negative_fraction DOUBLE bfe0000000000000 DOUBLE DOUBLE bfe0000000000000 BITS:bfe0000000000000 +float_byte_signaling_nan FLOAT 7f800001 BYTE BYTE 0 +float_short_signaling_nan FLOAT 7f800001 SHORT SHORT 0 +float_int_signaling_nan FLOAT 7f800001 INT INT NULL +float_long_signaling_nan FLOAT 7f800001 LONG LONG NULL +float_float_signaling_nan FLOAT 7f800001 FLOAT FLOAT NULL +float_double_signaling_nan FLOAT 7f800001 DOUBLE DOUBLE NULL +double_byte_signaling_nan DOUBLE 7ff0000000000001 BYTE BYTE 0 +double_short_signaling_nan DOUBLE 7ff0000000000001 SHORT SHORT 0 +double_int_signaling_nan DOUBLE 7ff0000000000001 INT INT NULL +double_long_signaling_nan DOUBLE 7ff0000000000001 LONG LONG NULL +double_float_signaling_nan DOUBLE 7ff0000000000001 FLOAT FLOAT NULL +double_double_signaling_nan DOUBLE 7ff0000000000001 DOUBLE DOUBLE NULL +double_float_half_min_subnormal DOUBLE 3690000000000000 FLOAT FLOAT 00000000 BITS:00000000 +double_float_above_half_min_subnormal DOUBLE 3690000000000001 FLOAT FLOAT 00000001 BITS:00000001 +double_float_negative_half_min_subnormal DOUBLE b690000000000000 FLOAT FLOAT 80000000 BITS:80000000 +double_float_float_min_normal DOUBLE 3810000000000000 FLOAT FLOAT 00800000 BITS:00800000 +double_float_below_float_min_normal DOUBLE 380fffffffffffff FLOAT FLOAT 00800000 BITS:00800000 +double_float_float_overflow_tie DOUBLE 47effffff0000000 FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_below_float_overflow DOUBLE 47efffffefffffff FLOAT FLOAT 7f7fffff BITS:7f7fffff +double_float_tie_one_even_down DOUBLE 3ff0000010000000 FLOAT FLOAT 3f800000 BITS:3f800000 +double_float_tie_one_even_up DOUBLE 3ff0000030000000 FLOAT FLOAT 3f800002 BITS:3f800002 +double_float_subnormal_normal_tie DOUBLE 380fffffe0000000 FLOAT FLOAT 00800000 BITS:00800000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv new file mode 100644 index 000000000..cdb47cb18 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv @@ -0,0 +1,49 @@ +# case_id input_type input_raw_bits target_type expected_wire_type expected_utf8_hex expected_sql_value +float_string_pos_zero FLOAT 00000000 STRING VARCHAR 302e30 0.0 +float_string_neg_zero FLOAT 80000000 STRING VARCHAR 2d302e30 -0.0 +float_string_one FLOAT 3f800000 STRING VARCHAR 312e30 1.0 +float_string_point_one FLOAT 3dcccccd STRING VARCHAR 302e3130303030303030313439303131363132 0.10000000149011612 +float_string_min_subnormal FLOAT 00000001 STRING VARCHAR 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_string_max_finite FLOAT 7f7fffff STRING VARCHAR 332e34303238323334363633383532383836453338 3.4028234663852886E38 +float_string_pos_infinity FLOAT 7f800000 STRING VARCHAR 496e66696e697479 Infinity +float_string_neg_infinity FLOAT ff800000 STRING VARCHAR 2d496e66696e697479 -Infinity +float_string_qnan FLOAT 7fc00002 STRING VARCHAR +float_string_snan_negative FLOAT ff800001 STRING VARCHAR +float_string_1e23 FLOAT 65a96816 STRING VARCHAR 392e393939393939373738313936333038453232 9.999999778196308E22 +float_varchar_pos_zero FLOAT 00000000 VARCHAR VARCHAR 302e30 0.0 +float_varchar_neg_zero FLOAT 80000000 VARCHAR VARCHAR 2d302e30 -0.0 +float_varchar_point_one FLOAT 3dcccccd VARCHAR VARCHAR 302e3130303030303030313439303131363132 0.10000000149011612 +float_varchar_min_subnormal FLOAT 00000001 VARCHAR VARCHAR 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_varchar_pos_infinity FLOAT 7f800000 VARCHAR VARCHAR 496e66696e697479 Infinity +float_varchar_qnan FLOAT 7fc00002 VARCHAR VARCHAR +float_symbol_pos_zero FLOAT 00000000 SYMBOL SYMBOL 302e30 0.0 +float_symbol_neg_zero FLOAT 80000000 SYMBOL SYMBOL 2d302e30 -0.0 +float_symbol_point_one FLOAT 3dcccccd SYMBOL SYMBOL 302e3130303030303030313439303131363132 0.10000000149011612 +float_symbol_min_subnormal FLOAT 00000001 SYMBOL SYMBOL 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_symbol_neg_infinity FLOAT ff800000 SYMBOL SYMBOL 2d496e66696e697479 -Infinity +float_symbol_snan_negative FLOAT ff800001 SYMBOL SYMBOL +double_string_pos_zero DOUBLE 0000000000000000 STRING VARCHAR 302e30 0.0 +double_string_neg_zero DOUBLE 8000000000000000 STRING VARCHAR 2d302e30 -0.0 +double_string_one DOUBLE 3ff0000000000000 STRING VARCHAR 312e30 1.0 +double_string_neg_two_half DOUBLE c004000000000000 STRING VARCHAR 2d322e35 -2.5 +double_string_1e23 DOUBLE 44b52d02c7e14af6 STRING VARCHAR 312e30453233 1.0E23 +double_string_min_subnormal DOUBLE 0000000000000001 STRING VARCHAR 352e30452d333234 5.0E-324 +double_string_max_finite DOUBLE 7fefffffffffffff STRING VARCHAR 312e3739373639333133343836323331353745333038 1.7976931348623157E308 +double_string_pos_infinity DOUBLE 7ff0000000000000 STRING VARCHAR 496e66696e697479 Infinity +double_string_neg_infinity DOUBLE fff0000000000000 STRING VARCHAR 2d496e66696e697479 -Infinity +double_string_qnan DOUBLE 7ff8000000000002 STRING VARCHAR +double_string_snan_negative DOUBLE fff0000000000001 STRING VARCHAR +double_varchar_pos_zero DOUBLE 0000000000000000 VARCHAR VARCHAR 302e30 0.0 +double_varchar_neg_zero DOUBLE 8000000000000000 VARCHAR VARCHAR 2d302e30 -0.0 +double_varchar_neg_two_half DOUBLE c004000000000000 VARCHAR VARCHAR 2d322e35 -2.5 +double_varchar_1e23 DOUBLE 44b52d02c7e14af6 VARCHAR VARCHAR 312e30453233 1.0E23 +double_varchar_min_subnormal DOUBLE 0000000000000001 VARCHAR VARCHAR 352e30452d333234 5.0E-324 +double_varchar_pos_infinity DOUBLE 7ff0000000000000 VARCHAR VARCHAR 496e66696e697479 Infinity +double_varchar_qnan DOUBLE 7ff8000000000002 VARCHAR VARCHAR +double_symbol_pos_zero DOUBLE 0000000000000000 SYMBOL SYMBOL 302e30 0.0 +double_symbol_neg_zero DOUBLE 8000000000000000 SYMBOL SYMBOL 2d302e30 -0.0 +double_symbol_neg_two_half DOUBLE c004000000000000 SYMBOL SYMBOL 2d322e35 -2.5 +double_symbol_1e23 DOUBLE 44b52d02c7e14af6 SYMBOL SYMBOL 312e30453233 1.0E23 +double_symbol_max_finite DOUBLE 7fefffffffffffff SYMBOL SYMBOL 312e3739373639333133343836323331353745333038 1.7976931348623157E308 +double_symbol_neg_infinity DOUBLE fff0000000000000 SYMBOL SYMBOL 2d496e66696e697479 -Infinity +double_symbol_snan_negative DOUBLE fff0000000000001 SYMBOL SYMBOL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv new file mode 100644 index 000000000..e203958cc --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv @@ -0,0 +1,25 @@ +# case_id input_type input_value target_type expected_value +byte_date_min BYTE -128 DATE -128 +byte_date_max BYTE 127 DATE 127 +byte_timestamp_min BYTE -128 TIMESTAMP -128 +byte_timestamp_max BYTE 127 TIMESTAMP 127 +byte_timestamp_ns_min BYTE -128 TIMESTAMP_NS -128 +byte_timestamp_ns_max BYTE 127 TIMESTAMP_NS 127 +short_date_min SHORT -32768 DATE -32768 +short_date_max SHORT 32767 DATE 32767 +short_timestamp_min SHORT -32768 TIMESTAMP -32768 +short_timestamp_max SHORT 32767 TIMESTAMP 32767 +short_timestamp_ns_min SHORT -32768 TIMESTAMP_NS -32768 +short_timestamp_ns_max SHORT 32767 TIMESTAMP_NS 32767 +int_date_null INT -2147483648 DATE +int_date_negative INT -1 DATE -1 +int_date_max INT 2147483647 DATE 2147483647 +int_timestamp_null INT -2147483648 TIMESTAMP +int_timestamp_negative INT -1 TIMESTAMP -1 +int_timestamp_max INT 2147483647 TIMESTAMP 2147483647 +int_timestamp_ns_null INT -2147483648 TIMESTAMP_NS +int_timestamp_ns_negative INT -1 TIMESTAMP_NS -1 +int_timestamp_ns_max INT 2147483647 TIMESTAMP_NS 2147483647 +long_date_null LONG -9223372036854775808 DATE +long_date_min_value LONG -9223372036854775807 DATE -9223372036854775807 +long_date_max LONG 9223372036854775807 DATE 9223372036854775807 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv new file mode 100644 index 000000000..8044b25f9 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv @@ -0,0 +1,17 @@ +# case_id input_kind input outcome expected_hex expected_text +typed_null INT 0 NULL 00000000 +typed_low INT 1 VALUE 00000001 0.0.0.1 +typed_int_min INT -2147483648 VALUE 80000000 128.0.0.0 +typed_high_bit INT -1062731519 VALUE c0a80101 192.168.1.1 +typed_all_bits INT -1 VALUE ffffffff 255.255.255.255 +text_private TEXT 10.20.30.40 VALUE 0a141e28 10.20.30.40 +text_normalized TEXT .....1.2.3.4...... VALUE 01020304 1.2.3.4 +text_normalized_null TEXT .0.0.0.0. NULL 00000000 +text_null_literal TEXT null INVALID - - +text_null_literal_upper TEXT NULL INVALID - - +text_zero_literal TEXT 0.0.0.0 INVALID - - +text_empty TEXT INVALID - - +text_bad_octet TEXT 1.2.3.256 INVALID - - +text_bad_character TEXT 1.2.3.x INVALID - - +text_single_dot TEXT . INVALID - - +text_only_dots TEXT .... INVALID - - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv new file mode 100644 index 000000000..ecf131bd5 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv @@ -0,0 +1,40 @@ +# case_id input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +8_positive_max 9 DECIMAL8 2 1 VALUE 000000000000005a - - - 9.0 +8_positive_over 10 DECIMAL8 2 1 INVALID - - - - - +8_negative_max -9 DECIMAL8 2 1 VALUE ffffffffffffffa6 - - - -9.0 +8_negative_over -10 DECIMAL8 2 1 INVALID - - - - - +16_positive_max 99 DECIMAL16 4 2 VALUE 00000000000026ac - - - 99.00 +16_positive_over 100 DECIMAL16 4 2 INVALID - - - - - +16_negative_max -99 DECIMAL16 4 2 VALUE ffffffffffffd954 - - - -99.00 +16_negative_over -100 DECIMAL16 4 2 INVALID - - - - - +32_positive_max 999999 DECIMAL32 9 3 VALUE 000000003b9ac618 - - - 999999.000 +32_positive_over 1000000 DECIMAL32 9 3 INVALID - - - - - +32_negative_max -999999 DECIMAL32 9 3 VALUE ffffffffc46539e8 - - - -999999.000 +32_negative_over -1000000 DECIMAL32 9 3 INVALID - - - - - +64_positive_max 99999999999999999 DECIMAL64 18 1 VALUE 0de0b6b3a763fff6 - - - 99999999999999999.0 +64_positive_over 100000000000000000 DECIMAL64 18 1 INVALID - - - - - +64_negative_max -99999999999999999 DECIMAL64 18 1 VALUE f21f494c589c000a - - - -99999999999999999.0 +64_negative_over -100000000000000000 DECIMAL64 18 1 INVALID - - - - - +128_positive_max 999999999999999999 DECIMAL128 38 20 VALUE 9dc2c4129cf00000 4b3b4ca85a86c474 - - 999999999999999999.00000000000000000000 +128_positive_over 1000000000000000000 DECIMAL128 38 20 INVALID - - - - - +128_negative_max -999999999999999999 DECIMAL128 38 20 VALUE 623d3bed63100000 b4c4b357a5793b8b - - -999999999999999999.00000000000000000000 +128_negative_over -1000000000000000000 DECIMAL128 38 20 INVALID - - - - - +256_positive_max 999999999999999999 DECIMAL256 75 57 VALUE b600000000000000 9ff45edd38440d7c 8b27fbc1cea32b80 0235fadd81c2822b 999999999999999999.000000000000000000000000000000000000000000000000000000000 +256_positive_over 1000000000000000000 DECIMAL256 75 57 INVALID - - - - - +256_negative_max -999999999999999999 DECIMAL256 75 57 VALUE 4a00000000000000 600ba122c7bbf283 74d8043e315cd47f fdca05227e3d7dd4 -999999999999999999.000000000000000000000000000000000000000000000000000000000 +256_negative_over -1000000000000000000 DECIMAL256 75 57 INVALID - - - - - +long_max_to_128 9223372036854775807 DECIMAL128 38 19 VALUE 7538dcfb76180000 4563918244f3ffff - - 9223372036854775807.0000000000000000000 +long_min_plus_one_to_128 -9223372036854775807 DECIMAL128 38 19 VALUE 8ac7230489e80000 ba9c6e7dbb0c0000 - - -9223372036854775807.0000000000000000000 +long_max_to_256 9223372036854775807 DECIMAL256 76 57 VALUE b600000000000000 b90234ab79b5257c cd3668f47a8b47cf 14643e5ae44d12b8 9223372036854775807.000000000000000000000000000000000000000000000000000000000 +long_min_plus_one_to_256 -9223372036854775807 DECIMAL256 76 57 VALUE 4a00000000000000 46fdcb54864ada83 32c9970b8574b830 eb9bc1a51bb2ed47 -9223372036854775807.000000000000000000000000000000000000000000000000000000000 +long_max_256_storage_over 9223372036854775807 DECIMAL256 76 58 INVALID - - - - - +long_min_plus_one_256_storage_over -9223372036854775807 DECIMAL256 76 58 INVALID - - - - - +zero_64_max_scale 0 DECIMAL64 18 18 VALUE 0000000000000000 - - - 0.000000000000000000 +zero_128_max_scale 0 DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +zero_256_max_scale 0 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +long_min_null_8 -9223372036854775808 DECIMAL8 2 1 NULL - - - - NULL +long_min_null_16 -9223372036854775808 DECIMAL16 4 2 NULL - - - - NULL +long_min_null_32 -9223372036854775808 DECIMAL32 9 3 NULL - - - - NULL +long_min_null_64 -9223372036854775808 DECIMAL64 18 1 NULL - - - - NULL +long_min_null_128 -9223372036854775808 DECIMAL128 38 20 NULL - - - - NULL +long_min_null_256 -9223372036854775808 DECIMAL256 76 58 NULL - - - - NULL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv new file mode 100644 index 000000000..53af275f1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv @@ -0,0 +1,49 @@ +# case_id input target expected +# means the target's missing-value representation; BYTE/SHORT materialize it as zero. +long_min_byte -9223372036854775808 BYTE +long_min_short -9223372036854775808 SHORT +long_min_int -9223372036854775808 INT +long_min_long -9223372036854775808 LONG +long_min_float -9223372036854775808 FLOAT +long_min_double -9223372036854775808 DOUBLE +byte_min -128 BYTE -128 +byte_max 127 BYTE 127 +byte_zero 0 BYTE 0 +byte_below -129 BYTE +byte_above 128 BYTE +short_min -32768 SHORT -32768 +short_max 32767 SHORT 32767 +short_zero 0 SHORT 0 +short_below -32769 SHORT +short_above 32768 SHORT +int_min -2147483648 INT +int_min_plus_one -2147483647 INT -2147483647 +int_max 2147483647 INT 2147483647 +int_zero 0 INT 0 +int_below -2147483649 INT +int_above 2147483648 INT +zero_long 0 LONG 0 +minus_one_long -1 LONG -1 +long_max 9223372036854775807 LONG 9223372036854775807 +long_min_plus_one -9223372036854775807 LONG -9223372036854775807 +zero_float 0 FLOAT 0x00000000 +minus_one_float -1 FLOAT 0xbf800000 +precision_float 2147483647 FLOAT 0x4f000000 +float_exact_below 16777215 FLOAT 0x4b7fffff +float_exact_boundary 16777216 FLOAT 0x4b800000 +float_round_down 16777217 FLOAT 0x4b800000 +float_round_even 16777218 FLOAT 0x4b800001 +float_round_up 16777219 FLOAT 0x4b800002 +float_negative_round -16777217 FLOAT 0xcb800000 +long_min_plus_one_float -9223372036854775807 FLOAT 0xdf000000 +long_max_float 9223372036854775807 FLOAT 0x5f000000 +zero_double 0 DOUBLE 0x0000000000000000 +minus_one_double -1 DOUBLE 0xbff0000000000000 +precision_double 9007199254740993 DOUBLE 0x4340000000000000 +double_exact_below 9007199254740991 DOUBLE 0x433fffffffffffff +double_exact_boundary 9007199254740992 DOUBLE 0x4340000000000000 +double_next_even 9007199254740994 DOUBLE 0x4340000000000001 +double_round_up 9007199254740995 DOUBLE 0x4340000000000002 +double_negative_round -9007199254740993 DOUBLE 0xc340000000000000 +long_min_plus_one_double -9223372036854775807 DOUBLE 0xc3e0000000000000 +long_max_double 9223372036854775807 DOUBLE 0x43e0000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv new file mode 100644 index 000000000..23967eb15 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv @@ -0,0 +1,19 @@ +# case_id input_long target_type expected_wire_type expected_utf8_hex expected_sql_value +long_string_min -9223372036854775808 STRING VARCHAR +long_string_min_plus_1 -9223372036854775807 STRING VARCHAR 2d39323233333732303336383534373735383037 -9223372036854775807 +long_string_neg_1 -1 STRING VARCHAR 2d31 -1 +long_string_zero 0 STRING VARCHAR 30 0 +long_string_one 1 STRING VARCHAR 31 1 +long_string_max 9223372036854775807 STRING VARCHAR 39323233333732303336383534373735383037 9223372036854775807 +long_varchar_min -9223372036854775808 VARCHAR VARCHAR +long_varchar_min_plus_1 -9223372036854775807 VARCHAR VARCHAR 2d39323233333732303336383534373735383037 -9223372036854775807 +long_varchar_neg_1 -1 VARCHAR VARCHAR 2d31 -1 +long_varchar_zero 0 VARCHAR VARCHAR 30 0 +long_varchar_one 1 VARCHAR VARCHAR 31 1 +long_varchar_max 9223372036854775807 VARCHAR VARCHAR 39323233333732303336383534373735383037 9223372036854775807 +long_symbol_min -9223372036854775808 SYMBOL SYMBOL +long_symbol_min_plus_1 -9223372036854775807 SYMBOL SYMBOL 2d39323233333732303336383534373735383037 -9223372036854775807 +long_symbol_neg_1 -1 SYMBOL SYMBOL 2d31 -1 +long_symbol_zero 0 SYMBOL SYMBOL 30 0 +long_symbol_one 1 SYMBOL SYMBOL 31 1 +long_symbol_max 9223372036854775807 SYMBOL SYMBOL 39323233333732303336383534373735383037 9223372036854775807 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv new file mode 100644 index 000000000..cdd0db95a --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv @@ -0,0 +1,13 @@ +# case_id input_value target_type expected_wire_type expected_wire_value expected_sql_null +long_timestamp_min -9223372036854775808 TIMESTAMP TIMESTAMP true +long_timestamp_min_plus_one -9223372036854775807 TIMESTAMP TIMESTAMP -9223372036854775807 false +long_timestamp_negative_one -1 TIMESTAMP TIMESTAMP -1 false +long_timestamp_zero 0 TIMESTAMP TIMESTAMP 0 false +long_timestamp_one 1 TIMESTAMP TIMESTAMP 1 false +long_timestamp_max 9223372036854775807 TIMESTAMP TIMESTAMP 9223372036854775807 false +long_timestamp_nanos_min -9223372036854775808 TIMESTAMP_NANOS TIMESTAMP_NANOS true +long_timestamp_nanos_min_plus_one -9223372036854775807 TIMESTAMP_NANOS TIMESTAMP_NANOS -9223372036854775807 false +long_timestamp_nanos_negative_one -1 TIMESTAMP_NANOS TIMESTAMP_NANOS -1 false +long_timestamp_nanos_zero 0 TIMESTAMP_NANOS TIMESTAMP_NANOS 0 false +long_timestamp_nanos_one 1 TIMESTAMP_NANOS TIMESTAMP_NANOS 1 false +long_timestamp_nanos_max 9223372036854775807 TIMESTAMP_NANOS TIMESTAMP_NANOS 9223372036854775807 false diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv new file mode 100644 index 000000000..9b45988c2 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv @@ -0,0 +1,9 @@ +# case_id input_l0 input_l1 input_l2 input_l3 outcome expected_text +zero 0 0 0 0 VALUE 0x00 +low_one 1 0 0 0 VALUE 0x01 +second_limb 1 2 0 0 VALUE 0x020000000000000001 +limb_order 1 2 3 4 VALUE 0x04000000000000000300000000000000020000000000000001 +all_ones -1 -1 -1 -1 VALUE 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +partial_min -9223372036854775808 -9223372036854775808 -9223372036854775808 0 VALUE 0x800000000000000080000000000000008000000000000000 +high_sign_bit 0 0 0 -9223372036854775808 VALUE 0x8000000000000000000000000000000000000000000000000000000000000000 +all_min_null -9223372036854775808 -9223372036854775808 -9223372036854775808 -9223372036854775808 NULL - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv new file mode 100644 index 000000000..caa4173ee --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv @@ -0,0 +1,61 @@ +# case_id source_type source_scale source_ll_hex source_lh_hex source_hl_hex source_hh_hex target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +64_to_8 DECIMAL64 1 000000000000000c - - - DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +64_to_16 DECIMAL64 1 000000000000000c - - - DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +64_to_32 DECIMAL64 1 000000000000000c - - - DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +64_to_64 DECIMAL64 1 000000000000000c - - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +64_to_128 DECIMAL64 1 000000000000000c - - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +64_to_256 DECIMAL64 1 000000000000000c - - - DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +128_to_8 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +128_to_16 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +128_to_32 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +128_to_64 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +128_to_128 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +128_to_256 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +256_to_8 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +256_to_16 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +256_to_32 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +256_to_64 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +256_to_128 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +256_to_256 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +64_zero DECIMAL64 2 0000000000000000 - - - DECIMAL64 18 2 VALUE 0000000000000000 - - - 0.00 +64_negative DECIMAL64 2 fffffffffffffff4 - - - DECIMAL64 18 2 VALUE fffffffffffffff4 - - - -0.12 +64_upscale DECIMAL64 1 000000000000000c - - - DECIMAL64 18 3 VALUE 00000000000004b0 - - - 1.200 +64_downscale_exact DECIMAL64 3 00000000000004b0 - - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +64_downscale_inexact DECIMAL64 3 00000000000004b1 - - - DECIMAL64 18 1 INVALID - - - - - +128_zero DECIMAL128 2 0000000000000000 0000000000000000 - - DECIMAL128 38 2 VALUE 0000000000000000 0000000000000000 - - 0.00 +128_negative DECIMAL128 2 fffffffffffffff4 ffffffffffffffff - - DECIMAL128 38 2 VALUE fffffffffffffff4 ffffffffffffffff - - -0.12 +128_upscale DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL128 38 3 VALUE 00000000000004b0 0000000000000000 - - 1.200 +128_downscale_exact DECIMAL128 3 00000000000004b0 0000000000000000 - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +128_downscale_inexact DECIMAL128 3 00000000000004b1 0000000000000000 - - DECIMAL128 38 1 INVALID - - - - - +256_zero DECIMAL256 2 0000000000000000 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 2 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.00 +256_negative DECIMAL256 2 fffffffffffffff4 ffffffffffffffff ffffffffffffffff ffffffffffffffff DECIMAL256 76 2 VALUE fffffffffffffff4 ffffffffffffffff ffffffffffffffff ffffffffffffffff -0.12 +256_upscale DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 3 VALUE 00000000000004b0 0000000000000000 0000000000000000 0000000000000000 1.200 +256_downscale_exact DECIMAL256 3 00000000000004b0 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +256_downscale_inexact DECIMAL256 3 00000000000004b1 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 INVALID - - - - - +8_precision_max DECIMAL64 0 0000000000000063 - - - DECIMAL8 2 0 VALUE 0000000000000063 - - - 99 +8_precision_overflow DECIMAL64 0 0000000000000064 - - - DECIMAL8 2 0 INVALID - - - - - +16_precision_max DECIMAL64 0 000000000000270f - - - DECIMAL16 4 0 VALUE 000000000000270f - - - 9999 +16_precision_overflow DECIMAL64 0 0000000000002710 - - - DECIMAL16 4 0 INVALID - - - - - +32_precision_max DECIMAL64 0 000000003b9ac9ff - - - DECIMAL32 9 0 VALUE 000000003b9ac9ff - - - 999999999 +32_precision_overflow DECIMAL64 0 000000003b9aca00 - - - DECIMAL32 9 0 INVALID - - - - - +64_precision_max DECIMAL64 0 0de0b6b3a763ffff - - - DECIMAL64 18 0 VALUE 0de0b6b3a763ffff - - - 999999999999999999 +64_precision_overflow DECIMAL64 0 0de0b6b3a7640000 - - - DECIMAL64 18 0 INVALID - - - - - +128_precision_max DECIMAL128 0 098a223fffffffff 4b3b4ca85a86c47a - - DECIMAL128 38 0 VALUE 098a223fffffffff 4b3b4ca85a86c47a - - 99999999999999999999999999999999999999 +128_precision_overflow DECIMAL128 0 098a224000000000 4b3b4ca85a86c47a - - DECIMAL128 38 0 INVALID - - - - - +256_precision_max DECIMAL256 0 ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 DECIMAL256 76 0 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_precision_overflow DECIMAL256 0 0000000000000000 7775a5f171951000 0764b4abe8652979 161bcca7119915b5 DECIMAL256 76 0 INVALID - - - - - +8_precision_negative_max DECIMAL64 0 ffffffffffffff9d - - - DECIMAL8 2 0 VALUE ffffffffffffff9d - - - -99 +8_precision_negative_overflow DECIMAL64 0 ffffffffffffff9c - - - DECIMAL8 2 0 INVALID - - - - - +16_precision_negative_max DECIMAL64 0 ffffffffffffd8f1 - - - DECIMAL16 4 0 VALUE ffffffffffffd8f1 - - - -9999 +16_precision_negative_overflow DECIMAL64 0 ffffffffffffd8f0 - - - DECIMAL16 4 0 INVALID - - - - - +32_precision_negative_max DECIMAL64 0 ffffffffc4653601 - - - DECIMAL32 9 0 VALUE ffffffffc4653601 - - - -999999999 +32_precision_negative_overflow DECIMAL64 0 ffffffffc4653600 - - - DECIMAL32 9 0 INVALID - - - - - +64_precision_negative_max DECIMAL64 0 f21f494c589c0001 - - - DECIMAL64 18 0 VALUE f21f494c589c0001 - - - -999999999999999999 +64_precision_negative_overflow DECIMAL64 0 f21f494c589c0000 - - - DECIMAL64 18 0 INVALID - - - - - +128_precision_negative_max DECIMAL128 0 f675ddc000000001 b4c4b357a5793b85 - - DECIMAL128 38 0 VALUE f675ddc000000001 b4c4b357a5793b85 - - -99999999999999999999999999999999999999 +128_precision_negative_overflow DECIMAL128 0 f675ddc000000000 b4c4b357a5793b85 - - DECIMAL128 38 0 INVALID - - - - - +256_precision_negative_max DECIMAL256 0 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a DECIMAL256 76 0 VALUE 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a -9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_precision_negative_overflow DECIMAL256 0 0000000000000000 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a DECIMAL256 76 0 INVALID - - - - - +64_zero_max_scale DECIMAL64 18 0000000000000000 - - - DECIMAL64 18 18 VALUE 0000000000000000 - - - 0.000000000000000000 +128_zero_max_scale DECIMAL128 38 0000000000000000 0000000000000000 - - DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +256_zero_max_scale DECIMAL256 76 0000000000000000 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv new file mode 100644 index 000000000..40af5fb0f --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv @@ -0,0 +1,25 @@ +# case_id source_type source_scale source_ll_hex source_lh_hex source_hl_hex source_hh_hex target_type input_form expected_text +64_string_positive_trailing DECIMAL64 2 0000000000003034 - - - STRING VALUE 123.40 +64_varchar_negative_fraction DECIMAL64 3 ffffffffffffff83 - - - VARCHAR VALUE -0.125 +64_string_precision_max DECIMAL64 0 0de0b6b3a763ffff - - - STRING VALUE 999999999999999999 +64_varchar_negative_max_scale DECIMAL64 18 f21f494c589c0001 - - - VARCHAR VALUE -0.999999999999999999 +64_string_leading_fraction DECIMAL64 18 0000000000000001 - - - STRING VALUE 0.000000000000000001 +64_varchar_zero_max_scale DECIMAL64 18 0000000000000000 - - - VARCHAR VALUE 0.000000000000000000 +128_string_positive_trailing DECIMAL128 2 0000000000003034 0000000000000000 - - STRING VALUE 123.40 +128_varchar_negative_fraction DECIMAL128 3 ffffffffffffff83 ffffffffffffffff - - VARCHAR VALUE -0.125 +128_string_precision_max DECIMAL128 0 098a223fffffffff 4b3b4ca85a86c47a - - STRING VALUE 99999999999999999999999999999999999999 +128_varchar_negative_max_scale DECIMAL128 38 f675ddc000000001 b4c4b357a5793b85 - - VARCHAR VALUE -0.99999999999999999999999999999999999999 +128_string_leading_fraction DECIMAL128 38 0000000000000001 0000000000000000 - - STRING VALUE 0.00000000000000000000000000000000000001 +128_varchar_zero_max_scale DECIMAL128 38 0000000000000000 0000000000000000 - - VARCHAR VALUE 0.00000000000000000000000000000000000000 +256_string_positive_trailing DECIMAL256 2 0000000000003034 0000000000000000 0000000000000000 0000000000000000 STRING VALUE 123.40 +256_varchar_negative_fraction DECIMAL256 3 ffffffffffffff83 ffffffffffffffff ffffffffffffffff ffffffffffffffff VARCHAR VALUE -0.125 +256_string_precision_max DECIMAL256 0 ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 STRING VALUE 9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_varchar_negative_max_scale DECIMAL256 76 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a VARCHAR VALUE -0.9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_string_leading_fraction DECIMAL256 76 0000000000000001 0000000000000000 0000000000000000 0000000000000000 STRING VALUE 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +256_varchar_zero_max_scale DECIMAL256 76 0000000000000000 0000000000000000 0000000000000000 0000000000000000 VARCHAR VALUE 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +64_string_java_null DECIMAL64 0 - - - - STRING JAVA_NULL - +64_varchar_null_value DECIMAL64 0 - - - - VARCHAR NULL_VALUE - +128_string_java_null DECIMAL128 0 - - - - STRING JAVA_NULL - +128_varchar_null_value DECIMAL128 0 - - - - VARCHAR NULL_VALUE - +256_string_java_null DECIMAL256 0 - - - - STRING JAVA_NULL - +256_varchar_null_value DECIMAL256 0 - - - - VARCHAR NULL_VALUE - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv new file mode 100644 index 000000000..484515cc0 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv @@ -0,0 +1,16 @@ +# case_id input_bits precision_bits expected_value_hex expected_text +zero_1 0 1 0 0 +one_1 1 1 1 1 +zero_5 0 5 0 00000 +masked_5 -1 5 1f 11111 +max_7 127 7 7f 1111111 +max_8 255 8 ff 11111111 +max_15 32767 15 7fff 111111111111111 +max_16 65535 16 ffff 1111111111111111 +leading_zero_20 74565 20 12345 00010010001101000101 +pattern_20 703710 20 abcde 10101011110011011110 +max_31 2147483647 31 7fffffff 1111111111111111111111111111111 +max_32 4294967295 32 ffffffff 11111111111111111111111111111111 +max_35 34359738367 35 7ffffffff 11111111111111111111111111111111111 +masked_20 1152921504606846975 20 fffff 11111111111111111111 +max_60 1152921504606846975 60 fffffffffffffff 111111111111111111111111111111111111111111111111111111111111 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv new file mode 100644 index 000000000..89f5b126d --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv @@ -0,0 +1,31 @@ +# case_id input_type input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +dec8_byte_pos BYTE 9 DECIMAL8 2 1 VALUE 000000000000005a - - - 9.0 +dec8_short_neg SHORT -9 DECIMAL8 2 1 VALUE ffffffffffffffa6 - - - -9.0 +dec8_int_pos INT 5 DECIMAL8 2 1 VALUE 0000000000000032 - - - 5.0 +dec8_int_null INT -2147483648 DECIMAL8 2 1 NULL - - - - NULL +dec16_byte_pos BYTE 99 DECIMAL16 4 2 VALUE 00000000000026ac - - - 99.00 +dec16_short_neg SHORT -99 DECIMAL16 4 2 VALUE ffffffffffffd954 - - - -99.00 +dec16_int_pos INT 42 DECIMAL16 4 2 VALUE 0000000000001068 - - - 42.00 +dec16_int_null INT -2147483648 DECIMAL16 4 2 NULL - - - - NULL +dec32_byte_min BYTE -128 DECIMAL32 9 3 VALUE fffffffffffe0c00 - - - -128.000 +dec32_short_max SHORT 32767 DECIMAL32 9 3 VALUE 0000000001f3fc18 - - - 32767.000 +dec32_int_neg INT -999999 DECIMAL32 9 3 VALUE ffffffffc46539e8 - - - -999999.000 +dec32_int_null INT -2147483648 DECIMAL32 9 3 NULL - - - - NULL +dec64_byte_max BYTE 127 DECIMAL64 18 2 VALUE 000000000000319c - - - 127.00 +dec64_short_min SHORT -32768 DECIMAL64 18 2 VALUE ffffffffffce0000 - - - -32768.00 +dec64_int_max INT 2147483647 DECIMAL64 18 2 VALUE 00000031ffffff9c - - - 2147483647.00 +dec64_int_null INT -2147483648 DECIMAL64 18 2 NULL - - - - NULL +dec128_byte_min BYTE -128 DECIMAL128 38 20 VALUE 1c50e94e78000000 fffffffffffffd4a - - -128.00000000000000000000 +dec128_short_max SHORT 32767 DECIMAL128 38 20 VALUE 434f535a9cf00000 000000000002b5de - - 32767.00000000000000000000 +dec128_int_min_plus_one INT -2147483647 DECIMAL128 38 20 VALUE ba3f5e2d63100000 fffffffd4a1c50ee - - -2147483647.00000000000000000000 +dec128_int_null INT -2147483648 DECIMAL128 38 20 NULL - - - - NULL +dec256_byte_max BYTE 127 DECIMAL256 76 57 VALUE b600000000000000 12e7deee9f226721 3b75de2e84789384 0000000000000014 127.000000000000000000000000000000000000000000000000000000000 +dec256_short_min SHORT -32768 DECIMAL256 76 57 VALUE 0000000000000000 1a55bcda92be5b00 c1a51bb2ed470a01 ffffffffffffeb9b -32768.000000000000000000000000000000000000000000000000000000000 +dec256_int_max INT 2147483647 DECIMAL256 76 57 VALUE b600000000000000 5727a1ed1eb5257c bb8496032d64c038 0000000014643e5a 2147483647.000000000000000000000000000000000000000000000000000000000 +dec256_int_null INT -2147483648 DECIMAL256 76 57 NULL - - - - NULL +byte_min_scale_zero BYTE -128 DECIMAL16 3 0 VALUE ffffffffffffff80 - - - -128 +short_min_scale_zero SHORT -32768 DECIMAL32 5 0 VALUE ffffffffffff8000 - - - -32768 +int_min_plus_one_scale_zero INT -2147483647 DECIMAL64 10 0 VALUE ffffffff80000001 - - - -2147483647 +byte_precision_overflow BYTE 10 DECIMAL8 2 1 INVALID - - - - - +short_precision_overflow SHORT 100 DECIMAL16 4 2 INVALID - - - - - +int_precision_overflow INT 1000000 DECIMAL32 9 3 INVALID - - - - - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv new file mode 100644 index 000000000..e429960b1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv @@ -0,0 +1,66 @@ +# case_id input_type input target expected +# FLOAT and DOUBLE expectations are exact raw IEEE-754 bits. INT_MIN is a stable source NULL. +byte_min_byte BYTE -128 BYTE -128 +byte_min_short BYTE -128 SHORT -128 +byte_min_int BYTE -128 INT -128 +byte_min_long BYTE -128 LONG -128 +byte_min_float BYTE -128 FLOAT 0xc3000000 +byte_min_double BYTE -128 DOUBLE 0xc060000000000000 +byte_zero_byte BYTE 0 BYTE 0 +byte_zero_short BYTE 0 SHORT 0 +byte_zero_int BYTE 0 INT 0 +byte_zero_long BYTE 0 LONG 0 +byte_zero_float BYTE 0 FLOAT 0x00000000 +byte_zero_double BYTE 0 DOUBLE 0x0000000000000000 +byte_max_byte BYTE 127 BYTE 127 +byte_max_short BYTE 127 SHORT 127 +byte_max_int BYTE 127 INT 127 +byte_max_long BYTE 127 LONG 127 +byte_max_float BYTE 127 FLOAT 0x42fe0000 +byte_max_double BYTE 127 DOUBLE 0x405fc00000000000 +short_min_byte SHORT -32768 BYTE +short_min_short SHORT -32768 SHORT -32768 +short_min_int SHORT -32768 INT -32768 +short_min_long SHORT -32768 LONG -32768 +short_min_float SHORT -32768 FLOAT 0xc7000000 +short_min_double SHORT -32768 DOUBLE 0xc0e0000000000000 +short_below_byte SHORT -129 BYTE +short_byte_min SHORT -128 BYTE -128 +short_byte_max SHORT 127 BYTE 127 +short_above_byte SHORT 128 BYTE +short_max_byte SHORT 32767 BYTE +short_max_short SHORT 32767 SHORT 32767 +short_max_int SHORT 32767 INT 32767 +short_max_long SHORT 32767 LONG 32767 +short_max_float SHORT 32767 FLOAT 0x46fffe00 +short_max_double SHORT 32767 DOUBLE 0x40dfffc000000000 +int_min_byte INT -2147483648 BYTE +int_min_short INT -2147483648 SHORT +int_min_int INT -2147483648 INT +int_min_long INT -2147483648 LONG +int_min_float INT -2147483648 FLOAT +int_min_double INT -2147483648 DOUBLE +int_min_plus_one_byte INT -2147483647 BYTE +int_min_plus_one_short INT -2147483647 SHORT +int_min_plus_one_int INT -2147483647 INT -2147483647 +int_min_plus_one_long INT -2147483647 LONG -2147483647 +int_min_plus_one_float INT -2147483647 FLOAT 0xcf000000 +int_min_plus_one_double INT -2147483647 DOUBLE 0xc1dfffffffc00000 +int_below_short INT -32769 SHORT +int_short_min INT -32768 SHORT -32768 +int_below_byte INT -129 BYTE +int_byte_min INT -128 BYTE -128 +int_byte_max INT 127 BYTE 127 +int_above_byte INT 128 BYTE +int_128_short INT 128 SHORT 128 +int_short_max INT 32767 SHORT 32767 +int_above_short INT 32768 SHORT +int_32768_int INT 32768 INT 32768 +int_float_round INT 16777217 FLOAT 0x4b800000 +int_double_exact INT 16777217 DOUBLE 0x4170000010000000 +int_max_byte INT 2147483647 BYTE +int_max_short INT 2147483647 SHORT +int_max_int INT 2147483647 INT 2147483647 +int_max_long INT 2147483647 LONG 2147483647 +int_max_float INT 2147483647 FLOAT 0x4f000000 +int_max_double INT 2147483647 DOUBLE 0x41dfffffffc00000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv new file mode 100644 index 000000000..32b77a1b5 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv @@ -0,0 +1,31 @@ +# case_id input_type input_value target_type expected_utf8_hex expected_sql_value +byte_string_min BYTE -128 STRING 2d313238 -128 +byte_string_max BYTE 127 STRING 313237 127 +short_string_min SHORT -32768 STRING 2d3332373638 -32768 +short_string_max SHORT 32767 STRING 3332373637 32767 +int_string_null INT -2147483648 STRING +int_string_min_value INT -2147483647 STRING 2d32313437343833363437 -2147483647 +int_string_neg_one INT -1 STRING 2d31 -1 +int_string_zero INT 0 STRING 30 0 +int_string_one INT 1 STRING 31 1 +int_string_max INT 2147483647 STRING 32313437343833363437 2147483647 +byte_varchar_min BYTE -128 VARCHAR 2d313238 -128 +byte_varchar_max BYTE 127 VARCHAR 313237 127 +short_varchar_min SHORT -32768 VARCHAR 2d3332373638 -32768 +short_varchar_max SHORT 32767 VARCHAR 3332373637 32767 +int_varchar_null INT -2147483648 VARCHAR +int_varchar_min_value INT -2147483647 VARCHAR 2d32313437343833363437 -2147483647 +int_varchar_neg_one INT -1 VARCHAR 2d31 -1 +int_varchar_zero INT 0 VARCHAR 30 0 +int_varchar_one INT 1 VARCHAR 31 1 +int_varchar_max INT 2147483647 VARCHAR 32313437343833363437 2147483647 +byte_symbol_min BYTE -128 SYMBOL 2d313238 -128 +byte_symbol_max BYTE 127 SYMBOL 313237 127 +short_symbol_min SHORT -32768 SYMBOL 2d3332373638 -32768 +short_symbol_max SHORT 32767 SYMBOL 3332373637 32767 +int_symbol_null INT -2147483648 SYMBOL +int_symbol_min_value INT -2147483647 SYMBOL 2d32313437343833363437 -2147483647 +int_symbol_neg_one INT -1 SYMBOL 2d31 -1 +int_symbol_zero INT 0 SYMBOL 30 0 +int_symbol_one INT 1 SYMBOL 31 1 +int_symbol_max INT 2147483647 SYMBOL 32313437343833363437 2147483647 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv new file mode 100644 index 000000000..0b31a9fdd --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv @@ -0,0 +1,19 @@ +# case_id input bits outcome value_hex +bit1_max z 1 VALUE 1 +bit7_max zz 7 VALUE 7f +bit8_max zz 8 VALUE ff +bit15_max zzz 15 VALUE 7fff +bit16_max zzzz 16 VALUE ffff +bit31_max zzzzzzz 31 VALUE 7fffffff +bit32_max zzzzzzz 32 VALUE ffffffff +bit60_max zzzzzzzzzzzz 60 VALUE fffffffffffffff +uppercase U 5 VALUE 1a +zero 0 5 VALUE 0 +whole_prefix_consumed 10 5 VALUE 1 +suffix_after_cap zzzzzzzzzzzz! 60 VALUE fffffffffffffff +unicode_after_cap zzzzzzzzzzzzé 60 VALUE fffffffffffffff +source_null 8 NULL +empty 8 NULL +too_short zzzzzzzzzzz 60 INVALID +invalid_consumed a 5 INVALID +unicode_consumed é 5 INVALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv new file mode 100644 index 000000000..943b815d9 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv @@ -0,0 +1,24 @@ +# case_id input outcome l0_hex l1_hex l2_hex l3_hex +zero 0x00 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 +byte_max 0xff VALUE 00000000000000ff 0000000000000000 0000000000000000 0000000000000000 +uppercase_digits 0xABCD VALUE 000000000000abcd 0000000000000000 0000000000000000 0000000000000000 +leading_zeroes 0x0001 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 +two_limbs 0x0123456789abcdeffedcba9876543210 VALUE fedcba9876543210 0123456789abcdef 0000000000000000 0000000000000000 +four_limbs 0x0123456789abcdeffedcba9876543210112233445566778899aabbccddeeff00 VALUE 99aabbccddeeff00 1122334455667788 fedcba9876543210 0123456789abcdef +all_ones 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff VALUE ffffffffffffffff ffffffffffffffff ffffffffffffffff ffffffffffffffff +single_min_limb 0x8000000000000000 VALUE 8000000000000000 0000000000000000 0000000000000000 0000000000000000 +partial_high_group 0x00112233445566778899 VALUE 2233445566778899 0000000000000011 0000000000000000 0000000000000000 +three_limbs 0x111111111111111122222222222222223333333333333333 VALUE 3333333333333333 2222222222222222 1111111111111111 0000000000000000 +source_null NULL +empty_string INVALID +empty_digits 0x INVALID +odd_digits 0x0 INVALID +uppercase_prefix 0X00 INVALID +leading_sign -0x00 INVALID +leading_space 0x00 INVALID +internal_underscore 0x0_0 INVALID +non_hex 0xgg INVALID +non_ascii 0xéé INVALID +trailing_space 0x00 INVALID +too_long 0x000000000000000000000000000000000000000000000000000000000000000000 INVALID +null_sentinel 0x8000000000000000800000000000000080000000000000008000000000000000 INVALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv new file mode 100644 index 000000000..9be92bbd1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv @@ -0,0 +1,81 @@ +# case_id input_utf16_hex expected_wire_value expected_sql_value +true_case_00 0074007200750065 1 true +true_case_01 0054007200750065 1 true +true_case_02 0074005200750065 1 true +true_case_03 0054005200750065 1 true +true_case_04 0074007200550065 1 true +true_case_05 0054007200550065 1 true +true_case_06 0074005200550065 1 true +true_case_07 0054005200550065 1 true +true_case_08 0074007200750045 1 true +true_case_09 0054007200750045 1 true +true_case_10 0074005200750045 1 true +true_case_11 0054005200750045 1 true +true_case_12 0074007200550045 1 true +true_case_13 0054007200550045 1 true +true_case_14 0074005200550045 1 true +true_case_15 0054005200550045 1 true +false_case_00 00660061006c00730065 0 false +false_case_01 00460061006c00730065 0 false +false_case_02 00660041006c00730065 0 false +false_case_03 00460041006c00730065 0 false +false_case_04 00660061004c00730065 0 false +false_case_05 00460061004c00730065 0 false +false_case_06 00660041004c00730065 0 false +false_case_07 00460041004c00730065 0 false +false_case_08 00660061006c00530065 0 false +false_case_09 00460061006c00530065 0 false +false_case_10 00660041006c00530065 0 false +false_case_11 00460041006c00530065 0 false +false_case_12 00660061004c00530065 0 false +false_case_13 00460061004c00530065 0 false +false_case_14 00660041004c00530065 0 false +false_case_15 00460041004c00530065 0 false +false_case_16 00660061006c00730045 0 false +false_case_17 00460061006c00730045 0 false +false_case_18 00660041006c00730045 0 false +false_case_19 00460041006c00730045 0 false +false_case_20 00660061004c00730045 0 false +false_case_21 00460061004c00730045 0 false +false_case_22 00660041004c00730045 0 false +false_case_23 00460041004c00730045 0 false +false_case_24 00660061006c00530045 0 false +false_case_25 00460061006c00530045 0 false +false_case_26 00660041006c00530045 0 false +false_case_27 00460041006c00530045 0 false +false_case_28 00660061004c00530045 0 false +false_case_29 00460061004c00530045 0 false +false_case_30 00660041004c00530045 0 false +false_case_31 00460041004c00530045 0 false +digit_zero 0030 0 false +digit_one 0031 1 true +explicit_null false +empty +leading_space 00200074007200750065 +trailing_space 00740072007500650020 +leading_tab 00090074007200750065 +trailing_lf 0074007200750065000a +double_zero 00300030 +zero_one 00300031 +one_zero 00310030 +plus_one 002b0031 +minus_zero 002d0030 +yes 007900650073 +truth 00740072007500740068 +falsehood 00660061006c007300650068006f006f0064 +unicode_capital_t 0166007200750065 +fullwidth_one ff11 +nul_true 00000074007200750065 +lone_high d800 +lone_low dc00 +true_nul 00740072007500650000 +single_t 0074 +single_f 0066 +no 006e006f +null_word 006e0075006c006c +digit_two 0032 +minus_one 002d0031 +true_question 0074007200750065003f +spaces_only 00200020 +false_long_s 00660061006c017f0065 +true_embedded_high 00740072d8000065 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv new file mode 100644 index 000000000..d7c321ebb --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv @@ -0,0 +1,54 @@ +case_id input_kind input outcome expected_raw feature +java_null TEXT NULL java_null +empty TEXT INVALID empty +raw_zero TEXT 0 VALID 0 raw_millis +raw_negative TEXT -1 VALID -1 raw_millis +raw_suffix TEXT 1577836800000L VALID 1577836800000 raw_suffix +raw_underscore TEXT 1_000 VALID 1000 raw_underscore +raw_null_sentinel TEXT -9223372036854775808 NULL raw_null_sentinel +raw_max TEXT 9223372036854775807 VALID 9223372036854775807 raw_boundary +raw_overflow TEXT 9223372036854775808 INVALID raw_overflow +epoch_date TEXT 1970-01-01 VALID 0 pg_date +pre_epoch_date TEXT 1969-12-31 VALID -86400000 pg_date +leap_date TEXT 2000-02-29 VALID 951782400000 pg_date +non_leap_date TEXT 1900-02-29 INVALID calendar_validation +date_time_utc TEXT 2020-01-01 00:00:00UTC VALID 1577836800000 pg_datetime +date_zone_utc TEXT 2020-01-01 UTC VALID 1577836800000 pg_date_zone +fraction_one TEXT 2020-01-01 00:00:00.1UTC VALID 1577836800100 greedy_millis +fraction_two TEXT 2020-01-01 00:00:00.12UTC VALID 1577836800120 greedy_millis +fraction_three TEXT 2020-01-01 00:00:00.123UTC VALID 1577836800123 greedy_millis +fraction_negative TEXT 2020-01-01 00:00:00.-1UTC VALID 1577836799990 greedy_millis +fraction_four TEXT 2020-01-01 00:00:00.1234UTC INVALID greedy_millis +iso_z TEXT 2020-01-01T00:00:00.123Z VALID 1577836800123 utc_format +iso_named_utc TEXT 2020-01-01T00:00:00.123UTC VALID 1577836800123 utc_format +iso_offset TEXT 2020-01-01T00:00:00.123+02:30 VALID 1577827800123 utc_format +iso_without_millis TEXT 2020-01-01T00:00:00Z INVALID utc_format +iso_two_millis TEXT 2020-01-01T00:00:00.12Z INVALID utc_format +iso_four_millis TEXT 2020-01-01T00:00:00.1234Z INVALID utc_format +offset_datetime TEXT 2020-01-01 00:00:00+02:00 VALID 1577829600000 numeric_zone +offset_date TEXT 2020-01-01 +02:00 VALID 1577829600000 numeric_zone +berlin_winter TEXT 2020-01-01 12:00:00Europe/Berlin VALID 1577876400000 named_zone +berlin_summer TEXT 2020-07-01 12:00:00Europe/Berlin VALID 1593597600000 named_zone +berlin_gap TEXT 2020-03-29 02:30:00Europe/Berlin VALID 1585441800000 zone_transition +berlin_overlap TEXT 2020-10-25 02:30:00Europe/Berlin VALID 1603589400000 zone_transition +hour_24 TEXT 2020-01-01 24:59:59UTC VALID 1577840399000 hour_24_wrap +hour_25 TEXT 2020-01-01 25:00:00UTC INVALID time_validation +minute_60 TEXT 2020-01-01 00:60:00UTC INVALID time_validation +second_60 TEXT 2020-01-01 00:00:60UTC INVALID time_validation +year_zero TEXT 0000-01-01 VALID -62167219200000 proleptic_year +negative_year TEXT -0001-01-01 VALID -62198755200000 proleptic_year +max_int_year TEXT 2147483647-01-01 VALID -6019000092841406464 year_wrap +min_int_year TEXT -2147483648-01-01 VALID 6018875726867006464 year_wrap +negative_year_named_zone_wrap TEXT -1000000000-01-01 00:00:00Europe/London INVALID server_unchecked_guard +year_overflow TEXT 2147483648-01-01 INVALID year_overflow +plus_year TEXT +1-01-01 INVALID strict_year +underscore_year TEXT 2_0-01-01 INVALID strict_year +bad_month TEXT 2020-13-01 INVALID calendar_validation +bad_day TEXT 2020-02-30 INVALID calendar_validation +missing_zone TEXT 2020-01-01 00:00:00 INVALID zone_required +unknown_zone TEXT 2020-01-01 Mars/Base INVALID zone_validation +trailing_date_space TEXT 2020-01-01 INVALID no_tail +trailing_zone_space TEXT 2020-01-01 UTC INVALID no_tail +raw_trailing_text TEXT 1_000x INVALID no_tail +unicode_digits UTF16_HEX ff11,ff12,ff13 INVALID ascii_only +plain_text TEXT not-a-date INVALID invalid_text diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv new file mode 100644 index 000000000..2fc76ec03 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv @@ -0,0 +1,56 @@ +# case_id input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +8_precision_max 9.9 DECIMAL8 2 1 VALUE 0000000000000063 - - - 9.9 +8_precision_over 10.0 DECIMAL8 2 1 INVALID - - - - - +8_precision_loss 0.01 DECIMAL8 2 1 INVALID - - - - - +8_java_null DECIMAL8 2 1 NULL - - - - NULL +16_precision_max 99.99 DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +16_precision_over 100.00 DECIMAL16 4 2 INVALID - - - - - +16_precision_loss 0.001 DECIMAL16 4 2 INVALID - - - - - +16_java_null DECIMAL16 4 2 NULL - - - - NULL +32_precision_max 999999.999 DECIMAL32 9 3 VALUE 000000003b9ac9ff - - - 999999.999 +32_precision_over 1000000.000 DECIMAL32 9 3 INVALID - - - - - +32_precision_loss 0.0001 DECIMAL32 9 3 INVALID - - - - - +32_java_null DECIMAL32 9 3 NULL - - - - NULL +64_precision_max 99999999999999999.9 DECIMAL64 18 1 VALUE 0de0b6b3a763ffff - - - 99999999999999999.9 +64_precision_over 100000000000000000.0 DECIMAL64 18 1 INVALID - - - - - +64_precision_loss 0.01 DECIMAL64 18 1 INVALID - - - - - +64_java_null DECIMAL64 18 1 NULL - - - - NULL +128_precision_max 999999999999999999.99999999999999999999 DECIMAL128 38 20 VALUE 098a223fffffffff 4b3b4ca85a86c47a - - 999999999999999999.99999999999999999999 +128_precision_over 1000000000000000000.0 DECIMAL128 38 20 INVALID - - - - - +128_precision_loss 0.000000000000000000001 DECIMAL128 38 20 INVALID - - - - - +128_java_null DECIMAL128 38 20 NULL - - - - NULL +256_precision_max 9999999999999999999.999999999999999999999999999999999999999999999999999999999 DECIMAL256 76 57 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 9999999999999999999.999999999999999999999999999999999999999999999999999999999 +256_precision_over 10000000000000000000 DECIMAL256 76 57 INVALID - - - - - +256_precision_loss 1e-58 DECIMAL256 76 57 INVALID - - - - - +256_java_null DECIMAL256 76 57 NULL - - - - NULL +8_leading_space_plus_zeros +00.5 DECIMAL8 2 1 VALUE 0000000000000005 - - - 0.5 +8_zero_exponent_after_leading_zeros 0000.e80 DECIMAL8 1 0 VALUE 0000000000000000 - - - 0 +8_full_scale_fraction .5 DECIMAL8 1 1 VALUE 0000000000000005 - - - 0.5 +8_nan NaN DECIMAL8 2 1 NULL - - - - NULL +8_empty DECIMAL8 2 1 INVALID - - - - - +16_point_without_fraction 123. DECIMAL16 4 1 VALUE 00000000000004ce - - - 123.0 +16_infinity Infinity DECIMAL16 4 1 NULL - - - - NULL +16_point_only . DECIMAL16 4 1 INVALID - - - - - +32_positive_exponent 1.23e3 DECIMAL32 9 2 VALUE 000000000001e078 - - - 1230.00 +32_positive_nan +NaN DECIMAL32 9 2 NULL - - - - NULL +32_multiple_points 1.2.3 DECIMAL32 9 2 INVALID - - - - - +64_negative_exponent -2.5e-2 DECIMAL64 18 3 VALUE ffffffffffffffe7 - - - -0.025 +64_incomplete_exponent 1e DECIMAL64 18 3 INVALID - - - - - +64_exponent_underscore 1e1_0 DECIMAL64 18 2 VALUE 000000e8d4a51000 - - - 10000000000.00 +64_nan_decimal_suffix NaNm DECIMAL64 18 3 NULL - - - - NULL +128_trailing_zeros_relaxed 1.2300 DECIMAL128 38 2 VALUE 000000000000007b 0000000000000000 - - 1.23 +64_double_suffix -7.5d DECIMAL64 18 2 VALUE fffffffffffffd12 - - - -7.50 +128_float_suffix 123.0f DECIMAL128 38 4 VALUE 000000000012c4b0 0000000000000000 - - 123.0000 +128_decimal_suffix_strict_loss 1.2300M DECIMAL128 38 2 INVALID - - - - - +128_float_then_decimal_suffix_strict_loss 1.2300fm DECIMAL128 38 2 INVALID - - - - - +128_decimal_then_float_suffix_invalid 1.2300mf DECIMAL128 38 2 INVALID - - - - - +128_nan NaN DECIMAL128 38 4 NULL - - - - NULL +128_nan_junk NaNjunk DECIMAL128 38 4 NULL - - - - NULL +256_negative_infinity -Infinity DECIMAL256 76 8 NULL - - - - NULL +256_infinity_junk Infinitygarbage DECIMAL256 76 8 NULL - - - - NULL +256_lowercase_nan nan DECIMAL256 76 8 INVALID - - - - - +256_upper_exponent 7.89E-3 DECIMAL256 76 5 VALUE 0000000000000315 0000000000000000 0000000000000000 0000000000000000 0.00789 +256_exponent_int_max 1e2147483647 DECIMAL256 76 0 INVALID - - - - - +256_exponent_int_min 1e-2147483648 DECIMAL256 76 0 INVALID - - - - - +256_full_scale_fraction 0.9999999999999999999999999999999999999999999999999999999999999999999999999999 DECIMAL256 76 76 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 0.9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_full_scale_zero 0 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv new file mode 100644 index 000000000..5f7c243bf --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv @@ -0,0 +1,433 @@ +# Seven fields: case_id, input_utf16_hex, target_type, expected_wire_type, expected_wire_bits, expected_sql_value, outcome +# Input is lowercase UTF-16 code-unit hex; empty is a present empty string and is explicit null. +# Wire bits are lowercase fixed-width hexadecimal. means bitmap null/no value bytes; paired fields mean local rejection. +# BITS: means stored floating raw bits. Parsed INT/LONG minima and floating NaN have present wire bits but SQL NULL. +diag_001 BYTE BYTE 0 VALID +diag_002 BYTE BYTE INVALID +diag_003 0030 BYTE BYTE 00 0 VALID +diag_004 002d0030 BYTE BYTE 00 0 VALID +diag_005 002b0030 BYTE BYTE 00 0 VALID +diag_006 00300030 BYTE BYTE 00 0 VALID +diag_007 0031 BYTE BYTE 01 1 VALID +diag_008 002b0031 BYTE BYTE 01 1 VALID +diag_009 002d0031 BYTE BYTE ff -1 VALID +diag_010 003100320037 BYTE BYTE 7f 127 VALID +diag_011 003100320038 BYTE BYTE INVALID +diag_012 002d003100320038 BYTE BYTE 80 -128 VALID +diag_013 002d003100320039 BYTE BYTE INVALID +diag_014 00330032003700360037 BYTE BYTE INVALID +diag_015 00330032003700360038 BYTE BYTE INVALID +diag_016 002d00330032003700360038 BYTE BYTE INVALID +diag_017 002d00330032003700360039 BYTE BYTE INVALID +diag_018 0032003100340037003400380033003600340037 BYTE BYTE INVALID +diag_019 0032003100340037003400380033003600340038 BYTE BYTE INVALID +diag_020 002d0032003100340037003400380033003600340038 BYTE BYTE INVALID +diag_021 002d0032003100340037003400380033003600340039 BYTE BYTE INVALID +diag_022 0039003200320033003300370032003000330036003800350034003700370035003800300037 BYTE BYTE INVALID +diag_023 0039003200320033003300370032003000330036003800350034003700370035003800300038 BYTE BYTE INVALID +diag_024 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 BYTE BYTE INVALID +diag_025 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 BYTE BYTE INVALID +diag_026 0031002e0030 BYTE BYTE INVALID +diag_027 0031002e0035 BYTE BYTE INVALID +diag_028 003100650032 BYTE BYTE INVALID +diag_029 00310065002d0032 BYTE BYTE INVALID +diag_030 0031005f0030 BYTE BYTE 0a 10 VALID +diag_031 002d0031005f0030 BYTE BYTE f6 -10 VALID +diag_032 005f0031 BYTE BYTE INVALID +diag_033 0031005f BYTE BYTE INVALID +diag_034 0031005f005f0030 BYTE BYTE INVALID +diag_035 0031007f0030 BYTE BYTE INVALID +diag_036 0031004c BYTE BYTE INVALID +diag_037 0031006c BYTE BYTE INVALID +diag_038 00200031 BYTE BYTE INVALID +diag_039 00310020 BYTE BYTE INVALID +diag_040 00090031 BYTE BYTE INVALID +diag_041 0031000a BYTE BYTE INVALID +diag_042 0030007800310030 BYTE BYTE INVALID +diag_043 004e0061004e BYTE BYTE INVALID +diag_044 006e0061006e BYTE BYTE INVALID +diag_045 0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_046 002b0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_047 002d0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_048 0069006e0066 BYTE BYTE INVALID +diag_049 00310065003300300039 BYTE BYTE INVALID +diag_050 00310065002d0034003000300030 BYTE BYTE INVALID +diag_051 00310038003400340036003700340034003000370033003700300039003500350031003600310035 BYTE BYTE INVALID +diag_052 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 BYTE BYTE INVALID +diag_053 0000 BYTE BYTE INVALID +diag_054 00000031 BYTE BYTE INVALID +diag_055 00310000 BYTE BYTE INVALID +diag_056 003100000032 BYTE BYTE INVALID +diag_057 0661 BYTE BYTE INVALID +diag_058 00a00031 BYTE BYTE INVALID +diag_059 d800 BYTE BYTE INVALID +diag_060 0031d800 BYTE BYTE INVALID +diag_061 dc00 BYTE BYTE INVALID +diag_062 SHORT SHORT 0 VALID +diag_063 SHORT SHORT INVALID +diag_064 0030 SHORT SHORT 0000 0 VALID +diag_065 002d0030 SHORT SHORT 0000 0 VALID +diag_066 002b0030 SHORT SHORT INVALID +diag_067 00300030 SHORT SHORT 0000 0 VALID +diag_068 0031 SHORT SHORT 0001 1 VALID +diag_069 002b0031 SHORT SHORT INVALID +diag_070 002d0031 SHORT SHORT ffff -1 VALID +diag_071 003100320037 SHORT SHORT 007f 127 VALID +diag_072 003100320038 SHORT SHORT 0080 128 VALID +diag_073 002d003100320038 SHORT SHORT ff80 -128 VALID +diag_074 002d003100320039 SHORT SHORT ff7f -129 VALID +diag_075 00330032003700360037 SHORT SHORT 7fff 32767 VALID +diag_076 00330032003700360038 SHORT SHORT INVALID +diag_077 002d00330032003700360038 SHORT SHORT 8000 -32768 VALID +diag_078 002d00330032003700360039 SHORT SHORT INVALID +diag_079 0032003100340037003400380033003600340037 SHORT SHORT INVALID +diag_080 0032003100340037003400380033003600340038 SHORT SHORT INVALID +diag_081 002d0032003100340037003400380033003600340038 SHORT SHORT INVALID +diag_082 002d0032003100340037003400380033003600340039 SHORT SHORT INVALID +diag_083 0039003200320033003300370032003000330036003800350034003700370035003800300037 SHORT SHORT INVALID +diag_084 0039003200320033003300370032003000330036003800350034003700370035003800300038 SHORT SHORT INVALID +diag_085 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 SHORT SHORT INVALID +diag_086 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 SHORT SHORT INVALID +diag_087 0031002e0030 SHORT SHORT INVALID +diag_088 0031002e0035 SHORT SHORT INVALID +diag_089 003100650032 SHORT SHORT INVALID +diag_090 00310065002d0032 SHORT SHORT INVALID +diag_091 0031005f0030 SHORT SHORT INVALID +diag_092 002d0031005f0030 SHORT SHORT INVALID +diag_093 005f0031 SHORT SHORT INVALID +diag_094 0031005f SHORT SHORT INVALID +diag_095 0031005f005f0030 SHORT SHORT INVALID +diag_096 0031007f0030 SHORT SHORT INVALID +diag_097 0031004c SHORT SHORT INVALID +diag_098 0031006c SHORT SHORT INVALID +diag_099 00200031 SHORT SHORT INVALID +diag_100 00310020 SHORT SHORT INVALID +diag_101 00090031 SHORT SHORT INVALID +diag_102 0031000a SHORT SHORT INVALID +diag_103 0030007800310030 SHORT SHORT INVALID +diag_104 004e0061004e SHORT SHORT INVALID +diag_105 006e0061006e SHORT SHORT INVALID +diag_106 0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_107 002b0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_108 002d0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_109 0069006e0066 SHORT SHORT INVALID +diag_110 00310065003300300039 SHORT SHORT INVALID +diag_111 00310065002d0034003000300030 SHORT SHORT INVALID +diag_112 00310038003400340036003700340034003000370033003700300039003500350031003600310035 SHORT SHORT INVALID +diag_113 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 SHORT SHORT INVALID +diag_114 0000 SHORT SHORT INVALID +diag_115 00000031 SHORT SHORT INVALID +diag_116 00310000 SHORT SHORT INVALID +diag_117 003100000032 SHORT SHORT INVALID +diag_118 0661 SHORT SHORT INVALID +diag_119 00a00031 SHORT SHORT INVALID +diag_120 d800 SHORT SHORT INVALID +diag_121 0031d800 SHORT SHORT INVALID +diag_122 dc00 SHORT SHORT INVALID +diag_123 INT INT NULL VALID +diag_124 INT INT INVALID +diag_125 0030 INT INT 00000000 0 VALID +diag_126 002d0030 INT INT 00000000 0 VALID +diag_127 002b0030 INT INT 00000000 0 VALID +diag_128 00300030 INT INT 00000000 0 VALID +diag_129 0031 INT INT 00000001 1 VALID +diag_130 002b0031 INT INT 00000001 1 VALID +diag_131 002d0031 INT INT ffffffff -1 VALID +diag_132 003100320037 INT INT 0000007f 127 VALID +diag_133 003100320038 INT INT 00000080 128 VALID +diag_134 002d003100320038 INT INT ffffff80 -128 VALID +diag_135 002d003100320039 INT INT ffffff7f -129 VALID +diag_136 00330032003700360037 INT INT 00007fff 32767 VALID +diag_137 00330032003700360038 INT INT 00008000 32768 VALID +diag_138 002d00330032003700360038 INT INT ffff8000 -32768 VALID +diag_139 002d00330032003700360039 INT INT ffff7fff -32769 VALID +diag_140 0032003100340037003400380033003600340037 INT INT 7fffffff 2147483647 VALID +diag_141 0032003100340037003400380033003600340038 INT INT INVALID +diag_142 002d0032003100340037003400380033003600340038 INT INT 80000000 NULL VALID +diag_143 002d0032003100340037003400380033003600340039 INT INT INVALID +diag_144 0039003200320033003300370032003000330036003800350034003700370035003800300037 INT INT INVALID +diag_145 0039003200320033003300370032003000330036003800350034003700370035003800300038 INT INT INVALID +diag_146 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 INT INT INVALID +diag_147 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 INT INT INVALID +diag_148 0031002e0030 INT INT INVALID +diag_149 0031002e0035 INT INT INVALID +diag_150 003100650032 INT INT INVALID +diag_151 00310065002d0032 INT INT INVALID +diag_152 0031005f0030 INT INT 0000000a 10 VALID +diag_153 002d0031005f0030 INT INT fffffff6 -10 VALID +diag_154 005f0031 INT INT INVALID +diag_155 0031005f INT INT INVALID +diag_156 0031005f005f0030 INT INT INVALID +diag_157 0031007f0030 INT INT INVALID +diag_158 0031004c INT INT INVALID +diag_159 0031006c INT INT INVALID +diag_160 00200031 INT INT INVALID +diag_161 00310020 INT INT INVALID +diag_162 00090031 INT INT INVALID +diag_163 0031000a INT INT INVALID +diag_164 0030007800310030 INT INT INVALID +diag_165 004e0061004e INT INT INVALID +diag_166 006e0061006e INT INT INVALID +diag_167 0049006e00660069006e006900740079 INT INT INVALID +diag_168 002b0049006e00660069006e006900740079 INT INT INVALID +diag_169 002d0049006e00660069006e006900740079 INT INT INVALID +diag_170 0069006e0066 INT INT INVALID +diag_171 00310065003300300039 INT INT INVALID +diag_172 00310065002d0034003000300030 INT INT INVALID +diag_173 00310038003400340036003700340034003000370033003700300039003500350031003600310035 INT INT INVALID +diag_174 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 INT INT INVALID +diag_175 0000 INT INT INVALID +diag_176 00000031 INT INT INVALID +diag_177 00310000 INT INT INVALID +diag_178 003100000032 INT INT INVALID +diag_179 0661 INT INT INVALID +diag_180 00a00031 INT INT INVALID +diag_181 d800 INT INT INVALID +diag_182 0031d800 INT INT INVALID +diag_183 dc00 INT INT INVALID +diag_184 LONG LONG NULL VALID +diag_185 LONG LONG INVALID +diag_186 0030 LONG LONG 0000000000000000 0 VALID +diag_187 002d0030 LONG LONG 0000000000000000 0 VALID +diag_188 002b0030 LONG LONG INVALID +diag_189 00300030 LONG LONG 0000000000000000 0 VALID +diag_190 0031 LONG LONG 0000000000000001 1 VALID +diag_191 002b0031 LONG LONG INVALID +diag_192 002d0031 LONG LONG ffffffffffffffff -1 VALID +diag_193 003100320037 LONG LONG 000000000000007f 127 VALID +diag_194 003100320038 LONG LONG 0000000000000080 128 VALID +diag_195 002d003100320038 LONG LONG ffffffffffffff80 -128 VALID +diag_196 002d003100320039 LONG LONG ffffffffffffff7f -129 VALID +diag_197 00330032003700360037 LONG LONG 0000000000007fff 32767 VALID +diag_198 00330032003700360038 LONG LONG 0000000000008000 32768 VALID +diag_199 002d00330032003700360038 LONG LONG ffffffffffff8000 -32768 VALID +diag_200 002d00330032003700360039 LONG LONG ffffffffffff7fff -32769 VALID +diag_201 0032003100340037003400380033003600340037 LONG LONG 000000007fffffff 2147483647 VALID +diag_202 0032003100340037003400380033003600340038 LONG LONG 0000000080000000 2147483648 VALID +diag_203 002d0032003100340037003400380033003600340038 LONG LONG ffffffff80000000 -2147483648 VALID +diag_204 002d0032003100340037003400380033003600340039 LONG LONG ffffffff7fffffff -2147483649 VALID +diag_205 0039003200320033003300370032003000330036003800350034003700370035003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +diag_206 0039003200320033003300370032003000330036003800350034003700370035003800300038 LONG LONG INVALID +diag_207 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 LONG LONG 8000000000000000 NULL VALID +diag_208 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 LONG LONG INVALID +diag_209 0031002e0030 LONG LONG INVALID +diag_210 0031002e0035 LONG LONG INVALID +diag_211 003100650032 LONG LONG INVALID +diag_212 00310065002d0032 LONG LONG INVALID +diag_213 0031005f0030 LONG LONG 000000000000000a 10 VALID +diag_214 002d0031005f0030 LONG LONG fffffffffffffff6 -10 VALID +diag_215 005f0031 LONG LONG INVALID +diag_216 0031005f LONG LONG INVALID +diag_217 0031005f005f0030 LONG LONG INVALID +diag_218 0031007f0030 LONG LONG 000000000000000a 10 VALID +diag_219 0031004c LONG LONG 0000000000000001 1 VALID +diag_220 0031006c LONG LONG 0000000000000001 1 VALID +diag_221 00200031 LONG LONG INVALID +diag_222 00310020 LONG LONG INVALID +diag_223 00090031 LONG LONG INVALID +diag_224 0031000a LONG LONG INVALID +diag_225 0030007800310030 LONG LONG INVALID +diag_226 004e0061004e LONG LONG INVALID +diag_227 006e0061006e LONG LONG INVALID +diag_228 0049006e00660069006e006900740079 LONG LONG INVALID +diag_229 002b0049006e00660069006e006900740079 LONG LONG INVALID +diag_230 002d0049006e00660069006e006900740079 LONG LONG INVALID +diag_231 0069006e0066 LONG LONG INVALID +diag_232 00310065003300300039 LONG LONG INVALID +diag_233 00310065002d0034003000300030 LONG LONG INVALID +diag_234 00310038003400340036003700340034003000370033003700300039003500350031003600310035 LONG LONG INVALID +diag_235 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 LONG LONG INVALID +diag_236 0000 LONG LONG INVALID +diag_237 00000031 LONG LONG INVALID +diag_238 00310000 LONG LONG INVALID +diag_239 003100000032 LONG LONG INVALID +diag_240 0661 LONG LONG INVALID +diag_241 00a00031 LONG LONG INVALID +diag_242 d800 LONG LONG INVALID +diag_243 0031d800 LONG LONG INVALID +diag_244 dc00 LONG LONG INVALID +diag_245 FLOAT FLOAT NULL VALID +diag_246 FLOAT FLOAT INVALID +diag_247 0030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_248 002d0030 FLOAT FLOAT 80000000 BITS:80000000 VALID +diag_249 002b0030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_250 00300030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_251 0031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_252 002b0031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_253 002d0031 FLOAT FLOAT bf800000 BITS:bf800000 VALID +diag_254 003100320037 FLOAT FLOAT 42fe0000 BITS:42fe0000 VALID +diag_255 003100320038 FLOAT FLOAT 43000000 BITS:43000000 VALID +diag_256 002d003100320038 FLOAT FLOAT c3000000 BITS:c3000000 VALID +diag_257 002d003100320039 FLOAT FLOAT c3010000 BITS:c3010000 VALID +diag_258 00330032003700360037 FLOAT FLOAT 46fffe00 BITS:46fffe00 VALID +diag_259 00330032003700360038 FLOAT FLOAT 47000000 BITS:47000000 VALID +diag_260 002d00330032003700360038 FLOAT FLOAT c7000000 BITS:c7000000 VALID +diag_261 002d00330032003700360039 FLOAT FLOAT c7000100 BITS:c7000100 VALID +diag_262 0032003100340037003400380033003600340037 FLOAT FLOAT 4f000000 BITS:4f000000 VALID +diag_263 0032003100340037003400380033003600340038 FLOAT FLOAT 4f000000 BITS:4f000000 VALID +diag_264 002d0032003100340037003400380033003600340038 FLOAT FLOAT cf000000 BITS:cf000000 VALID +diag_265 002d0032003100340037003400380033003600340039 FLOAT FLOAT cf000000 BITS:cf000000 VALID +diag_266 0039003200320033003300370032003000330036003800350034003700370035003800300037 FLOAT FLOAT 5f000000 BITS:5f000000 VALID +diag_267 0039003200320033003300370032003000330036003800350034003700370035003800300038 FLOAT FLOAT 5f000000 BITS:5f000000 VALID +diag_268 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 FLOAT FLOAT df000000 BITS:df000000 VALID +diag_269 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 FLOAT FLOAT df000000 BITS:df000000 VALID +diag_270 0031002e0030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_271 0031002e0035 FLOAT FLOAT 3fc00000 BITS:3fc00000 VALID +diag_272 003100650032 FLOAT FLOAT 42c80000 BITS:42c80000 VALID +diag_273 00310065002d0032 FLOAT FLOAT 3c23d70a BITS:3c23d70a VALID +diag_274 0031005f0030 FLOAT FLOAT INVALID +diag_275 002d0031005f0030 FLOAT FLOAT INVALID +diag_276 005f0031 FLOAT FLOAT INVALID +diag_277 0031005f FLOAT FLOAT INVALID +diag_278 0031005f005f0030 FLOAT FLOAT INVALID +diag_279 0031007f0030 FLOAT FLOAT INVALID +diag_280 0031004c FLOAT FLOAT INVALID +diag_281 0031006c FLOAT FLOAT INVALID +diag_282 00200031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_283 00310020 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_284 00090031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_285 0031000a FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_286 0030007800310030 FLOAT FLOAT INVALID +diag_287 004e0061004e FLOAT FLOAT 7fc00000 NULL VALID +diag_288 006e0061006e FLOAT FLOAT INVALID +diag_289 0049006e00660069006e006900740079 FLOAT FLOAT 7f800000 BITS:7f800000 VALID +diag_290 002b0049006e00660069006e006900740079 FLOAT FLOAT 7f800000 BITS:7f800000 VALID +diag_291 002d0049006e00660069006e006900740079 FLOAT FLOAT ff800000 BITS:ff800000 VALID +diag_292 0069006e0066 FLOAT FLOAT INVALID +diag_293 00310065003300300039 FLOAT FLOAT INVALID +diag_294 00310065002d0034003000300030 FLOAT FLOAT INVALID +diag_295 00310038003400340036003700340034003000370033003700300039003500350031003600310035 FLOAT FLOAT 5f800000 BITS:5f800000 VALID +diag_296 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 FLOAT FLOAT INVALID +diag_297 0000 FLOAT FLOAT INVALID +diag_298 00000031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_299 00310000 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_300 003100000032 FLOAT FLOAT INVALID +diag_301 0661 FLOAT FLOAT INVALID +diag_302 00a00031 FLOAT FLOAT INVALID +diag_303 d800 FLOAT FLOAT INVALID +diag_304 0031d800 FLOAT FLOAT INVALID +diag_305 dc00 FLOAT FLOAT INVALID +diag_306 DOUBLE DOUBLE NULL VALID +diag_307 DOUBLE DOUBLE INVALID +diag_308 0030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_309 002d0030 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 VALID +diag_310 002b0030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_311 00300030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_312 0031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_313 002b0031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_314 002d0031 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 VALID +diag_315 003100320037 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 VALID +diag_316 003100320038 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 VALID +diag_317 002d003100320038 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 VALID +diag_318 002d003100320039 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 VALID +diag_319 00330032003700360037 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 VALID +diag_320 00330032003700360038 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 VALID +diag_321 002d00330032003700360038 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 VALID +diag_322 002d00330032003700360039 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 VALID +diag_323 0032003100340037003400380033003600340037 DOUBLE DOUBLE 41dfffffffc00000 BITS:41dfffffffc00000 VALID +diag_324 0032003100340037003400380033003600340038 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 VALID +diag_325 002d0032003100340037003400380033003600340038 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 VALID +diag_326 002d0032003100340037003400380033003600340039 DOUBLE DOUBLE c1e0000000200000 BITS:c1e0000000200000 VALID +diag_327 0039003200320033003300370032003000330036003800350034003700370035003800300037 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 VALID +diag_328 0039003200320033003300370032003000330036003800350034003700370035003800300038 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 VALID +diag_329 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 VALID +diag_330 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 VALID +diag_331 0031002e0030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_332 0031002e0035 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 VALID +diag_333 003100650032 DOUBLE DOUBLE 4059000000000000 BITS:4059000000000000 VALID +diag_334 00310065002d0032 DOUBLE DOUBLE 3f847ae147ae147b BITS:3f847ae147ae147b VALID +diag_335 0031005f0030 DOUBLE DOUBLE INVALID +diag_336 002d0031005f0030 DOUBLE DOUBLE INVALID +diag_337 005f0031 DOUBLE DOUBLE INVALID +diag_338 0031005f DOUBLE DOUBLE INVALID +diag_339 0031005f005f0030 DOUBLE DOUBLE INVALID +diag_340 0031007f0030 DOUBLE DOUBLE INVALID +diag_341 0031004c DOUBLE DOUBLE INVALID +diag_342 0031006c DOUBLE DOUBLE INVALID +diag_343 00200031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_344 00310020 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_345 00090031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_346 0031000a DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_347 0030007800310030 DOUBLE DOUBLE INVALID +diag_348 004e0061004e DOUBLE DOUBLE 7ff8000000000000 NULL VALID +diag_349 006e0061006e DOUBLE DOUBLE INVALID +diag_350 0049006e00660069006e006900740079 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 VALID +diag_351 002b0049006e00660069006e006900740079 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 VALID +diag_352 002d0049006e00660069006e006900740079 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 VALID +diag_353 0069006e0066 DOUBLE DOUBLE INVALID +diag_354 00310065003300300039 DOUBLE DOUBLE INVALID +diag_355 00310065002d0034003000300030 DOUBLE DOUBLE INVALID +diag_356 00310038003400340036003700340034003000370033003700300039003500350031003600310035 DOUBLE DOUBLE 43f0000000000000 BITS:43f0000000000000 VALID +diag_357 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 DOUBLE DOUBLE 4a511b0ec57e649a BITS:4a511b0ec57e649a VALID +diag_358 0000 DOUBLE DOUBLE INVALID +diag_359 00000031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_360 00310000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_361 003100000032 DOUBLE DOUBLE INVALID +diag_362 0661 DOUBLE DOUBLE INVALID +diag_363 00a00031 DOUBLE DOUBLE INVALID +diag_364 d800 DOUBLE DOUBLE INVALID +diag_365 0031d800 DOUBLE DOUBLE INVALID +diag_366 dc00 DOUBLE DOUBLE INVALID +wrap_pos_21e18 00320031003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_neg_21e18 002d00320031003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_25e18 00320035003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_42e18 00340032003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_63e18 00360033003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +overflow_u64_max 00310038003400340036003700340034003000370033003700300039003500350031003600310035 LONG LONG INVALID +overflow_u64 00310038003400340036003700340034003000370033003700300039003500350031003600310036 LONG LONG INVALID +float_hex_one 00300078003100700030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_f 003000780031007000300066 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_fraction_one 003000780031002e003000700030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_decimal_one_f 00310066 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_decimal_one_d 00310064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_hex_one 00300078003100700030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_f 003000780031007000300066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_fraction_one 003000780031002e003000700030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_decimal_one_f 00310066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_decimal_one_d 00310064 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +long_negative_suffix 002d0031004c LONG LONG ffffffffffffffff -1 VALID +long_separator_suffix 0031005f0030004c LONG LONG 000000000000000a 10 VALID +long_separator_before_suffix 0031005f004c LONG LONG INVALID +long_leading_zero_max 0030003000300039003200320033003300370032003000330036003800350034003700370035003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_max_suffix 0039003200320033003300370032003000330036003800350034003700370035003800300037006c LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_max_plus_one_suffix 0039003200320033003300370032003000330036003800350034003700370035003800300038004c LONG LONG INVALID +long_grouped_max 0039005f003200320033005f003300370032005f003000330036005f003800350034005f003700370035005f003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_del_grouped 0039007f003200320033007f003300370032007f003000330036007f003800350034007f003700370035007f003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +float_hex_one_d 003000780031007000300064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_upper_f 003000780031007000300046 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_upper_d 003000780031007000300044 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_upper_xp 00300058003100500030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_hex_one_d 003000780031007000300064 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_upper_f 003000780031007000300046 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_upper_d 003000780031007000300044 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_upper_xp 00300058003100500030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +float_min_subnormal 0031002e00340065002d00340035 FLOAT FLOAT 00000001 BITS:00000001 VALID +float_half_subnormal_underflow 00370065002d00340036 FLOAT FLOAT INVALID +float_max_finite 0033002e0034003000320038003200330035006500330038 FLOAT FLOAT 7f7fffff BITS:7f7fffff VALID +float_above_max_finite 0033002e0034003000320038003200330036006500330038 FLOAT FLOAT INVALID +double_min_subnormal 0034002e00390065002d003300320034 DOUBLE DOUBLE 0000000000000001 BITS:0000000000000001 VALID +double_half_subnormal_underflow 0032002e00340065002d003300320034 DOUBLE DOUBLE INVALID +double_max_finite 0031002e00370039003700360039003300310033003400380036003200330031003500370065003300300038 DOUBLE DOUBLE 7fefffffffffffff BITS:7fefffffffffffff VALID +double_above_max_finite 0031002e00370039003700360039003300310033003400380036003200330031003500390065003300300038 DOUBLE DOUBLE INVALID +float_cross_suffix_trailing_space 003100640020 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_cross_suffix_internal_space 003100200064 FLOAT FLOAT INVALID +float_mixed_suffix_chain 003100660064 FLOAT FLOAT INVALID +float_nan_cross_suffix 004e0061004e0064 FLOAT FLOAT INVALID +float_infinity_cross_suffix 0049006e00660069006e0069007400790064 FLOAT FLOAT INVALID +float_negative_cross_suffix 002d00310064 FLOAT FLOAT bf800000 BITS:bf800000 VALID +float_decimal_point_cross_suffix 0031002e0064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_missing_digits_cross_suffix 002e0064 FLOAT FLOAT INVALID +float_exponent_cross_suffix 0031006500300064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_leading_nul_cross_suffix 000000310064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_trailing_nul_cross_suffix 003100640000 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_cross_suffix_trailing_space 003100660020 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_cross_suffix_internal_space 003100200066 DOUBLE DOUBLE INVALID +double_mixed_suffix_chain 003100640066 DOUBLE DOUBLE INVALID +double_nan_cross_suffix 004e0061004e0066 DOUBLE DOUBLE INVALID +double_infinity_cross_suffix 0049006e00660069006e0069007400790066 DOUBLE DOUBLE INVALID +double_negative_cross_suffix 002d00310066 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 VALID +double_decimal_point_cross_suffix 0031002e0066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_missing_digits_cross_suffix 002e0066 DOUBLE DOUBLE INVALID +double_exponent_cross_suffix 0031006500300066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_leading_nul_cross_suffix 000000310066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_trailing_nul_cross_suffix 003100660000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv new file mode 100644 index 000000000..f48385efc --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv @@ -0,0 +1,8 @@ +case_id target input_kind input feature +runtime_two_digit_year TIMESTAMP TEXT 24-01-02 03:04:05.006Z reference_century +runtime_greedy_negative_year TIMESTAMP TEXT -1-01-01 00:00:00.000Z greedy_year_reference +runtime_zone_gap TIMESTAMP TEXT 2024-03-31T02:30:00Europe/Berlin tzdb_gap +runtime_zone_overlap TIMESTAMP TEXT 2024-10-27T02:30:00europe/berlin tzdb_overlap_case +runtime_zone_alias TIMESTAMP TEXT 2024-01-02T03:04:05EST zone_alias +runtime_historic_zone TIMESTAMP TEXT 1880-01-01T00:00:00Europe/Berlin tzdb_history +runtime_wrapped_zone TIMESTAMP TEXT 3509450-07-01 00:00:00.000Europe/Berlin parsed_year_rules diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv new file mode 100644 index 000000000..2cf2dcf70 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv @@ -0,0 +1,111 @@ +case_id target input_kind input outcome expected_raw feature +micro_pg_millis_z TIMESTAMP TEXT 2024-01-02 03:04:05.006Z VALID 1704164645006000 format_pg +micro_fraction_1 TIMESTAMP TEXT 2024-01-02T03:04:05.1Z VALID 1704164645100000 fraction_1 +micro_fraction_2 TIMESTAMP TEXT 2024-01-02T03:04:05.12Z VALID 1704164645120000 fraction_2 +micro_fraction_3 TIMESTAMP TEXT 2024-01-02T03:04:05.123Z VALID 1704164645123000 fraction_3 +micro_fraction_4 TIMESTAMP TEXT 2024-01-02T03:04:05.1234Z VALID 1704164645234000 fraction_4_compiled_behavior +micro_fraction_6 TIMESTAMP TEXT 2024-01-02T03:04:05.123456Z VALID 1704164645123456 fraction_6 +micro_seconds_z TIMESTAMP TEXT 2024-01-02T03:04:05Z VALID 1704164645000000 format_seconds +micro_hour TIMESTAMP TEXT 2024-01-02T03 VALID 1704164400000000 format_hour +micro_day TIMESTAMP TEXT 2024-01-02 VALID 1704153600000000 format_day +micro_week TIMESTAMP TEXT 2024-W01 VALID 1704067200000000 format_week +micro_month TIMESTAMP TEXT 2024-01 VALID 1704067200000000 format_month +micro_year TIMESTAMP TEXT 2024 VALID 1704067200000000 format_year +micro_negative_year TIMESTAMP TEXT -0001-01-01 VALID -62198755200000000 signed_year +micro_hour24 TIMESTAMP TEXT 2024-01-02T24:01:00Z VALID 1704153660000000 hour24_modulo +micro_numeric_zone TIMESTAMP TEXT 2024-01-02T03:04:05+01:30 VALID 1704159245000000 numeric_zone +micro_utc_offset TIMESTAMP TEXT 2024-01-02T03:04:05UTC+01:30 VALID 1704159245000000 utc_offset +micro_epoch_negative TIMESTAMP TEXT 1969-12-31T23:59:59.999999Z VALID -1 negative +micro_epoch_zero TIMESTAMP TEXT 1970-01-01T00:00:00Z VALID 0 zero +micro_parsed_long_min TIMESTAMP TEXT -292278-01-01 00:00:00.000Z VALID -9223372036854775808 parsed_sentinel_present_sql_null +micro_null TIMESTAMP TEXT NULL null +micro_bad_day TIMESTAMP TEXT 2024-02-30 INVALID calendar +micro_bad_week TIMESTAMP TEXT 2024-W54 INVALID week +micro_bad_fraction_empty TIMESTAMP TEXT 2024-01-02T03:04:05.Z INVALID fraction +micro_bad_fraction_5 TIMESTAMP TEXT 2024-01-02T03:04:05.12345Z INVALID fraction +micro_bad_fraction_7 TIMESTAMP TEXT 2024-01-02T03:04:05.1234567Z INVALID fraction +micro_bad_plus_year TIMESTAMP TEXT +2024-01-01 INVALID signed_year +micro_signed_hour TIMESTAMP TEXT 2024-01-01T-0:00:00Z VALID 1704067200000000 signed_fixed_hour +micro_signed_minute_second TIMESTAMP TEXT 2024-01-01T00:-0:-0Z VALID 1704067200000000 signed_fixed_fields +micro_signed_month TIMESTAMP TEXT 2024-+1-01 VALID 1704067200000000 signed_fixed_month +micro_negative_fraction TIMESTAMP TEXT 2024-01-01T00:00:00.-01Z VALID 1704067199999000 signed_fraction +micro_split_negative_fraction TIMESTAMP TEXT 2024-01-01T00:00:00.123-01Z VALID 1704067200122999 signed_micro_fraction +micro_nested_negative_year TIMESTAMP TEXT --001-01-01T00:00:00Z VALID -62135596800000000 nested_fixed_year +micro_negative_week TIMESTAMP TEXT 2024-W-1 VALID 1704067200000000 signed_week +micro_int_min_year TIMESTAMP TEXT -2147483648-01-01 00:00:00.000Z VALID -9223372036854775808 year_micros_saturation +micro_malformed_high TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,d800,0031 INVALID utf16_replacement +micro_malformed_low TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,dc00,0031 INVALID utf16_replacement +micro_empty TIMESTAMP TEXT INVALID empty +micro_trailing_junk TIMESTAMP TEXT 2024-01-01T00:00:00Zjunk INVALID trailing_junk +micro_nonascii_zone_lookalike TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,0030,0031,002d,0030,0031,0054,0030,0030,003a,0030,0030,003a,0030,0030,0045,0075,0072,006f,0070,0065,002f,0042,0065,0072,006c,0131,006e INVALID unicode_zone_lookalike +micro_fixed_year_plus TIMESTAMP TEXT +2024 INVALID fixed_year_sign +micro_fixed_year_underscore TIMESTAMP TEXT 2024_01_01 INVALID separator +micro_reachable_max TIMESTAMP TEXT 44135800-09-20 06:17:51.147Z VALID 9223372036854775800 micro_wrapped_lattice_max +micro_reachable_max_successor_wrap TIMESTAMP TEXT 44135800-09-20 06:17:51.148Z VALID -9223372036854774816 micro_unchecked_wrap +micro_nonwrapped_near_max TIMESTAMP TEXT 294247-01-10 04:00:54.775Z VALID 9223372036854775000 micro_near_max +micro_saturation_plus_millis TIMESTAMP TEXT -292278-01-01 00:00:00.001Z VALID -9223372036854774808 year_saturation_plus +micro_reachable_min_plus_eight TIMESTAMP TEXT 28937395-06-09 13:30:22.805Z VALID -9223372036854775800 micro_wrapped_lattice_min +nano_pg_millis_z TIMESTAMP_NANOS TEXT 2024-01-02 03:04:05.006Z VALID 1704164645006000000 format_pg +nano_fraction_1 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1Z VALID 1704164645100000000 fraction_1 +nano_fraction_2 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.12Z VALID 1704164645120000000 fraction_2 +nano_fraction_3 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.123Z VALID 1704164645123000000 fraction_3 +nano_fraction_4 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1234Z VALID 1704164645234000000 fraction_4_compiled_behavior +nano_fraction_6 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.123456Z VALID 1704164645123456000 fraction_6 +nano_seconds_z TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05Z VALID 1704164645000000000 format_seconds +nano_hour TIMESTAMP_NANOS TEXT 2024-01-02T03 VALID 1704164400000000000 format_hour +nano_day TIMESTAMP_NANOS TEXT 2024-01-02 VALID 1704153600000000000 format_day +nano_week TIMESTAMP_NANOS TEXT 2024-W01 VALID 1704067200000000000 format_week +nano_month TIMESTAMP_NANOS TEXT 2024-01 VALID 1704067200000000000 format_month +nano_year TIMESTAMP_NANOS TEXT 2024 VALID 1704067200000000000 format_year +nano_negative_year TIMESTAMP_NANOS TEXT -0001-01-01 INVALID nano_overflow_after_parse +nano_hour24 TIMESTAMP_NANOS TEXT 2024-01-02T24:01:00Z VALID 1704153660000000000 hour24_modulo +nano_numeric_zone TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05+01:30 VALID 1704159245000000000 numeric_zone +nano_utc_offset TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05UTC+01:30 VALID 1704159245000000000 utc_offset +nano_epoch_negative TIMESTAMP_NANOS TEXT 1969-12-31T23:59:59.999999Z VALID -1000 negative +nano_epoch_zero TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00Z VALID 0 zero +nano_parsed_long_min TIMESTAMP_NANOS TEXT -292278-01-01 00:00:00.000Z INVALID nano_overflow_after_parse +nano_null TIMESTAMP_NANOS TEXT NULL null +nano_bad_day TIMESTAMP_NANOS TEXT 2024-02-30 INVALID calendar +nano_bad_week TIMESTAMP_NANOS TEXT 2024-W54 INVALID week +nano_bad_fraction_empty TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.Z INVALID fraction +nano_bad_fraction_5 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.12345Z INVALID fraction +nano_bad_fraction_7 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1234567Z INVALID fraction +nano_bad_plus_year TIMESTAMP_NANOS TEXT +2024-01-01 INVALID signed_year +nano_signed_hour TIMESTAMP_NANOS TEXT 2024-01-01T-0:00:00Z VALID 1704067200000000000 signed_fixed_hour +nano_signed_minute_second TIMESTAMP_NANOS TEXT 2024-01-01T00:-0:-0Z VALID 1704067200000000000 signed_fixed_fields +nano_signed_month TIMESTAMP_NANOS TEXT 2024-+1-01 VALID 1704067200000000000 signed_fixed_month +nano_negative_fraction TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00.-01Z VALID 1704067199999000000 signed_fraction +nano_split_negative_fraction TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00.123-01Z VALID 1704067200122999000 signed_micro_fraction +nano_nested_negative_year TIMESTAMP_NANOS TEXT --001-01-01T00:00:00Z INVALID nano_overflow_after_parse +nano_negative_week TIMESTAMP_NANOS TEXT 2024-W-1 VALID 1704067200000000000 signed_week +nano_int_min_year TIMESTAMP_NANOS TEXT -2147483648-01-01 00:00:00.000Z INVALID nano_overflow_after_parse +nano_malformed_high TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,d800,0031 INVALID utf16_replacement +nano_malformed_low TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,dc00,0031 INVALID utf16_replacement +nano_empty TIMESTAMP_NANOS TEXT INVALID empty +nano_trailing_junk TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00Zjunk INVALID trailing_junk +nano_nonascii_zone_lookalike TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,0030,0031,002d,0030,0031,0054,0030,0030,003a,0030,0030,003a,0030,0030,0045,0075,0072,006f,0070,0065,002f,0042,0065,0072,006c,0131,006e INVALID unicode_zone_lookalike +nano_fixed_year_plus TIMESTAMP_NANOS TEXT +2024 INVALID fixed_year_sign +nano_fixed_year_underscore TIMESTAMP_NANOS TEXT 2024_01_01 INVALID separator +nano_reachable_max TIMESTAMP_NANOS TEXT 44135800-09-20 06:17:51.147Z INVALID nano_overflow_after_parse +nano_reachable_max_successor_wrap TIMESTAMP_NANOS TEXT 44135800-09-20 06:17:51.148Z INVALID nano_overflow_after_parse +nano_nonwrapped_near_max TIMESTAMP_NANOS TEXT 294247-01-10 04:00:54.775Z INVALID nano_overflow_after_parse +nano_saturation_plus_millis TIMESTAMP_NANOS TEXT -292278-01-01 00:00:00.001Z INVALID nano_overflow_after_parse +nano_reachable_min_plus_eight TIMESTAMP_NANOS TEXT 28937395-06-09 13:30:22.805Z INVALID nano_overflow_after_parse +nano_upper_bound TIMESTAMP_NANOS TEXT 2262-04-11T23:47:16.854775Z VALID 9223372036854775000 nano_upper_bound +nano_upper_bound_plus_one TIMESTAMP_NANOS TEXT 2262-04-11T23:47:16.854776Z INVALID nano_overflow +nano_lower_bound TIMESTAMP_NANOS TEXT 1677-09-21T00:12:43.145225Z VALID -9223372036854775000 nano_lower_bound +nano_lower_bound_minus_one TIMESTAMP_NANOS TEXT 1677-09-21T00:12:43.145224Z INVALID nano_overflow +micro_fraction_fixed_plus TIMESTAMP TEXT 1970-01-01T00:00:00.+12Z VALID 12000 fixed_fraction_priority +micro_fraction_fixed_underscore TIMESTAMP TEXT 1970-01-01T00:00:00.1_2Z VALID 12000 fixed_fraction_priority +micro_fraction_fixed_numeric_zone TIMESTAMP TEXT 1970-01-01T00:00:00.10000 VALID 100000 fixed_fraction_priority +micro_fraction_extra_zone_tail TIMESTAMP TEXT 1970-01-01T00:00:00.10000Z INVALID fixed_fraction_priority +micro_fraction_split_signed TIMESTAMP TEXT 1970-01-01T00:00:00.1_2+34Z VALID 12034 fixed_fraction_priority +nano_fraction_fixed_plus TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.+12Z VALID 12000000 fixed_fraction_priority +nano_fraction_fixed_underscore TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.1_2Z VALID 12000000 fixed_fraction_priority +nano_fraction_fixed_numeric_zone TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.10000 VALID 100000000 fixed_fraction_priority +nano_fraction_extra_zone_tail TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.10000Z INVALID fixed_fraction_priority +nano_fraction_split_signed TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.1_2+34Z VALID 12034000 fixed_fraction_priority +micro_iso_week_retained_leap_1297 TIMESTAMP TEXT 1297-W01 VALID -21237724800000000 iso_week_source_leap +micro_iso_week_retained_leap_1697 TIMESTAMP TEXT 1697-W01 VALID -8614944000000000 iso_week_source_leap +nano_iso_week_retained_leap_1297 TIMESTAMP_NANOS TEXT 1297-W01 INVALID nano_overflow_after_parse +nano_iso_week_retained_leap_1697 TIMESTAMP_NANOS TEXT 1697-W01 VALID -8614944000000000000 iso_week_source_leap diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv new file mode 100644 index 000000000..f9f4decd0 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv @@ -0,0 +1,91 @@ +# case_id input_type target_type input_utf16_hex expected_wire_type expected_utf8_hex expected_sql_utf16_hex +string_string_ascii STRING STRING 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +string_string_lower STRING STRING 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +string_string_controls STRING STRING 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +string_string_empty STRING STRING VARCHAR +string_string_spaces STRING STRING 002000610020 VARCHAR 206120 002000610020 +string_string_nul STRING STRING 006100000062 VARCHAR 610062 006100000062 +string_string_bmp STRING STRING 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +string_string_pair STRING STRING d83dde00 VARCHAR f09f9880 d83dde00 +string_string_high STRING STRING d800 VARCHAR 3f 003f +string_string_low STRING STRING dc00 VARCHAR 3f 003f +string_string_high_ascii STRING STRING d8000041 VARCHAR 3f41 003f0041 +string_string_high_high_low STRING STRING d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +string_string_low_high_ascii STRING STRING dc00d8000041 VARCHAR 3f3f41 003f003f0041 +string_string_question STRING STRING 003f VARCHAR 3f 003f +string_string_null STRING STRING VARCHAR +string_varchar_ascii STRING VARCHAR 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +string_varchar_lower STRING VARCHAR 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +string_varchar_controls STRING VARCHAR 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +string_varchar_empty STRING VARCHAR VARCHAR +string_varchar_spaces STRING VARCHAR 002000610020 VARCHAR 206120 002000610020 +string_varchar_nul STRING VARCHAR 006100000062 VARCHAR 610062 006100000062 +string_varchar_bmp STRING VARCHAR 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +string_varchar_pair STRING VARCHAR d83dde00 VARCHAR f09f9880 d83dde00 +string_varchar_high STRING VARCHAR d800 VARCHAR 3f 003f +string_varchar_low STRING VARCHAR dc00 VARCHAR 3f 003f +string_varchar_high_ascii STRING VARCHAR d8000041 VARCHAR 3f41 003f0041 +string_varchar_high_high_low STRING VARCHAR d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +string_varchar_low_high_ascii STRING VARCHAR dc00d8000041 VARCHAR 3f3f41 003f003f0041 +string_varchar_question STRING VARCHAR 003f VARCHAR 3f 003f +string_varchar_null STRING VARCHAR VARCHAR +string_symbol_ascii STRING SYMBOL 00510075006500730074 SYMBOL 5175657374 00510075006500730074 +string_symbol_lower STRING SYMBOL 00710075006500730074 SYMBOL 7175657374 00710075006500730074 +string_symbol_controls STRING SYMBOL 00610009000d000a0062 SYMBOL 61090d0a62 00610009000d000a0062 +string_symbol_empty STRING SYMBOL SYMBOL +string_symbol_spaces STRING SYMBOL 002000610020 SYMBOL 206120 002000610020 +string_symbol_nul STRING SYMBOL 006100000062 SYMBOL 610062 006100000062 +string_symbol_bmp STRING SYMBOL 00e94e2d SYMBOL c3a9e4b8ad 00e94e2d +string_symbol_pair STRING SYMBOL d83dde00 SYMBOL f09f9880 d83dde00 +string_symbol_high STRING SYMBOL d800 SYMBOL 3f 003f +string_symbol_low STRING SYMBOL dc00 SYMBOL 3f 003f +string_symbol_high_ascii STRING SYMBOL d8000041 SYMBOL 3f41 003f0041 +string_symbol_high_high_low STRING SYMBOL d800d83dde00 SYMBOL 3ff09f9880 003fd83dde00 +string_symbol_low_high_ascii STRING SYMBOL dc00d8000041 SYMBOL 3f3f41 003f003f0041 +string_symbol_question STRING SYMBOL 003f SYMBOL 3f 003f +string_symbol_null STRING SYMBOL SYMBOL +symbol_string_ascii SYMBOL STRING 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +symbol_string_lower SYMBOL STRING 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +symbol_string_controls SYMBOL STRING 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +symbol_string_empty SYMBOL STRING VARCHAR +symbol_string_spaces SYMBOL STRING 002000610020 VARCHAR 206120 002000610020 +symbol_string_nul SYMBOL STRING 006100000062 VARCHAR 610062 006100000062 +symbol_string_bmp SYMBOL STRING 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +symbol_string_pair SYMBOL STRING d83dde00 VARCHAR f09f9880 d83dde00 +symbol_string_high SYMBOL STRING d800 VARCHAR 3f 003f +symbol_string_low SYMBOL STRING dc00 VARCHAR 3f 003f +symbol_string_high_ascii SYMBOL STRING d8000041 VARCHAR 3f41 003f0041 +symbol_string_high_high_low SYMBOL STRING d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +symbol_string_low_high_ascii SYMBOL STRING dc00d8000041 VARCHAR 3f3f41 003f003f0041 +symbol_string_question SYMBOL STRING 003f VARCHAR 3f 003f +symbol_string_null SYMBOL STRING VARCHAR +symbol_varchar_ascii SYMBOL VARCHAR 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +symbol_varchar_lower SYMBOL VARCHAR 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +symbol_varchar_controls SYMBOL VARCHAR 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +symbol_varchar_empty SYMBOL VARCHAR VARCHAR +symbol_varchar_spaces SYMBOL VARCHAR 002000610020 VARCHAR 206120 002000610020 +symbol_varchar_nul SYMBOL VARCHAR 006100000062 VARCHAR 610062 006100000062 +symbol_varchar_bmp SYMBOL VARCHAR 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +symbol_varchar_pair SYMBOL VARCHAR d83dde00 VARCHAR f09f9880 d83dde00 +symbol_varchar_high SYMBOL VARCHAR d800 VARCHAR 3f 003f +symbol_varchar_low SYMBOL VARCHAR dc00 VARCHAR 3f 003f +symbol_varchar_high_ascii SYMBOL VARCHAR d8000041 VARCHAR 3f41 003f0041 +symbol_varchar_high_high_low SYMBOL VARCHAR d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +symbol_varchar_low_high_ascii SYMBOL VARCHAR dc00d8000041 VARCHAR 3f3f41 003f003f0041 +symbol_varchar_question SYMBOL VARCHAR 003f VARCHAR 3f 003f +symbol_varchar_null SYMBOL VARCHAR VARCHAR +symbol_symbol_ascii SYMBOL SYMBOL 00510075006500730074 SYMBOL 5175657374 00510075006500730074 +symbol_symbol_lower SYMBOL SYMBOL 00710075006500730074 SYMBOL 7175657374 00710075006500730074 +symbol_symbol_controls SYMBOL SYMBOL 00610009000d000a0062 SYMBOL 61090d0a62 00610009000d000a0062 +symbol_symbol_empty SYMBOL SYMBOL SYMBOL +symbol_symbol_spaces SYMBOL SYMBOL 002000610020 SYMBOL 206120 002000610020 +symbol_symbol_nul SYMBOL SYMBOL 006100000062 SYMBOL 610062 006100000062 +symbol_symbol_bmp SYMBOL SYMBOL 00e94e2d SYMBOL c3a9e4b8ad 00e94e2d +symbol_symbol_pair SYMBOL SYMBOL d83dde00 SYMBOL f09f9880 d83dde00 +symbol_symbol_high SYMBOL SYMBOL d800 SYMBOL 3f 003f +symbol_symbol_low SYMBOL SYMBOL dc00 SYMBOL 3f 003f +symbol_symbol_high_ascii SYMBOL SYMBOL d8000041 SYMBOL 3f41 003f0041 +symbol_symbol_high_high_low SYMBOL SYMBOL d800d83dde00 SYMBOL 3ff09f9880 003fd83dde00 +symbol_symbol_low_high_ascii SYMBOL SYMBOL dc00d8000041 SYMBOL 3f3f41 003f003f0041 +symbol_symbol_question SYMBOL SYMBOL 003f SYMBOL 3f 003f +symbol_symbol_null SYMBOL SYMBOL SYMBOL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv new file mode 100644 index 000000000..050717e7d --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv @@ -0,0 +1,103 @@ +# case_id input_kind input source_unit target_type expected_wire_value expected_sql_value +millis_micro_zero LONG 0 MILLIS TIMESTAMP 0 0 +millis_micro_neg_one LONG -1 MILLIS TIMESTAMP -1000 -1000 +millis_micro_pos_one LONG 1 MILLIS TIMESTAMP 1000 1000 +millis_micro_low_valid LONG -9223372036854775 MILLIS TIMESTAMP -9223372036854775000 -9223372036854775000 +millis_micro_low_invalid LONG -9223372036854776 MILLIS TIMESTAMP +millis_micro_high_valid LONG 9223372036854775 MILLIS TIMESTAMP 9223372036854775000 9223372036854775000 +millis_micro_high_invalid LONG 9223372036854776 MILLIS TIMESTAMP +millis_micro_long_min LONG -9223372036854775808 MILLIS TIMESTAMP +millis_nano_zero LONG 0 MILLIS TIMESTAMP_NS 0 0 +millis_nano_neg_one LONG -1 MILLIS TIMESTAMP_NS -1000000 -1000000 +millis_nano_pos_one LONG 1 MILLIS TIMESTAMP_NS 1000000 1000000 +millis_nano_low_valid LONG -9223372036854 MILLIS TIMESTAMP_NS -9223372036854000000 -9223372036854000000 +millis_nano_low_invalid LONG -9223372036855 MILLIS TIMESTAMP_NS +millis_nano_high_valid LONG 9223372036854 MILLIS TIMESTAMP_NS 9223372036854000000 9223372036854000000 +millis_nano_high_invalid LONG 9223372036855 MILLIS TIMESTAMP_NS +millis_nano_long_min LONG -9223372036854775808 MILLIS TIMESTAMP_NS +seconds_micro_zero LONG 0 SECONDS TIMESTAMP 0 0 +seconds_micro_neg_one LONG -1 SECONDS TIMESTAMP -1000000 -1000000 +seconds_micro_pos_one LONG 1 SECONDS TIMESTAMP 1000000 1000000 +seconds_micro_low_valid LONG -9223372036854 SECONDS TIMESTAMP -9223372036854000000 -9223372036854000000 +seconds_micro_low_invalid LONG -9223372036855 SECONDS TIMESTAMP +seconds_micro_high_valid LONG 9223372036854 SECONDS TIMESTAMP 9223372036854000000 9223372036854000000 +seconds_micro_high_invalid LONG 9223372036855 SECONDS TIMESTAMP +seconds_micro_long_min LONG -9223372036854775808 SECONDS TIMESTAMP +seconds_nano_zero LONG 0 SECONDS TIMESTAMP_NS 0 0 +seconds_nano_neg_one LONG -1 SECONDS TIMESTAMP_NS -1000000000 -1000000000 +seconds_nano_pos_one LONG 1 SECONDS TIMESTAMP_NS 1000000000 1000000000 +seconds_nano_low_valid LONG -9223372036 SECONDS TIMESTAMP_NS -9223372036000000000 -9223372036000000000 +seconds_nano_low_invalid LONG -9223372037 SECONDS TIMESTAMP_NS +seconds_nano_high_valid LONG 9223372036 SECONDS TIMESTAMP_NS 9223372036000000000 9223372036000000000 +seconds_nano_high_invalid LONG 9223372037 SECONDS TIMESTAMP_NS +seconds_nano_long_min LONG -9223372036854775808 SECONDS TIMESTAMP_NS +minutes_micro_zero LONG 0 MINUTES TIMESTAMP 0 0 +minutes_micro_neg_one LONG -1 MINUTES TIMESTAMP -60000000 -60000000 +minutes_micro_pos_one LONG 1 MINUTES TIMESTAMP 60000000 60000000 +minutes_micro_low_valid LONG -153722867280 MINUTES TIMESTAMP -9223372036800000000 -9223372036800000000 +minutes_micro_low_invalid LONG -153722867281 MINUTES TIMESTAMP +minutes_micro_high_valid LONG 153722867280 MINUTES TIMESTAMP 9223372036800000000 9223372036800000000 +minutes_micro_high_invalid LONG 153722867281 MINUTES TIMESTAMP +minutes_micro_long_min LONG -9223372036854775808 MINUTES TIMESTAMP +minutes_nano_zero LONG 0 MINUTES TIMESTAMP_NS 0 0 +minutes_nano_neg_one LONG -1 MINUTES TIMESTAMP_NS -60000000000 -60000000000 +minutes_nano_pos_one LONG 1 MINUTES TIMESTAMP_NS 60000000000 60000000000 +minutes_nano_low_valid LONG -153722867 MINUTES TIMESTAMP_NS -9223372020000000000 -9223372020000000000 +minutes_nano_low_invalid LONG -153722868 MINUTES TIMESTAMP_NS +minutes_nano_high_valid LONG 153722867 MINUTES TIMESTAMP_NS 9223372020000000000 9223372020000000000 +minutes_nano_high_invalid LONG 153722868 MINUTES TIMESTAMP_NS +minutes_nano_long_min LONG -9223372036854775808 MINUTES TIMESTAMP_NS +hours_micro_zero LONG 0 HOURS TIMESTAMP 0 0 +hours_micro_neg_one LONG -1 HOURS TIMESTAMP -3600000000 -3600000000 +hours_micro_pos_one LONG 1 HOURS TIMESTAMP 3600000000 3600000000 +hours_micro_low_valid LONG -2562047788 HOURS TIMESTAMP -9223372036800000000 -9223372036800000000 +hours_micro_low_invalid LONG -2562047789 HOURS TIMESTAMP +hours_micro_high_valid LONG 2562047788 HOURS TIMESTAMP 9223372036800000000 9223372036800000000 +hours_micro_high_invalid LONG 2562047789 HOURS TIMESTAMP +hours_micro_long_min LONG -9223372036854775808 HOURS TIMESTAMP +hours_nano_zero LONG 0 HOURS TIMESTAMP_NS 0 0 +hours_nano_neg_one LONG -1 HOURS TIMESTAMP_NS -3600000000000 -3600000000000 +hours_nano_pos_one LONG 1 HOURS TIMESTAMP_NS 3600000000000 3600000000000 +hours_nano_low_valid LONG -2562047 HOURS TIMESTAMP_NS -9223369200000000000 -9223369200000000000 +hours_nano_low_invalid LONG -2562048 HOURS TIMESTAMP_NS +hours_nano_high_valid LONG 2562047 HOURS TIMESTAMP_NS 9223369200000000000 9223369200000000000 +hours_nano_high_invalid LONG 2562048 HOURS TIMESTAMP_NS +hours_nano_long_min LONG -9223372036854775808 HOURS TIMESTAMP_NS +days_micro_zero LONG 0 DAYS TIMESTAMP 0 0 +days_micro_neg_one LONG -1 DAYS TIMESTAMP -86400000000 -86400000000 +days_micro_pos_one LONG 1 DAYS TIMESTAMP 86400000000 86400000000 +days_micro_low_valid LONG -106751991 DAYS TIMESTAMP -9223372022400000000 -9223372022400000000 +days_micro_low_invalid LONG -106751992 DAYS TIMESTAMP +days_micro_high_valid LONG 106751991 DAYS TIMESTAMP 9223372022400000000 9223372022400000000 +days_micro_high_invalid LONG 106751992 DAYS TIMESTAMP +days_micro_long_min LONG -9223372036854775808 DAYS TIMESTAMP +days_nano_zero LONG 0 DAYS TIMESTAMP_NS 0 0 +days_nano_neg_one LONG -1 DAYS TIMESTAMP_NS -86400000000000 -86400000000000 +days_nano_pos_one LONG 1 DAYS TIMESTAMP_NS 86400000000000 86400000000000 +days_nano_low_valid LONG -106751 DAYS TIMESTAMP_NS -9223286400000000000 -9223286400000000000 +days_nano_low_invalid LONG -106752 DAYS TIMESTAMP_NS +days_nano_high_valid LONG 106751 DAYS TIMESTAMP_NS 9223286400000000000 9223286400000000000 +days_nano_high_invalid LONG 106752 DAYS TIMESTAMP_NS +days_nano_long_min LONG -9223372036854775808 DAYS TIMESTAMP_NS +instant_micro_zero INSTANT 1970-01-01T00:00:00Z - TIMESTAMP 0 0 +instant_micro_pos_submicro INSTANT 1970-01-01T00:00:00.000000999Z - TIMESTAMP 0 0 +instant_micro_neg_submicro INSTANT 1969-12-31T23:59:59.999999999Z - TIMESTAMP -1 -1 +instant_micro_min INSTANT -290308-12-21T19:59:05.224192Z - TIMESTAMP -9223372036854775808 NULL +instant_micro_min_fraction INSTANT -290308-12-21T19:59:05.224192999Z - TIMESTAMP -9223372036854775808 NULL +instant_micro_min_plus_one INSTANT -290308-12-21T19:59:05.224193Z - TIMESTAMP -9223372036854775807 -9223372036854775807 +instant_micro_below_min INSTANT -290308-12-21T19:59:05.224191999Z - TIMESTAMP +instant_micro_max INSTANT +294247-01-10T04:00:54.775807Z - TIMESTAMP 9223372036854775807 9223372036854775807 +instant_micro_max_fraction INSTANT +294247-01-10T04:00:54.775807999Z - TIMESTAMP 9223372036854775807 9223372036854775807 +instant_micro_above_max INSTANT +294247-01-10T04:00:54.775808Z - TIMESTAMP +instant_nano_zero INSTANT 1970-01-01T00:00:00Z - TIMESTAMP_NS 0 0 +instant_nano_pos_submicro INSTANT 1970-01-01T00:00:00.000000999Z - TIMESTAMP_NS 999 999 +instant_nano_neg_submicro INSTANT 1969-12-31T23:59:59.999999999Z - TIMESTAMP_NS -1 -1 +instant_nano_min INSTANT 1677-09-21T00:12:43.145224192Z - TIMESTAMP_NS -9223372036854775808 NULL +instant_nano_min_plus_one INSTANT 1677-09-21T00:12:43.145224193Z - TIMESTAMP_NS -9223372036854775807 -9223372036854775807 +instant_nano_below_min INSTANT 1677-09-21T00:12:43.145224191Z - TIMESTAMP_NS +instant_nano_max INSTANT 2262-04-11T23:47:16.854775807Z - TIMESTAMP_NS 9223372036854775807 9223372036854775807 +instant_nano_above_max INSTANT 2262-04-11T23:47:16.854775808Z - TIMESTAMP_NS +instant_min_micro INSTANT -1000000000-01-01T00:00:00Z - TIMESTAMP +instant_max_micro INSTANT +1000000000-12-31T23:59:59.999999999Z - TIMESTAMP +instant_min_nano INSTANT -1000000000-01-01T00:00:00Z - TIMESTAMP_NS +instant_max_nano INSTANT +1000000000-12-31T23:59:59.999999999Z - TIMESTAMP_NS diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv new file mode 100644 index 000000000..69ceaa4bd --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv @@ -0,0 +1,64 @@ +# case_id input_kind unit value nanos outcome expected_text +micro_min LONG MICROS -9223372036854775808 VALUE +micro_min_plus_one LONG MICROS -9223372036854775807 VALUE -290308-01-01T19:59:05.224Z +micro_minus_1000 LONG MICROS -1000 VALUE 1969-12-31T23:59:59.999Z +micro_minus_999 LONG MICROS -999 VALUE 1969-12-31T23:59:59.999Z +micro_minus_one LONG MICROS -1 VALUE 1969-12-31T23:59:59.999Z +micro_zero LONG MICROS 0 VALUE 1970-01-01T00:00:00.000Z +micro_one LONG MICROS 1 VALUE 1970-01-01T00:00:00.000Z +micro_999 LONG MICROS 999 VALUE 1970-01-01T00:00:00.000Z +micro_1000 LONG MICROS 1000 VALUE 1970-01-01T00:00:00.001Z +micro_max LONG MICROS 9223372036854775807 VALUE 294247-01-10T04:00:54.775Z +micro_year_minus_one LONG MICROS -62198755200000000 VALUE 000-1-01-01T00:00:00.000Z +micro_year_zero LONG MICROS -62167219200000000 VALUE 0001-01-01T00:00:00.000Z +micro_year_one LONG MICROS -62135596800000000 VALUE 0001-01-01T00:00:00.000Z +micro_year_99 LONG MICROS -59042995200000000 VALUE 0099-01-01T00:00:00.000Z +micro_year_100 LONG MICROS -59011459200000000 VALUE 0100-01-01T00:00:00.000Z +micro_year_9999 LONG MICROS 253370764800000000 VALUE 9999-01-01T00:00:00.000Z +micro_year_10000 LONG MICROS 253402300800000000 VALUE 10000-01-01T00:00:00.000Z +nano_min LONG NANOS -9223372036854775808 VALUE 1677-09-21T00:12:43.145Z +nano_min_plus_one LONG NANOS -9223372036854775807 VALUE 1677-09-21T00:12:43.145Z +nano_minus_1000 LONG NANOS -1000 VALUE 1969-12-31T23:59:59.999Z +nano_minus_999 LONG NANOS -999 VALUE 1970-01-01T00:00:00.000Z +nano_minus_one LONG NANOS -1 VALUE 1970-01-01T00:00:00.000Z +nano_zero LONG NANOS 0 VALUE 1970-01-01T00:00:00.000Z +nano_one LONG NANOS 1 VALUE 1970-01-01T00:00:00.000Z +nano_999 LONG NANOS 999 VALUE 1970-01-01T00:00:00.000Z +nano_1000 LONG NANOS 1000 VALUE 1970-01-01T00:00:00.000Z +nano_max LONG NANOS 9223372036854775807 VALUE 2262-04-11T23:47:16.854Z +millis_negative LONG MILLIS -1 VALUE 1969-12-31T23:59:59.999Z +millis_positive LONG MILLIS 1 VALUE 1970-01-01T00:00:00.001Z +seconds_negative LONG SECONDS -1 VALUE 1969-12-31T23:59:59.000Z +seconds_positive LONG SECONDS 1 VALUE 1970-01-01T00:00:01.000Z +minutes_positive LONG MINUTES 1 VALUE 1970-01-01T00:01:00.000Z +hours_negative LONG HOURS -1 VALUE 1969-12-31T23:00:00.000Z +days_positive LONG DAYS 1 VALUE 1970-01-02T00:00:00.000Z +days_overflow LONG DAYS 9223372036854775807 INVALID +millis_max_accepted LONG MILLIS 9223372036854775 VALUE 294247-01-10T04:00:54.775Z +millis_positive_overflow LONG MILLIS 9223372036854776 INVALID +millis_min_accepted LONG MILLIS -9223372036854775 VALUE -290308-01-01T19:59:05.225Z +millis_negative_overflow LONG MILLIS -9223372036854776 INVALID +seconds_max_accepted LONG SECONDS 9223372036854 VALUE 294247-01-10T04:00:54.000Z +seconds_positive_overflow LONG SECONDS 9223372036855 INVALID +seconds_min_accepted LONG SECONDS -9223372036854 VALUE -290308-01-01T19:59:06.000Z +seconds_negative_overflow LONG SECONDS -9223372036855 INVALID +minutes_max_accepted LONG MINUTES 153722867280 VALUE 294247-01-10T04:00:00.000Z +minutes_positive_overflow LONG MINUTES 153722867281 INVALID +minutes_min_accepted LONG MINUTES -153722867280 VALUE -290308-01-01T20:00:00.000Z +minutes_negative_overflow LONG MINUTES -153722867281 INVALID +hours_max_accepted LONG HOURS 2562047788 VALUE 294247-01-10T04:00:00.000Z +hours_positive_overflow LONG HOURS 2562047789 INVALID +hours_min_accepted LONG HOURS -2562047788 VALUE -290308-01-01T20:00:00.000Z +hours_negative_overflow LONG HOURS -2562047789 INVALID +days_max_accepted LONG DAYS 106751991 VALUE 294247-01-10T00:00:00.000Z +days_positive_overflow LONG DAYS 106751992 INVALID +days_min_accepted LONG DAYS -106751991 VALUE -290308-01-01T00:00:00.000Z +days_negative_overflow LONG DAYS -106751992 INVALID +instant_negative_fraction INSTANT -1 999999999 VALUE 1969-12-31T23:59:59.999Z +instant_epoch INSTANT 0 0 VALUE 1970-01-01T00:00:00.000Z +instant_micro_max INSTANT 9223372036854 775807000 VALUE 294247-01-10T04:00:54.775Z +instant_micro_min INSTANT -9223372036855 224192000 VALUE +instant_above_micro_max INSTANT 9223372036854 775808000 INVALID +instant_below_micro_min INSTANT -9223372036855 224191000 INVALID +instant_micro_min_plus_one INSTANT -9223372036855 224193000 VALUE -290308-01-01T19:59:05.224Z +instant_micro_max_fraction INSTANT 9223372036854 775807999 VALUE 294247-01-10T04:00:54.775Z diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv new file mode 100644 index 000000000..f55c4c584 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv @@ -0,0 +1,49 @@ +# case_id input source_unit target_type expected_wire_value expected_sql_value +micro_micro_zero 0 MICROS TIMESTAMP 0 0 +micro_micro_neg_1001 -1001 MICROS TIMESTAMP -1001 -1001 +micro_micro_neg_1000 -1000 MICROS TIMESTAMP -1000 -1000 +micro_micro_neg_999 -999 MICROS TIMESTAMP -999 -999 +micro_micro_neg_one -1 MICROS TIMESTAMP -1 -1 +micro_micro_pos_one 1 MICROS TIMESTAMP 1 1 +micro_micro_pos_999 999 MICROS TIMESTAMP 999 999 +micro_micro_pos_1000 1000 MICROS TIMESTAMP 1000 1000 +micro_micro_pos_1001 1001 MICROS TIMESTAMP 1001 1001 +micro_micro_min -9223372036854775808 MICROS TIMESTAMP -9223372036854775808 NULL +micro_micro_max 9223372036854775807 MICROS TIMESTAMP 9223372036854775807 9223372036854775807 +nano_nano_zero 0 NANOS TIMESTAMP_NS 0 0 +nano_nano_neg_1001 -1001 NANOS TIMESTAMP_NS -1001 -1001 +nano_nano_neg_1000 -1000 NANOS TIMESTAMP_NS -1000 -1000 +nano_nano_neg_999 -999 NANOS TIMESTAMP_NS -999 -999 +nano_nano_neg_one -1 NANOS TIMESTAMP_NS -1 -1 +nano_nano_pos_one 1 NANOS TIMESTAMP_NS 1 1 +nano_nano_pos_999 999 NANOS TIMESTAMP_NS 999 999 +nano_nano_pos_1000 1000 NANOS TIMESTAMP_NS 1000 1000 +nano_nano_pos_1001 1001 NANOS TIMESTAMP_NS 1001 1001 +nano_nano_min -9223372036854775808 NANOS TIMESTAMP_NS -9223372036854775808 NULL +nano_nano_max 9223372036854775807 NANOS TIMESTAMP_NS 9223372036854775807 9223372036854775807 +nano_micro_pos_fraction 999 NANOS TIMESTAMP 0 0 +nano_micro_zero 0 NANOS TIMESTAMP 0 0 +nano_micro_pos_one 1 NANOS TIMESTAMP 0 0 +nano_micro_pos_exact 1000 NANOS TIMESTAMP 1 1 +nano_micro_pos_over 1001 NANOS TIMESTAMP 1 1 +nano_micro_neg_fraction -999 NANOS TIMESTAMP 0 0 +nano_micro_neg_one -1 NANOS TIMESTAMP 0 0 +nano_micro_neg_exact -1000 NANOS TIMESTAMP -1 -1 +nano_micro_neg_over -1001 NANOS TIMESTAMP -1 -1 +nano_micro_min -9223372036854775808 NANOS TIMESTAMP -9223372036854775 -9223372036854775 +nano_micro_max 9223372036854775807 NANOS TIMESTAMP 9223372036854775 9223372036854775 +micro_nano_min_valid -9223372036854775 MICROS TIMESTAMP_NS -9223372036854775000 -9223372036854775000 +micro_nano_zero 0 MICROS TIMESTAMP_NS 0 0 +micro_nano_neg_one -1 MICROS TIMESTAMP_NS -1000 -1000 +micro_nano_neg_999 -999 MICROS TIMESTAMP_NS -999000 -999000 +micro_nano_neg_1000 -1000 MICROS TIMESTAMP_NS -1000000 -1000000 +micro_nano_neg_1001 -1001 MICROS TIMESTAMP_NS -1001000 -1001000 +micro_nano_below_min -9223372036854776 MICROS TIMESTAMP_NS +micro_nano_max_valid 9223372036854775 MICROS TIMESTAMP_NS 9223372036854775000 9223372036854775000 +micro_nano_pos_one 1 MICROS TIMESTAMP_NS 1000 1000 +micro_nano_pos_999 999 MICROS TIMESTAMP_NS 999000 999000 +micro_nano_pos_1000 1000 MICROS TIMESTAMP_NS 1000000 1000000 +micro_nano_pos_1001 1001 MICROS TIMESTAMP_NS 1001000 1001000 +micro_nano_above_max 9223372036854776 MICROS TIMESTAMP_NS +micro_nano_min -9223372036854775808 MICROS TIMESTAMP_NS +micro_nano_max 9223372036854775807 MICROS TIMESTAMP_NS diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv new file mode 100644 index 000000000..789bd6277 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv @@ -0,0 +1,21 @@ +# case_id input expected +null +empty +nil 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +all_fs ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff +upper_mixed 01234567-89AB-cDeF-0123-456789aBCDef 01234567-89ab-cdef-0123-456789abcdef +distinct_limbs 01234567-89ab-cdef-fedc-ba9876543210 01234567-89ab-cdef-fedc-ba9876543210 +both_min_null 80000000-0000-0000-8000-000000000000 +high_min_only 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +low_min_only 00000000-0000-0000-8000-000000000000 00000000-0000-0000-8000-000000000000 +bad_hex_high_first g1234567-89ab-cdef-0123-456789abcdef +bad_hex_high_second 01234567-x9ab-cdef-0123-456789abcdef +bad_hex_high_third 01234567-89ab-xdef-0123-456789abcdef +bad_hex_low_first 01234567-89ab-cdef-x123-456789abcdef +bad_hex_low_second 01234567-89ab-cdef-0123-x56789abcdef +short_java_uuid 1-1-1-1-1 +leading_space 01234567-89ab-cdef-0123-456789abcdef +trailing_space 01234567-89ab-cdef-0123-456789abcdef +unicode_hex ０1234567-89ab-cdef-0123-456789abcdef +wrong_dash 01234567_89ab-cdef-0123-456789abcdef +missing_dash 0123456789ab-cdef-0123-456789abcdef diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv new file mode 100644 index 000000000..4ad4ad323 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv @@ -0,0 +1,17 @@ +# case_id input_lo input_hi target_type expected_wire_type expected_utf8_hex expected_sql_value +uuid_string_asym -4941174770667877871 -6850330615249940744 STRING VARCHAR 61306565626339392d396330622d346566382d626236642d366262396264333830613131 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +uuid_string_zero 0 0 STRING VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303030 00000000-0000-0000-0000-000000000000 +uuid_string_one 1 0 STRING VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303031 00000000-0000-0000-0000-000000000001 +uuid_string_ones -1 -1 STRING VARCHAR 66666666666666662d666666662d666666662d666666662d666666666666666666666666 ffffffff-ffff-ffff-ffff-ffffffffffff +uuid_string_lo_min -9223372036854775808 0 STRING VARCHAR 30303030303030302d303030302d303030302d383030302d303030303030303030303030 00000000-0000-0000-8000-000000000000 +uuid_string_hi_min 0 -9223372036854775808 STRING VARCHAR 38303030303030302d303030302d303030302d303030302d303030303030303030303030 80000000-0000-0000-0000-000000000000 +uuid_string_both_min -9223372036854775808 -9223372036854775808 STRING VARCHAR +uuid_string_mixed 81985529216486895 -81985529216486896 STRING VARCHAR 66656463626139382d373635342d333231302d303132332d343536373839616263646566 fedcba98-7654-3210-0123-456789abcdef +uuid_varchar_asym -4941174770667877871 -6850330615249940744 VARCHAR VARCHAR 61306565626339392d396330622d346566382d626236642d366262396264333830613131 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +uuid_varchar_zero 0 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303030 00000000-0000-0000-0000-000000000000 +uuid_varchar_one 1 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303031 00000000-0000-0000-0000-000000000001 +uuid_varchar_ones -1 -1 VARCHAR VARCHAR 66666666666666662d666666662d666666662d666666662d666666666666666666666666 ffffffff-ffff-ffff-ffff-ffffffffffff +uuid_varchar_lo_min -9223372036854775808 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d383030302d303030303030303030303030 00000000-0000-0000-8000-000000000000 +uuid_varchar_hi_min 0 -9223372036854775808 VARCHAR VARCHAR 38303030303030302d303030302d303030302d303030302d303030303030303030303030 80000000-0000-0000-0000-000000000000 +uuid_varchar_both_min -9223372036854775808 -9223372036854775808 VARCHAR VARCHAR +uuid_varchar_mixed 81985529216486895 -81985529216486896 VARCHAR VARCHAR 66656463626139382d373635342d333231302d303132332d343536373839616263646566 fedcba98-7654-3210-0123-456789abcdef