From 71de5b2d12552b53d78fd378e94f58ab3862852a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 12 Nov 2026 17:21:48 +0000 Subject: [PATCH 1/5] Add query options and details to the query client QwpQueryClient and the Query facade can now use the query-details flags of QUERY_REQUEST, which the server advertises with the CAP_QUERY_DETAILS bit of SERVER_INFO (questdb PR #8356): - QwpQueryOptions sets a row window (skipRows, maxRows) and asks for the row count of the whole result set (countRows), execution statistics (stats) and the error position (errorPosition). It also carries resetSymbolDict. QwpQueryClient.execute() and Query.options() take it; the client reads it once, when the query starts, so a failover replay sends the same options. - The handler receives the answers through new default overloads: onEnd and onExecDone with a QwpQueryDetails (the row count, the compile, execute, count and client-wait times, and the JIT flag), and onError with the position. Each delegates to the next simpler overload, so existing handlers keep working. The client now calls the detailed overloads for every terminal event, client-side failures included, so one override sees them all. - QueryException.getPosition() carries the position to callers of the facade's Completion.await(). A query without options is byte-identical on the wire. Against a server without CAP_QUERY_DETAILS, the client leaves out the count, the statistics and the position, and fails a query that sets a row window before sending it, since that server would stream every row. After a failover reconnect the client checks the new server again. The I/O thread decodes the trailers the request asked for: row_count and the statistics block on RESULT_END, the statistics block on EXEC_DONE, and the position on QUERY_ERROR. A RESULT_END or EXEC_DONE without a requested trailer is a terminal transport error, like any other truncated terminator. A QUERY_ERROR without the position is the baseline frame the server sends for a request whose flags it could not read; it reports no position. RESULT_END and EXEC_DONE now share one bounds-checked varint reader, which also rejects data bits above bit 63 on a tenth byte, as the result batch decoder already does. --- README.md | 56 ++ .../main/java/io/questdb/client/Query.java | 17 +- .../io/questdb/client/QueryException.java | 25 +- .../client/cutlass/qwp/client/QueryEvent.java | 33 + .../qwp/client/QwpColumnBatchHandler.java | 69 ++- .../cutlass/qwp/client/QwpEgressIoThread.java | 275 ++++++--- .../cutlass/qwp/client/QwpEgressMsgKind.java | 67 +++ .../cutlass/qwp/client/QwpQueryClient.java | 221 +++++-- .../cutlass/qwp/client/QwpQueryDetails.java | 165 +++++ .../cutlass/qwp/client/QwpQueryOptions.java | 258 ++++++++ .../io/questdb/client/impl/QueryImpl.java | 57 +- .../io/questdb/client/impl/QueryLease.java | 7 + .../client/test/QuestDBQueryOptionsTest.java | 222 +++++++ .../cutlass/qwp/client/QueryEventTest.java | 75 ++- .../qwp/client/QwpColumnBatchHandlerTest.java | 63 +- .../cutlass/qwp/client/QwpEgressFrames.java | 304 ++++++++++ .../qwp/client/QwpEgressMockServer.java | 119 ++++ .../client/QwpEgressTerminatorDecodeTest.java | 567 ++++++++++++++++++ .../QwpQueryClientQueryDetailsTest.java | 518 ++++++++++++++++ .../client/QwpQueryClientQueryFlagsTest.java | 222 ++++++- .../qwp/client/QwpQueryOptionsTest.java | 164 +++++ .../qwp/client/RecordingQueryHandler.java | 148 +++++ 22 files changed, 3505 insertions(+), 147 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java create mode 100644 core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java diff --git a/README.md b/README.md index 7aaf10c95..232e64fbd 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,62 @@ try (Query q = db.borrowQuery()) { > A borrowed `Query` handle is **single-flight**: submit one query at a time on it, then submit again or close it. To > run queries concurrently, borrow one handle per concurrent query (up to `query_pool_max`). +### Page Through a Result + +`QwpQueryOptions` asks the server for one page of the result set, and for what the `/exec` HTTP endpoint reports +besides the rows: the row count of the whole result set, execution statistics, and the position of an error in the SQL +text. The handler receives them through the detailed overloads of its callbacks, and a failed `await()` throws a +`QueryException` whose `getPosition()` carries the position. The options need a QuestDB server that supports them; an +older server fails a query that sets `skipRows` or `maxRows`, and runs the others without the details. + +```java +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; + +QwpQueryOptions page = new QwpQueryOptions() + .skipRows(100) // leave out the first 100 rows + .maxRows(50) // stream at most 50 rows + .countRows(true) // count the rows of the whole result set + .stats(true) // report compile and execution times + .errorPosition(true); // report where in the SQL text an error is + +QwpColumnBatchHandler pageHandler = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + // the rows of the page + } + + @Override + public void onEnd(long totalRows) { + // not called: the overload below receives the end of the result + } + + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + System.out.println(totalRows + " of " + details.getRowCount() + " rows in " + + details.getExecuteNanos() / 1_000 + " us"); + } + + @Override + public void onError(byte status, String message) { + // not called: the overload below receives the error + } + + @Override + public void onError(long requestId, byte status, String message, int position) { + System.err.println("error at position " + position + ": " + message); + } +}; + +try (Query q = db.borrowQuery()) { + q.sql("SELECT * FROM trades ORDER BY ts DESC") + .options(page) + .handler(pageHandler) + .submit() + .await(); +} +``` + ### Custom Pool Sizing and Timeouts Use the builder to override the defaults. `senderPoolSize`/`queryPoolSize` set a fixed pool size; `*Min`/`*Max` allow diff --git a/core/src/main/java/io/questdb/client/Query.java b/core/src/main/java/io/questdb/client/Query.java index 539d57117..634134327 100644 --- a/core/src/main/java/io/questdb/client/Query.java +++ b/core/src/main/java/io/questdb/client/Query.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.QwpBindSetter; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; import java.io.Closeable; @@ -43,9 +44,9 @@ * creates one small lease handle per borrow (often scalar-replaced by the JIT * when used with try-with-resources). *

- * Lifecycle: configure with {@link #sql}, optional {@link #binds}, and - * {@link #handler}, then call {@link #submit()} to obtain a {@link Completion} - * and {@code await()} it before the next {@link #submit()}. + * Lifecycle: configure with {@link #sql}, optional {@link #binds} and + * {@link #options}, and {@link #handler}, then call {@link #submit()} to obtain + * a {@link Completion} and {@code await()} it before the next {@link #submit()}. *

* Thread safety: not thread-safe and single-flight -- one in-flight query per * handle. To run queries concurrently, borrow one handle per concurrent query. @@ -100,6 +101,16 @@ public interface Query extends Closeable { */ Query handler(QwpColumnBatchHandler handler); + /** + * Sets the per-query options: a row window that pages through the result + * set, and what the server reports besides the rows -- the row count of the + * whole result set, execution statistics and the error position. See + * {@link QwpQueryOptions}. The handle copies the values, so the instance can + * be reused right away; they apply to every {@link #submit()} until changed. + * Pass null to drop them. + */ + Query options(QwpQueryOptions options); + /** * Sets the SQL text. The buffer is not retained past {@link #submit()}. */ diff --git a/core/src/main/java/io/questdb/client/QueryException.java b/core/src/main/java/io/questdb/client/QueryException.java index a3fdedbd1..4fbff9928 100644 --- a/core/src/main/java/io/questdb/client/QueryException.java +++ b/core/src/main/java/io/questdb/client/QueryException.java @@ -24,6 +24,9 @@ package io.questdb.client; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; + /** * Thrown from {@link Completion#await()} / {@link Completion#await(long, java.util.concurrent.TimeUnit)} * when the server reported an error for the corresponding {@link Query}, @@ -32,20 +35,40 @@ *

* The original wire-level status byte is exposed via {@link #getStatus()} so * callers can distinguish cancellation from schema errors etc. without - * string-matching the message. + * string-matching the message. When the query asked for the error position + * with {@link QwpQueryOptions#errorPosition(boolean)}, {@link #getPosition()} + * points at the offending character of the SQL text. */ public class QueryException extends RuntimeException { + private final int position; private final byte status; public QueryException(byte status, String message) { + this(status, message, QwpEgressMsgKind.NO_ERROR_POSITION); + } + + public QueryException(byte status, String message, int position) { super(message); this.status = status; + this.position = position; } public QueryException(byte status, String message, Throwable cause) { super(message, cause); this.status = status; + this.position = QwpEgressMsgKind.NO_ERROR_POSITION; + } + + /** + * Returns the {@code char} index into the SQL text the server-reported + * error points at, or {@link QwpEgressMsgKind#NO_ERROR_POSITION} when the + * query did not ask for it with {@link QwpQueryOptions#errorPosition(boolean)}, + * the server did not report it, or the error does not come from the SQL + * text. + */ + public int getPosition() { + return position; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java index d3083e4c7..75b617354 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java @@ -45,8 +45,12 @@ public class QueryEvent { */ public static final int KIND_TRANSPORT_ERROR = 4; + // valid for KIND_END and KIND_EXEC_DONE: the row count and statistics trailers + public final QwpQueryDetails details = new QwpQueryDetails(); public QwpBatchBuffer buffer; // valid for KIND_BATCH (must be released to pool by consumer) public String errorMessage; // valid for KIND_ERROR + // valid for KIND_ERROR: the position trailer, NO_ERROR_POSITION when absent + public int errorPosition = QwpEgressMsgKind.NO_ERROR_POSITION; public byte errorStatus; // valid for KIND_ERROR public int kind; public short opType; // valid for KIND_EXEC_DONE (matches CompiledQuery.SELECT/INSERT/etc.) @@ -63,14 +67,20 @@ public QueryEvent asEnd(long totalRows) { this.kind = KIND_END; this.buffer = null; this.totalRows = totalRows; + this.details.clear(); return this; } public QueryEvent asError(byte status, String message) { + return asError(status, message, QwpEgressMsgKind.NO_ERROR_POSITION); + } + + public QueryEvent asError(byte status, String message, int position) { this.kind = KIND_ERROR; this.buffer = null; this.errorStatus = status; this.errorMessage = message; + this.errorPosition = position; return this; } @@ -79,6 +89,7 @@ public QueryEvent asExecDone(short opType, long rowsAffected) { this.buffer = null; this.opType = opType; this.rowsAffected = rowsAffected; + this.details.clear(); return this; } @@ -87,6 +98,7 @@ public QueryEvent asTransportError(byte status, String message) { this.buffer = null; this.errorStatus = status; this.errorMessage = message; + this.errorPosition = QwpEgressMsgKind.NO_ERROR_POSITION; return this; } @@ -102,9 +114,30 @@ public void reset() { this.kind = -1; this.buffer = null; this.errorMessage = null; + this.errorPosition = QwpEgressMsgKind.NO_ERROR_POSITION; this.errorStatus = 0; this.opType = 0; this.rowsAffected = 0; this.totalRows = 0; + this.details.clear(); + } + + /** + * Records the {@code row_count} trailer of a {@code RESULT_END} on + * {@link #details}. Call after {@link #asEnd(long)}. + */ + public QueryEvent withRowCount(long rowCount) { + this.details.setRowCount(rowCount); + return this; + } + + /** + * Records the statistics trailer of a {@code RESULT_END} or + * {@code EXEC_DONE} on {@link #details}. Call after {@link #asEnd(long)} or + * {@link #asExecDone(short, long)}. + */ + public QueryEvent withStats(long compileNanos, long executeNanos, long countNanos, long clientWaitNanos, boolean isJitCompiled) { + this.details.setStats(compileNanos, executeNanos, countNanos, clientWaitNanos, isJitCompiled); + return this; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java index 6237e1de0..937c3c2a3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java @@ -27,11 +27,20 @@ /** * Callback interface for consuming a streamed QWP egress query result. *

- * Invoked by {@link QwpQueryClient#execute(String, QwpColumnBatchHandler)}: + * Invoked by {@link QwpQueryClient#execute(CharSequence, QwpColumnBatchHandler)}: * once per {@code RESULT_BATCH} frame via {@link #onBatch(QwpColumnBatch)}, * then exactly once via either {@link #onEnd(long)} or * {@link #onError(byte, String)} (or {@link #onExecDone} for non-SELECT queries). *

+ * The client calls the most detailed overload of each terminal callback -- + * {@link #onEnd(long, long, QwpQueryDetails)}, + * {@link #onError(long, byte, String, int)} and + * {@link #onExecDone(long, short, long, QwpQueryDetails)} -- and each overload + * delegates by default to the next simpler one, down to the abstract + * {@link #onEnd(long)} and {@link #onError(byte, String)} and the no-op + * {@link #onExecDone(short, long)}. Override the overload that carries what you + * need; the simpler ones then do not fire for that event. + *

* The {@link QwpColumnBatch} passed to {@link #onBatch} is valid only for the * duration of that call. Copy any values you need to retain. *

@@ -71,6 +80,23 @@ default void onEnd(long requestId, long totalRows) { onEnd(totalRows); } + /** + * Details-aware overload of {@link #onEnd(long, long)}. Receives what the + * query reported besides its rows on the request of {@link QwpQueryOptions}: + * the row count of the whole result set and execution statistics. The + * default delegates to {@link #onEnd(long, long)} for backwards + * compatibility; override this method when you need the details. + * + * @param requestId client-assigned request id (matches {@link QwpColumnBatch#requestId()}) + * @param totalRows server-reported number of rows streamed, which a row window + * of {@link QwpQueryOptions} bounds + * @param details the row count and statistics the server reported; never null, + * and valid only for the duration of this call + */ + default void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + onEnd(requestId, totalRows); + } + /** * Invoked exactly once if the query fails at any point, instead of * {@link #onEnd} / {@link #onExecDone}. @@ -100,6 +126,29 @@ default void onError(long requestId, byte status, String message) { onError(status, message); } + /** + * Position-aware overload of {@link #onError(long, byte, String)}. When the + * query asked for the error position with + * {@link QwpQueryOptions#errorPosition(boolean)} and the server reported it, + * {@code message} is the bare error message and {@code position} the + * {@code char} index into the SQL text the error points at. Otherwise + * {@code message} is what the server sent, which may start with the + * position in brackets, and {@code position} is + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}, as it is for errors that do + * not come from the SQL text, such as a cancellation or a transport + * failure. The default delegates to {@link #onError(long, byte, String)} for + * backwards compatibility; override this method when you need the position. + * + * @param requestId client-assigned request id, or {@code -1} if no request was in flight + * @param status one of the QWP status codes (e.g., {@code STATUS_PARSE_ERROR}) + * @param message server-supplied error message (may be empty) + * @param position {@code char} index into the SQL text, or + * {@link QwpEgressMsgKind#NO_ERROR_POSITION} + */ + default void onError(long requestId, byte status, String message, int position) { + onError(requestId, status, message); + } + /** * Invoked when {@link QwpQueryClient#execute} has transparently reconnected * to another endpoint after a transport failure and is about to re-submit @@ -163,4 +212,22 @@ default void onExecDone(short opType, long rowsAffected) { default void onExecDone(long requestId, short opType, long rowsAffected) { onExecDone(opType, rowsAffected); } + + /** + * Details-aware overload of {@link #onExecDone(long, short, long)}. Receives + * the execution statistics the query reported on the request of + * {@link QwpQueryOptions#stats(boolean)}; a statement that produces no rows + * reports no row count. The default delegates to + * {@link #onExecDone(long, short, long)} for backwards compatibility; + * override this method when you need the statistics. + * + * @param requestId client-assigned request id (matches {@link QwpColumnBatch#requestId()}) + * @param opType server-side opType constant (CompiledQuery.SELECT / INSERT / UPDATE / CREATE_TABLE / etc.) + * @param rowsAffected rows inserted / updated / deleted; 0 for pure DDL + * @param details the statistics the server reported; never null, and valid + * only for the duration of this call + */ + default void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + onExecDone(requestId, opType, rowsAffected); + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java index 153b57e96..31e8bb1d5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -113,6 +113,10 @@ public class QwpEgressIoThread implements Runnable, WebSocketFrameHandler { private volatile boolean closed; private boolean creditEnabled; private boolean currentQueryDone; + // query_flags of the in-flight request as it went on the wire, after the + // client dropped the bits the server does not advertise. Tells the + // terminator decoders which trailers to read. I/O thread only. + private long currentQueryFlags; private long currentRequestId = -1L; private volatile boolean shutdown; @@ -164,14 +168,30 @@ public QwpEgressIoThread(WebSocketClient wsClient, int bufferPoolSize, TerminalF } } + /** + * Decodes a QUERY_ERROR payload of a request without + * {@link QwpEgressMsgKind#QUERY_FLAG_ERROR_POSITION} into a + * {@link QueryEvent}. Visible for testing. See + * {@link #decodeError(long, int, long)}. + */ + public static QueryEvent decodeError(long payload, int payloadLen) { + return decodeError(payload, payloadLen, 0); + } + /** * Decodes a QUERY_ERROR payload into a {@link QueryEvent}. Visible for testing. + * Body: {@code msg_kind(1) + request_id(8) + status(1) + msg_len(u16) + msg}, + * then {@code position(i32)} when {@code queryFlags} has + * {@link QwpEgressMsgKind#QUERY_FLAG_ERROR_POSITION}. Bytes after the known + * fields are ignored. + *

* Bound-checks msgLen against the actual payload: a hostile or buggy server * can encode msgLen=0xFFFF with a tiny payload, which would otherwise read * up to 65 KiB of native memory beyond the frame and surface it to the user - * callback as a String. + * callback as a String. A frame that is too short for what it claims + * decodes to a {@code STATUS_INTERNAL_ERROR} event that names the defect. */ - public static QueryEvent decodeError(long payload, int payloadLen) { + public static QueryEvent decodeError(long payload, int payloadLen, long queryFlags) { long payloadEnd = payload + payloadLen; long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; if (p + 1 + 2 > payloadEnd) { @@ -189,7 +209,63 @@ public static QueryEvent decodeError(long payload, int payloadLen) { for (int i = 0; i < msgLen; i++) { bytes[i] = Unsafe.getUnsafe().getByte(p + i); } - return new QueryEvent().asError(status, new String(bytes, StandardCharsets.UTF_8)); + String message = new String(bytes, StandardCharsets.UTF_8); + p += msgLen; + // A frame that ends with the message is a baseline frame, which the + // server also sends for a request whose flags it could not read. + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION) == 0 || p == payloadEnd) { + return new QueryEvent().asError(status, message); + } + if (p + Integer.BYTES > payloadEnd) { + return new QueryEvent().asError(WebSocketResponse.STATUS_INTERNAL_ERROR, + "QUERY_ERROR frame truncated mid position [status=" + status + ", message=" + message + ']'); + } + return new QueryEvent().asError(status, message, Unsafe.getUnsafe().getInt(p)); + } + + /** + * Decodes an EXEC_DONE payload into a {@link QueryEvent}. Visible for + * testing. Body: {@code msg_kind(1) + request_id(8) + op_type(1) + + * rows_affected(varint)}, then the statistics block when {@code queryFlags} + * has {@link QwpEgressMsgKind#QUERY_FLAG_STATS}. Statements that produce no + * rows carry no {@code row_count}, whatever the flags. Bytes after the known + * fields are ignored. + * + * @throws QwpDecodeException if the frame ends inside a field the flags call + * for, or a varint does not fit in 64 bits + */ + public static QueryEvent decodeExecDone(long payload, int payloadLen, long queryFlags) throws QwpDecodeException { + TerminatorReader rd = new TerminatorReader("EXEC_DONE", payload + QwpConstants.HEADER_SIZE + 1 + 8, payload + payloadLen); + byte opType = rd.int8("op_type"); + QueryEvent ev = new QueryEvent().asExecDone(opType, rd.varint("rows_affected")); + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + readStats(rd, ev); + } + return ev; + } + + /** + * Decodes a RESULT_END payload into a {@link QueryEvent}. Visible for + * testing. Body: {@code msg_kind(1) + request_id(8) + final_seq(varint) + + * total_rows(varint)}, then {@code row_count(varint)} when + * {@code queryFlags} has {@link QwpEgressMsgKind#QUERY_FLAG_COUNT_ROWS}, then + * the statistics block when it has {@link QwpEgressMsgKind#QUERY_FLAG_STATS}. + * Bytes after the known fields are ignored. + * + * @throws QwpDecodeException if the frame ends inside a field the flags call + * for, or a varint does not fit in 64 bits + */ + public static QueryEvent decodeResultEnd(long payload, int payloadLen, long queryFlags) throws QwpDecodeException { + TerminatorReader rd = new TerminatorReader("RESULT_END", payload + QwpConstants.HEADER_SIZE + 1 + 8, payload + payloadLen); + rd.varint("final_seq"); + QueryEvent ev = new QueryEvent().asEnd(rd.varint("total_rows")); + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS) != 0) { + ev.withRowCount(rd.varint("row_count")); + } + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + readStats(rd, ev); + } + return ev; } @Override @@ -316,6 +392,7 @@ public void run() { // Per-query state; accessed only from the I/O thread. currentQueryDone = false; currentRequestId = req.requestId; + currentQueryFlags = req.queryFlags; creditEnabled = req.initialCredit > 0L; // The schema rides the first RESULT_BATCH (batch_seq == 0) of each // query; invalidate any schema left from the prior query so a @@ -380,6 +457,12 @@ public void shutdown() { * during {@link #sendQueryRequest} and does not retain a reference after * the send completes. {@code bindCount} is the number of binds the * payload contains; zero when the user supplied no binds. + *

+ * {@code queryFlags} are the bits the request carries, which the server + * must advertise; zero sends no {@code query_flags} trailer. {@code skipRows} + * goes on the wire only with {@link QwpEgressMsgKind#QUERY_FLAG_SKIP_ROWS} + * and {@code maxRows} only with {@link QwpEgressMsgKind#QUERY_FLAG_MAX_ROWS}; + * both must be non-negative. */ public void submitQuery( CharSequence sql, @@ -388,7 +471,9 @@ public void submitQuery( int bindCount, long bindPayloadPtr, long bindPayloadLen, - long queryFlags + long queryFlags, + long skipRows, + long maxRows ) throws InterruptedException { pendingRequest.sql = sql; pendingRequest.requestId = requestId; @@ -397,6 +482,8 @@ public void submitQuery( pendingRequest.bindPayloadPtr = bindPayloadPtr; pendingRequest.bindPayloadLen = bindPayloadLen; pendingRequest.queryFlags = queryFlags; + pendingRequest.skipRows = skipRows; + pendingRequest.maxRows = maxRows; requests.put(pendingRequest); } @@ -407,6 +494,22 @@ public QueryEvent takeEvent() throws InterruptedException { return events.take(); } + /** + * Reads the statistics block of {@link QwpEgressMsgKind#QUERY_FLAG_STATS}: + * {@code compile_ns, execute_ns, count_ns, client_wait_ns} as varints and + * the {@code stats_flags} byte. Flag bits other than + * {@link QwpEgressMsgKind#STATS_FLAG_JIT_COMPILED} are reserved and ignored. + */ + private static void readStats(TerminatorReader rd, QueryEvent ev) throws QwpDecodeException { + long compileNanos = rd.varint("compile_ns"); + long executeNanos = rd.varint("execute_ns"); + long countNanos = rd.varint("count_ns"); + long clientWaitNanos = rd.varint("client_wait_ns"); + byte statsFlags = rd.int8("stats_flags"); + ev.withStats(compileNanos, executeNanos, countNanos, clientWaitNanos, + (statsFlags & QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED) != 0); + } + /** * Takes a pre-allocated {@link QueryEvent} from the pool for the I/O thread * to populate before offering to {@link #events}. Falls back to a fresh @@ -424,104 +527,35 @@ private QueryEvent borrowEvent() { } private void decodeAndEmitError(long payload, int payloadLen) { - QueryEvent ev = decodeError(payload, payloadLen); + QueryEvent ev = decodeError(payload, payloadLen, currentQueryFlags); events.offer(ev); } /** - * EXEC_DONE body: msg_kind(1) + requestId(8) + op_type(1) + rows_affected(varint). - * Parses all fields, surfaces as a {@link QueryEvent#KIND_EXEC_DONE} event. + * Surfaces an EXEC_DONE as a {@link QueryEvent#KIND_EXEC_DONE} event. A + * frame that ends inside a field, or carries a varint that does not fit in + * 64 bits, is a terminal transport error rather than an EXEC_DONE with a + * partial value. */ private void decodeAndEmitExecDone(long payload, int payloadLen) { - long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; - long limit = payload + payloadLen; - if (p + 1 > limit) { - emitTerminalTransportError("EXEC_DONE frame truncated before op_type"); - return; - } - byte opType = Unsafe.getUnsafe().getByte(p++); - long rowsAffected = 0; - int shift = 0; - boolean terminated = false; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - rowsAffected |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - terminated = true; - break; - } - shift += 7; - if (shift > 63) { - emitTerminalTransportError("EXEC_DONE rows_affected varint overflow"); - return; - } - } - // The loop also exits when {@code p == limit} on a frame that ended - // mid-varint without a terminator byte. Without this guard, a buggy or - // hostile server could truncate the frame and the partial value would - // surface to the user handler as if the EXEC_DONE completed normally. - if (!terminated) { - emitTerminalTransportError("EXEC_DONE frame truncated mid rows_affected varint"); - return; + try { + events.offer(decodeExecDone(payload, payloadLen, currentQueryFlags)); + } catch (QwpDecodeException e) { + emitTerminalTransportError(e.getMessage()); } - events.offer(new QueryEvent().asExecDone(opType, rowsAffected)); } /** - * RESULT_END body: msg_kind(1) + requestId(8) + final_seq(varint) + total_rows(varint). - * We only need total_rows. Both varint loops cap at 10 bytes so a hostile - * server cannot drive {@code shift} past 63 (where Java's {@code <<} masks - * the count to 6 bits and silently wraps high bytes back into the low bits, - * producing a wildly wrong total). A frame that ends mid-varint surfaces - * as a terminal transport error rather than a partial-value RESULT_END. + * Surfaces a RESULT_END as a {@link QueryEvent#KIND_END} event. A frame that + * ends inside a field, or carries a varint that does not fit in 64 bits, is + * a terminal transport error rather than a RESULT_END with a partial value. */ private void decodeAndEmitResultEnd(long payload, int payloadLen) { - long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; - long limit = payload + payloadLen; - int seqBytes = 0; - boolean seqTerminated = false; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - if ((b & 0x80) == 0) { - seqTerminated = true; - break; - } - if (++seqBytes > 9) { - // Continuation bit set on byte 10 of the final_seq varint -- - // malformed. Surface as terminal so the caller cannot proceed - // with a desynced cursor. - emitTerminalTransportError("RESULT_END final_seq varint overflow"); - return; - } - } - if (!seqTerminated) { - emitTerminalTransportError("RESULT_END frame truncated mid final_seq varint"); - return; - } - long total = 0; - int shift = 0; - boolean totalTerminated = false; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - total |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - totalTerminated = true; - break; - } - shift += 7; - if (shift > 63) { - // Same overflow guard decodeAndEmitExecDone uses; without it, - // byte 10 of total_rows could land in the sign bit and bytes - // 11+ would silently wrap. - emitTerminalTransportError("RESULT_END total_rows varint overflow"); - return; - } - } - if (!totalTerminated) { - emitTerminalTransportError("RESULT_END frame truncated mid total_rows varint"); - return; + try { + events.offer(decodeResultEnd(payload, payloadLen, currentQueryFlags)); + } catch (QwpDecodeException e) { + emitTerminalTransportError(e.getMessage()); } - events.offer(new QueryEvent().asEnd(total)); } /** @@ -721,9 +755,16 @@ private void sendQueryRequest(QueryRequest req) { sendScratch.putBlockOfBytes(req.bindPayloadPtr, req.bindPayloadLen); } // Optional query_flags trailer; omitted when zero so a baseline frame - // stays byte-identical and the server defaults the flags to 0. + // stays byte-identical and the server defaults the flags to 0. The + // row-window values follow the flags in ascending order of their bits. if (req.queryFlags != 0) { sendScratch.putVarint(req.queryFlags); + if ((req.queryFlags & QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS) != 0) { + sendScratch.putVarint(req.skipRows); + } + if ((req.queryFlags & QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS) != 0) { + sendScratch.putVarint(req.maxRows); + } } wsClient.sendBinary(sendScratch.getBufferPtr(), sendScratch.getPosition()); sendScratch.reset(); @@ -783,8 +824,62 @@ private static final class QueryRequest { long bindPayloadLen; long bindPayloadPtr; long initialCredit; + long maxRows; long queryFlags; long requestId; + long skipRows; CharSequence sql; } + + /** + * Bounds-checked cursor over the body of a terminator frame (RESULT_END, + * EXEC_DONE). Every read fails with a {@link QwpDecodeException} that names + * the frame and the field, rather than read past the frame or return a + * partial value. + */ + private static final class TerminatorReader { + private final String frame; + private final long limit; + private long p; + + TerminatorReader(String frame, long p, long limit) { + this.frame = frame; + this.p = p; + this.limit = limit; + } + + byte int8(String field) throws QwpDecodeException { + if (p >= limit) { + throw new QwpDecodeException(frame + " frame truncated before " + field); + } + return Unsafe.getUnsafe().getByte(p++); + } + + /** + * Reads an unsigned LEB128 varint of at most 10 bytes. On byte 10 only + * bit 0 of the data still fits in a long: a hostile server must not + * drive the shift past 63, where Java's {@code <<} masks the count to + * 6 bits and wraps the high bytes back into the low bits, producing a + * wildly wrong value. + */ + long varint(String field) throws QwpDecodeException { + long value = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + if (shift == 63 && (b & 0x7E) != 0) { + throw new QwpDecodeException(frame + ' ' + field + " varint overflow"); + } + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return value; + } + shift += 7; + if (shift > 63) { + throw new QwpDecodeException(frame + ' ' + field + " varint overflow"); + } + } + throw new QwpDecodeException(frame + " frame truncated mid " + field + " varint"); + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java index e9b2c8934..29ec7b0fc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java @@ -38,6 +38,16 @@ public final class QwpEgressMsgKind { */ public static final byte CACHE_RESET = 0x17; public static final byte CANCEL = 0x14; + /** + * {@code SERVER_INFO.capabilities} bit: the server honours the query-details + * flags of {@code QUERY_REQUEST.query_flags} -- {@link #QUERY_FLAG_ERROR_POSITION}, + * {@link #QUERY_FLAG_STATS}, {@link #QUERY_FLAG_COUNT_ROWS}, + * {@link #QUERY_FLAG_SKIP_ROWS} and {@link #QUERY_FLAG_MAX_ROWS}. The client + * sends those bits only when this bit is set: a server without it ignores + * them and answers with the baseline frames. Mirrors the server-side constant + * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_DETAILS}. + */ + public static final int CAP_QUERY_DETAILS = 0x00000008; /** * {@code SERVER_INFO.capabilities} bit: the server parses the optional * {@code query_flags:varint} trailer on {@code QUERY_REQUEST}. The client @@ -60,13 +70,65 @@ public final class QwpEgressMsgKind { * {@code request_id:u64, op_type:u8, rows_affected:varint}. */ public static final byte EXEC_DONE = 0x16; + /** + * Error position that points nowhere in the SQL text: the position of an + * error that does not come from the SQL text, such as a cancellation, and + * of every error when the position was not requested or not reported. + */ + public static final int NO_ERROR_POSITION = -1; public static final byte QUERY_ERROR = 0x13; + /** + * {@code QUERY_REQUEST.query_flags} bit: report the row count of the whole + * result set, whatever {@link #QUERY_FLAG_SKIP_ROWS} and + * {@link #QUERY_FLAG_MAX_ROWS} leave out of the stream. {@code RESULT_END} + * then carries {@code row_count:varint} after {@code total_rows}. Statements + * that do not produce rows ignore it. Sent only when the server advertised + * {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_COUNT_ROWS = 0x08; + /** + * {@code QUERY_REQUEST.query_flags} bit: report the error position. A + * {@code QUERY_ERROR} then carries the bare message, without the + * {@code [n]} prefix of the baseline frame, and ends with + * {@code position:i32}, the UTF-16 offset into the SQL text the error points + * at, or {@link #NO_ERROR_POSITION}. The server answers with the baseline + * frame when it could not read the request's flags. Sent only when the + * server advertised {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_ERROR_POSITION = 0x02; + /** + * {@code QUERY_REQUEST.query_flags} bit: stream at most {@code max_rows} + * rows. The request carries {@code max_rows:varint} after + * {@code query_flags}, and after {@code skip_rows} when + * {@link #QUERY_FLAG_SKIP_ROWS} is set too. Zero streams only the schema. + * Statements that do not produce rows ignore it. Sent only when the server + * advertised {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_MAX_ROWS = 0x20; /** * {@code QUERY_REQUEST.query_flags} bit: reset the connection-scoped SYMBOL * dict before this query, scoping it to the query. Sent only when the server * advertised {@link #CAP_QUERY_FLAGS}. */ public static final int QUERY_FLAG_RESET_DICT = 0x01; + /** + * {@code QUERY_REQUEST.query_flags} bit: leave out the first + * {@code skip_rows} rows of the result set. The request carries + * {@code skip_rows:varint} right after {@code query_flags}. Statements that + * do not produce rows ignore it. Sent only when the server advertised + * {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_SKIP_ROWS = 0x10; + /** + * {@code QUERY_REQUEST.query_flags} bit: report execution statistics. + * {@code RESULT_END} and {@code EXEC_DONE} then end with + * {@code compile_ns:varint, execute_ns:varint, count_ns:varint, + * client_wait_ns:varint, stats_flags:u8}, after {@code row_count} when + * {@link #QUERY_FLAG_COUNT_ROWS} is set too. See + * {@link #STATS_FLAG_JIT_COMPILED}. Sent only when the server advertised + * {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_STATS = 0x04; public static final byte QUERY_REQUEST = 0x10; /** * Reset mask bit: clear the connection-scoped SYMBOL dict. @@ -105,6 +167,11 @@ public final class QwpEgressMsgKind { * {@link #CACHE_RESET}; SERVER_INFO lives at 0x18. */ public static final byte SERVER_INFO = 0x18; + /** + * {@code stats_flags} bit of the {@link #QUERY_FLAG_STATS} block: the query + * filter runs as JIT-compiled native code. + */ + public static final byte STATS_FLAG_JIT_COMPILED = 0x01; private QwpEgressMsgKind() { } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index dbd34fec5..2f4efb57b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -100,6 +100,13 @@ * truncated / unknown frame, decoder out of sync, I/O thread interrupt) all * surface with {@link WebSocketResponse#STATUS_INTERNAL_ERROR} and the * specific cause in the message. + *

+ * Per-query options: + * {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler, QwpQueryOptions)} + * pages through a result set with a row window, and asks the server for what + * the {@code /exec} HTTP endpoint reports besides the rows: the row count of + * the whole result set, execution statistics and the error position. See + * {@link QwpQueryOptions}. */ public class QwpQueryClient implements QuietCloseable { @@ -156,6 +163,15 @@ public class QwpQueryClient implements QuietCloseable { // Warn once per process when an application-supplied token provider is used over ws:: (spec section 9). private static final AtomicBoolean CLEARTEXT_PROVIDER_WARNED = new AtomicBoolean(); private static final Logger LOG = LoggerFactory.getLogger(QwpQueryClient.class); + // query_flags bits a server honours only when it advertises CAP_QUERY_DETAILS + private static final int QUERY_DETAILS_FLAGS = QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION + | QwpEgressMsgKind.QUERY_FLAG_STATS + | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS + | QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + // query_flags bits of the row window, which change what the server streams + private static final int ROW_WINDOW_QUERY_FLAGS = QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; // Reusable typed bind-value sink. Populated on the user thread by the // {@link QwpBindSetter} passed to execute(); the pre-encoded bytes are // handed to the I/O thread via QueryRequest. Allocated once per client to @@ -756,6 +772,15 @@ public void execute(CharSequence sql, QwpColumnBatchHandler handler, boolean res execute(sql, null, handler, resetSymbolDict); } + /** + * As {@link #execute(CharSequence, QwpColumnBatchHandler)}, with per-query + * {@code options}. See + * {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler, QwpQueryOptions)}. + */ + public void execute(CharSequence sql, QwpColumnBatchHandler handler, QwpQueryOptions options) { + execute(sql, null, handler, options); + } + /** * Executes {@code sql} with typed bind parameters supplied by the given * {@link QwpBindSetter}. The setter runs on the calling thread before the @@ -779,20 +804,35 @@ public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler * {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}; otherwise it is silently ignored. */ public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, boolean resetSymbolDict) { - if (!executing.compareAndSet(false, true)) { - throw new IllegalStateException( - "QwpQueryClient.execute called while another execute is in flight; one query at a time per client"); - } - // Drop any cancel latched between calls (e.g., a watchdog that fired - // while no request was in flight, or a previous pooled user that - // released the client without ever calling execute). Failover retries - // inside this execute() must still honor a fresh cancel, so the latch - // is intentionally NOT cleared inside executeOnce(). - pendingCancel = false; - try { - executeImpl(sql, binds, handler, resetSymbolDict); - } finally { - executing.set(false); + executeExclusive(sql, binds, handler, resetSymbolDict ? QwpEgressMsgKind.QUERY_FLAG_RESET_DICT : 0, 0, Long.MAX_VALUE); + } + + /** + * As {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler)}, + * with per-query {@code options}: a row window that pages through the + * result set, and what the server reports besides the rows -- the row count + * of the whole result set and execution statistics, which reach + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)} and + * {@link QwpColumnBatchHandler#onExecDone(long, short, long, QwpQueryDetails)}, + * and the error position, which reaches + * {@link QwpColumnBatchHandler#onError(long, byte, String, int)}. The client + * reads {@code options} when the call starts; null runs the query without + * options. + *

+ * The options need a server that advertises + * {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. Against any other, a row + * window fails the query through + * {@link QwpColumnBatchHandler#onError(long, byte, String, int)} with + * {@link WebSocketResponse#STATUS_INTERNAL_ERROR} before it is sent, since + * the server would stream every row; the other options are left out and the + * query runs without them. After a failover reconnect the client checks the + * new server again. + */ + public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, QwpQueryOptions options) { + if (options == null) { + executeExclusive(sql, binds, handler, 0, 0, Long.MAX_VALUE); + } else { + executeExclusive(sql, binds, handler, options.toQueryFlags(), options.getSkipRows(), options.getMaxRows()); } } @@ -1367,6 +1407,23 @@ private static boolean matchesTarget(byte role, String target) { return true; } + /** + * Returns the bits of {@code queryFlags} a server with {@code capabilities} + * honours: {@link QwpEgressMsgKind#QUERY_FLAG_RESET_DICT} needs + * {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}, and the query-details bits need + * {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. + */ + private static long resolveQueryFlags(long queryFlags, int capabilities) { + long resolved = queryFlags; + if ((capabilities & QwpEgressMsgKind.CAP_QUERY_FLAGS) == 0) { + resolved &= ~QwpEgressMsgKind.QUERY_FLAG_RESET_DICT; + } + if ((capabilities & QwpEgressMsgKind.CAP_QUERY_DETAILS) == 0) { + resolved &= ~QUERY_DETAILS_FLAGS; + } + return resolved; + } + private static TokenProviderSpec validateConfigAndResolveTokenProvider(ConfigView view, boolean tls) { view.getHostPorts("addr", DEFAULT_WS_PORT, (h, p) -> { }); @@ -1699,7 +1756,44 @@ private void connectWalk() { throw failure; } - private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, boolean resetSymbolDict) { + /** + * Runs one {@link #execute} at a time on this client. {@code queryFlags} are + * the {@code query_flags} bits the caller asks for, before + * {@link #resolveQueryFlags} drops those the server does not advertise. + */ + private void executeExclusive( + CharSequence sql, + QwpBindSetter binds, + QwpColumnBatchHandler handler, + long queryFlags, + long skipRows, + long maxRows + ) { + if (!executing.compareAndSet(false, true)) { + throw new IllegalStateException( + "QwpQueryClient.execute called while another execute is in flight; one query at a time per client"); + } + // Drop any cancel latched between calls (e.g., a watchdog that fired + // while no request was in flight, or a previous pooled user that + // released the client without ever calling execute). Failover retries + // inside this execute() must still honor a fresh cancel, so the latch + // is intentionally NOT cleared inside executeOnce(). + pendingCancel = false; + try { + executeImpl(sql, binds, handler, queryFlags, skipRows, maxRows); + } finally { + executing.set(false); + } + } + + private void executeImpl( + CharSequence sql, + QwpBindSetter binds, + QwpColumnBatchHandler handler, + long queryFlags, + long skipRows, + long maxRows + ) { if (closedFlag.get() || isShutdownRequested) { throw new IllegalStateException("QwpQueryClient is closed"); } @@ -1715,7 +1809,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa connect(); } catch (RuntimeException e) { handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, - "reconnect failed: " + e.getMessage()); + "reconnect failed: " + e.getMessage(), QwpEgressMsgKind.NO_ERROR_POSITION); return; } } @@ -1735,24 +1829,27 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa int attempt = 0; while (true) { if (isShutdownRequested) { - handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "query client shut down"); + handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "query client shut down", + QwpEgressMsgKind.NO_ERROR_POSITION); return; } attempt++; FailoverProbeHandler probe = new FailoverProbeHandler(handler); - executeOnce(sql, binds, probe, resetSymbolDict); + executeOnce(sql, binds, probe, queryFlags, skipRows, maxRows); if (!probe.transportFailureIntercepted) { return; } if (isShutdownRequested) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); + handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down", + QwpEgressMsgKind.NO_ERROR_POSITION); return; } if (!failoverEnabled) { // With failover off the transport failure stays latched: every later execute() reports it. healthTracker.connectionLost(); healthTracker.failed(); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, probe.interceptedMessage); + handler.onError(probe.interceptedRequestId, probe.interceptedStatus, probe.interceptedMessage, + QwpEgressMsgKind.NO_ERROR_POSITION); return; } if (attempt >= failoverMaxAttempts || System.nanoTime() - failoverDeadlineNanos >= 0) { @@ -1769,7 +1866,8 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa + (attempt == 1 ? "" : "s") + " (" + failovers + " failover reconnect" + (failovers == 1 ? "" : "s") + "); last error: " - + probe.interceptedMessage); + + probe.interceptedMessage, + QwpEgressMsgKind.NO_ERROR_POSITION); return; } int failedIndex = currentEndpointIndex; @@ -1803,7 +1901,8 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa + (attempt == 1 ? "" : "s") + " (" + failovers + " failover reconnect" + (failovers == 1 ? "" : "s") + "); last error: " - + probe.interceptedMessage); + + probe.interceptedMessage, + QwpEgressMsgKind.NO_ERROR_POSITION); return; } if (delay > remaining) { @@ -1816,7 +1915,8 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa Thread.currentThread().interrupt(); handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "failover interrupted while backing off after attempt " - + attempt + "; last error: " + probe.interceptedMessage); + + attempt + "; last error: " + probe.interceptedMessage, + QwpEgressMsgKind.NO_ERROR_POSITION); return; } } @@ -1839,24 +1939,28 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa "auth-rejected during failover reconnect [host=" + authErr.getHost() + ':' + authErr.getPort() + ", status=" + authErr.getStatusCode() - + ", last error: " + probe.interceptedMessage + ']'); + + ", last error: " + probe.interceptedMessage + ']', + QwpEgressMsgKind.NO_ERROR_POSITION); return; } catch (QwpCredentialUnavailableException credentialErr) { // The token provider could not supply a credential; no endpoint was contacted. handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "failover reconnect failed: " + credentialErr.getMessage() - + " [last error: " + probe.interceptedMessage + ']'); + + " [last error: " + probe.interceptedMessage + ']', + QwpEgressMsgKind.NO_ERROR_POSITION); return; } catch (RuntimeException reconnectErr) { handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "failover reconnect failed after " + attempt + " attempt" + (attempt == 1 ? "" : "s") + " [last error: " + probe.interceptedMessage + ", reconnect error: " - + reconnectErr.getMessage() + ']'); + + reconnectErr.getMessage() + ']', + QwpEgressMsgKind.NO_ERROR_POSITION); return; } if (isShutdownRequested) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); + handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down", + QwpEgressMsgKind.NO_ERROR_POSITION); return; } handler.onFailoverReset(probe.interceptedRequestId, serverInfo); @@ -1868,7 +1972,14 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa * the user's handler in a {@link FailoverProbeHandler} so that the outer * loop can intercept transport failures before they reach the user. */ - private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHandler probe, boolean resetSymbolDict) { + private void executeOnce( + CharSequence sql, + QwpBindSetter binds, + FailoverProbeHandler probe, + long queryFlags, + long skipRows, + long maxRows + ) { // Cache the I/O thread reference at entry: close() may null the field while // we are inside this loop, so reading the field per-iteration would NPE // exactly when the user is mid-execute() and close() races. The queue and @@ -1876,7 +1987,8 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan // before close() returns. QwpEgressIoThread io = ioThread; if (io == null) { - probe.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "QwpQueryClient is closed"); + probe.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "QwpQueryClient is closed", + QwpEgressMsgKind.NO_ERROR_POSITION); return; } GenerationListener listener = currentGenerationListener; @@ -1885,6 +1997,19 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan probe.markTransportFailure(-1L, tf.status, tf.message); return; } + // Resolved per attempt: a failover reconnect may land on a server that + // advertises other capabilities. + final QwpServerInfo info = serverInfo; + final int capabilities = info != null ? info.getCapabilities() : 0; + final long wireQueryFlags = resolveQueryFlags(queryFlags, capabilities); + if ((wireQueryFlags & ROW_WINDOW_QUERY_FLAGS) != (queryFlags & ROW_WINDOW_QUERY_FLAGS)) { + // A server that does not know the row window would stream every + // row of the result set. Like a bind error, this is deterministic + // on this server, not a transport failure. + probe.deliverFinal(-1L, "server does not support skipRows and maxRows [capabilities=0x" + + Integer.toHexString(capabilities) + "]; they need a server that advertises CAP_QUERY_DETAILS"); + return; + } bindValues.reset(); if (binds != null) { try { @@ -1917,7 +2042,7 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan } try { io.submitQuery(sql, requestId, initialCreditBytes, bindValues.count(), bindValues.bufferPtr(), bindValues.bufferLen(), - resolveQueryFlags(resetSymbolDict)); + wireQueryFlags, skipRows, maxRows); boolean isBatchSkipped = false; while (true) { QueryEvent ev = io.takeEvent(); @@ -1942,20 +2067,21 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan if (isBatchSkipped) { probe.deliverFinal(requestId, "query client shut down"); } else { - probe.onEnd(requestId, ev.totalRows); + probe.onEnd(requestId, ev.totalRows, ev.details); } return; case QueryEvent.KIND_EXEC_DONE: - probe.onExecDone(requestId, ev.opType, ev.rowsAffected); + probe.onExecDone(requestId, ev.opType, ev.rowsAffected, ev.details); return; case QueryEvent.KIND_ERROR: - probe.onError(requestId, ev.errorStatus, ev.errorMessage); + probe.onError(requestId, ev.errorStatus, ev.errorMessage, ev.errorPosition); return; case QueryEvent.KIND_TRANSPORT_ERROR: probe.markTransportFailure(requestId, ev.errorStatus, ev.errorMessage); return; default: - probe.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind); + probe.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind, + QwpEgressMsgKind.NO_ERROR_POSITION); return; } } finally { @@ -2199,16 +2325,6 @@ private String resolveAuthorizationHeader() { return authorizationHeader; } - private long resolveQueryFlags(boolean resetSymbolDict) { - if (!resetSymbolDict) { - return 0L; - } - QwpServerInfo info = serverInfo; - return info != null && (info.getCapabilities() & QwpEgressMsgKind.CAP_QUERY_FLAGS) != 0 - ? QwpEgressMsgKind.QUERY_FLAG_RESET_DICT - : 0L; - } - private void runUpgradeWithTimeout(Endpoint ep, String authHeader) { // Connect first, OUTSIDE the upgrade try. A connect-phase failure -- // including a connect_timeout overage flagged via flagAsTimeout() -- must @@ -2312,6 +2428,11 @@ public void onEnd(long requestId, long totalRows) { delegate.onEnd(requestId, totalRows); } + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + delegate.onEnd(requestId, totalRows, details); + } + @Override public void onError(byte status, String message) { // Server-emitted QUERY_ERROR. Pass straight through. Transport @@ -2324,6 +2445,11 @@ public void onError(long requestId, byte status, String message) { delegate.onError(requestId, status, message); } + @Override + public void onError(long requestId, byte status, String message, int position) { + delegate.onError(requestId, status, message, position); + } + @Override public void onExecDone(short opType, long rowsAffected) { delegate.onExecDone(opType, rowsAffected); @@ -2334,6 +2460,11 @@ public void onExecDone(long requestId, short opType, long rowsAffected) { delegate.onExecDone(requestId, opType, rowsAffected); } + @Override + public void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + delegate.onExecDone(requestId, opType, rowsAffected, details); + } + @Override public void onFailoverReset(QwpServerInfo newNode) { delegate.onFailoverReset(newNode); @@ -2350,7 +2481,7 @@ public void onFailoverReset(long requestId, QwpServerInfo newNode) { * errors, interrupts) so they don't trigger a spurious failover. */ void deliverFinal(long requestId, String message) { - delegate.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, message); + delegate.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, message, QwpEgressMsgKind.NO_ERROR_POSITION); } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java new file mode 100644 index 000000000..0b683c7aa --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java @@ -0,0 +1,165 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * What a successful query reported besides its rows, on the request of + * {@link QwpQueryOptions}: the row count of the whole result set and execution + * statistics. Passed to + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)} and + * {@link QwpColumnBatchHandler#onExecDone(long, short, long, QwpQueryDetails)}, + * never null. + *

+ * The instance is valid only for the duration of that call; the client reuses + * it afterwards. Copy any values you need to retain. + */ +public final class QwpQueryDetails { + + private long clientWaitNanos = -1; + private long compileNanos = -1; + private long countNanos = -1; + private long executeNanos = -1; + private boolean hasRowCount; + private boolean hasStats; + private boolean isJitCompiled; + private long rowCount = -1; + + /** + * Returns how long, in nanoseconds, the query waited for this client to + * read its results or grant credit, or -1 when {@link #hasStats()} is false. + * Part of {@link #getExecuteNanos()}. + */ + public long getClientWaitNanos() { + return clientWaitNanos; + } + + /** + * Returns how long, in nanoseconds, the server took to compile the SQL text, + * or -1 when {@link #hasStats()} is false. Zero when the server reused a + * compiled query from its cache. Part of {@link #getExecuteNanos()}. + */ + public long getCompileNanos() { + return compileNanos; + } + + /** + * Returns how long, in nanoseconds, the server took to count the rows the + * row window left out, or -1 when {@link #hasStats()} is false. Zero when + * the row count was not requested, or needed no counting. Part of + * {@link #getExecuteNanos()}. + */ + public long getCountNanos() { + return countNanos; + } + + /** + * Returns how long, in nanoseconds, the query ran on the server, from the + * arrival of the request to its last frame, or -1 when {@link #hasStats()} + * is false. Includes the compile, count and client-wait times. + */ + public long getExecuteNanos() { + return executeNanos; + } + + /** + * Returns the number of rows in the whole result set, including the rows + * the row window left out of the stream, or -1 when {@link #hasRowCount()} + * is false. + */ + public long getRowCount() { + return rowCount; + } + + /** + * Returns true when the server reported the row count of the whole result + * set: the query asked for it with {@link QwpQueryOptions#countRows(boolean)}, + * the server advertises {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}, and the + * statement produced rows. + */ + public boolean hasRowCount() { + return hasRowCount; + } + + /** + * Returns true when the server reported execution statistics: the query + * asked for them with {@link QwpQueryOptions#stats(boolean)} and the server + * advertises {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. + */ + public boolean hasStats() { + return hasStats; + } + + /** + * Returns true when the query filter ran as JIT-compiled native code; false + * when it did not, or when {@link #hasStats()} is false. + */ + public boolean isJitCompiled() { + return isJitCompiled; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("QwpQueryDetails{"); + if (hasRowCount) { + sb.append("rowCount=").append(rowCount); + } + if (hasStats) { + if (hasRowCount) { + sb.append(", "); + } + sb.append("compileNanos=").append(compileNanos) + .append(", executeNanos=").append(executeNanos) + .append(", countNanos=").append(countNanos) + .append(", clientWaitNanos=").append(clientWaitNanos) + .append(", jitCompiled=").append(isJitCompiled); + } + return sb.append('}').toString(); + } + + void clear() { + clientWaitNanos = -1; + compileNanos = -1; + countNanos = -1; + executeNanos = -1; + hasRowCount = false; + hasStats = false; + isJitCompiled = false; + rowCount = -1; + } + + void setRowCount(long rowCount) { + this.rowCount = rowCount; + this.hasRowCount = true; + } + + void setStats(long compileNanos, long executeNanos, long countNanos, long clientWaitNanos, boolean isJitCompiled) { + this.compileNanos = compileNanos; + this.executeNanos = executeNanos; + this.countNanos = countNanos; + this.clientWaitNanos = clientWaitNanos; + this.isJitCompiled = isJitCompiled; + this.hasStats = true; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java new file mode 100644 index 000000000..93f5034ee --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java @@ -0,0 +1,258 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Per-query options of + * {@link QwpQueryClient#execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler, QwpQueryOptions)} + * and {@link io.questdb.client.Query#options(QwpQueryOptions)}: a row window + * that pages through the result set, and what the server reports besides the + * rows -- the row count of the whole result set, execution statistics and the + * position of an error in the SQL text. A new instance has every option off, + * and a query run with it is the same on the wire as one run without options. + *

+ * The row window, the row count, the statistics and the error position need a + * server that advertises {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. Against a + * server without it, the client fails a query that sets a row window, since the + * server would stream every row, and runs a query that asks only for the row + * count, the statistics or the error position without them: the handler then + * sees {@link QwpQueryDetails#hasRowCount()} and + * {@link QwpQueryDetails#hasStats()} false and the position + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}. + *

+ * The client reads the options when the query starts, so one instance can be + * reused for any number of queries, and changing it does not affect a query + * already running. Not thread-safe. + */ +public final class QwpQueryOptions { + + private boolean isErrorPositionRequested; + private boolean isRowCountRequested; + private boolean isStatsRequested; + private boolean isSymbolDictResetRequested; + private long maxRows = Long.MAX_VALUE; + private long skipRows; + + /** + * Resets every option to its default: no row window, nothing reported + * besides the rows, and the connection-scoped SYMBOL dictionary kept. + * + * @return this instance + */ + public QwpQueryOptions clear() { + isErrorPositionRequested = false; + isRowCountRequested = false; + isStatsRequested = false; + isSymbolDictResetRequested = false; + maxRows = Long.MAX_VALUE; + skipRows = 0; + return this; + } + + /** + * Copies every option of {@code other} into this instance, or resets them + * when {@code other} is null. + * + * @return this instance + */ + public QwpQueryOptions copyFrom(QwpQueryOptions other) { + if (other == null) { + return clear(); + } + isErrorPositionRequested = other.isErrorPositionRequested; + isRowCountRequested = other.isRowCountRequested; + isStatsRequested = other.isStatsRequested; + isSymbolDictResetRequested = other.isSymbolDictResetRequested; + maxRows = other.maxRows; + skipRows = other.skipRows; + return this; + } + + /** + * Asks the server to count the rows of the whole result set, whatever the + * row window leaves out of the stream, as the {@code count} of the + * {@code /exec} HTTP endpoint does. The count reaches + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)}. + * Statements that do not produce rows report no count. When the row window + * ends before the result set does, the server reads the rest of the result + * set to count it, without sending it. + * + * @return this instance + */ + public QwpQueryOptions countRows(boolean isRequested) { + isRowCountRequested = isRequested; + return this; + } + + /** + * Asks the server for the position of an error in the SQL text, as the + * {@code position} of the {@code /exec} HTTP endpoint. A failed query then + * reaches {@link QwpColumnBatchHandler#onError(long, byte, String, int)} + * with the bare error message, without the {@code [position]} prefix the + * server puts in front of it otherwise, and the position as a {@code char} + * index into the SQL text. + * + * @return this instance + */ + public QwpQueryOptions errorPosition(boolean isRequested) { + isErrorPositionRequested = isRequested; + return this; + } + + /** + * Returns the most rows the query streams, {@link Long#MAX_VALUE} when the + * stream is unbounded. + */ + public long getMaxRows() { + return maxRows; + } + + /** + * Returns how many rows the query leaves out at the start of the result set. + */ + public long getSkipRows() { + return skipRows; + } + + public boolean isErrorPositionRequested() { + return isErrorPositionRequested; + } + + public boolean isRowCountRequested() { + return isRowCountRequested; + } + + public boolean isStatsRequested() { + return isStatsRequested; + } + + public boolean isSymbolDictResetRequested() { + return isSymbolDictResetRequested; + } + + /** + * Streams at most {@code rows} rows of the result set, after the rows + * {@link #skipRows(long)} leaves out, like the {@code limit} of the + * {@code /exec} HTTP endpoint. Zero streams no rows, only the schema. + * {@link Long#MAX_VALUE}, the default, leaves the stream unbounded. + * Statements that do not produce rows ignore it. + * + * @param rows the most rows to stream, zero or more + * @return this instance + * @throws IllegalArgumentException if {@code rows} is negative + */ + public QwpQueryOptions maxRows(long rows) { + if (rows < 0) { + throw new IllegalArgumentException("maxRows must be >= 0 [maxRows=" + rows + ']'); + } + maxRows = rows; + return this; + } + + /** + * Asks the server to reset the connection-scoped SYMBOL dictionary before + * the query, scoping the dictionary to its symbols. Needs a server that + * advertises {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}; against any other the + * dictionary is kept. + * + * @return this instance + */ + public QwpQueryOptions resetSymbolDict(boolean isRequested) { + isSymbolDictResetRequested = isRequested; + return this; + } + + /** + * Leaves out the first {@code rows} rows of the result set, like the lower + * bound of the {@code limit} of the {@code /exec} HTTP endpoint. Statements + * that do not produce rows ignore it. + * + * @param rows the rows to leave out, zero or more; zero, the default, + * leaves out none + * @return this instance + * @throws IllegalArgumentException if {@code rows} is negative + */ + public QwpQueryOptions skipRows(long rows) { + if (rows < 0) { + throw new IllegalArgumentException("skipRows must be >= 0 [skipRows=" + rows + ']'); + } + skipRows = rows; + return this; + } + + /** + * Asks the server for execution statistics -- the compile, execute, count + * and client-wait times, and whether the filter is JIT-compiled -- as the + * {@code timings} and {@code explain} of the {@code /exec} HTTP endpoint + * report them. They reach + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)} or + * {@link QwpColumnBatchHandler#onExecDone(long, short, long, QwpQueryDetails)}; + * a failed query reports none. + * + * @return this instance + */ + public QwpQueryOptions stats(boolean isRequested) { + isStatsRequested = isRequested; + return this; + } + + @Override + public String toString() { + return "QwpQueryOptions{skipRows=" + skipRows + + ", maxRows=" + maxRows + + ", countRows=" + isRowCountRequested + + ", stats=" + isStatsRequested + + ", errorPosition=" + isErrorPositionRequested + + ", resetSymbolDict=" + isSymbolDictResetRequested + + '}'; + } + + /** + * Returns the {@code query_flags} bits these options ask for, before the + * client drops the ones the server does not advertise. + */ + long toQueryFlags() { + long flags = 0; + if (isSymbolDictResetRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_RESET_DICT; + } + if (isErrorPositionRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION; + } + if (isStatsRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_STATS; + } + if (isRowCountRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS; + } + if (skipRows > 0) { + flags |= QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS; + } + if (maxRows != Long.MAX_VALUE) { + flags |= QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + } + return flags; + } +} diff --git a/core/src/main/java/io/questdb/client/impl/QueryImpl.java b/core/src/main/java/io/questdb/client/impl/QueryImpl.java index 82088fa8b..40a79b931 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QueryImpl.java @@ -29,7 +29,10 @@ import io.questdb.client.cutlass.qwp.client.QwpBindValues; import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; import io.questdb.client.std.str.StringSink; @@ -39,7 +42,7 @@ /** * Reusable per-{@link QueryWorker} query state: the configured SQL, optional - * binds, handler, terminal-event signalling, and a wrapping + * binds and options, handler, terminal-event signalling, and a wrapping * {@link QwpColumnBatchHandler} that forwards callbacks to the user handler and * signals completion on terminal events. One instance is pre-allocated per * worker in the constructor and reused across every borrow. @@ -68,12 +71,16 @@ final class QueryImpl { private final Condition doneCondition; private final ReentrantLock doneLock = new ReentrantLock(); + // The handle's own copy of the user's options: Query.options() copies into it, + // so the user may reuse the instance, and execute() reads it on the worker thread. + private final QwpQueryOptions options = new QwpQueryOptions(); private final StringSink sqlBuffer = new StringSink(); private final QueryWorker worker; private final QwpBindSetter wireBinds = this::applyBinds; private final WrappingHandler wrappingHandler = new WrappingHandler(); private volatile boolean done = true; private volatile String resultMessage; + private volatile int resultPosition = QwpEgressMsgKind.NO_ERROR_POSITION; private volatile byte resultStatus; private volatile Throwable unexpectedError; private QwpBindSetter userBinds; @@ -91,6 +98,7 @@ void abandon(long gen) { } userBinds = null; userHandler = null; + options.clear(); sqlBuffer.clear(); } @@ -207,6 +215,11 @@ void setHandler(long gen, QwpColumnBatchHandler handler) { this.userHandler = handler; } + void setOptions(long gen, QwpQueryOptions options) { + checkLive(gen); + this.options.copyFrom(options); + } + void setSql(long gen, CharSequence sql) { checkLive(gen); sqlBuffer.clear(); @@ -231,6 +244,7 @@ void submit(long gen) { done = false; resultStatus = 0; resultMessage = null; + resultPosition = QwpEgressMsgKind.NO_ERROR_POSITION; unexpectedError = null; } finally { doneLock.unlock(); @@ -296,6 +310,10 @@ private void rejectHandlerReentry(String op) { } private void signalDone(byte status, String message, Throwable unexpected) { + signalDone(status, message, QwpEgressMsgKind.NO_ERROR_POSITION, unexpected); + } + + private void signalDone(byte status, String message, int position, Throwable unexpected) { doneLock.lock(); try { if (done) { @@ -303,6 +321,7 @@ private void signalDone(byte status, String message, Throwable unexpected) { } this.resultStatus = status; this.resultMessage = message; + this.resultPosition = position; this.unexpectedError = unexpected; this.done = true; doneCondition.signalAll(); @@ -317,7 +336,7 @@ private void throwIfFailed() { throw new QueryException(resultStatus, resultMessage, unexpected); } if (resultStatus != 0) { - throw new QueryException(resultStatus, resultMessage); + throw new QueryException(resultStatus, resultMessage, resultPosition); } } @@ -332,9 +351,11 @@ private void throwIfFailed() { void resetForBorrow() { userBinds = null; userHandler = null; + options.clear(); sqlBuffer.clear(); resultStatus = 0; resultMessage = null; + resultPosition = QwpEgressMsgKind.NO_ERROR_POSITION; unexpectedError = null; done = true; } @@ -344,8 +365,9 @@ void runOn(QwpQueryClient client) { // reads chars and writes UTF-8 bytes straight into the send buffer. // sqlBuffer is stable for the duration of execute(): the calling // worker thread is blocked here until a terminal event arrives, and - // sql(...) cannot be invoked again until done==true. - client.execute(sqlBuffer, wireBinds, wrappingHandler); + // sql(...) cannot be invoked again until done==true. The same holds for + // options, which execute() reads once, when it starts. + client.execute(sqlBuffer, wireBinds, wrappingHandler, options); } /** @@ -381,6 +403,15 @@ public void onEnd(long requestId, long totalRows) { } } + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + try { + userHandler.onEnd(requestId, totalRows, details); + } finally { + signalDone((byte) 0, null, null); + } + } + @Override public void onError(byte status, String message) { try { @@ -399,6 +430,15 @@ public void onError(long requestId, byte status, String message) { } } + @Override + public void onError(long requestId, byte status, String message, int position) { + try { + userHandler.onError(requestId, status, message, position); + } finally { + signalDone(status, message, position, null); + } + } + @Override public void onExecDone(short opType, long rowsAffected) { try { @@ -417,6 +457,15 @@ public void onExecDone(long requestId, short opType, long rowsAffected) { } } + @Override + public void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + try { + userHandler.onExecDone(requestId, opType, rowsAffected, details); + } finally { + signalDone((byte) 0, null, null); + } + } + @Override public void onFailoverReset(QwpServerInfo newNode) { userHandler.onFailoverReset(newNode); diff --git a/core/src/main/java/io/questdb/client/impl/QueryLease.java b/core/src/main/java/io/questdb/client/impl/QueryLease.java index 4b07e9654..398596c40 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryLease.java +++ b/core/src/main/java/io/questdb/client/impl/QueryLease.java @@ -28,6 +28,7 @@ import io.questdb.client.Query; import io.questdb.client.cutlass.qwp.client.QwpBindSetter; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; import java.util.concurrent.TimeUnit; @@ -97,6 +98,12 @@ public boolean isDone() { return impl.isDone(generation); } + @Override + public Query options(QwpQueryOptions options) { + impl.setOptions(generation, options); + return this; + } + @Override public Query sql(CharSequence sql) { impl.setSql(generation, sql); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java b/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java new file mode 100644 index 000000000..0e5207c03 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java @@ -0,0 +1,222 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.Query; +import io.questdb.client.QueryException; +import io.questdb.client.QuestDB; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.cutlass.qwp.client.QwpEgressFrames; +import io.questdb.client.test.cutlass.qwp.client.QwpEgressMockServer; +import io.questdb.client.test.cutlass.qwp.client.RecordingQueryHandler; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +/** + * {@link Query#options(QwpQueryOptions)} on the pooled facade: the options + * reach the wire, the details the server reports reach the handler, the + * error position reaches {@link QueryException#getPosition()}, and the handle + * keeps its own copy of the options for its lifetime only. + */ +public class QuestDBQueryOptionsTest { + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; + private static final int WINDOW_FLAGS = QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + + @Test + public void testAbandonDropsOptions() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server); + Query q = db.borrowQuery()) { + q.sql("SELECT x").options(new QwpQueryOptions().maxRows(1)).handler(new RecordingQueryHandler()); + q.abandon(); + q.sql("SELECT x").handler(new RecordingQueryHandler()).submit().await(); + Assert.assertFalse(server.getRequests().get(0).hasQueryFlags); + server.assertNoFailure(); + } + }); + } + + @Test + public void testErrorPositionReachesQueryException() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server); + Query q = db.borrowQuery()) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("FAIL").options(new QwpQueryOptions().errorPosition(true)).handler(handler); + try { + q.submit().await(); + Assert.fail("the query must fail"); + } catch (QueryException e) { + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, e.getStatus()); + Assert.assertEquals("Invalid column: nope", e.getMessage()); + Assert.assertEquals(7, e.getPosition()); + } + Assert.assertEquals(7, handler.errorPosition); + + q.options(null); + try { + q.submit().await(); + Assert.fail("the query must fail"); + } catch (QueryException e) { + Assert.assertEquals("[7] Invalid column: nope", e.getMessage()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + } + + // a failure after one with a position reports its own + q.sql("SELECT x").options(new QwpQueryOptions().maxRows(1)); + server.close(); + try { + q.submit().await(); + Assert.fail("the query must fail"); + } catch (QueryException e) { + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testOptionsAreCopiedAndKeptAcrossSubmits() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model)) { + try (QuestDB db = connect(server)) { + try (Query q = db.borrowQuery()) { + QwpQueryOptions options = new QwpQueryOptions().skipRows(1).maxRows(2).countRows(true).stats(true); + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("SELECT x").options(options).handler(handler); + // the handle copied the options: changing them does not reach it + options.clear().maxRows(9); + q.submit().await(); + Assert.assertEquals(Arrays.asList(1L, 2L), handler.values); + Assert.assertEquals(2, handler.totalRows); + Assert.assertEquals(10, handler.rowCount); + Assert.assertTrue(handler.hasStats); + Assert.assertEquals(123, handler.compileNanos); + + // the next submit on the handle keeps them + handler.values.clear(); + q.submit().await(); + Assert.assertEquals(Arrays.asList(1L, 2L), handler.values); + + // until they are replaced, or dropped + q.options(options); + q.submit().await(); + q.options(null); + q.submit().await(); + // the lease ends with options set + q.options(new QwpQueryOptions().stats(true).skipRows(3)); + } + // a new borrow of the same pooled client starts without options + try (Query q = db.borrowQuery()) { + q.sql("SELECT x").handler(new RecordingQueryHandler()).submit().await(); + } + } + List requests = server.getRequests(); + Assert.assertEquals(5, requests.size()); + for (int i = 0; i < 2; i++) { + QwpEgressFrames.QueryRequest r = requests.get(i); + Assert.assertEquals(WINDOW_FLAGS | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS, + r.queryFlags); + Assert.assertEquals(1, r.skipRows); + Assert.assertEquals(2, r.maxRows); + } + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, requests.get(2).queryFlags); + Assert.assertEquals(9, requests.get(2).maxRows); + Assert.assertFalse(requests.get(3).hasQueryFlags); + Assert.assertFalse(requests.get(4).hasQueryFlags); + server.assertNoFailure(); + } + }); + } + + @Test + public void testStatsReachExecDone() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server); + Query q = db.borrowQuery()) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("INSERT INTO t VALUES (1)").options(new QwpQueryOptions().stats(true)).handler(handler).submit().await(); + Assert.assertEquals(1, handler.execDoneCount); + Assert.assertEquals(4, handler.rowsAffected); + Assert.assertTrue(handler.hasStats); + Assert.assertEquals(123, handler.compileNanos); + Assert.assertFalse(handler.isJitCompiled); + server.assertNoFailure(); + } + }); + } + + private static QuestDB connect(QwpEgressMockServer server) { + return QuestDB.connect("ws::addr=" + server.getAddress() + + ";sender_pool_min=0;query_pool_min=1;query_pool_max=1;failover=off;auth_timeout_ms=2000;"); + } + + /** + * A 10-row table {@code x = 0..9} in one batch per query; an + * {@code INSERT} that reports 4 rows; anything else fails at position 7. + * Trailers follow the flags of the request, as on the server. + */ + private static byte[][] model(QwpEgressFrames.QueryRequest r) { + boolean hasStats = r.has(QwpEgressMsgKind.QUERY_FLAG_STATS); + if (r.sql.startsWith("SELECT")) { + long first = Math.max(0, Math.min(r.skipRows, 10)); + long count = Math.min(r.maxRows < 0 ? Long.MAX_VALUE : r.maxRows, 10 - first); + long[] values = new long[(int) count]; + for (int i = 0; i < count; i++) { + values[i] = first + i; + } + QwpEgressFrames.Body end = QwpEgressFrames.resultEnd(r.requestId, 0, count); + if (r.has(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS)) { + end.varint(10); + } + if (hasStats) { + end.stats(123, 456, 0, 0, 0); + } + return new byte[][]{QwpEgressFrames.resultBatchOfLongs(r.requestId, 0, values), end.toFrame()}; + } + if (r.sql.startsWith("INSERT")) { + QwpEgressFrames.Body done = QwpEgressFrames.execDone(r.requestId, 2, 4); + if (hasStats) { + done.stats(123, 456, 0, 0, 0); + } + return new byte[][]{done.toFrame()}; + } + String message = "Invalid column: nope"; + if (r.has(QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION)) { + return new byte[][]{QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, message).i32(7).toFrame()}; + } + return new byte[][]{QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, "[7] " + message).toFrame()}; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java index 9867ee957..535ddf731 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.client.QueryEvent; import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; import org.junit.Assert; import org.junit.Test; @@ -50,6 +52,57 @@ public void testFreshEventDefaults() { Assert.assertEquals(0, ev.opType); Assert.assertEquals(0L, ev.rowsAffected); Assert.assertEquals(0L, ev.totalRows); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + assertNoDetails(ev.details); + } + + @Test + public void testAsEndAndAsExecDoneDropStaleDetails() { + QueryEvent ev = new QueryEvent().asEnd(1).withRowCount(9).withStats(1, 2, 3, 4, true); + ev.asEnd(2); + assertNoDetails(ev.details); + ev.withRowCount(9).withStats(1, 2, 3, 4, true).asExecDone((short) 1, 2); + assertNoDetails(ev.details); + } + + @Test + public void testAsErrorWithPosition() { + QueryEvent ev = new QueryEvent(); + Assert.assertSame(ev, ev.asError((byte) 0x05, "Invalid column: nope", 7)); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals((byte) 0x05, ev.errorStatus); + Assert.assertEquals("Invalid column: nope", ev.errorMessage); + Assert.assertEquals(7, ev.errorPosition); + // the builders without a position drop the stale one + ev.asError((byte) 0x05, "boom"); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + ev.asError((byte) 0x05, "boom", 3).asTransportError((byte) 0x06, "socket reset"); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + } + + @Test + public void testWithRowCountAndStats() { + QueryEvent ev = new QueryEvent().asEnd(5); + Assert.assertSame(ev, ev.withRowCount(0)); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(0, ev.details.getRowCount()); + Assert.assertFalse(ev.details.hasStats()); + Assert.assertEquals("QwpQueryDetails{rowCount=0}", ev.details.toString()); + + Assert.assertSame(ev, ev.withStats(10, 20, 30, 40, true)); + Assert.assertTrue(ev.details.hasStats()); + Assert.assertEquals(10, ev.details.getCompileNanos()); + Assert.assertEquals(20, ev.details.getExecuteNanos()); + Assert.assertEquals(30, ev.details.getCountNanos()); + Assert.assertEquals(40, ev.details.getClientWaitNanos()); + Assert.assertTrue(ev.details.isJitCompiled()); + + QueryEvent statsOnly = new QueryEvent().asExecDone((short) 2, 1).withStats(0, 0, 0, 0, false); + Assert.assertFalse(statsOnly.details.hasRowCount()); + Assert.assertTrue(statsOnly.details.hasStats()); + Assert.assertFalse(statsOnly.details.isJitCompiled()); + Assert.assertEquals("QwpQueryDetails{compileNanos=0, executeNanos=0, countNanos=0, clientWaitNanos=0, " + + "jitCompiled=false}", statsOnly.details.toString()); } @Test @@ -132,17 +185,25 @@ public void testKindConstantsAreDistinct() { @Test public void testResetClearsEverythingFromError() { QueryEvent ev = new QueryEvent(); - ev.asError((byte) 0x05, "boom"); + ev.asError((byte) 0x05, "boom", 12); ev.reset(); Assert.assertEquals(-1, ev.kind); Assert.assertNull(ev.buffer); Assert.assertNull(ev.errorMessage); Assert.assertEquals(0, ev.errorStatus); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); Assert.assertEquals(0, ev.opType); Assert.assertEquals(0L, ev.rowsAffected); Assert.assertEquals(0L, ev.totalRows); } + @Test + public void testResetClearsDetails() { + QueryEvent ev = new QueryEvent().asEnd(5).withRowCount(9).withStats(1, 2, 3, 4, true); + ev.reset(); + assertNoDetails(ev.details); + } + @Test public void testResetClearsEverythingFromBatch() { QueryEvent ev = new QueryEvent(); @@ -163,4 +224,16 @@ public void testResetClearsEverythingFromExecDone() { Assert.assertEquals(0, ev.opType); Assert.assertEquals(0L, ev.rowsAffected); } + + private static void assertNoDetails(QwpQueryDetails details) { + Assert.assertFalse(details.hasRowCount()); + Assert.assertEquals(-1, details.getRowCount()); + Assert.assertFalse(details.hasStats()); + Assert.assertEquals(-1, details.getCompileNanos()); + Assert.assertEquals(-1, details.getExecuteNanos()); + Assert.assertEquals(-1, details.getCountNanos()); + Assert.assertEquals(-1, details.getClientWaitNanos()); + Assert.assertFalse(details.isJitCompiled()); + Assert.assertEquals("QwpQueryDetails{}", details.toString()); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java index d4cdce6de..22a233373 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java @@ -24,11 +24,17 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.qwp.client.QueryEvent; import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * The two default methods on {@link QwpColumnBatchHandler} * ({@link QwpColumnBatchHandler#onFailoverReset(QwpServerInfo) onFailoverReset} @@ -40,6 +46,61 @@ */ public class QwpColumnBatchHandlerTest { + @Test + public void testDetailedOverloadsDelegateToSimplerOnes() { + // Each detailed overload delegates to the next simpler one, so a handler + // that overrides only the abstract methods sees every terminal event. + List calls = new ArrayList<>(); + QwpColumnBatchHandler handler = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + calls.add("onEnd " + totalRows); + } + + @Override + public void onError(byte status, String message) { + calls.add("onError " + status + ' ' + message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + calls.add("onExecDone " + opType + ' ' + rowsAffected); + } + }; + QueryEvent event = new QueryEvent().asEnd(0).withRowCount(9).withStats(1, 2, 3, 4, true); + handler.onEnd(1, 5, event.details); + handler.onError(2, (byte) 5, "Invalid column: nope", 7); + handler.onExecDone(3, (short) 2, 4, event.details); + Assert.assertEquals(Arrays.asList("onEnd 5", "onError 5 Invalid column: nope", "onExecDone 2 4"), calls); + + // and the correlation-aware overloads get the request id + calls.clear(); + QwpColumnBatchHandler correlating = new MinimalHandler() { + @Override + public void onEnd(long requestId, long totalRows) { + calls.add("onEnd " + requestId + ' ' + totalRows); + } + + @Override + public void onError(long requestId, byte status, String message) { + calls.add("onError " + requestId + ' ' + status + ' ' + message); + } + + @Override + public void onExecDone(long requestId, short opType, long rowsAffected) { + calls.add("onExecDone " + requestId + ' ' + opType + ' ' + rowsAffected); + } + }; + correlating.onEnd(1, 5, event.details); + correlating.onError(2, (byte) 5, "Invalid column: nope", 7); + correlating.onExecDone(3, (short) 2, 4, event.details); + Assert.assertEquals(Arrays.asList("onEnd 1 5", "onError 2 5 Invalid column: nope", "onExecDone 3 2 4"), calls); + } + @Test public void testDefaultMethodsAreNoOpAndDoNotThrow() { QwpColumnBatchHandler handler = new MinimalHandler(); @@ -56,7 +117,7 @@ public void testDefaultMethodsAreNoOpAndDoNotThrow() { * lost their {@code default} modifier this class would no longer compile, * so this test doubles as a compile-time guard on the interface contract. */ - private static final class MinimalHandler implements QwpColumnBatchHandler { + private static class MinimalHandler implements QwpColumnBatchHandler { @Override public void onBatch(QwpColumnBatch batch) { } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java new file mode 100644 index 000000000..a0e08acf2 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java @@ -0,0 +1,304 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import org.junit.Assert; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +/** + * Builds QWP egress server frames byte by byte, and parses the + * {@code QUERY_REQUEST} a client sends, so tests spell out the wire layout + * they exercise. Encodes the layouts of the server's + * {@code QwpEgressFrameWriter} and decodes the one of its + * {@code QwpEgressRequestDecoder}. + */ +public final class QwpEgressFrames { + + private QwpEgressFrames() { + } + + /** + * Starts a frame body with {@code msg_kind} and {@code request_id}, the + * prelude of every per-query server frame. + */ + public static Body body(byte msgKind, long requestId) { + return new Body().u8(msgKind).i64(requestId); + } + + /** + * {@code EXEC_DONE} body up to {@code rows_affected}; append the trailers + * the test needs. + */ + public static Body execDone(long requestId, int opType, long rowsAffected) { + return body(QwpEgressMsgKind.EXEC_DONE, requestId).u8(opType).varint(rowsAffected); + } + + /** + * Parses a {@code QUERY_REQUEST} payload. Supports the LONG binds tests + * send; the {@code query_flags} trailer and the row-window values follow + * the binds. + */ + public static QueryRequest parseQueryRequest(byte[] payload) { + Reader rd = new Reader(payload); + Assert.assertEquals("frame must be a QUERY_REQUEST", QwpEgressMsgKind.QUERY_REQUEST, (byte) rd.u8()); + long requestId = rd.i64(); + String sql = rd.utf8((int) rd.varint()); + long initialCredit = rd.varint(); + int bindCount = (int) rd.varint(); + long[] longBinds = new long[bindCount]; + for (int i = 0; i < bindCount; i++) { + Assert.assertEquals("tests bind LONG values only", QwpConstants.TYPE_LONG, (byte) rd.u8()); + Assert.assertEquals("tests bind non-NULL values only", 0, rd.u8()); + longBinds[i] = rd.i64(); + } + boolean hasQueryFlags = rd.remaining() > 0; + long queryFlags = hasQueryFlags ? rd.varint() : 0; + long skipRows = (queryFlags & QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS) != 0 ? rd.varint() : -1; + long maxRows = (queryFlags & QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS) != 0 ? rd.varint() : -1; + return new QueryRequest(payload, requestId, sql, initialCredit, longBinds, hasQueryFlags, queryFlags, skipRows, maxRows, rd.remaining()); + } + + /** + * {@code QUERY_ERROR} body up to the message; append the position trailer + * when the test needs it. + */ + public static Body queryError(long requestId, byte status, String message) { + byte[] msg = message.getBytes(StandardCharsets.UTF_8); + return body(QwpEgressMsgKind.QUERY_ERROR, requestId).u8(status).u16(msg.length).bytes(msg); + } + + /** + * {@code RESULT_BATCH} carrying one LONG column {@code x}; the schema rides + * the first batch of a query only. + */ + public static byte[] resultBatchOfLongs(long requestId, long batchSeq, long... values) { + Body b = body(QwpEgressMsgKind.RESULT_BATCH, requestId) + .varint(batchSeq) + .varint(0) // table name length + .varint(values.length); + if (batchSeq == 0) { + b.varint(1).varint(1).u8('x').u8(QwpConstants.TYPE_LONG); + } + b.u8(0); // no NULLs + for (long v : values) { + b.i64(v); + } + return b.toFrame(); + } + + /** + * {@code RESULT_END} body up to {@code total_rows}; append the trailers the + * test needs. + */ + public static Body resultEnd(long requestId, long finalSeq, long totalRows) { + return body(QwpEgressMsgKind.RESULT_END, requestId).varint(finalSeq).varint(totalRows); + } + + /** + * Growable little-endian frame body. {@link #toFrame()} prepends the + * 12-byte QWP message header. + */ + public static final class Body { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + public Body bytes(byte[] bytes) { + out.write(bytes, 0, bytes.length); + return this; + } + + public Body i32(int v) { + for (int i = 0; i < 4; i++) { + out.write(v >>> (8 * i)); + } + return this; + } + + public Body i64(long v) { + for (int i = 0; i < 8; i++) { + out.write((int) (v >>> (8 * i))); + } + return this; + } + + public int size() { + return out.size(); + } + + /** + * Appends the {@code QUERY_FLAG_STATS} block: four varint durations and + * the {@code stats_flags} byte. + */ + public Body stats(long compileNanos, long executeNanos, long countNanos, long clientWaitNanos, int statsFlags) { + return varint(compileNanos).varint(executeNanos).varint(countNanos).varint(clientWaitNanos).u8(statsFlags); + } + + public byte[] toBody() { + return out.toByteArray(); + } + + public byte[] toFrame() { + byte[] body = out.toByteArray(); + return new Body() + .i32(QwpConstants.MAGIC_MESSAGE) + .u8(QwpConstants.VERSION) + .u8(0) // flags + .u16(0) // table_count + .i32(body.length) + .bytes(body) + .toBody(); + } + + public Body u16(int v) { + out.write(v); + out.write(v >>> 8); + return this; + } + + public Body u8(int v) { + out.write(v); + return this; + } + + /** + * Unsigned LEB128, the encoding of the server's {@code QwpVarint}. + */ + public Body varint(long v) { + while ((v & ~0x7FL) != 0) { + out.write((int) ((v & 0x7F) | 0x80)); + v >>>= 7; + } + out.write((int) v); + return this; + } + } + + /** + * A {@code QUERY_REQUEST} as the server decodes it. {@link #skipRows} and + * {@link #maxRows} are -1 when the request does not carry them. + */ + public static final class QueryRequest { + public final boolean hasQueryFlags; + public final long initialCredit; + public final long[] longBinds; + public final long maxRows; + // the raw payload, as the client sent it + public final byte[] payload; + public final long queryFlags; + public final long requestId; + public final long skipRows; + public final String sql; + public final int trailingBytes; + + QueryRequest( + byte[] payload, + long requestId, + String sql, + long initialCredit, + long[] longBinds, + boolean hasQueryFlags, + long queryFlags, + long skipRows, + long maxRows, + int trailingBytes + ) { + this.payload = payload; + this.requestId = requestId; + this.sql = sql; + this.initialCredit = initialCredit; + this.longBinds = longBinds; + this.hasQueryFlags = hasQueryFlags; + this.queryFlags = queryFlags; + this.skipRows = skipRows; + this.maxRows = maxRows; + this.trailingBytes = trailingBytes; + } + + public boolean has(int queryFlag) { + return (queryFlags & queryFlag) != 0; + } + + @Override + public String toString() { + return "QueryRequest{requestId=" + requestId + + ", sql='" + sql + '\'' + + ", hasQueryFlags=" + hasQueryFlags + + ", queryFlags=0x" + Long.toHexString(queryFlags) + + ", skipRows=" + skipRows + + ", maxRows=" + maxRows + + ", trailingBytes=" + trailingBytes + + '}'; + } + } + + private static final class Reader { + private final byte[] buf; + private int pos; + + Reader(byte[] buf) { + this.buf = buf; + } + + long i64() { + Assert.assertTrue("request ends inside an i64", remaining() >= 8); + long v = 0; + for (int i = 0; i < 8; i++) { + v |= (long) (buf[pos++] & 0xFF) << (8 * i); + } + return v; + } + + int remaining() { + return buf.length - pos; + } + + int u8() { + Assert.assertTrue("request ends before the next byte", remaining() > 0); + return buf[pos++] & 0xFF; + } + + String utf8(int len) { + Assert.assertTrue("request ends inside the SQL text", remaining() >= len); + String s = new String(buf, pos, len, StandardCharsets.UTF_8); + pos += len; + return s; + } + + long varint() { + long v = 0; + for (int shift = 0; ; shift += 7) { + int b = u8(); + v |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return v; + } + Assert.assertTrue("varint longer than 10 bytes", shift < 63); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java new file mode 100644 index 000000000..6b0aa1db8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java @@ -0,0 +1,119 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Egress endpoint that advertises the given {@code SERVER_INFO} capabilities, + * records every {@code QUERY_REQUEST} it receives, and answers each with the + * frames a test-supplied {@link Responder} builds from it. A responder that + * returns null drops the connection instead, the way a node that dies + * mid-query does. + */ +public final class QwpEgressMockServer implements AutoCloseable { + private final AtomicReference failure = new AtomicReference<>(); + private final List requests = new CopyOnWriteArrayList<>(); + private final TestWebSocketServer server; + + public QwpEgressMockServer(int capabilities, Responder responder) throws IOException, InterruptedException { + server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length == 0 || data[0] != QwpEgressMsgKind.QUERY_REQUEST) { + return; + } + byte[][] frames; + try { + QwpEgressFrames.QueryRequest request = QwpEgressFrames.parseQueryRequest(data); + requests.add(request); + frames = responder.respond(request); + } catch (Throwable th) { + // A broken request or responder must fail the test, not hang it. + failure.compareAndSet(null, th); + frames = null; + } + if (frames == null) { + // ClientHandler.close() joins this read thread, so close from another one + Thread dropper = new Thread(client::close, "mock-egress-drop"); + dropper.setDaemon(true); + dropper.start(); + return; + } + try { + for (byte[] frame : frames) { + client.sendBinary(frame); + } + } catch (IOException e) { + // surfaces to the client as a transport error + } + } + }); + server.setSendServerInfo(true); + server.setCapabilities(capabilities); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + } + + /** + * Fails when parsing a request or building an answer threw on the server. + */ + public void assertNoFailure() { + Throwable th = failure.get(); + if (th != null) { + throw new AssertionError("mock egress server failed", th); + } + } + + @Override + public void close() { + server.close(); + } + + public String getAddress() { + return "localhost:" + server.getPort(); + } + + public List getRequests() { + return requests; + } + + @FunctionalInterface + public interface Responder { + /** + * Returns the frames that answer {@code request}, or null to drop the + * connection. + */ + byte[][] respond(QwpEgressFrames.QueryRequest request); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java new file mode 100644 index 000000000..50a3a7641 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java @@ -0,0 +1,567 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QueryEvent; +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Decoding of the trailers the query-details flags add to the terminator + * frames: {@code row_count} and the statistics block on {@code RESULT_END}, + * the statistics block on {@code EXEC_DONE}, and {@code position} on + * {@code QUERY_ERROR}. The frames are built byte by byte in the layout of the + * server's {@code QwpEgressFrameWriter}, and decoded through the same static + * decoders the I/O thread runs, with the flags the request carried. + */ +public class QwpEgressTerminatorDecodeTest { + private static final int ALL_FLAGS = QwpEgressMsgKind.QUERY_FLAG_RESET_DICT + | QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION + | QwpEgressMsgKind.QUERY_FLAG_STATS + | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS + | QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + private static final int PRELUDE = QwpConstants.HEADER_SIZE + 1 + 8; + private static final long[] VARINT_VALUES = { + 0, 1, 127, 128, 16_383, 16_384, 1_000_000_000L, Long.MAX_VALUE - 1, Long.MAX_VALUE + }; + + @Test + public void testExecDoneCarriesNoRowCount() throws Exception { + // COUNT_ROWS does not apply to statements without rows: the statistics + // follow rows_affected directly, and a frame without trailers is whole. + TestUtils.assertMemoryLeak(() -> { + byte[] withStats = QwpEgressFrames.execDone(3, 2, 30).stats(11, 22, 0, 44, 1).toFrame(); + QueryEvent ev = decodeExecDone(withStats, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(30, ev.rowsAffected); + assertStats(ev.details, 11, 22, 0, 44, true); + Assert.assertFalse(ev.details.hasRowCount()); + Assert.assertEquals(-1, ev.details.getRowCount()); + + ev = decodeExecDone(QwpEgressFrames.execDone(3, 2, 30).toFrame(), QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertEquals(30, ev.rowsAffected); + assertNoDetails(ev.details); + }); + } + + @Test + public void testExecDoneStats() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (long v : VARINT_VALUES) { + for (int jit = 0; jit < 2; jit++) { + byte[] frame = QwpEgressFrames.execDone(3, 7, v).stats(v, Long.MAX_VALUE - v, v / 2, v / 3, jit).toFrame(); + QueryEvent ev = decodeExecDone(frame, QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(QueryEvent.KIND_EXEC_DONE, ev.kind); + Assert.assertEquals(7, ev.opType); + Assert.assertEquals(v, ev.rowsAffected); + assertStats(ev.details, v, Long.MAX_VALUE - v, v / 2, v / 3, jit == 1); + Assert.assertFalse(ev.details.hasRowCount()); + } + } + }); + } + + @Test + public void testExecDoneTruncatedAtEveryByte() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List fields = new ArrayList<>(); + QwpEgressFrames.Body body = new QwpEgressFrames.Body(); + body.u8(QwpEgressMsgKind.EXEC_DONE).i64(3); + appendField(body, fields, "op_type", b -> b.u8(2)); + appendField(body, fields, "rows_affected", b -> b.varint(16_384)); + appendStatsFields(body, fields); + assertTruncatedAtEveryByte(body.toFrame(), fields, "EXEC_DONE", QwpEgressMsgKind.QUERY_FLAG_STATS, true); + }); + } + + @Test + public void testExecDoneVarintOverflow() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (byte[] bad : overflowingVarints()) { + byte[] frame = new QwpEgressFrames.Body().u8(QwpEgressMsgKind.EXEC_DONE).i64(3).u8(2).bytes(bad).toFrame(); + assertDecodeFails("EXEC_DONE rows_affected varint overflow", frame, 0, true); + } + for (String field : new String[]{"compile_ns", "execute_ns", "count_ns", "client_wait_ns"}) { + for (byte[] bad : overflowingVarints()) { + byte[] frame = statsWithBadField(QwpEgressFrames.execDone(3, 2, 30), field, bad).toFrame(); + assertDecodeFails("EXEC_DONE " + field + " varint overflow", frame, QwpEgressMsgKind.QUERY_FLAG_STATS, true); + } + } + }); + } + + @Test + public void testExecDoneWithoutFlagsIgnoresTrailers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.execDone(3, 2, 30).stats(11, 22, 33, 44, 1).toFrame(); + QueryEvent ev = decodeExecDone(frame, 0); + Assert.assertEquals(2, ev.opType); + Assert.assertEquals(30, ev.rowsAffected); + assertNoDetails(ev.details); + }); + } + + @Test + public void testQueryErrorEmptyMessageWithPosition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, "").i32(3).toFrame(); + QueryEvent ev = decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals("", ev.errorMessage); + Assert.assertEquals(3, ev.errorPosition); + }); + } + + @Test + public void testQueryErrorMalformedWithPositionRequested() throws Exception { + // the guards that predate the position trailer hold when it is requested + TestUtils.assertMemoryLeak(() -> { + byte[] truncated = new QwpEgressFrames.Body().u8(QwpEgressMsgKind.QUERY_ERROR).i64(5).u8(5).u8(4).toFrame(); + QueryEvent ev = decodeError(truncated, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, ev.errorStatus); + Assert.assertEquals("QUERY_ERROR frame truncated before msg_len", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + + // msg_len claims the message and the position, and one byte more + byte[] overrun = new QwpEgressFrames.Body() + .u8(QwpEgressMsgKind.QUERY_ERROR).i64(5).u8(5).u16(9).bytes("boom".getBytes()).i32(7) + .toFrame(); + ev = decodeError(overrun, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, ev.errorStatus); + Assert.assertEquals("QUERY_ERROR msg_len 9 exceeds frame remainder 8", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + }); + } + + @Test + public void testQueryErrorPosition() throws Exception { + // 'é' and '€' are one char each but 2 and 3 UTF-8 bytes, and the emoji is + // two chars and 4 bytes: the message survives the trailer whole + final String message = "table does not exist [table=é€\uD83D\uDE00]"; + TestUtils.assertMemoryLeak(() -> { + for (int position : new int[]{QwpEgressMsgKind.NO_ERROR_POSITION, 0, 1, 14, 65_535, Integer.MAX_VALUE}) { + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, message).i32(position).toFrame(); + for (long flags : new long[]{QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION, ALL_FLAGS}) { + QueryEvent ev = decodeError(frame, flags); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals(message, ev.errorMessage); + Assert.assertEquals(position, ev.errorPosition); + } + } + }); + } + + @Test + public void testQueryErrorPositionFollowedByMoreBytes() throws Exception { + // bytes after the known fields are ignored, as a newer server may add some + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.queryError(5, QwpConstants.STATUS_CANCELLED, "cancelled by client") + .i32(QwpEgressMsgKind.NO_ERROR_POSITION).u8(1).u8(2).u8(3).toFrame(); + QueryEvent ev = decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QwpConstants.STATUS_CANCELLED, ev.errorStatus); + Assert.assertEquals("cancelled by client", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + + frame = QwpEgressFrames.queryError(5, QwpConstants.STATUS_CANCELLED, "cancelled by client") + .i32(42).u8(1).toFrame(); + Assert.assertEquals(42, decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION).errorPosition); + }); + } + + @Test + public void testQueryErrorPositionTrailerCutShort() throws Exception { + // 1 to 3 bytes after the message: neither a baseline frame nor a whole + // position, so the event names the defect and keeps what the server said + TestUtils.assertMemoryLeak(() -> { + for (int extra = 1; extra < Integer.BYTES; extra++) { + QwpEgressFrames.Body body = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, "Invalid column: nope"); + for (int i = 0; i < extra; i++) { + body.u8(7); + } + QueryEvent ev = decodeError(body.toFrame(), QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, ev.errorStatus); + Assert.assertEquals("QUERY_ERROR frame truncated mid position [status=5, message=Invalid column: nope]", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + } + }); + } + + @Test + public void testQueryErrorTrailerIgnoredWhenNotRequested() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, "[14] table does not exist") + .i32(14).toFrame(); + for (long flags : new long[]{0, ALL_FLAGS & ~QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION}) { + QueryEvent ev = decodeError(frame, flags); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals("[14] table does not exist", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + } + // the decoder of a request without flags + QueryEvent ev = decodeBaselineError(frame); + Assert.assertEquals("[14] table does not exist", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + }); + } + + @Test + public void testQueryErrorWithoutTrailerWhenPositionRequested() throws Exception { + // the server answers a request it could not decode, or one it rejects + // while another query streams, with the baseline frame + TestUtils.assertMemoryLeak(() -> { + String message = "Phase 1 egress supports a single in-flight query per connection"; + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, message).toFrame(); + QueryEvent ev = decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals(message, ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + }); + } + + @Test + public void testResultEndRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (long rowCount : VARINT_VALUES) { + byte[] frame = QwpEgressFrames.resultEnd(7, 0, 5).varint(rowCount).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertEquals(QueryEvent.KIND_END, ev.kind); + Assert.assertEquals(5, ev.totalRows); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(rowCount, ev.details.getRowCount()); + Assert.assertFalse(ev.details.hasStats()); + } + }); + } + + @Test + public void testResultEndRowCountThenStats() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.resultEnd(7, 2, 50).varint(100).stats(1_500, 900_000, 300, 0, 0).toFrame(); + for (long flags : new long[]{QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS, ALL_FLAGS}) { + QueryEvent ev = decodeResultEnd(frame, flags); + Assert.assertEquals(50, ev.totalRows); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(100, ev.details.getRowCount()); + assertStats(ev.details, 1_500, 900_000, 300, 0, false); + Assert.assertEquals("QwpQueryDetails{rowCount=100, compileNanos=1500, executeNanos=900000, " + + "countNanos=300, clientWaitNanos=0, jitCompiled=false}", ev.details.toString()); + } + }); + } + + @Test + public void testResultEndStats() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (long v : VARINT_VALUES) { + for (int jit = 0; jit < 2; jit++) { + byte[] frame = QwpEgressFrames.resultEnd(7, 3, 40).stats(v, Long.MAX_VALUE - v, v / 2, v / 3, jit).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(40, ev.totalRows); + assertStats(ev.details, v, Long.MAX_VALUE - v, v / 2, v / 3, jit == 1); + Assert.assertFalse(ev.details.hasRowCount()); + Assert.assertEquals(-1, ev.details.getRowCount()); + } + } + }); + } + + @Test + public void testResultEndTrailersFollowedByMoreBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.resultEnd(7, 2, 50).varint(100).stats(1, 2, 3, 4, 1).u8(0xFF).u8(0x80).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(100, ev.details.getRowCount()); + assertStats(ev.details, 1, 2, 3, 4, true); + }); + } + + @Test + public void testResultEndTruncatedAtEveryByte() throws Exception { + // multi-byte varints in every field, so cuts land inside each of them + TestUtils.assertMemoryLeak(() -> { + List fields = new ArrayList<>(); + QwpEgressFrames.Body body = new QwpEgressFrames.Body(); + body.u8(QwpEgressMsgKind.RESULT_END).i64(7); + appendField(body, fields, "final_seq", b -> b.varint(300)); + appendField(body, fields, "total_rows", b -> b.varint(16_384)); + appendField(body, fields, "row_count", b -> b.varint(1_000_000)); + appendStatsFields(body, fields); + assertTruncatedAtEveryByte(body.toFrame(), fields, "RESULT_END", + QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS, false); + }); + } + + @Test + public void testResultEndVarintOverflow() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (byte[] bad : overflowingVarints()) { + byte[] frame = QwpEgressFrames.body(QwpEgressMsgKind.RESULT_END, 7).bytes(bad).varint(5).toFrame(); + assertDecodeFails("RESULT_END final_seq varint overflow", frame, 0, false); + frame = QwpEgressFrames.body(QwpEgressMsgKind.RESULT_END, 7).varint(0).bytes(bad).toFrame(); + assertDecodeFails("RESULT_END total_rows varint overflow", frame, 0, false); + frame = QwpEgressFrames.resultEnd(7, 0, 5).bytes(bad).toFrame(); + assertDecodeFails("RESULT_END row_count varint overflow", frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS, false); + } + for (String field : new String[]{"compile_ns", "execute_ns", "count_ns", "client_wait_ns"}) { + for (byte[] bad : overflowingVarints()) { + byte[] frame = statsWithBadField(QwpEgressFrames.resultEnd(7, 0, 5), field, bad).toFrame(); + assertDecodeFails("RESULT_END " + field + " varint overflow", frame, QwpEgressMsgKind.QUERY_FLAG_STATS, false); + } + } + }); + } + + @Test + public void testResultEndVarintUsesAll64Bits() throws Exception { + // a tenth byte of 0x01 sets bit 63, the last bit a long holds + TestUtils.assertMemoryLeak(() -> { + byte[] bit63 = {(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}; + byte[] frame = QwpEgressFrames.resultEnd(7, 0, 5).bytes(bit63).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(Long.MIN_VALUE, ev.details.getRowCount()); + }); + } + + @Test + public void testResultEndWithoutFlagsIgnoresTrailers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] baseline = QwpEgressFrames.resultEnd(7, 2, 30).toFrame(); + byte[] withTrailers = QwpEgressFrames.resultEnd(7, 2, 30).varint(100).stats(1, 2, 3, 4, 1).toFrame(); + for (byte[] frame : new byte[][]{baseline, withTrailers}) { + for (long flags : new long[]{0, ALL_FLAGS & ~(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS)}) { + QueryEvent ev = decodeResultEnd(frame, flags); + Assert.assertEquals(QueryEvent.KIND_END, ev.kind); + Assert.assertEquals(30, ev.totalRows); + assertNoDetails(ev.details); + } + } + // COUNT_ROWS alone reads the row count and leaves what follows alone + QueryEvent ev = decodeResultEnd(withTrailers, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertEquals(100, ev.details.getRowCount()); + Assert.assertFalse(ev.details.hasStats()); + }); + } + + @Test + public void testStatsFlagsReservedBitsIgnored() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int[] statsFlags = {0x00, 0x01, 0x02, 0x80, 0x81, 0xFE, 0xFF}; + for (int flags : statsFlags) { + byte[] frame = QwpEgressFrames.resultEnd(7, 0, 1).stats(1, 2, 3, 4, flags).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertTrue(ev.details.hasStats()); + Assert.assertEquals("stats_flags 0x" + Integer.toHexString(flags), + (flags & QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED) != 0, ev.details.isJitCompiled()); + } + }); + } + + private static void appendField(QwpEgressFrames.Body body, List fields, String field, FieldWriter writer) { + int before = body.size(); + writer.write(body); + for (int i = before; i < body.size(); i++) { + fields.add(field); + } + } + + private static void appendStatsFields(QwpEgressFrames.Body body, List fields) { + appendField(body, fields, "compile_ns", b -> b.varint(128)); + appendField(body, fields, "execute_ns", b -> b.varint(Long.MAX_VALUE)); + appendField(body, fields, "count_ns", b -> b.varint(0)); + appendField(body, fields, "client_wait_ns", b -> b.varint(200)); + appendField(body, fields, "stats_flags", b -> b.u8(QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED)); + } + + private static void assertDecodeFails(String expectedMessage, byte[] frame, long flags, boolean isExecDone) throws Exception { + try { + if (isExecDone) { + decodeExecDone(frame, flags); + } else { + decodeResultEnd(frame, flags); + } + Assert.fail("decode must fail: " + expectedMessage); + } catch (QwpDecodeException e) { + Assert.assertEquals(expectedMessage, e.getMessage()); + } + } + + private static void assertNoDetails(QwpQueryDetails details) { + Assert.assertFalse(details.hasRowCount()); + Assert.assertEquals(-1, details.getRowCount()); + Assert.assertFalse(details.hasStats()); + Assert.assertEquals(-1, details.getCompileNanos()); + Assert.assertEquals(-1, details.getExecuteNanos()); + Assert.assertEquals(-1, details.getCountNanos()); + Assert.assertEquals(-1, details.getClientWaitNanos()); + Assert.assertFalse(details.isJitCompiled()); + Assert.assertEquals("QwpQueryDetails{}", details.toString()); + } + + private static void assertStats( + QwpQueryDetails details, + long compileNanos, + long executeNanos, + long countNanos, + long clientWaitNanos, + boolean isJitCompiled + ) { + Assert.assertTrue(details.hasStats()); + Assert.assertEquals(compileNanos, details.getCompileNanos()); + Assert.assertEquals(executeNanos, details.getExecuteNanos()); + Assert.assertEquals(countNanos, details.getCountNanos()); + Assert.assertEquals(clientWaitNanos, details.getClientWaitNanos()); + Assert.assertEquals(isJitCompiled, details.isJitCompiled()); + } + + /** + * Cuts {@code frame} after every byte from the end of the prelude to the + * last but one: each cut lands in the field {@code fields} names for the + * first missing byte, and the decode fails naming that field. + */ + private static void assertTruncatedAtEveryByte( + byte[] frame, + List fields, + String frameName, + long flags, + boolean isExecDone + ) throws Exception { + Assert.assertEquals(PRELUDE + fields.size(), frame.length); + for (int len = PRELUDE; len < frame.length; len++) { + String field = fields.get(len - PRELUDE); + boolean isByteField = "op_type".equals(field) || "stats_flags".equals(field); + String expected = frameName + " frame truncated " + (isByteField ? "before " + field : "mid " + field + " varint"); + assertDecodeFails(expected, Arrays.copyOf(frame, len), flags, isExecDone); + } + // the whole frame decodes + if (isExecDone) { + decodeExecDone(frame, flags); + } else { + decodeResultEnd(frame, flags); + } + } + + private static QueryEvent decodeBaselineError(byte[] frame) { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeError(buf, frame.length); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static QueryEvent decodeError(byte[] frame, long flags) { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeError(buf, frame.length, flags); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static QueryEvent decodeExecDone(byte[] frame, long flags) throws QwpDecodeException { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeExecDone(buf, frame.length, flags); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static QueryEvent decodeResultEnd(byte[] frame, long flags) throws QwpDecodeException { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeResultEnd(buf, frame.length, flags); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + /** + * Varints that do not fit in 64 bits: a continuation bit on the tenth + * byte, whose data bit still fits, and data bits above bit 63 on the + * tenth byte. + */ + private static byte[][] overflowingVarints() { + byte[] continued = new byte[11]; + Arrays.fill(continued, (byte) 0x80); + continued[9] = (byte) 0x81; + continued[10] = 0x00; + byte[] highBits = new byte[10]; + Arrays.fill(highBits, (byte) 0x80); + highBits[9] = 0x02; + byte[] allHighBits = new byte[10]; + Arrays.fill(allHighBits, (byte) 0xFF); + allHighBits[9] = 0x7F; + return new byte[][]{continued, highBits, allHighBits}; + } + + /** + * Appends a statistics block whose {@code field} carries {@code bad} and + * whose other fields are well-formed. + */ + private static QwpEgressFrames.Body statsWithBadField(QwpEgressFrames.Body body, String field, byte[] bad) { + String[] names = {"compile_ns", "execute_ns", "count_ns", "client_wait_ns"}; + for (String name : names) { + if (name.equals(field)) { + body.bytes(bad); + } else { + body.varint(1); + } + } + return body.u8(0); + } + + private static long toNative(byte[] frame) { + long buf = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(buf + i, frame[i]); + } + return buf; + } + + @FunctionalInterface + private interface FieldWriter { + void write(QwpEgressFrames.Body body); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java new file mode 100644 index 000000000..dc0109029 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java @@ -0,0 +1,518 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * End-to-end coverage of {@link QwpQueryOptions} through + * {@link QwpQueryClient#execute}: the I/O thread decodes the trailers the + * request asked for and the handler receives them. The mock server models the + * server's {@code QwpEgressUpgradeProcessor}: a 10-row table {@code x = 0..9} + * streamed in batches of up to 3 rows, cut by the row window; trailers only + * for the flags the request carries; the bare message and the position of an + * error only under {@code QUERY_FLAG_ERROR_POSITION}. The questdb repo checks + * the same against a live server. + */ +public class QwpQueryClientQueryDetailsTest { + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; + private static final long CLIENT_WAIT_NANOS = 44; + private static final long COMPILE_NANOS = 11; + private static final long COUNT_NANOS = 33; + private static final String ERROR_MESSAGE = "Invalid column: nope"; + private static final int ERROR_POSITION = 7; + private static final long EXECUTE_NANOS = 22; + private static final int OP_INSERT = 2; + private static final long ROWS_AFFECTED = 5; + private static final int TABLE_ROWS = 10; + + @Test + public void testDetailsAbsentWithoutCapability() throws Exception { + // the client leaves out what the server cannot report, and the server + // answers with the baseline frames + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(QwpEgressMsgKind.CAP_QUERY_FLAGS, + QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true); + + RecordingQueryHandler select = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", select, options); + Assert.assertEquals(1, select.endCount); + Assert.assertEquals(TABLE_ROWS, select.totalRows); + select.assertNoDetails(); + + RecordingQueryHandler insert = new RecordingQueryHandler(); + client.execute("INSERT INTO t VALUES (1)", insert, options); + Assert.assertEquals(1, insert.execDoneCount); + insert.assertNoDetails(); + + RecordingQueryHandler failed = new RecordingQueryHandler(); + client.execute("FAIL", failed, options); + failed.assertOnlyError(); + Assert.assertEquals("[7] " + ERROR_MESSAGE, failed.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, failed.errorPosition); + + for (QwpEgressFrames.QueryRequest r : server.getRequests()) { + Assert.assertFalse(r.toString(), r.hasQueryFlags); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testDetailsDoNotLeakToTheNextQuery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true).maxRows(2); + for (int i = 0; i < 2; i++) { + RecordingQueryHandler detailed = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", detailed, options); + Assert.assertEquals(Arrays.asList(0L, 1L), detailed.values); + Assert.assertEquals(TABLE_ROWS, detailed.rowCount); + Assert.assertTrue(detailed.hasStats); + + RecordingQueryHandler plain = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", plain); + Assert.assertEquals(TABLE_ROWS, plain.values.size()); + Assert.assertEquals(TABLE_ROWS, plain.totalRows); + plain.assertNoDetails(); + + RecordingQueryHandler failed = new RecordingQueryHandler(); + client.execute("FAIL", failed, options); + Assert.assertEquals(ERROR_POSITION, failed.errorPosition); + + RecordingQueryHandler plainFailed = new RecordingQueryHandler(); + client.execute("FAIL", plainFailed); + Assert.assertEquals("[7] " + ERROR_MESSAGE, plainFailed.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, plainFailed.errorPosition); + + RecordingQueryHandler plainInsert = new RecordingQueryHandler(); + client.execute("INSERT INTO t VALUES (1)", plainInsert); + plainInsert.assertNoDetails(); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testErrorPositionReachesOnError() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("FAIL", handler, new QwpQueryOptions().errorPosition(true)); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, handler.errorStatus); + Assert.assertEquals(ERROR_MESSAGE, handler.errorMessage); + Assert.assertEquals(ERROR_POSITION, handler.errorPosition); + server.assertNoFailure(); + } + }); + } + + @Test + public void testFailoverChecksTheNewServerAgain() throws Exception { + // the first node dies mid-query; the replay goes to a node without + // CAP_QUERY_DETAILS, which gets no detail flags and fails a row window + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer dying = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpEgressMockServer old = new QwpEgressMockServer(QwpEgressMsgKind.CAP_QUERY_FLAGS, + QwpQueryClientQueryDetailsTest::model)) { + String config = "ws::addr=" + dying.getAddress() + ',' + old.getAddress() + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(config)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, new QwpQueryOptions().stats(true).countRows(true)); + Assert.assertEquals(1, handler.failoverResetCount); + Assert.assertEquals(1, handler.endCount); + Assert.assertEquals(TABLE_ROWS, handler.values.size()); + handler.assertNoDetails(); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_STATS | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS, + dying.getRequests().get(0).queryFlags); + Assert.assertFalse(old.getRequests().get(0).hasQueryFlags); + } + try (QwpQueryClient client = QwpQueryClient.fromConfig(config)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, new QwpQueryOptions().maxRows(2)); + Assert.assertEquals(1, handler.failoverResetCount); + handler.assertOnlyError(); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("server does not support skipRows and maxRows [capabilities=0x2]; " + + "they need a server that advertises CAP_QUERY_DETAILS", handler.errorMessage); + Assert.assertEquals(2, dying.getRequests().size()); + Assert.assertEquals(2, dying.getRequests().get(1).maxRows); + Assert.assertEquals("the replay must not reach the old node", 1, old.getRequests().size()); + } + dying.assertNoFailure(); + old.assertNoFailure(); + } + }); + } + + @Test + public void testFailoverReplaysTheOptionsTheQueryStartedWith() throws Exception { + // the client copies the options when execute() starts: a change during + // the query does not reach the replay after a failover + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer dying = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpEgressMockServer healthy = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model)) { + String config = "ws::addr=" + dying.getAddress() + ',' + healthy.getAddress() + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(config)) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().skipRows(4).maxRows(3).countRows(true).stats(true); + RecordingQueryHandler handler = new RecordingQueryHandler() { + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + super.onFailoverReset(requestId, newNode); + options.clear(); + } + }; + client.execute("SELECT x FROM t", handler, options); + Assert.assertEquals(1, handler.failoverResetCount); + Assert.assertEquals(Arrays.asList(4L, 5L, 6L), handler.values); + Assert.assertEquals(3, handler.totalRows); + Assert.assertEquals(TABLE_ROWS, handler.rowCount); + Assert.assertTrue(handler.hasStats); + QwpEgressFrames.QueryRequest replay = healthy.getRequests().get(0); + Assert.assertEquals(4, replay.skipRows); + Assert.assertEquals(3, replay.maxRows); + } + dying.assertNoFailure(); + healthy.assertNoFailure(); + } + }); + } + + @Test + public void testFlaggedRequestAnsweredWithBaselineError() throws Exception { + // the server answers a request it could not decode with the baseline + // frame, although the request asked for the position + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> new byte[][]{ + QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, + "incomplete varint: buffer underflow").toFrame() + }); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, new QwpQueryOptions().errorPosition(true).maxRows(1)); + handler.assertOnlyError(); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, handler.errorStatus); + Assert.assertEquals("incomplete varint: buffer underflow", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertTrue("the connection stays usable", client.isConnected()); + + RecordingQueryHandler next = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", next, new QwpQueryOptions().errorPosition(true)); + Assert.assertEquals(1, next.errorCount); + server.assertNoFailure(); + } + }); + } + + @Test + public void testLegacyHandlerOverloadsStillFire() throws Exception { + // a handler written before the details overloads gets every callback + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true).skipRows(1); + List calls = new ArrayList<>(); + QwpColumnBatchHandler legacy = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + calls.add("onEnd " + totalRows); + } + + @Override + public void onError(byte status, String message) { + calls.add("onError " + status + ' ' + message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + calls.add("onExecDone " + opType + ' ' + rowsAffected); + } + }; + client.execute("SELECT x FROM t", legacy, options); + client.execute("INSERT INTO t VALUES (1)", legacy, options); + client.execute("FAIL", legacy, options); + Assert.assertEquals(Arrays.asList( + "onEnd 9", + "onExecDone 2 5", + "onError 5 " + ERROR_MESSAGE + ), calls); + + List requestIds = new ArrayList<>(); + QwpColumnBatchHandler correlating = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + Assert.fail(); + } + + @Override + public void onEnd(long requestId, long totalRows) { + requestIds.add("onEnd " + requestId); + } + + @Override + public void onError(byte status, String message) { + Assert.fail(); + } + + @Override + public void onError(long requestId, byte status, String message) { + requestIds.add("onError " + requestId); + } + + @Override + public void onExecDone(long requestId, short opType, long rowsAffected) { + requestIds.add("onExecDone " + requestId); + } + }; + client.execute("SELECT x FROM t", correlating, options); + client.execute("INSERT INTO t VALUES (1)", correlating, options); + client.execute("FAIL", correlating, options); + List requests = server.getRequests(); + Assert.assertEquals(Arrays.asList( + "onEnd " + requests.get(3).requestId, + "onExecDone " + requests.get(4).requestId, + "onError " + requests.get(5).requestId + ), requestIds); + server.assertNoFailure(); + } + }); + } + + @Test + public void testMissingTrailerIsTransportFailure() throws Exception { + // A server that advertises CAP_QUERY_DETAILS must send the trailers the + // request asks for; a terminator without them is a broken stream. + Object[][] cases = { + {"SELECT x FROM t", new QwpQueryOptions().countRows(true), "RESULT_END frame truncated mid row_count varint"}, + {"SELECT x FROM t", new QwpQueryOptions().stats(true), "RESULT_END frame truncated mid compile_ns varint"}, + {"INSERT INTO t VALUES (1)", new QwpQueryOptions().stats(true), "EXEC_DONE frame truncated mid compile_ns varint"}, + }; + for (Object[] c : cases) { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, + r -> model(r, 0)); + QwpQueryClient client = newClient(server, "failover=off;")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute((String) c[0], handler, (QwpQueryOptions) c[1]); + handler.assertOnlyError(); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals(c[2], handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + server.assertNoFailure(); + } + }); + } + } + + @Test + public void testReentrantExecuteWithOptionsRejected() throws Exception { + // one query at a time: an execute() from inside a handler fails, its + // exception leaves the outer execute(), and the client stays usable + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().maxRows(1).stats(true); + RecordingQueryHandler reentrant = new RecordingQueryHandler() { + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + super.onEnd(requestId, totalRows, details); + client.execute("SELECT x FROM t", new RecordingQueryHandler(), options); + } + }; + try { + client.execute("SELECT x FROM t", reentrant, options); + Assert.fail("a nested execute() must throw"); + } catch (IllegalStateException e) { + Assert.assertEquals("QwpQueryClient.execute called while another execute is in flight; " + + "one query at a time per client", e.getMessage()); + } + Assert.assertEquals(1, reentrant.endCount); + Assert.assertEquals("the nested query must not reach the server", 1, server.getRequests().size()); + + RecordingQueryHandler next = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", next, options); + Assert.assertEquals(Arrays.asList(0L), next.values); + Assert.assertTrue(next.hasStats); + server.assertNoFailure(); + } + }); + } + + @Test + public void testRowWindowCountAndStatsReachOnEnd() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + // {skip, max, expected first value, expected rows} + long[][] windows = {{3, 4, 3, 4}, {0, 10, 0, 10}, {8, 5, 8, 2}, {10, 1, 0, 0}, {0, 0, 0, 0}, {1, Long.MAX_VALUE, 1, 9}}; + for (long[] w : windows) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, + new QwpQueryOptions().skipRows(w[0]).maxRows(w[1]).countRows(true).stats(true)); + String msg = "skip=" + w[0] + ", max=" + w[1]; + Assert.assertEquals(msg, 1, handler.endCount); + Assert.assertEquals(msg, w[3], handler.values.size()); + for (int i = 0; i < w[3]; i++) { + Assert.assertEquals(msg, w[2] + i, (long) handler.values.get(i)); + } + Assert.assertEquals(msg, w[3], handler.totalRows); + Assert.assertTrue(msg, handler.hasRowCount); + Assert.assertEquals(msg, TABLE_ROWS, handler.rowCount); + Assert.assertTrue(msg, handler.hasStats); + Assert.assertEquals(COMPILE_NANOS, handler.compileNanos); + Assert.assertEquals(EXECUTE_NANOS, handler.executeNanos); + Assert.assertEquals(COUNT_NANOS, handler.countNanos); + Assert.assertEquals(CLIENT_WAIT_NANOS, handler.clientWaitNanos); + Assert.assertTrue(handler.isJitCompiled); + Assert.assertEquals(server.getRequests().get(server.getRequests().size() - 1).requestId, handler.endRequestId); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testStatsReachOnExecDone() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + // the row options do not apply to a statement without rows + client.execute("INSERT INTO t VALUES (1)", handler, + new QwpQueryOptions().stats(true).countRows(true).skipRows(1).maxRows(1)); + Assert.assertEquals(1, handler.execDoneCount); + Assert.assertEquals(OP_INSERT, handler.opType); + Assert.assertEquals(ROWS_AFFECTED, handler.rowsAffected); + Assert.assertFalse(handler.hasRowCount); + Assert.assertTrue(handler.hasStats); + Assert.assertEquals(COMPILE_NANOS, handler.compileNanos); + Assert.assertEquals(EXECUTE_NANOS, handler.executeNanos); + Assert.assertEquals(COUNT_NANOS, handler.countNanos); + Assert.assertEquals(CLIENT_WAIT_NANOS, handler.clientWaitNanos); + Assert.assertTrue(handler.isJitCompiled); + server.assertNoFailure(); + } + }); + } + + private static byte[][] model(QwpEgressFrames.QueryRequest r) { + return model(r, r.queryFlags); + } + + /** + * Answers {@code r} the way the server does, with the trailers of + * {@code trailerFlags}: a {@code SELECT} streams the row window of the + * 10-row table, an {@code INSERT} reports {@value #ROWS_AFFECTED} rows, and + * anything else fails at position {@value #ERROR_POSITION}. + */ + private static byte[][] model(QwpEgressFrames.QueryRequest r, long trailerFlags) { + List frames = new ArrayList<>(); + if (r.sql.startsWith("SELECT")) { + long skip = r.skipRows < 0 ? 0 : r.skipRows; + long max = r.maxRows < 0 ? Long.MAX_VALUE : r.maxRows; + long first = Math.min(skip, TABLE_ROWS); + long end = first + Math.min(max, TABLE_ROWS - first); + long seq = 0; + long x = first; + do { + long n = Math.min(3, end - x); + long[] values = new long[(int) n]; + for (int i = 0; i < n; i++) { + values[i] = x++; + } + frames.add(QwpEgressFrames.resultBatchOfLongs(r.requestId, seq++, values)); + } while (x < end); + QwpEgressFrames.Body end0 = QwpEgressFrames.resultEnd(r.requestId, seq - 1, end - first); + if ((trailerFlags & QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS) != 0) { + end0.varint(TABLE_ROWS); + } + if ((trailerFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + end0.stats(COMPILE_NANOS, EXECUTE_NANOS, COUNT_NANOS, CLIENT_WAIT_NANOS, QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED); + } + frames.add(end0.toFrame()); + } else if (r.sql.startsWith("INSERT")) { + QwpEgressFrames.Body done = QwpEgressFrames.execDone(r.requestId, OP_INSERT, ROWS_AFFECTED); + if ((trailerFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + done.stats(COMPILE_NANOS, EXECUTE_NANOS, COUNT_NANOS, CLIENT_WAIT_NANOS, QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED); + } + frames.add(done.toFrame()); + } else if ((r.queryFlags & QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION) != 0) { + frames.add(QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, ERROR_MESSAGE) + .i32(ERROR_POSITION).toFrame()); + } else { + frames.add(QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, + "[" + ERROR_POSITION + "] " + ERROR_MESSAGE).toFrame()); + } + return frames.toArray(new byte[0][]); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server, String extraConfig) { + return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() + ";auth_timeout_ms=2000;" + extraConfig); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java index 07a1643ca..c65d008b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java @@ -28,8 +28,11 @@ import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -37,6 +40,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -58,9 +62,16 @@ * The decode -> dict-reset effect on the server, and the end-to-end honouring * of the flag, are covered against a live server in the questdb repo * (QwpEgressQueryFlagsResetTest, QwpEgressCacheResetWireTest). + *

+ * The {@link QwpQueryOptions} tests pin the query-details bits and the + * row-window values the server's {@code QwpEgressRequestDecoder} reads after + * the flags: each option sets its own bit, {@code skip_rows} and + * {@code max_rows} follow the flags in that order, a server without + * {@link QwpEgressMsgKind#CAP_QUERY_DETAILS} gets none of them, and a row + * window it would ignore fails the query before it is sent. */ public class QwpQueryClientQueryFlagsTest { - + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; private static final QwpColumnBatchHandler NOOP_HANDLER = new QwpColumnBatchHandler() { @Override public void onBatch(QwpColumnBatch batch) { @@ -74,6 +85,52 @@ public void onEnd(long totalRows) { public void onError(byte status, String message) { } }; + private static final String SQL = "SELECT 1"; + + @Test + public void testDetailOptionsDroppedWithoutCapability() throws Exception { + // the query runs without what the server cannot report + QwpQueryOptions options = new QwpQueryOptions().errorPosition(true).stats(true).countRows(true).resetSymbolDict(true); + QwpEgressFrames.QueryRequest r = executeOnce(QwpEgressMsgKind.CAP_QUERY_FLAGS, options); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, r.queryFlags); + Assert.assertEquals(0, r.trailingBytes); + + r = executeOnce(0, options); + Assert.assertFalse("a server without CAP_QUERY_FLAGS gets no trailer", r.hasQueryFlags); + Assert.assertEquals(0, r.trailingBytes); + } + + @Test + public void testEachDetailOptionSendsItsFlag() throws Exception { + Object[][] cases = { + {new QwpQueryOptions().errorPosition(true), QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION}, + {new QwpQueryOptions().stats(true), QwpEgressMsgKind.QUERY_FLAG_STATS}, + {new QwpQueryOptions().countRows(true), QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS}, + {new QwpQueryOptions().resetSymbolDict(true), QwpEgressMsgKind.QUERY_FLAG_RESET_DICT}, + }; + for (Object[] c : cases) { + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, (QwpQueryOptions) c[0]); + Assert.assertEquals(c[0].toString(), (long) (int) c[1], r.queryFlags); + Assert.assertEquals(-1, r.skipRows); + Assert.assertEquals(-1, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + } + + @Test + public void testEveryOptionOnTheWire() throws Exception { + QwpQueryOptions options = new QwpQueryOptions() + .resetSymbolDict(true).errorPosition(true).stats(true).countRows(true) + .skipRows(300).maxRows(16_384); + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, options); + Assert.assertEquals(0x3F, r.queryFlags); + Assert.assertEquals(300, r.skipRows); + Assert.assertEquals(16_384, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + // flags, skip_rows, max_rows: the last bytes of the request, in that order + byte[] tail = new QwpEgressFrames.Body().varint(0x3F).varint(300).varint(16_384).toBody(); + Assert.assertArrayEquals(tail, Arrays.copyOfRange(r.payload, r.payload.length - tail.length, r.payload.length)); + } @Test public void testFlagDroppedWhenServerLacksCapability() throws Exception { @@ -94,6 +151,79 @@ public void testLegacyTwoArgOverloadSendsNoTrailer() throws Exception { Assert.assertEquals(-1L, queryFlagsTrailer(frame)); } + @Test + public void testMaxRowsOnTheWire() throws Exception { + // zero streams only the schema, so it goes on the wire like any bound + for (long maxRows : new long[]{0, 1, 127, 128, 16_383, 16_384, Long.MAX_VALUE - 1}) { + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().maxRows(maxRows)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(-1, r.skipRows); + Assert.assertEquals(maxRows, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + // the unbounded default sends nothing + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().maxRows(5).maxRows(Long.MAX_VALUE)); + Assert.assertFalse(r.hasQueryFlags); + Assert.assertEquals(0, r.trailingBytes); + } + + @Test + public void testOptionsFollowBinds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t WHERE x > $1 AND x < $2", binds -> binds.setLong(0, -7).setLong(1, Long.MAX_VALUE), + handler, new QwpQueryOptions().countRows(true).skipRows(2).maxRows(3)); + Assert.assertEquals(1, handler.execDoneCount); + server.assertNoFailure(); + QwpEgressFrames.QueryRequest r = server.getRequests().get(0); + Assert.assertArrayEquals(new long[]{-7, Long.MAX_VALUE}, r.longBinds); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(2, r.skipRows); + Assert.assertEquals(3, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + }); + } + + @Test + public void testOptionsWithNothingSetAreTheBaseline() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler); + client.execute(SQL, handler, new QwpQueryOptions()); + client.execute(SQL, handler, (QwpQueryOptions) null); + client.execute(SQL, null, handler, new QwpQueryOptions().skipRows(5).maxRows(5).clear()); + Assert.assertEquals(4, handler.execDoneCount); + server.assertNoFailure(); + List requests = server.getRequests(); + Assert.assertEquals(4, requests.size()); + byte[] baseline = requests.get(0).payload; + zeroRequestId(baseline); + for (int i = 1; i < requests.size(); i++) { + byte[] payload = requests.get(i).payload; + zeroRequestId(payload); + Assert.assertArrayEquals("request " + i, baseline, payload); + } + } + }); + } + + @Test + public void testResetDictDroppedWithoutQueryFlagsCapability() throws Exception { + QwpEgressFrames.QueryRequest r = executeOnce(QwpEgressMsgKind.CAP_QUERY_DETAILS, + new QwpQueryOptions().resetSymbolDict(true).stats(true).maxRows(9)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_STATS | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(9, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + @Test public void testResetFlagAppendsTrailerWhenCapabilityAdvertised() throws Exception { byte[] frame = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", NOOP_HANDLER, true)); @@ -113,6 +243,59 @@ public void testResetTrailerIsTheOnlyDifferenceFromBaseline() throws Exception { Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, on[on.length - 1]); } + @Test + public void testRowWindowFailsWithoutCapability() throws Exception { + // A server without CAP_QUERY_DETAILS would stream every row. The query + // fails before it is sent, and the connection stays usable. + QwpQueryOptions[] windows = { + new QwpQueryOptions().skipRows(1), + new QwpQueryOptions().maxRows(0), + new QwpQueryOptions().skipRows(10).maxRows(10).stats(true), + }; + for (int capabilities : new int[]{0, QwpEgressMsgKind.CAP_QUERY_FLAGS}) { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(capabilities, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + for (QwpQueryOptions window : windows) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, binds -> Assert.fail("binds of a refused query must not be encoded"), handler, window); + handler.assertOnlyError(); + Assert.assertEquals(-1, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("server does not support skipRows and maxRows [capabilities=0x" + + Integer.toHexString(capabilities) + "]; they need a server that advertises CAP_QUERY_DETAILS", + handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + } + Assert.assertEquals("a refused query must not reach the server", 0, server.getRequests().size()); + + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, new QwpQueryOptions().stats(true)); + Assert.assertEquals(1, handler.execDoneCount); + Assert.assertEquals(1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + } + + @Test + public void testSkipRowsOnTheWire() throws Exception { + for (long skipRows : new long[]{1, 127, 128, 16_383, 16_384, Long.MAX_VALUE}) { + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().skipRows(skipRows)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS, r.queryFlags); + Assert.assertEquals(skipRows, r.skipRows); + Assert.assertEquals(-1, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + // skipping no rows sends nothing + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().skipRows(5).skipRows(0).maxRows(10)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(-1, r.skipRows); + Assert.assertEquals(10, r.maxRows); + } + private static byte[] buildExecDone(byte[] queryRequest) { int bodyLen = 1 + 8 + 1 + 1; // msg_kind + request_id + op_type + rows_affected varint byte[] frame = new byte[QwpConstants.HEADER_SIZE + bodyLen]; @@ -129,6 +312,43 @@ private static byte[] buildExecDone(byte[] queryRequest) { return frame; } + /** + * Answers a request with an {@code EXEC_DONE}, carrying the statistics + * block when the request asked for it, as the server does. + */ + private static byte[][] execDone(QwpEgressFrames.QueryRequest r) { + QwpEgressFrames.Body body = QwpEgressFrames.execDone(r.requestId, 0, 0); + if (r.has(QwpEgressMsgKind.QUERY_FLAG_STATS)) { + body.stats(0, 0, 0, 0, 0); + } + return new byte[][]{body.toFrame()}; + } + + /** + * Runs {@link #SQL} once with {@code options} against a server that + * advertises {@code capabilities}, and returns the request it received. + */ + private static QwpEgressFrames.QueryRequest executeOnce(int capabilities, QwpQueryOptions options) throws Exception { + AtomicReference request = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(capabilities, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, options); + Assert.assertEquals("the query must complete", 1, handler.execDoneCount); + server.assertNoFailure(); + Assert.assertEquals(1, server.getRequests().size()); + request.set(server.getRequests().get(0)); + } + }); + return request.get(); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server) { + return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() + ";auth_timeout_ms=2000;failover=off;"); + } + /** * Parses the {@code QUERY_REQUEST} the client emitted and returns the * decoded {@code query_flags} trailer, or {@code -1} when no trailer was diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java new file mode 100644 index 000000000..56b5c5bf9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java @@ -0,0 +1,164 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.QueryException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import org.junit.Assert; +import org.junit.Test; + +/** + * State of {@link QwpQueryOptions}: defaults, setters, validation, reset and + * copy. What the options put on the wire is covered by + * {@link QwpQueryClientQueryFlagsTest}. + */ +public class QwpQueryOptionsTest { + + @Test + public void testClearRestoresDefaults() { + QwpQueryOptions options = allSet(); + Assert.assertSame(options, options.clear()); + assertDefaults(options); + } + + @Test + public void testCopyFrom() { + QwpQueryOptions source = allSet(); + QwpQueryOptions target = new QwpQueryOptions(); + Assert.assertSame(target, target.copyFrom(source)); + Assert.assertEquals(source.toString(), target.toString()); + assertAllSet(target); + + // the copy is independent of its source + source.clear(); + assertAllSet(target); + + Assert.assertSame(target, target.copyFrom(target)); + assertAllSet(target); + + // null copies the defaults + Assert.assertSame(target, target.copyFrom(null)); + assertDefaults(target); + } + + @Test + public void testDefaults() { + QwpQueryOptions options = new QwpQueryOptions(); + assertDefaults(options); + Assert.assertEquals("QwpQueryOptions{skipRows=0, maxRows=9223372036854775807, countRows=false, " + + "stats=false, errorPosition=false, resetSymbolDict=false}", options.toString()); + } + + @Test + public void testNegativeRowsRejected() { + QwpQueryOptions options = new QwpQueryOptions().skipRows(3).maxRows(4); + for (long rows : new long[]{-1, -128, Long.MIN_VALUE}) { + try { + options.skipRows(rows); + Assert.fail("skipRows(" + rows + ") must throw"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("skipRows must be >= 0 [skipRows=" + rows + ']', e.getMessage()); + } + try { + options.maxRows(rows); + Assert.fail("maxRows(" + rows + ") must throw"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("maxRows must be >= 0 [maxRows=" + rows + ']', e.getMessage()); + } + // a rejected value leaves the previous one + Assert.assertEquals(3, options.getSkipRows()); + Assert.assertEquals(4, options.getMaxRows()); + } + } + + @Test + public void testQueryExceptionPosition() { + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, new QueryException((byte) 5, "x").getPosition()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, + new QueryException((byte) 0, "x", new IllegalStateException()).getPosition()); + QueryException e = new QueryException((byte) 5, "table does not exist", 14); + Assert.assertEquals(5, e.getStatus()); + Assert.assertEquals("table does not exist", e.getMessage()); + Assert.assertEquals(14, e.getPosition()); + } + + @Test + public void testRowBounds() { + QwpQueryOptions options = new QwpQueryOptions(); + for (long rows : new long[]{0, 1, 127, 128, Long.MAX_VALUE - 1, Long.MAX_VALUE}) { + Assert.assertSame(options, options.skipRows(rows)); + Assert.assertEquals(rows, options.getSkipRows()); + Assert.assertSame(options, options.maxRows(rows)); + Assert.assertEquals(rows, options.getMaxRows()); + } + } + + @Test + public void testSetters() { + QwpQueryOptions options = new QwpQueryOptions(); + Assert.assertSame(options, options.countRows(true)); + Assert.assertTrue(options.isRowCountRequested()); + Assert.assertSame(options, options.stats(true)); + Assert.assertTrue(options.isStatsRequested()); + Assert.assertSame(options, options.errorPosition(true)); + Assert.assertTrue(options.isErrorPositionRequested()); + Assert.assertSame(options, options.resetSymbolDict(true)); + Assert.assertTrue(options.isSymbolDictResetRequested()); + Assert.assertEquals("QwpQueryOptions{skipRows=0, maxRows=9223372036854775807, countRows=true, " + + "stats=true, errorPosition=true, resetSymbolDict=true}", options.toString()); + + options.countRows(false).stats(false).errorPosition(false).resetSymbolDict(false); + assertDefaults(options); + } + + private static QwpQueryOptions allSet() { + return new QwpQueryOptions() + .skipRows(100) + .maxRows(50) + .countRows(true) + .stats(true) + .errorPosition(true) + .resetSymbolDict(true); + } + + private static void assertAllSet(QwpQueryOptions options) { + Assert.assertEquals(100, options.getSkipRows()); + Assert.assertEquals(50, options.getMaxRows()); + Assert.assertTrue(options.isRowCountRequested()); + Assert.assertTrue(options.isStatsRequested()); + Assert.assertTrue(options.isErrorPositionRequested()); + Assert.assertTrue(options.isSymbolDictResetRequested()); + } + + private static void assertDefaults(QwpQueryOptions options) { + Assert.assertEquals(0, options.getSkipRows()); + Assert.assertEquals(Long.MAX_VALUE, options.getMaxRows()); + Assert.assertFalse(options.isRowCountRequested()); + Assert.assertFalse(options.isStatsRequested()); + Assert.assertFalse(options.isErrorPositionRequested()); + Assert.assertFalse(options.isSymbolDictResetRequested()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java new file mode 100644 index 000000000..69991f28e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java @@ -0,0 +1,148 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.List; + +/** + * Records what the most detailed callbacks of {@link QwpColumnBatchHandler} + * receive, copying the {@link QwpQueryDetails} values out while they are + * valid. The simpler overloads fail: the client must call the detailed ones. + * Batches must carry a single LONG column. + */ +public class RecordingQueryHandler implements QwpColumnBatchHandler { + public final List values = new ArrayList<>(); + public int batchCount; + public long clientWaitNanos; + public long compileNanos; + public long countNanos; + public int endCount; + public long endRequestId; + public int errorCount; + public String errorMessage; + public int errorPosition; + public long errorRequestId; + public byte errorStatus; + public int execDoneCount; + public long executeNanos; + public int failoverResetCount; + public boolean hasRowCount; + public boolean hasStats; + public boolean isJitCompiled; + public short opType; + public long rowCount; + public long rowsAffected; + public long totalRows; + + public void assertNoDetails() { + Assert.assertFalse("row count", hasRowCount); + Assert.assertEquals(-1, rowCount); + Assert.assertFalse("stats", hasStats); + Assert.assertEquals(-1, compileNanos); + Assert.assertEquals(-1, executeNanos); + Assert.assertEquals(-1, countNanos); + Assert.assertEquals(-1, clientWaitNanos); + Assert.assertFalse(isJitCompiled); + } + + public void assertOnlyError() { + Assert.assertEquals("errors", 1, errorCount); + Assert.assertEquals("ends", 0, endCount); + Assert.assertEquals("exec dones", 0, execDoneCount); + } + + @Override + public void onBatch(QwpColumnBatch batch) { + batchCount++; + for (int row = 0; row < batch.getRowCount(); row++) { + values.add(batch.getLongValue(0, row)); + } + } + + @Override + public void onEnd(long totalRows) { + Assert.fail("the client must call onEnd(requestId, totalRows, details)"); + } + + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + endCount++; + endRequestId = requestId; + this.totalRows = totalRows; + copy(details); + } + + @Override + public void onError(byte status, String message) { + Assert.fail("the client must call onError(requestId, status, message, position)"); + } + + @Override + public void onError(long requestId, byte status, String message, int position) { + errorCount++; + errorRequestId = requestId; + errorStatus = status; + errorMessage = message; + errorPosition = position; + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + Assert.fail("the client must call onExecDone(requestId, opType, rowsAffected, details)"); + } + + @Override + public void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + execDoneCount++; + this.opType = opType; + this.rowsAffected = rowsAffected; + copy(details); + } + + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + failoverResetCount++; + values.clear(); + } + + private void copy(QwpQueryDetails details) { + Assert.assertNotNull("details must never be null", details); + hasRowCount = details.hasRowCount(); + rowCount = details.getRowCount(); + hasStats = details.hasStats(); + compileNanos = details.getCompileNanos(); + executeNanos = details.getExecuteNanos(); + countNanos = details.getCountNanos(); + clientWaitNanos = details.getClientWaitNanos(); + isJitCompiled = details.isJitCompiled(); + } +} From f29249d7b9515769b5acc154777ddf606cdf9467 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 12 Nov 2026 17:57:25 +0000 Subject: [PATCH 2/5] Cover the client-side failure paths of execute() The query-details change routes every failure that execute() detects itself to the position-aware onError overload, but the tests written for it reached none of those paths. QwpQueryClientFailurePathsTest now drives each one with a handler that fails on any other overload: - a failover reconnect that fails, and the reconnect of the next query that fails after it; - a stop request between two attempts, from onFailoverReset; - a stop request while the failover reconnect waits for SERVER_INFO; - a stop request right after a transport failure; - a close() between two attempts, from onFailoverReset. A stop request can reach the point right after a transport failure only through a race with another thread, so QwpQueryClient gains setAfterTransportFailureHookForTest(), a barrier that runs there, like setBeforeCloseHookForTest(). TestWebSocketServer gains a hook that runs before SERVER_INFO, and its rejected egress upgrades now drop the connection, as its documentation says, instead of leaving the client to wait out its auth timeout. The default branch of the event switch in executeOnce() is unreachable, as the I/O thread emits only the kinds it handles; an assertion now says so. A comment explains the null guard on serverInfo, which only a connect() racing the query from another thread can trip. --- .../cutlass/qwp/client/QwpQueryClient.java | 20 +- .../qwp/client/QwpEgressMockServer.java | 24 ++ .../QwpQueryClientFailurePathsTest.java | 242 ++++++++++++++++++ .../qwp/websocket/TestWebSocketServer.java | 21 ++ 4 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index 2f4efb57b..0f999c179 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -188,6 +188,11 @@ public class QwpQueryClient implements QuietCloseable { private final Random failoverRandom = new Random(); // Connection health (design/qwp-token-provider-spec.md, section 8.4). private final QwpConnectionHealthTracker healthTracker = new QwpConnectionHealthTracker(); + // Deterministic barrier used by shutdown tests. Null in production. execute() + // runs it after an attempt intercepted a transport failure and before it + // checks for a stop request, the window a requestShutdown() from another + // thread can otherwise only hit by chance. + private volatile Runnable afterTransportFailureHook; private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; // Deterministic lifecycle barrier used by facade shutdown tests. Null in @@ -1016,6 +1021,11 @@ public void seedFailoverRandomForTest(long seed) { } } + @TestOnly + public void setAfterTransportFailureHookForTest(Runnable hook) { + afterTransportFailureHook = hook; + } + @TestOnly public void setBeforeCloseHookForTest(Runnable hook) { beforeCloseHook = hook; @@ -1839,6 +1849,10 @@ private void executeImpl( if (!probe.transportFailureIntercepted) { return; } + Runnable hook = afterTransportFailureHook; + if (hook != null) { + hook.run(); + } if (isShutdownRequested) { handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down", QwpEgressMsgKind.NO_ERROR_POSITION); @@ -1998,7 +2012,9 @@ private void executeOnce( return; } // Resolved per attempt: a failover reconnect may land on a server that - // advertises other capabilities. + // advertises other capabilities. Every bound connection has its + // SERVER_INFO; only a connect() racing this query from another thread + // could clear it, and then no flag goes out. final QwpServerInfo info = serverInfo; final int capabilities = info != null ? info.getCapabilities() : 0; final long wireQueryFlags = resolveQueryFlags(queryFlags, capabilities); @@ -2080,6 +2096,8 @@ private void executeOnce( probe.markTransportFailure(requestId, ev.errorStatus, ev.errorMessage); return; default: + // Unreachable: the I/O thread emits only the kinds above. + assert false : "unknown event kind " + ev.kind; probe.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind, QwpEgressMsgKind.NO_ERROR_POSITION); return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java index 6b0aa1db8..630cd3b05 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java @@ -108,6 +108,30 @@ public List getRequests() { return requests; } + /** + * Number of WebSocket upgrades the server completed: the initial connect + * and each reconnect. + */ + public int handshakeCount() { + return server.handshakeCount(); + } + + /** + * Runs {@code hook} on each connection right before the server sends + * {@code SERVER_INFO}; see {@link TestWebSocketServer#setBeforeServerInfoHook}. + */ + public void setBeforeServerInfoHook(Runnable hook) { + server.setBeforeServerInfoHook(hook); + } + + /** + * While set, the server drops every new egress connection before the + * upgrade completes, so connects and failover reconnects fail. + */ + public void setRejectUpgrades(boolean isRejecting) { + server.setRejectReadUpgrade(isRejecting); + } + @FunctionalInterface public interface Responder { /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java new file mode 100644 index 000000000..a00aaa20b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java @@ -0,0 +1,242 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * The failures {@link QwpQueryClient#execute} detects itself -- a failover + * reconnect that fails, a stop request or a {@code close()} between attempts -- + * reach the handler through the position-aware + * {@code onError(requestId, status, message, position)} with + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}, like the server's errors, so a + * handler that overrides only that overload sees them all. Each query asks for + * the details; the first one dies with its connection, which sends + * {@code execute()} down its failover path. + */ +public class QwpQueryClientFailurePathsTest { + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; + private static final QwpQueryOptions OPTIONS = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true); + private static final String SQL = "SELECT x FROM t"; + + @Test(timeout = 30_000) + public void testCloseBetweenAttemptsReportsClosed() throws Exception { + // close() from the failover callback: the next attempt finds no I/O + // thread and ends the query instead of touching the freed client + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler() { + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + super.onFailoverReset(requestId, newNode); + client.close(); + } + }; + client.execute(SQL, handler, OPTIONS); + Assert.assertEquals(1, handler.failoverResetCount); + handler.assertOnlyError(); + Assert.assertEquals(-1, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("QwpQueryClient is closed", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertFalse(client.wasLastCloseTimedOut()); + Assert.assertEquals("the closed client must not replay the query", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testFailedReconnectsReportNoPosition() throws Exception { + // The server dies with the first query and then refuses connections: the + // failover reconnect fails, and so does the reconnect of the next query. + // Once it accepts connections again, the client recovers. + TestUtils.assertMemoryLeak(() -> { + AtomicReference serverRef = new AtomicReference<>(); + AtomicInteger queries = new AtomicInteger(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> { + if (queries.getAndIncrement() == 0) { + serverRef.get().setRejectUpgrades(true); + return null; + } + return answer(r); + }); + QwpQueryClient client = newClient(server)) { + serverRef.set(server); + client.connect(); + + RecordingQueryHandler failover = new RecordingQueryHandler(); + client.execute(SQL, failover, OPTIONS); + failover.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, failover.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, failover.errorStatus); + TestUtils.assertContains(failover.errorMessage, "failover reconnect failed after 1 attempt [last error: "); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, failover.errorPosition); + Assert.assertFalse(client.isConnected()); + + RecordingQueryHandler reconnect = new RecordingQueryHandler(); + client.execute(SQL, reconnect, OPTIONS); + reconnect.assertOnlyError(); + Assert.assertEquals(-1, reconnect.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, reconnect.errorStatus); + Assert.assertTrue(reconnect.errorMessage, reconnect.errorMessage.startsWith("reconnect failed: ")); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, reconnect.errorPosition); + + server.setRejectUpgrades(false); + RecordingQueryHandler recovered = new RecordingQueryHandler(); + client.execute(SQL, recovered, OPTIONS); + Assert.assertEquals(1, recovered.endCount); + Assert.assertEquals(Collections.singletonList(42L), recovered.values); + Assert.assertEquals(1, recovered.rowCount); + Assert.assertTrue(recovered.hasStats); + Assert.assertEquals(2, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testShutdownAfterTransportFailureReportsNoPosition() throws Exception { + // a stop request that lands right after the connection dies ends the + // query instead of failing over + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpQueryClient client = newClient(server)) { + client.connect(); + client.setAfterTransportFailureHookForTest(client::requestShutdown); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(0, handler.failoverResetCount); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("query client shut down", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("no failover reconnect after the stop request", 1, server.handshakeCount()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testShutdownBetweenAttemptsReportsNoPosition() throws Exception { + // a stop request from the failover callback ends the query before the + // replay + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler() { + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + super.onFailoverReset(requestId, newNode); + client.requestShutdown(); + } + }; + client.execute(SQL, handler, OPTIONS); + Assert.assertEquals(1, handler.failoverResetCount); + handler.assertOnlyError(); + Assert.assertEquals(-1, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("query client shut down", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("the query must not be replayed", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testShutdownDuringReconnectReportsNoPosition() throws Exception { + // the stop request arrives while the failover reconnect waits for + // SERVER_INFO: the reconnect completes, then the query ends + TestUtils.assertMemoryLeak(() -> { + AtomicReference clientRef = new AtomicReference<>(); + AtomicInteger serverInfos = new AtomicInteger(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server)) { + clientRef.set(client); + server.setBeforeServerInfoHook(() -> { + if (serverInfos.incrementAndGet() == 2) { + clientRef.get().requestShutdown(); + } + }); + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(0, handler.failoverResetCount); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("query client shut down", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals(2, server.handshakeCount()); + Assert.assertEquals("the query must not be replayed", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + /** + * One LONG row, x = 42, and the trailers the request asks for. + */ + private static byte[][] answer(QwpEgressFrames.QueryRequest r) { + QwpEgressFrames.Body end = QwpEgressFrames.resultEnd(r.requestId, 0, 1); + if (r.has(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS)) { + end.varint(1); + } + if (r.has(QwpEgressMsgKind.QUERY_FLAG_STATS)) { + end.stats(1, 2, 3, 4, 0); + } + return new byte[][]{QwpEgressFrames.resultBatchOfLongs(r.requestId, 0, 42), end.toFrame()}; + } + + /** + * Drops the connection on the first query, then answers like + * {@link #answer}. + */ + private static QwpEgressMockServer.Responder dropFirstQuery() { + AtomicInteger queries = new AtomicInteger(); + return r -> queries.getAndIncrement() == 0 ? null : answer(r); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server) { + return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;"); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 102922d6b..e63637b1d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -112,6 +112,10 @@ public class TestWebSocketServer implements Closeable { // connections carry only ACK frames. private volatile boolean sendServerInfo; private volatile int capabilities; + // Runs on the connection's thread right before it sends SERVER_INFO, while + // the client waits for the frame inside connect() or a failover reconnect. + // Null for none. Set via setBeforeServerInfoHook(). + private volatile Runnable beforeServerInfoHook; // When true, the server fails the WebSocket upgrade on the egress read path // (/read...) by dropping the connection before the 101, while still serving // the ingest write path (/write...) normally. Lets one server + one cluster @@ -434,6 +438,16 @@ public void setSendServerInfo(boolean sendServerInfo) { this.sendServerInfo = sendServerInfo; } + /** + * Runs {@code hook} on each egress connection right before the server sends + * its {@code SERVER_INFO} frame, which the client awaits inside + * {@code connect()} or a failover reconnect. Lets a test act at a fixed + * point of the client's connect sequence. Null removes the hook. + */ + public void setBeforeServerInfoHook(Runnable hook) { + this.beforeServerInfoHook = hook; + } + // Advertised SERVER_INFO capabilities. CAP_ZONE is unsupported here: the // frame builder emits no zone_id trailer. public void setCapabilities(int capabilities) { @@ -792,6 +806,9 @@ private boolean performHandshake() throws IOException { // query pool's connect fails fast, while ingest write-path upgrades // still complete on this same server. if (rejectReadUpgrade && isReadPath(requestPath)) { + // Drop the connection before the 101, so the client fails its + // upgrade at once instead of waiting out its auth timeout. + close(); return false; } @@ -941,6 +958,10 @@ void start() { // connection parses every inbound frame as an ACK and // would fail on it. if (sendServerInfo && isReadPath(requestPath)) { + Runnable hook = beforeServerInfoHook; + if (hook != null) { + hook.run(); + } sendBinary(buildServerInfoFrame(roleByte(advertisedRole), capabilities)); } From dfa1eea62cd7e004e75585ab2a514be6238ed70a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 13 Nov 2026 11:28:00 +0000 Subject: [PATCH 3/5] Correct the error position docs for cancels The javadoc of QwpColumnBatchHandler.onError(long, byte, String, int) and of QwpEgressMsgKind.NO_ERROR_POSITION said that a cancellation reports no error position. Only a CANCEL this client sends does: the server reports position 0 for a query timeout and for a cancellation on the server, such as CANCEL QUERY, as the /exec HTTP endpoint does. The javadoc now says so. --- .../client/cutlass/qwp/client/QwpColumnBatchHandler.java | 9 ++++++--- .../client/cutlass/qwp/client/QwpEgressMsgKind.java | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java index 937c3c2a3..ce809df61 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java @@ -135,9 +135,12 @@ default void onError(long requestId, byte status, String message) { * {@code message} is what the server sent, which may start with the * position in brackets, and {@code position} is * {@link QwpEgressMsgKind#NO_ERROR_POSITION}, as it is for errors that do - * not come from the SQL text, such as a cancellation or a transport - * failure. The default delegates to {@link #onError(long, byte, String)} for - * backwards compatibility; override this method when you need the position. + * not come from the SQL text, such as a {@code CANCEL} this client sent or a + * transport failure. A query timeout and a cancellation on the server, such + * as {@code CANCEL QUERY}, report position 0, as the {@code /exec} HTTP + * endpoint does. The default delegates to + * {@link #onError(long, byte, String)} for backwards compatibility; override + * this method when you need the position. * * @param requestId client-assigned request id, or {@code -1} if no request was in flight * @param status one of the QWP status codes (e.g., {@code STATUS_PARSE_ERROR}) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java index 29ec7b0fc..9bbd6f047 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java @@ -72,8 +72,11 @@ public final class QwpEgressMsgKind { public static final byte EXEC_DONE = 0x16; /** * Error position that points nowhere in the SQL text: the position of an - * error that does not come from the SQL text, such as a cancellation, and - * of every error when the position was not requested or not reported. + * error that does not come from the SQL text, such as a {@code CANCEL} the + * client sent, and of every error when the position was not requested or + * not reported. A query timeout and a cancellation on the server, such as + * {@code CANCEL QUERY}, report position 0, as the {@code /exec} HTTP endpoint + * does. */ public static final int NO_ERROR_POSITION = -1; public static final byte QUERY_ERROR = 0x13; From e56915c4fc0062f97857bb1759b83fe52689def8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 13 Nov 2026 12:22:37 +0000 Subject: [PATCH 4/5] Move query details off the timeout bits The query details took SERVER_INFO capability 0x08 and query flag 0x02, which per-query timeouts already use in released clients: c-questdb-client 7.2.0 and later, and the Python client 6.0.0 and later, read capability 0x08 as "the server honours a per-query timeout" and then send flag 0x02 followed by timeout_ms. A server with the query details would have read that flag as an error-position request, ignored the timeout and answered with a QUERY_ERROR those clients cannot decode. CAP_QUERY_DETAILS moves to 0x10, and the flags to 0x04 (ERROR_POSITION), 0x08 (STATS), 0x10 (COUNT_ROWS), 0x20 (SKIP_ROWS) and 0x40 (MAX_ROWS), as on the server. CAP_QUERY_TIMEOUT (0x08) and QUERY_FLAG_TIMEOUT (0x02) now name the reserved bits, with the values the timeout change gives them, and the docs say a flag's field goes after query_flags in ascending order of the flag bits, so timeout_ms comes ahead of skip_rows and max_rows. New tests pin the allocation: every capability and flag is a bit of its own and none takes the timeout bits. A server that advertises per-query timeouts without the query details gets none of the details, and a row window fails before it is sent. --- .../cutlass/qwp/client/QwpEgressMsgKind.java | 46 +++++++++++----- .../client/QwpQueryClientQueryFlagsTest.java | 54 +++++++++++++++++-- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java index 9bbd6f047..571e1d633 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java @@ -47,7 +47,7 @@ public final class QwpEgressMsgKind { * them and answers with the baseline frames. Mirrors the server-side constant * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_DETAILS}. */ - public static final int CAP_QUERY_DETAILS = 0x00000008; + public static final int CAP_QUERY_DETAILS = 0x00000010; /** * {@code SERVER_INFO.capabilities} bit: the server parses the optional * {@code query_flags:varint} trailer on {@code QUERY_REQUEST}. The client @@ -55,6 +55,15 @@ public final class QwpEgressMsgKind { * constant {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_FLAGS}. */ public static final int CAP_QUERY_FLAGS = 0x00000002; + /** + * {@code SERVER_INFO.capabilities} bit reserved for per-query timeouts, which + * this client does not send. Released clients (c-questdb-client 7.2.0 and + * later, and the Python client built on it) read the bit as "the server + * honours {@link #QUERY_FLAG_TIMEOUT}", so no other capability may take it. + * Mirrors the server-side constant + * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_TIMEOUT}. + */ + public static final int CAP_QUERY_TIMEOUT = 0x00000008; /** * {@code SERVER_INFO.capabilities} bit advertising that the frame ends with * an additional {@code zone_id:u16_len+utf8} field after {@code node_id}. @@ -88,7 +97,7 @@ public final class QwpEgressMsgKind { * that do not produce rows ignore it. Sent only when the server advertised * {@link #CAP_QUERY_DETAILS}. */ - public static final int QUERY_FLAG_COUNT_ROWS = 0x08; + public static final int QUERY_FLAG_COUNT_ROWS = 0x10; /** * {@code QUERY_REQUEST.query_flags} bit: report the error position. A * {@code QUERY_ERROR} then carries the bare message, without the @@ -98,16 +107,16 @@ public final class QwpEgressMsgKind { * frame when it could not read the request's flags. Sent only when the * server advertised {@link #CAP_QUERY_DETAILS}. */ - public static final int QUERY_FLAG_ERROR_POSITION = 0x02; + public static final int QUERY_FLAG_ERROR_POSITION = 0x04; /** * {@code QUERY_REQUEST.query_flags} bit: stream at most {@code max_rows} * rows. The request carries {@code max_rows:varint} after - * {@code query_flags}, and after {@code skip_rows} when - * {@link #QUERY_FLAG_SKIP_ROWS} is set too. Zero streams only the schema. - * Statements that do not produce rows ignore it. Sent only when the server - * advertised {@link #CAP_QUERY_DETAILS}. + * {@code query_flags}, behind the fields of the lower flag bits: after + * {@code skip_rows} when {@link #QUERY_FLAG_SKIP_ROWS} is set too. Zero + * streams only the schema. Statements that do not produce rows ignore it. + * Sent only when the server advertised {@link #CAP_QUERY_DETAILS}. */ - public static final int QUERY_FLAG_MAX_ROWS = 0x20; + public static final int QUERY_FLAG_MAX_ROWS = 0x40; /** * {@code QUERY_REQUEST.query_flags} bit: reset the connection-scoped SYMBOL * dict before this query, scoping it to the query. Sent only when the server @@ -117,11 +126,12 @@ public final class QwpEgressMsgKind { /** * {@code QUERY_REQUEST.query_flags} bit: leave out the first * {@code skip_rows} rows of the result set. The request carries - * {@code skip_rows:varint} right after {@code query_flags}. Statements that - * do not produce rows ignore it. Sent only when the server advertised + * {@code skip_rows:varint} after {@code query_flags}, behind the fields of + * the lower flag bits, of which this client sends none. Statements that do + * not produce rows ignore it. Sent only when the server advertised * {@link #CAP_QUERY_DETAILS}. */ - public static final int QUERY_FLAG_SKIP_ROWS = 0x10; + public static final int QUERY_FLAG_SKIP_ROWS = 0x20; /** * {@code QUERY_REQUEST.query_flags} bit: report execution statistics. * {@code RESULT_END} and {@code EXEC_DONE} then end with @@ -131,7 +141,19 @@ public final class QwpEgressMsgKind { * {@link #STATS_FLAG_JIT_COMPILED}. Sent only when the server advertised * {@link #CAP_QUERY_DETAILS}. */ - public static final int QUERY_FLAG_STATS = 0x04; + public static final int QUERY_FLAG_STATS = 0x08; + /** + * {@code QUERY_REQUEST.query_flags} bit reserved for per-query timeouts, which + * this client does not send. Released clients (c-questdb-client 7.2.0 and + * later, and the Python client built on it) send it, followed by a + * {@code timeout_ms:varint}, to a server that advertises + * {@link #CAP_QUERY_TIMEOUT}, so no other flag may take this bit. A flag that + * carries a field appends it after {@code query_flags} in ascending order of + * the flag bits, so {@code timeout_ms} goes ahead of {@code skip_rows} and + * {@code max_rows}. Mirrors the server-side constant + * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#QUERY_FLAG_TIMEOUT}. + */ + public static final int QUERY_FLAG_TIMEOUT = 0x02; public static final byte QUERY_REQUEST = 0x10; /** * Reset mask bit: clear the connection-scoped SYMBOL dict. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java index c65d008b2..c62b0fcc6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java @@ -95,6 +95,11 @@ public void testDetailOptionsDroppedWithoutCapability() throws Exception { Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, r.queryFlags); Assert.assertEquals(0, r.trailingBytes); + // a server with per-query timeouts, but without the query details + r = executeOnce(QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_TIMEOUT, options); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, r.queryFlags); + Assert.assertEquals(0, r.trailingBytes); + r = executeOnce(0, options); Assert.assertFalse("a server without CAP_QUERY_FLAGS gets no trailer", r.hasQueryFlags); Assert.assertEquals(0, r.trailingBytes); @@ -123,12 +128,13 @@ public void testEveryOptionOnTheWire() throws Exception { .resetSymbolDict(true).errorPosition(true).stats(true).countRows(true) .skipRows(300).maxRows(16_384); QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, options); - Assert.assertEquals(0x3F, r.queryFlags); + // every bit but 0x02, which per-query timeouts own + Assert.assertEquals(0x7D, r.queryFlags); Assert.assertEquals(300, r.skipRows); Assert.assertEquals(16_384, r.maxRows); Assert.assertEquals(0, r.trailingBytes); // flags, skip_rows, max_rows: the last bytes of the request, in that order - byte[] tail = new QwpEgressFrames.Body().varint(0x3F).varint(300).varint(16_384).toBody(); + byte[] tail = new QwpEgressFrames.Body().varint(0x7D).varint(300).varint(16_384).toBody(); Assert.assertArrayEquals(tail, Arrays.copyOfRange(r.payload, r.payload.length - tail.length, r.payload.length)); } @@ -252,7 +258,12 @@ public void testRowWindowFailsWithoutCapability() throws Exception { new QwpQueryOptions().maxRows(0), new QwpQueryOptions().skipRows(10).maxRows(10).stats(true), }; - for (int capabilities : new int[]{0, QwpEgressMsgKind.CAP_QUERY_FLAGS}) { + final int[] capabilitiesWithoutDetails = { + 0, + QwpEgressMsgKind.CAP_QUERY_FLAGS, + QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_TIMEOUT + }; + for (int capabilities : capabilitiesWithoutDetails) { TestUtils.assertMemoryLeak(() -> { try (QwpEgressMockServer server = new QwpEgressMockServer(capabilities, QwpQueryClientQueryFlagsTest::execDone); QwpQueryClient client = newClient(server)) { @@ -296,6 +307,43 @@ public void testSkipRowsOnTheWire() throws Exception { Assert.assertEquals(10, r.maxRows); } + /** + * Each capability and each query flag is a bit of its own, and the query + * details leave alone the per-query timeout bits that released clients + * already use: capability 0x08 and flag 0x02, followed by {@code timeout_ms} + * (c-questdb-client 7.2.0 and py-questdb-client 6.0.0). + */ + @Test + public void testWireBitsAreDistinctAndLeaveTimeoutBitsFree() { + Assert.assertEquals(0x08, QwpEgressMsgKind.CAP_QUERY_TIMEOUT); + Assert.assertEquals(0x02, QwpEgressMsgKind.QUERY_FLAG_TIMEOUT); + assertDistinctBits( + QwpEgressMsgKind.CAP_ZONE, + QwpEgressMsgKind.CAP_QUERY_FLAGS, + 0x04, // the server's CAP_COMPRESSION, which this client does not read + QwpEgressMsgKind.CAP_QUERY_TIMEOUT, + QwpEgressMsgKind.CAP_QUERY_DETAILS + ); + assertDistinctBits( + QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, + QwpEgressMsgKind.QUERY_FLAG_TIMEOUT, + QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION, + QwpEgressMsgKind.QUERY_FLAG_STATS, + QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS, + QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS, + QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS + ); + } + + private static void assertDistinctBits(int... bits) { + int union = 0; + for (int bit : bits) { + Assert.assertEquals("0x" + Integer.toHexString(bit) + " must be a single bit", 1, Integer.bitCount(bit)); + Assert.assertEquals("0x" + Integer.toHexString(bit) + " is taken twice", 0, union & bit); + union |= bit; + } + } + private static byte[] buildExecDone(byte[] queryRequest) { int bodyLen = 1 + 8 + 1 + 1; // msg_kind + request_id + op_type + rows_affected varint byte[] frame = new byte[QwpConstants.HEADER_SIZE + bodyLen]; From 70dd54eced1d2b88622c4b70d0ee1980f816f91e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 13 Nov 2026 19:30:13 +0000 Subject: [PATCH 5/5] Route client errors through one helper, tidy tests QwpQueryClient reported the failures execute() detects itself through eleven separate handler.onError() calls. Five of them -- an exhausted failover (two sites), an interrupted backoff, and a rejected or unavailable credential during a failover reconnect -- had no test that would notice a call to the 3-arg overload. A regression there would hide the error from a handler that overrides only onError(long, byte, String, int), as QwpColumnBatchHandler tells users to do. deliverClientError() now makes that call for every client-detected failure, deliverFinal() included, so any test that reaches one of them pins all of them. QwpQueryClientFailurePathsTest adds the exhausted failover, the rejected credential and the unavailable credential; QwpEgressMockServer gains setAuthorizationValidator() for the rejected credential. QuestDBQueryOptionsTest now checks that options() on a closed handle throws and leaves the next borrower's query alone. No test covered the stale-lease guard of the new Query.options() before. Test helper cleanup: - RecordingQueryHandler drops the unread batchCount and hosts the assertNoDetails(QwpQueryDetails) that QueryEventTest and QwpEgressTerminatorDecodeTest each carried a copy of. - QwpEgressFrames.QueryRequest drops the unread initialCredit. - testEachDetailOptionSendsItsFlag becomes testEachBooleanOptionSendsItsFlag: resetSymbolDict is not a query-details option. - The QueryException test moves from QwpQueryOptionsTest to its own QueryExceptionTest. - QwpEgressMockServer drops a connection on its read thread. The comment that sent the close to another thread claimed close() joins the read thread, which it skips when called on that thread. --- .../cutlass/qwp/client/QwpQueryClient.java | 68 ++++++------- .../client/test/QueryExceptionTest.java | 54 ++++++++++ .../client/test/QuestDBQueryOptionsTest.java | 27 +++++ .../cutlass/qwp/client/QueryEventTest.java | 21 +--- .../cutlass/qwp/client/QwpEgressFrames.java | 7 +- .../qwp/client/QwpEgressMockServer.java | 15 ++- .../client/QwpEgressTerminatorDecodeTest.java | 18 +--- .../QwpQueryClientFailurePathsTest.java | 99 ++++++++++++++++++- .../client/QwpQueryClientQueryFlagsTest.java | 2 +- .../qwp/client/QwpQueryOptionsTest.java | 13 --- .../qwp/client/RecordingQueryHandler.java | 21 +++- 11 files changed, 251 insertions(+), 94 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/QueryExceptionTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index 0f999c179..acc9a4a9c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -1401,6 +1401,20 @@ private static String defaultClientId() { return "questdb-java-egress/1.0.0"; } + /** + * Reports a failure that {@code execute()} detected itself, rather than one + * the server sent: a failed reconnect, an exhausted failover, a stop + * request, a bind error. Every such failure goes through this method, so it + * reaches the position-aware + * {@link QwpColumnBatchHandler#onError(long, byte, String, int)}, as the + * server's errors do, and a handler that overrides only that overload sees + * them all. The failure does not come from the SQL text, so its position is + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}. + */ + private static void deliverClientError(QwpColumnBatchHandler handler, long requestId, byte status, String message) { + handler.onError(requestId, status, message, QwpEgressMsgKind.NO_ERROR_POSITION); + } + @SuppressWarnings("BooleanMethodIsAlwaysInverted") private static boolean matchesTarget(byte role, String target) { if (TARGET_ANY.equals(target)) { @@ -1818,8 +1832,8 @@ private void executeImpl( try { connect(); } catch (RuntimeException e) { - handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, - "reconnect failed: " + e.getMessage(), QwpEgressMsgKind.NO_ERROR_POSITION); + deliverClientError(handler, -1L, WebSocketResponse.STATUS_INTERNAL_ERROR, + "reconnect failed: " + e.getMessage()); return; } } @@ -1839,8 +1853,7 @@ private void executeImpl( int attempt = 0; while (true) { if (isShutdownRequested) { - handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "query client shut down", - QwpEgressMsgKind.NO_ERROR_POSITION); + deliverClientError(handler, -1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "query client shut down"); return; } attempt++; @@ -1854,16 +1867,14 @@ private void executeImpl( hook.run(); } if (isShutdownRequested) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down", - QwpEgressMsgKind.NO_ERROR_POSITION); + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); return; } if (!failoverEnabled) { // With failover off the transport failure stays latched: every later execute() reports it. healthTracker.connectionLost(); healthTracker.failed(); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, probe.interceptedMessage, - QwpEgressMsgKind.NO_ERROR_POSITION); + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, probe.interceptedMessage); return; } if (attempt >= failoverMaxAttempts || System.nanoTime() - failoverDeadlineNanos >= 0) { @@ -1875,13 +1886,12 @@ private void executeImpl( healthTracker.failed(); } int failovers = Math.max(0, attempt - 1); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" + (attempt == 1 ? "" : "s") + " (" + failovers + " failover reconnect" + (failovers == 1 ? "" : "s") + "); last error: " - + probe.interceptedMessage, - QwpEgressMsgKind.NO_ERROR_POSITION); + + probe.interceptedMessage); return; } int failedIndex = currentEndpointIndex; @@ -1910,13 +1920,12 @@ private void executeImpl( long remaining = remainingNanos <= 0L ? 0L : remainingNanos / 1_000_000L; if (remainingNanos <= 0L) { int failovers = Math.max(0, attempt - 1); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" + (attempt == 1 ? "" : "s") + " (" + failovers + " failover reconnect" + (failovers == 1 ? "" : "s") + "); last error: " - + probe.interceptedMessage, - QwpEgressMsgKind.NO_ERROR_POSITION); + + probe.interceptedMessage); return; } if (delay > remaining) { @@ -1927,10 +1936,9 @@ private void executeImpl( Thread.sleep(delay); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "failover interrupted while backing off after attempt " - + attempt + "; last error: " + probe.interceptedMessage, - QwpEgressMsgKind.NO_ERROR_POSITION); + + attempt + "; last error: " + probe.interceptedMessage); return; } } @@ -1949,32 +1957,28 @@ private void executeImpl( // without recovery. Surface a distinct message so monitoring // can pull auth incidents apart from generic transport failures; // it names the failure class (design/qwp-token-provider-spec.md, 8.3). - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "auth-rejected during failover reconnect [host=" + authErr.getHost() + ':' + authErr.getPort() + ", status=" + authErr.getStatusCode() - + ", last error: " + probe.interceptedMessage + ']', - QwpEgressMsgKind.NO_ERROR_POSITION); + + ", last error: " + probe.interceptedMessage + ']'); return; } catch (QwpCredentialUnavailableException credentialErr) { // The token provider could not supply a credential; no endpoint was contacted. - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "failover reconnect failed: " + credentialErr.getMessage() - + " [last error: " + probe.interceptedMessage + ']', - QwpEgressMsgKind.NO_ERROR_POSITION); + + " [last error: " + probe.interceptedMessage + ']'); return; } catch (RuntimeException reconnectErr) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "failover reconnect failed after " + attempt + " attempt" + (attempt == 1 ? "" : "s") + " [last error: " + probe.interceptedMessage + ", reconnect error: " - + reconnectErr.getMessage() + ']', - QwpEgressMsgKind.NO_ERROR_POSITION); + + reconnectErr.getMessage() + ']'); return; } if (isShutdownRequested) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down", - QwpEgressMsgKind.NO_ERROR_POSITION); + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); return; } handler.onFailoverReset(probe.interceptedRequestId, serverInfo); @@ -2001,8 +2005,7 @@ private void executeOnce( // before close() returns. QwpEgressIoThread io = ioThread; if (io == null) { - probe.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "QwpQueryClient is closed", - QwpEgressMsgKind.NO_ERROR_POSITION); + probe.deliverFinal(-1L, "QwpQueryClient is closed"); return; } GenerationListener listener = currentGenerationListener; @@ -2098,8 +2101,7 @@ private void executeOnce( default: // Unreachable: the I/O thread emits only the kinds above. assert false : "unknown event kind " + ev.kind; - probe.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind, - QwpEgressMsgKind.NO_ERROR_POSITION); + probe.deliverFinal(requestId, "unknown event kind " + ev.kind); return; } } finally { @@ -2499,7 +2501,7 @@ public void onFailoverReset(long requestId, QwpServerInfo newNode) { * errors, interrupts) so they don't trigger a spurious failover. */ void deliverFinal(long requestId, String message) { - delegate.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, message, QwpEgressMsgKind.NO_ERROR_POSITION); + deliverClientError(delegate, requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, message); } /** diff --git a/core/src/test/java/io/questdb/client/test/QueryExceptionTest.java b/core/src/test/java/io/questdb/client/test/QueryExceptionTest.java new file mode 100644 index 000000000..6ae0313c4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QueryExceptionTest.java @@ -0,0 +1,54 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.QueryException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import org.junit.Assert; +import org.junit.Test; + +public class QueryExceptionTest { + + @Test + public void testPosition() { + QueryException e = new QueryException((byte) 5, "table does not exist", 14); + Assert.assertEquals(5, e.getStatus()); + Assert.assertEquals("table does not exist", e.getMessage()); + Assert.assertEquals(14, e.getPosition()); + Assert.assertNull(e.getCause()); + } + + @Test + public void testPositionAbsentByDefault() { + QueryException e = new QueryException((byte) 5, "x"); + Assert.assertEquals(5, e.getStatus()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + + IllegalStateException cause = new IllegalStateException(); + e = new QueryException((byte) 0, "x", cause); + Assert.assertSame(cause, e.getCause()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java b/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java index 0e5207c03..9282fb2b8 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java @@ -160,6 +160,33 @@ public void testOptionsAreCopiedAndKeptAcrossSubmits() throws Exception { }); } + @Test + public void testOptionsOnClosedHandleRejected() throws Exception { + // The pool holds one client, so the second borrow gets the client the + // closed handle had: its options must not reach that borrower's query. + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server)) { + Query closed = db.borrowQuery(); + closed.close(); + try (Query q = db.borrowQuery()) { + try { + closed.options(new QwpQueryOptions().maxRows(1)); + Assert.fail("a closed handle must reject options()"); + } catch (IllegalStateException e) { + Assert.assertEquals("query handle is not borrowed (closed or never leased)", e.getMessage()); + } + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("SELECT x").handler(handler).submit().await(); + Assert.assertEquals(10, handler.values.size()); + } + Assert.assertEquals(1, server.getRequests().size()); + Assert.assertFalse(server.getRequests().get(0).hasQueryFlags); + server.assertNoFailure(); + } + }); + } + @Test public void testStatsReachExecDone() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java index 535ddf731..488373c06 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java @@ -27,7 +27,6 @@ import io.questdb.client.cutlass.qwp.client.QueryEvent; import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; -import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; import org.junit.Assert; import org.junit.Test; @@ -53,16 +52,16 @@ public void testFreshEventDefaults() { Assert.assertEquals(0L, ev.rowsAffected); Assert.assertEquals(0L, ev.totalRows); Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); } @Test public void testAsEndAndAsExecDoneDropStaleDetails() { QueryEvent ev = new QueryEvent().asEnd(1).withRowCount(9).withStats(1, 2, 3, 4, true); ev.asEnd(2); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); ev.withRowCount(9).withStats(1, 2, 3, 4, true).asExecDone((short) 1, 2); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); } @Test @@ -201,7 +200,7 @@ public void testResetClearsEverythingFromError() { public void testResetClearsDetails() { QueryEvent ev = new QueryEvent().asEnd(5).withRowCount(9).withStats(1, 2, 3, 4, true); ev.reset(); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); } @Test @@ -224,16 +223,4 @@ public void testResetClearsEverythingFromExecDone() { Assert.assertEquals(0, ev.opType); Assert.assertEquals(0L, ev.rowsAffected); } - - private static void assertNoDetails(QwpQueryDetails details) { - Assert.assertFalse(details.hasRowCount()); - Assert.assertEquals(-1, details.getRowCount()); - Assert.assertFalse(details.hasStats()); - Assert.assertEquals(-1, details.getCompileNanos()); - Assert.assertEquals(-1, details.getExecuteNanos()); - Assert.assertEquals(-1, details.getCountNanos()); - Assert.assertEquals(-1, details.getClientWaitNanos()); - Assert.assertFalse(details.isJitCompiled()); - Assert.assertEquals("QwpQueryDetails{}", details.toString()); - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java index a0e08acf2..5ad80e279 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java @@ -69,7 +69,7 @@ public static QueryRequest parseQueryRequest(byte[] payload) { Assert.assertEquals("frame must be a QUERY_REQUEST", QwpEgressMsgKind.QUERY_REQUEST, (byte) rd.u8()); long requestId = rd.i64(); String sql = rd.utf8((int) rd.varint()); - long initialCredit = rd.varint(); + rd.varint(); // initial_credit int bindCount = (int) rd.varint(); long[] longBinds = new long[bindCount]; for (int i = 0; i < bindCount; i++) { @@ -81,7 +81,7 @@ public static QueryRequest parseQueryRequest(byte[] payload) { long queryFlags = hasQueryFlags ? rd.varint() : 0; long skipRows = (queryFlags & QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS) != 0 ? rd.varint() : -1; long maxRows = (queryFlags & QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS) != 0 ? rd.varint() : -1; - return new QueryRequest(payload, requestId, sql, initialCredit, longBinds, hasQueryFlags, queryFlags, skipRows, maxRows, rd.remaining()); + return new QueryRequest(payload, requestId, sql, longBinds, hasQueryFlags, queryFlags, skipRows, maxRows, rd.remaining()); } /** @@ -204,7 +204,6 @@ public Body varint(long v) { */ public static final class QueryRequest { public final boolean hasQueryFlags; - public final long initialCredit; public final long[] longBinds; public final long maxRows; // the raw payload, as the client sent it @@ -219,7 +218,6 @@ public static final class QueryRequest { byte[] payload, long requestId, String sql, - long initialCredit, long[] longBinds, boolean hasQueryFlags, long queryFlags, @@ -230,7 +228,6 @@ public static final class QueryRequest { this.payload = payload; this.requestId = requestId; this.sql = sql; - this.initialCredit = initialCredit; this.longBinds = longBinds; this.hasQueryFlags = hasQueryFlags; this.queryFlags = queryFlags; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java index 630cd3b05..f45582320 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java @@ -33,6 +33,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.ToIntFunction; /** * Egress endpoint that advertises the given {@code SERVER_INFO} capabilities, @@ -64,10 +65,7 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat frames = null; } if (frames == null) { - // ClientHandler.close() joins this read thread, so close from another one - Thread dropper = new Thread(client::close, "mock-egress-drop"); - dropper.setDaemon(true); - dropper.start(); + client.close(); return; } try { @@ -116,6 +114,15 @@ public int handshakeCount() { return server.handshakeCount(); } + /** + * Decides every new egress upgrade from its {@code Authorization} header: a + * positive return is the HTTP status the server rejects the upgrade with; + * see {@link TestWebSocketServer#setAuthorizationValidator}. Null removes it. + */ + public void setAuthorizationValidator(ToIntFunction validator) { + server.setAuthorizationValidator(validator); + } + /** * Runs {@code hook} on each connection right before the server sends * {@code SERVER_INFO}; see {@link TestWebSocketServer#setBeforeServerInfoHook}. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java index 50a3a7641..e337517d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java @@ -75,7 +75,7 @@ public void testExecDoneCarriesNoRowCount() throws Exception { ev = decodeExecDone(QwpEgressFrames.execDone(3, 2, 30).toFrame(), QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); Assert.assertEquals(30, ev.rowsAffected); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); }); } @@ -132,7 +132,7 @@ public void testExecDoneWithoutFlagsIgnoresTrailers() throws Exception { QueryEvent ev = decodeExecDone(frame, 0); Assert.assertEquals(2, ev.opType); Assert.assertEquals(30, ev.rowsAffected); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); }); } @@ -373,7 +373,7 @@ public void testResultEndWithoutFlagsIgnoresTrailers() throws Exception { QueryEvent ev = decodeResultEnd(frame, flags); Assert.assertEquals(QueryEvent.KIND_END, ev.kind); Assert.assertEquals(30, ev.totalRows); - assertNoDetails(ev.details); + RecordingQueryHandler.assertNoDetails(ev.details); } } // COUNT_ROWS alone reads the row count and leaves what follows alone @@ -426,18 +426,6 @@ private static void assertDecodeFails(String expectedMessage, byte[] frame, long } } - private static void assertNoDetails(QwpQueryDetails details) { - Assert.assertFalse(details.hasRowCount()); - Assert.assertEquals(-1, details.getRowCount()); - Assert.assertFalse(details.hasStats()); - Assert.assertEquals(-1, details.getCompileNanos()); - Assert.assertEquals(-1, details.getExecuteNanos()); - Assert.assertEquals(-1, details.getCountNanos()); - Assert.assertEquals(-1, details.getClientWaitNanos()); - Assert.assertFalse(details.isJitCompiled()); - Assert.assertEquals("QwpQueryDetails{}", details.toString()); - } - private static void assertStats( QwpQueryDetails details, long compileNanos, diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java index a00aaa20b..e42f08703 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.auth.TokenUnavailableException; import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; @@ -39,8 +40,9 @@ /** * The failures {@link QwpQueryClient#execute} detects itself -- a failover - * reconnect that fails, a stop request or a {@code close()} between attempts -- - * reach the handler through the position-aware + * reconnect that fails or meets a rejected or unavailable credential, a + * failover that runs out of attempts, a stop request or a {@code close()} + * between attempts -- reach the handler through the position-aware * {@code onError(requestId, status, message, position)} with * {@link QwpEgressMsgKind#NO_ERROR_POSITION}, like the server's errors, so a * handler that overrides only that overload sees them all. Each query asks for @@ -52,6 +54,34 @@ public class QwpQueryClientFailurePathsTest { private static final QwpQueryOptions OPTIONS = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true); private static final String SQL = "SELECT x FROM t"; + @Test(timeout = 30_000) + public void testAuthRejectedFailoverReportsNoPosition() throws Exception { + // the server dies with the first query and rejects the credential of + // the failover reconnect + TestUtils.assertMemoryLeak(() -> { + AtomicReference serverRef = new AtomicReference<>(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> { + serverRef.get().setAuthorizationValidator(authorization -> 401); + return null; + }); + QwpQueryClient client = newClient(server)) { + serverRef.set(server); + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertTrue(handler.errorMessage, + handler.errorMessage.startsWith("auth-rejected during failover reconnect [host=localhost:")); + TestUtils.assertContains(handler.errorMessage, ", status=401, last error: "); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("the failover reconnect must not complete", 1, server.handshakeCount()); + server.assertNoFailure(); + } + }); + } + @Test(timeout = 30_000) public void testCloseBetweenAttemptsReportsClosed() throws Exception { // close() from the failover callback: the next attempt finds no I/O @@ -81,6 +111,65 @@ public void onFailoverReset(long requestId, QwpServerInfo newNode) { }); } + @Test(timeout = 30_000) + public void testCredentialUnavailableFailoverReportsNoPosition() throws Exception { + // the token provider fails when the failover reconnect asks it for a + // credential, so the reconnect contacts no endpoint + TestUtils.assertMemoryLeak(() -> { + AtomicInteger tokenPulls = new AtomicInteger(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server).withBearerTokenProvider(() -> { + if (tokenPulls.incrementAndGet() > 1) { + throw TokenUnavailableException.retryable("IdP unreachable"); + } + return "TOKEN"; + })) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertTrue(handler.errorMessage, handler.errorMessage.startsWith("failover reconnect failed: ")); + TestUtils.assertContains(handler.errorMessage, "IdP unreachable"); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("the failover reconnect must contact no endpoint", 1, server.handshakeCount()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testExhaustedFailoverReportsNoPosition() throws Exception { + // failover_max_attempts=1 leaves no attempt for a failover: the query + // dies with its connection, and the next query hits the latched failure + TestUtils.assertMemoryLeak(() -> { + final String failure = "transport failure after 1 execute attempt (0 failover reconnects); last error: "; + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpQueryClient client = newClient(server, "failover_max_attempts=1;")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertTrue(handler.errorMessage, handler.errorMessage.startsWith(failure)); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + + RecordingQueryHandler latched = new RecordingQueryHandler(); + client.execute(SQL, latched, OPTIONS); + latched.assertOnlyError(); + Assert.assertEquals(-1, latched.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, latched.errorStatus); + Assert.assertTrue(latched.errorMessage, latched.errorMessage.startsWith(failure)); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, latched.errorPosition); + Assert.assertEquals("no failover reconnect", 1, server.handshakeCount()); + Assert.assertEquals("the latched failure sends nothing", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + @Test(timeout = 30_000) public void testFailedReconnectsReportNoPosition() throws Exception { // The server dies with the first query and then refuses connections: the @@ -236,7 +325,11 @@ private static QwpEgressMockServer.Responder dropFirstQuery() { } private static QwpQueryClient newClient(QwpEgressMockServer server) { + return newClient(server, ""); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server, String extraConfig) { return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() - + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;"); + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;" + extraConfig); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java index c62b0fcc6..41e7ad9ff 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java @@ -106,7 +106,7 @@ public void testDetailOptionsDroppedWithoutCapability() throws Exception { } @Test - public void testEachDetailOptionSendsItsFlag() throws Exception { + public void testEachBooleanOptionSendsItsFlag() throws Exception { Object[][] cases = { {new QwpQueryOptions().errorPosition(true), QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION}, {new QwpQueryOptions().stats(true), QwpEgressMsgKind.QUERY_FLAG_STATS}, diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java index 56b5c5bf9..9d50134cd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java @@ -24,8 +24,6 @@ package io.questdb.client.test.cutlass.qwp.client; -import io.questdb.client.QueryException; -import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import org.junit.Assert; import org.junit.Test; @@ -94,17 +92,6 @@ public void testNegativeRowsRejected() { } } - @Test - public void testQueryExceptionPosition() { - Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, new QueryException((byte) 5, "x").getPosition()); - Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, - new QueryException((byte) 0, "x", new IllegalStateException()).getPosition()); - QueryException e = new QueryException((byte) 5, "table does not exist", 14); - Assert.assertEquals(5, e.getStatus()); - Assert.assertEquals("table does not exist", e.getMessage()); - Assert.assertEquals(14, e.getPosition()); - } - @Test public void testRowBounds() { QwpQueryOptions options = new QwpQueryOptions(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java index 69991f28e..2a17cb524 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java @@ -37,11 +37,11 @@ * Records what the most detailed callbacks of {@link QwpColumnBatchHandler} * receive, copying the {@link QwpQueryDetails} values out while they are * valid. The simpler overloads fail: the client must call the detailed ones. - * Batches must carry a single LONG column. + * Batches must carry a single LONG column. Also hosts the assertion on a + * {@link QwpQueryDetails} that carries nothing, which the decoder tests share. */ public class RecordingQueryHandler implements QwpColumnBatchHandler { public final List values = new ArrayList<>(); - public int batchCount; public long clientWaitNanos; public long compileNanos; public long countNanos; @@ -63,6 +63,22 @@ public class RecordingQueryHandler implements QwpColumnBatchHandler { public long rowsAffected; public long totalRows; + /** + * Asserts that {@code details} carries neither a row count nor statistics: + * every getter returns its absent value. + */ + public static void assertNoDetails(QwpQueryDetails details) { + Assert.assertFalse(details.hasRowCount()); + Assert.assertEquals(-1, details.getRowCount()); + Assert.assertFalse(details.hasStats()); + Assert.assertEquals(-1, details.getCompileNanos()); + Assert.assertEquals(-1, details.getExecuteNanos()); + Assert.assertEquals(-1, details.getCountNanos()); + Assert.assertEquals(-1, details.getClientWaitNanos()); + Assert.assertFalse(details.isJitCompiled()); + Assert.assertEquals("QwpQueryDetails{}", details.toString()); + } + public void assertNoDetails() { Assert.assertFalse("row count", hasRowCount); Assert.assertEquals(-1, rowCount); @@ -82,7 +98,6 @@ public void assertOnlyError() { @Override public void onBatch(QwpColumnBatch batch) { - batchCount++; for (int row = 0; row < batch.getRowCount(); row++) { values.add(batch.getLongValue(0, row)); }