diff --git a/README.md b/README.md index 7aaf10c9..232e64fb 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,62 @@ try (Query q = db.borrowQuery()) { > A borrowed `Query` handle is **single-flight**: submit one query at a time on it, then submit again or close it. To > run queries concurrently, borrow one handle per concurrent query (up to `query_pool_max`). +### Page Through a Result + +`QwpQueryOptions` asks the server for one page of the result set, and for what the `/exec` HTTP endpoint reports +besides the rows: the row count of the whole result set, execution statistics, and the position of an error in the SQL +text. The handler receives them through the detailed overloads of its callbacks, and a failed `await()` throws a +`QueryException` whose `getPosition()` carries the position. The options need a QuestDB server that supports them; an +older server fails a query that sets `skipRows` or `maxRows`, and runs the others without the details. + +```java +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; + +QwpQueryOptions page = new QwpQueryOptions() + .skipRows(100) // leave out the first 100 rows + .maxRows(50) // stream at most 50 rows + .countRows(true) // count the rows of the whole result set + .stats(true) // report compile and execution times + .errorPosition(true); // report where in the SQL text an error is + +QwpColumnBatchHandler pageHandler = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + // the rows of the page + } + + @Override + public void onEnd(long totalRows) { + // not called: the overload below receives the end of the result + } + + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + System.out.println(totalRows + " of " + details.getRowCount() + " rows in " + + details.getExecuteNanos() / 1_000 + " us"); + } + + @Override + public void onError(byte status, String message) { + // not called: the overload below receives the error + } + + @Override + public void onError(long requestId, byte status, String message, int position) { + System.err.println("error at position " + position + ": " + message); + } +}; + +try (Query q = db.borrowQuery()) { + q.sql("SELECT * FROM trades ORDER BY ts DESC") + .options(page) + .handler(pageHandler) + .submit() + .await(); +} +``` + ### Custom Pool Sizing and Timeouts Use the builder to override the defaults. `senderPoolSize`/`queryPoolSize` set a fixed pool size; `*Min`/`*Max` allow diff --git a/core/src/main/java/io/questdb/client/Query.java b/core/src/main/java/io/questdb/client/Query.java index 539d5711..63413432 100644 --- a/core/src/main/java/io/questdb/client/Query.java +++ b/core/src/main/java/io/questdb/client/Query.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.QwpBindSetter; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; import java.io.Closeable; @@ -43,9 +44,9 @@ * creates one small lease handle per borrow (often scalar-replaced by the JIT * when used with try-with-resources). *

- * Lifecycle: configure with {@link #sql}, optional {@link #binds}, and - * {@link #handler}, then call {@link #submit()} to obtain a {@link Completion} - * and {@code await()} it before the next {@link #submit()}. + * Lifecycle: configure with {@link #sql}, optional {@link #binds} and + * {@link #options}, and {@link #handler}, then call {@link #submit()} to obtain + * a {@link Completion} and {@code await()} it before the next {@link #submit()}. *

* Thread safety: not thread-safe and single-flight -- one in-flight query per * handle. To run queries concurrently, borrow one handle per concurrent query. @@ -100,6 +101,16 @@ public interface Query extends Closeable { */ Query handler(QwpColumnBatchHandler handler); + /** + * Sets the per-query options: a row window that pages through the result + * set, and what the server reports besides the rows -- the row count of the + * whole result set, execution statistics and the error position. See + * {@link QwpQueryOptions}. The handle copies the values, so the instance can + * be reused right away; they apply to every {@link #submit()} until changed. + * Pass null to drop them. + */ + Query options(QwpQueryOptions options); + /** * Sets the SQL text. The buffer is not retained past {@link #submit()}. */ diff --git a/core/src/main/java/io/questdb/client/QueryException.java b/core/src/main/java/io/questdb/client/QueryException.java index a3fdedbd..4fbff992 100644 --- a/core/src/main/java/io/questdb/client/QueryException.java +++ b/core/src/main/java/io/questdb/client/QueryException.java @@ -24,6 +24,9 @@ package io.questdb.client; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; + /** * Thrown from {@link Completion#await()} / {@link Completion#await(long, java.util.concurrent.TimeUnit)} * when the server reported an error for the corresponding {@link Query}, @@ -32,20 +35,40 @@ *

* The original wire-level status byte is exposed via {@link #getStatus()} so * callers can distinguish cancellation from schema errors etc. without - * string-matching the message. + * string-matching the message. When the query asked for the error position + * with {@link QwpQueryOptions#errorPosition(boolean)}, {@link #getPosition()} + * points at the offending character of the SQL text. */ public class QueryException extends RuntimeException { + private final int position; private final byte status; public QueryException(byte status, String message) { + this(status, message, QwpEgressMsgKind.NO_ERROR_POSITION); + } + + public QueryException(byte status, String message, int position) { super(message); this.status = status; + this.position = position; } public QueryException(byte status, String message, Throwable cause) { super(message, cause); this.status = status; + this.position = QwpEgressMsgKind.NO_ERROR_POSITION; + } + + /** + * Returns the {@code char} index into the SQL text the server-reported + * error points at, or {@link QwpEgressMsgKind#NO_ERROR_POSITION} when the + * query did not ask for it with {@link QwpQueryOptions#errorPosition(boolean)}, + * the server did not report it, or the error does not come from the SQL + * text. + */ + public int getPosition() { + return position; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java index d3083e4c..75b61735 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java @@ -45,8 +45,12 @@ public class QueryEvent { */ public static final int KIND_TRANSPORT_ERROR = 4; + // valid for KIND_END and KIND_EXEC_DONE: the row count and statistics trailers + public final QwpQueryDetails details = new QwpQueryDetails(); public QwpBatchBuffer buffer; // valid for KIND_BATCH (must be released to pool by consumer) public String errorMessage; // valid for KIND_ERROR + // valid for KIND_ERROR: the position trailer, NO_ERROR_POSITION when absent + public int errorPosition = QwpEgressMsgKind.NO_ERROR_POSITION; public byte errorStatus; // valid for KIND_ERROR public int kind; public short opType; // valid for KIND_EXEC_DONE (matches CompiledQuery.SELECT/INSERT/etc.) @@ -63,14 +67,20 @@ public QueryEvent asEnd(long totalRows) { this.kind = KIND_END; this.buffer = null; this.totalRows = totalRows; + this.details.clear(); return this; } public QueryEvent asError(byte status, String message) { + return asError(status, message, QwpEgressMsgKind.NO_ERROR_POSITION); + } + + public QueryEvent asError(byte status, String message, int position) { this.kind = KIND_ERROR; this.buffer = null; this.errorStatus = status; this.errorMessage = message; + this.errorPosition = position; return this; } @@ -79,6 +89,7 @@ public QueryEvent asExecDone(short opType, long rowsAffected) { this.buffer = null; this.opType = opType; this.rowsAffected = rowsAffected; + this.details.clear(); return this; } @@ -87,6 +98,7 @@ public QueryEvent asTransportError(byte status, String message) { this.buffer = null; this.errorStatus = status; this.errorMessage = message; + this.errorPosition = QwpEgressMsgKind.NO_ERROR_POSITION; return this; } @@ -102,9 +114,30 @@ public void reset() { this.kind = -1; this.buffer = null; this.errorMessage = null; + this.errorPosition = QwpEgressMsgKind.NO_ERROR_POSITION; this.errorStatus = 0; this.opType = 0; this.rowsAffected = 0; this.totalRows = 0; + this.details.clear(); + } + + /** + * Records the {@code row_count} trailer of a {@code RESULT_END} on + * {@link #details}. Call after {@link #asEnd(long)}. + */ + public QueryEvent withRowCount(long rowCount) { + this.details.setRowCount(rowCount); + return this; + } + + /** + * Records the statistics trailer of a {@code RESULT_END} or + * {@code EXEC_DONE} on {@link #details}. Call after {@link #asEnd(long)} or + * {@link #asExecDone(short, long)}. + */ + public QueryEvent withStats(long compileNanos, long executeNanos, long countNanos, long clientWaitNanos, boolean isJitCompiled) { + this.details.setStats(compileNanos, executeNanos, countNanos, clientWaitNanos, isJitCompiled); + return this; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java index 6237e1de..ce809df6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java @@ -27,11 +27,20 @@ /** * Callback interface for consuming a streamed QWP egress query result. *

- * Invoked by {@link QwpQueryClient#execute(String, QwpColumnBatchHandler)}: + * Invoked by {@link QwpQueryClient#execute(CharSequence, QwpColumnBatchHandler)}: * once per {@code RESULT_BATCH} frame via {@link #onBatch(QwpColumnBatch)}, * then exactly once via either {@link #onEnd(long)} or * {@link #onError(byte, String)} (or {@link #onExecDone} for non-SELECT queries). *

+ * The client calls the most detailed overload of each terminal callback -- + * {@link #onEnd(long, long, QwpQueryDetails)}, + * {@link #onError(long, byte, String, int)} and + * {@link #onExecDone(long, short, long, QwpQueryDetails)} -- and each overload + * delegates by default to the next simpler one, down to the abstract + * {@link #onEnd(long)} and {@link #onError(byte, String)} and the no-op + * {@link #onExecDone(short, long)}. Override the overload that carries what you + * need; the simpler ones then do not fire for that event. + *

* The {@link QwpColumnBatch} passed to {@link #onBatch} is valid only for the * duration of that call. Copy any values you need to retain. *

@@ -71,6 +80,23 @@ default void onEnd(long requestId, long totalRows) { onEnd(totalRows); } + /** + * Details-aware overload of {@link #onEnd(long, long)}. Receives what the + * query reported besides its rows on the request of {@link QwpQueryOptions}: + * the row count of the whole result set and execution statistics. The + * default delegates to {@link #onEnd(long, long)} for backwards + * compatibility; override this method when you need the details. + * + * @param requestId client-assigned request id (matches {@link QwpColumnBatch#requestId()}) + * @param totalRows server-reported number of rows streamed, which a row window + * of {@link QwpQueryOptions} bounds + * @param details the row count and statistics the server reported; never null, + * and valid only for the duration of this call + */ + default void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + onEnd(requestId, totalRows); + } + /** * Invoked exactly once if the query fails at any point, instead of * {@link #onEnd} / {@link #onExecDone}. @@ -100,6 +126,32 @@ default void onError(long requestId, byte status, String message) { onError(status, message); } + /** + * Position-aware overload of {@link #onError(long, byte, String)}. When the + * query asked for the error position with + * {@link QwpQueryOptions#errorPosition(boolean)} and the server reported it, + * {@code message} is the bare error message and {@code position} the + * {@code char} index into the SQL text the error points at. Otherwise + * {@code message} is what the server sent, which may start with the + * position in brackets, and {@code position} is + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}, as it is for errors that do + * not come from the SQL text, such as a {@code CANCEL} this client sent or a + * transport failure. A query timeout and a cancellation on the server, such + * as {@code CANCEL QUERY}, report position 0, as the {@code /exec} HTTP + * endpoint does. The default delegates to + * {@link #onError(long, byte, String)} for backwards compatibility; override + * this method when you need the position. + * + * @param requestId client-assigned request id, or {@code -1} if no request was in flight + * @param status one of the QWP status codes (e.g., {@code STATUS_PARSE_ERROR}) + * @param message server-supplied error message (may be empty) + * @param position {@code char} index into the SQL text, or + * {@link QwpEgressMsgKind#NO_ERROR_POSITION} + */ + default void onError(long requestId, byte status, String message, int position) { + onError(requestId, status, message); + } + /** * Invoked when {@link QwpQueryClient#execute} has transparently reconnected * to another endpoint after a transport failure and is about to re-submit @@ -163,4 +215,22 @@ default void onExecDone(short opType, long rowsAffected) { default void onExecDone(long requestId, short opType, long rowsAffected) { onExecDone(opType, rowsAffected); } + + /** + * Details-aware overload of {@link #onExecDone(long, short, long)}. Receives + * the execution statistics the query reported on the request of + * {@link QwpQueryOptions#stats(boolean)}; a statement that produces no rows + * reports no row count. The default delegates to + * {@link #onExecDone(long, short, long)} for backwards compatibility; + * override this method when you need the statistics. + * + * @param requestId client-assigned request id (matches {@link QwpColumnBatch#requestId()}) + * @param opType server-side opType constant (CompiledQuery.SELECT / INSERT / UPDATE / CREATE_TABLE / etc.) + * @param rowsAffected rows inserted / updated / deleted; 0 for pure DDL + * @param details the statistics the server reported; never null, and valid + * only for the duration of this call + */ + default void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + onExecDone(requestId, opType, rowsAffected); + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java index 153b57e9..31e8bb1d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -113,6 +113,10 @@ public class QwpEgressIoThread implements Runnable, WebSocketFrameHandler { private volatile boolean closed; private boolean creditEnabled; private boolean currentQueryDone; + // query_flags of the in-flight request as it went on the wire, after the + // client dropped the bits the server does not advertise. Tells the + // terminator decoders which trailers to read. I/O thread only. + private long currentQueryFlags; private long currentRequestId = -1L; private volatile boolean shutdown; @@ -164,14 +168,30 @@ public QwpEgressIoThread(WebSocketClient wsClient, int bufferPoolSize, TerminalF } } + /** + * Decodes a QUERY_ERROR payload of a request without + * {@link QwpEgressMsgKind#QUERY_FLAG_ERROR_POSITION} into a + * {@link QueryEvent}. Visible for testing. See + * {@link #decodeError(long, int, long)}. + */ + public static QueryEvent decodeError(long payload, int payloadLen) { + return decodeError(payload, payloadLen, 0); + } + /** * Decodes a QUERY_ERROR payload into a {@link QueryEvent}. Visible for testing. + * Body: {@code msg_kind(1) + request_id(8) + status(1) + msg_len(u16) + msg}, + * then {@code position(i32)} when {@code queryFlags} has + * {@link QwpEgressMsgKind#QUERY_FLAG_ERROR_POSITION}. Bytes after the known + * fields are ignored. + *

* Bound-checks msgLen against the actual payload: a hostile or buggy server * can encode msgLen=0xFFFF with a tiny payload, which would otherwise read * up to 65 KiB of native memory beyond the frame and surface it to the user - * callback as a String. + * callback as a String. A frame that is too short for what it claims + * decodes to a {@code STATUS_INTERNAL_ERROR} event that names the defect. */ - public static QueryEvent decodeError(long payload, int payloadLen) { + public static QueryEvent decodeError(long payload, int payloadLen, long queryFlags) { long payloadEnd = payload + payloadLen; long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; if (p + 1 + 2 > payloadEnd) { @@ -189,7 +209,63 @@ public static QueryEvent decodeError(long payload, int payloadLen) { for (int i = 0; i < msgLen; i++) { bytes[i] = Unsafe.getUnsafe().getByte(p + i); } - return new QueryEvent().asError(status, new String(bytes, StandardCharsets.UTF_8)); + String message = new String(bytes, StandardCharsets.UTF_8); + p += msgLen; + // A frame that ends with the message is a baseline frame, which the + // server also sends for a request whose flags it could not read. + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION) == 0 || p == payloadEnd) { + return new QueryEvent().asError(status, message); + } + if (p + Integer.BYTES > payloadEnd) { + return new QueryEvent().asError(WebSocketResponse.STATUS_INTERNAL_ERROR, + "QUERY_ERROR frame truncated mid position [status=" + status + ", message=" + message + ']'); + } + return new QueryEvent().asError(status, message, Unsafe.getUnsafe().getInt(p)); + } + + /** + * Decodes an EXEC_DONE payload into a {@link QueryEvent}. Visible for + * testing. Body: {@code msg_kind(1) + request_id(8) + op_type(1) + + * rows_affected(varint)}, then the statistics block when {@code queryFlags} + * has {@link QwpEgressMsgKind#QUERY_FLAG_STATS}. Statements that produce no + * rows carry no {@code row_count}, whatever the flags. Bytes after the known + * fields are ignored. + * + * @throws QwpDecodeException if the frame ends inside a field the flags call + * for, or a varint does not fit in 64 bits + */ + public static QueryEvent decodeExecDone(long payload, int payloadLen, long queryFlags) throws QwpDecodeException { + TerminatorReader rd = new TerminatorReader("EXEC_DONE", payload + QwpConstants.HEADER_SIZE + 1 + 8, payload + payloadLen); + byte opType = rd.int8("op_type"); + QueryEvent ev = new QueryEvent().asExecDone(opType, rd.varint("rows_affected")); + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + readStats(rd, ev); + } + return ev; + } + + /** + * Decodes a RESULT_END payload into a {@link QueryEvent}. Visible for + * testing. Body: {@code msg_kind(1) + request_id(8) + final_seq(varint) + + * total_rows(varint)}, then {@code row_count(varint)} when + * {@code queryFlags} has {@link QwpEgressMsgKind#QUERY_FLAG_COUNT_ROWS}, then + * the statistics block when it has {@link QwpEgressMsgKind#QUERY_FLAG_STATS}. + * Bytes after the known fields are ignored. + * + * @throws QwpDecodeException if the frame ends inside a field the flags call + * for, or a varint does not fit in 64 bits + */ + public static QueryEvent decodeResultEnd(long payload, int payloadLen, long queryFlags) throws QwpDecodeException { + TerminatorReader rd = new TerminatorReader("RESULT_END", payload + QwpConstants.HEADER_SIZE + 1 + 8, payload + payloadLen); + rd.varint("final_seq"); + QueryEvent ev = new QueryEvent().asEnd(rd.varint("total_rows")); + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS) != 0) { + ev.withRowCount(rd.varint("row_count")); + } + if ((queryFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + readStats(rd, ev); + } + return ev; } @Override @@ -316,6 +392,7 @@ public void run() { // Per-query state; accessed only from the I/O thread. currentQueryDone = false; currentRequestId = req.requestId; + currentQueryFlags = req.queryFlags; creditEnabled = req.initialCredit > 0L; // The schema rides the first RESULT_BATCH (batch_seq == 0) of each // query; invalidate any schema left from the prior query so a @@ -380,6 +457,12 @@ public void shutdown() { * during {@link #sendQueryRequest} and does not retain a reference after * the send completes. {@code bindCount} is the number of binds the * payload contains; zero when the user supplied no binds. + *

+ * {@code queryFlags} are the bits the request carries, which the server + * must advertise; zero sends no {@code query_flags} trailer. {@code skipRows} + * goes on the wire only with {@link QwpEgressMsgKind#QUERY_FLAG_SKIP_ROWS} + * and {@code maxRows} only with {@link QwpEgressMsgKind#QUERY_FLAG_MAX_ROWS}; + * both must be non-negative. */ public void submitQuery( CharSequence sql, @@ -388,7 +471,9 @@ public void submitQuery( int bindCount, long bindPayloadPtr, long bindPayloadLen, - long queryFlags + long queryFlags, + long skipRows, + long maxRows ) throws InterruptedException { pendingRequest.sql = sql; pendingRequest.requestId = requestId; @@ -397,6 +482,8 @@ public void submitQuery( pendingRequest.bindPayloadPtr = bindPayloadPtr; pendingRequest.bindPayloadLen = bindPayloadLen; pendingRequest.queryFlags = queryFlags; + pendingRequest.skipRows = skipRows; + pendingRequest.maxRows = maxRows; requests.put(pendingRequest); } @@ -407,6 +494,22 @@ public QueryEvent takeEvent() throws InterruptedException { return events.take(); } + /** + * Reads the statistics block of {@link QwpEgressMsgKind#QUERY_FLAG_STATS}: + * {@code compile_ns, execute_ns, count_ns, client_wait_ns} as varints and + * the {@code stats_flags} byte. Flag bits other than + * {@link QwpEgressMsgKind#STATS_FLAG_JIT_COMPILED} are reserved and ignored. + */ + private static void readStats(TerminatorReader rd, QueryEvent ev) throws QwpDecodeException { + long compileNanos = rd.varint("compile_ns"); + long executeNanos = rd.varint("execute_ns"); + long countNanos = rd.varint("count_ns"); + long clientWaitNanos = rd.varint("client_wait_ns"); + byte statsFlags = rd.int8("stats_flags"); + ev.withStats(compileNanos, executeNanos, countNanos, clientWaitNanos, + (statsFlags & QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED) != 0); + } + /** * Takes a pre-allocated {@link QueryEvent} from the pool for the I/O thread * to populate before offering to {@link #events}. Falls back to a fresh @@ -424,104 +527,35 @@ private QueryEvent borrowEvent() { } private void decodeAndEmitError(long payload, int payloadLen) { - QueryEvent ev = decodeError(payload, payloadLen); + QueryEvent ev = decodeError(payload, payloadLen, currentQueryFlags); events.offer(ev); } /** - * EXEC_DONE body: msg_kind(1) + requestId(8) + op_type(1) + rows_affected(varint). - * Parses all fields, surfaces as a {@link QueryEvent#KIND_EXEC_DONE} event. + * Surfaces an EXEC_DONE as a {@link QueryEvent#KIND_EXEC_DONE} event. A + * frame that ends inside a field, or carries a varint that does not fit in + * 64 bits, is a terminal transport error rather than an EXEC_DONE with a + * partial value. */ private void decodeAndEmitExecDone(long payload, int payloadLen) { - long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; - long limit = payload + payloadLen; - if (p + 1 > limit) { - emitTerminalTransportError("EXEC_DONE frame truncated before op_type"); - return; - } - byte opType = Unsafe.getUnsafe().getByte(p++); - long rowsAffected = 0; - int shift = 0; - boolean terminated = false; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - rowsAffected |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - terminated = true; - break; - } - shift += 7; - if (shift > 63) { - emitTerminalTransportError("EXEC_DONE rows_affected varint overflow"); - return; - } - } - // The loop also exits when {@code p == limit} on a frame that ended - // mid-varint without a terminator byte. Without this guard, a buggy or - // hostile server could truncate the frame and the partial value would - // surface to the user handler as if the EXEC_DONE completed normally. - if (!terminated) { - emitTerminalTransportError("EXEC_DONE frame truncated mid rows_affected varint"); - return; + try { + events.offer(decodeExecDone(payload, payloadLen, currentQueryFlags)); + } catch (QwpDecodeException e) { + emitTerminalTransportError(e.getMessage()); } - events.offer(new QueryEvent().asExecDone(opType, rowsAffected)); } /** - * RESULT_END body: msg_kind(1) + requestId(8) + final_seq(varint) + total_rows(varint). - * We only need total_rows. Both varint loops cap at 10 bytes so a hostile - * server cannot drive {@code shift} past 63 (where Java's {@code <<} masks - * the count to 6 bits and silently wraps high bytes back into the low bits, - * producing a wildly wrong total). A frame that ends mid-varint surfaces - * as a terminal transport error rather than a partial-value RESULT_END. + * Surfaces a RESULT_END as a {@link QueryEvent#KIND_END} event. A frame that + * ends inside a field, or carries a varint that does not fit in 64 bits, is + * a terminal transport error rather than a RESULT_END with a partial value. */ private void decodeAndEmitResultEnd(long payload, int payloadLen) { - long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; - long limit = payload + payloadLen; - int seqBytes = 0; - boolean seqTerminated = false; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - if ((b & 0x80) == 0) { - seqTerminated = true; - break; - } - if (++seqBytes > 9) { - // Continuation bit set on byte 10 of the final_seq varint -- - // malformed. Surface as terminal so the caller cannot proceed - // with a desynced cursor. - emitTerminalTransportError("RESULT_END final_seq varint overflow"); - return; - } - } - if (!seqTerminated) { - emitTerminalTransportError("RESULT_END frame truncated mid final_seq varint"); - return; - } - long total = 0; - int shift = 0; - boolean totalTerminated = false; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - total |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - totalTerminated = true; - break; - } - shift += 7; - if (shift > 63) { - // Same overflow guard decodeAndEmitExecDone uses; without it, - // byte 10 of total_rows could land in the sign bit and bytes - // 11+ would silently wrap. - emitTerminalTransportError("RESULT_END total_rows varint overflow"); - return; - } - } - if (!totalTerminated) { - emitTerminalTransportError("RESULT_END frame truncated mid total_rows varint"); - return; + try { + events.offer(decodeResultEnd(payload, payloadLen, currentQueryFlags)); + } catch (QwpDecodeException e) { + emitTerminalTransportError(e.getMessage()); } - events.offer(new QueryEvent().asEnd(total)); } /** @@ -721,9 +755,16 @@ private void sendQueryRequest(QueryRequest req) { sendScratch.putBlockOfBytes(req.bindPayloadPtr, req.bindPayloadLen); } // Optional query_flags trailer; omitted when zero so a baseline frame - // stays byte-identical and the server defaults the flags to 0. + // stays byte-identical and the server defaults the flags to 0. The + // row-window values follow the flags in ascending order of their bits. if (req.queryFlags != 0) { sendScratch.putVarint(req.queryFlags); + if ((req.queryFlags & QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS) != 0) { + sendScratch.putVarint(req.skipRows); + } + if ((req.queryFlags & QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS) != 0) { + sendScratch.putVarint(req.maxRows); + } } wsClient.sendBinary(sendScratch.getBufferPtr(), sendScratch.getPosition()); sendScratch.reset(); @@ -783,8 +824,62 @@ private static final class QueryRequest { long bindPayloadLen; long bindPayloadPtr; long initialCredit; + long maxRows; long queryFlags; long requestId; + long skipRows; CharSequence sql; } + + /** + * Bounds-checked cursor over the body of a terminator frame (RESULT_END, + * EXEC_DONE). Every read fails with a {@link QwpDecodeException} that names + * the frame and the field, rather than read past the frame or return a + * partial value. + */ + private static final class TerminatorReader { + private final String frame; + private final long limit; + private long p; + + TerminatorReader(String frame, long p, long limit) { + this.frame = frame; + this.p = p; + this.limit = limit; + } + + byte int8(String field) throws QwpDecodeException { + if (p >= limit) { + throw new QwpDecodeException(frame + " frame truncated before " + field); + } + return Unsafe.getUnsafe().getByte(p++); + } + + /** + * Reads an unsigned LEB128 varint of at most 10 bytes. On byte 10 only + * bit 0 of the data still fits in a long: a hostile server must not + * drive the shift past 63, where Java's {@code <<} masks the count to + * 6 bits and wraps the high bytes back into the low bits, producing a + * wildly wrong value. + */ + long varint(String field) throws QwpDecodeException { + long value = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + if (shift == 63 && (b & 0x7E) != 0) { + throw new QwpDecodeException(frame + ' ' + field + " varint overflow"); + } + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return value; + } + shift += 7; + if (shift > 63) { + throw new QwpDecodeException(frame + ' ' + field + " varint overflow"); + } + } + throw new QwpDecodeException(frame + " frame truncated mid " + field + " varint"); + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java index e9b2c893..571e1d63 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java @@ -38,6 +38,16 @@ public final class QwpEgressMsgKind { */ public static final byte CACHE_RESET = 0x17; public static final byte CANCEL = 0x14; + /** + * {@code SERVER_INFO.capabilities} bit: the server honours the query-details + * flags of {@code QUERY_REQUEST.query_flags} -- {@link #QUERY_FLAG_ERROR_POSITION}, + * {@link #QUERY_FLAG_STATS}, {@link #QUERY_FLAG_COUNT_ROWS}, + * {@link #QUERY_FLAG_SKIP_ROWS} and {@link #QUERY_FLAG_MAX_ROWS}. The client + * sends those bits only when this bit is set: a server without it ignores + * them and answers with the baseline frames. Mirrors the server-side constant + * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_DETAILS}. + */ + public static final int CAP_QUERY_DETAILS = 0x00000010; /** * {@code SERVER_INFO.capabilities} bit: the server parses the optional * {@code query_flags:varint} trailer on {@code QUERY_REQUEST}. The client @@ -45,6 +55,15 @@ public final class QwpEgressMsgKind { * constant {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_FLAGS}. */ public static final int CAP_QUERY_FLAGS = 0x00000002; + /** + * {@code SERVER_INFO.capabilities} bit reserved for per-query timeouts, which + * this client does not send. Released clients (c-questdb-client 7.2.0 and + * later, and the Python client built on it) read the bit as "the server + * honours {@link #QUERY_FLAG_TIMEOUT}", so no other capability may take it. + * Mirrors the server-side constant + * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_TIMEOUT}. + */ + public static final int CAP_QUERY_TIMEOUT = 0x00000008; /** * {@code SERVER_INFO.capabilities} bit advertising that the frame ends with * an additional {@code zone_id:u16_len+utf8} field after {@code node_id}. @@ -60,13 +79,81 @@ public final class QwpEgressMsgKind { * {@code request_id:u64, op_type:u8, rows_affected:varint}. */ public static final byte EXEC_DONE = 0x16; + /** + * Error position that points nowhere in the SQL text: the position of an + * error that does not come from the SQL text, such as a {@code CANCEL} the + * client sent, and of every error when the position was not requested or + * not reported. A query timeout and a cancellation on the server, such as + * {@code CANCEL QUERY}, report position 0, as the {@code /exec} HTTP endpoint + * does. + */ + public static final int NO_ERROR_POSITION = -1; public static final byte QUERY_ERROR = 0x13; + /** + * {@code QUERY_REQUEST.query_flags} bit: report the row count of the whole + * result set, whatever {@link #QUERY_FLAG_SKIP_ROWS} and + * {@link #QUERY_FLAG_MAX_ROWS} leave out of the stream. {@code RESULT_END} + * then carries {@code row_count:varint} after {@code total_rows}. Statements + * that do not produce rows ignore it. Sent only when the server advertised + * {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_COUNT_ROWS = 0x10; + /** + * {@code QUERY_REQUEST.query_flags} bit: report the error position. A + * {@code QUERY_ERROR} then carries the bare message, without the + * {@code [n]} prefix of the baseline frame, and ends with + * {@code position:i32}, the UTF-16 offset into the SQL text the error points + * at, or {@link #NO_ERROR_POSITION}. The server answers with the baseline + * frame when it could not read the request's flags. Sent only when the + * server advertised {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_ERROR_POSITION = 0x04; + /** + * {@code QUERY_REQUEST.query_flags} bit: stream at most {@code max_rows} + * rows. The request carries {@code max_rows:varint} after + * {@code query_flags}, behind the fields of the lower flag bits: after + * {@code skip_rows} when {@link #QUERY_FLAG_SKIP_ROWS} is set too. Zero + * streams only the schema. Statements that do not produce rows ignore it. + * Sent only when the server advertised {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_MAX_ROWS = 0x40; /** * {@code QUERY_REQUEST.query_flags} bit: reset the connection-scoped SYMBOL * dict before this query, scoping it to the query. Sent only when the server * advertised {@link #CAP_QUERY_FLAGS}. */ public static final int QUERY_FLAG_RESET_DICT = 0x01; + /** + * {@code QUERY_REQUEST.query_flags} bit: leave out the first + * {@code skip_rows} rows of the result set. The request carries + * {@code skip_rows:varint} after {@code query_flags}, behind the fields of + * the lower flag bits, of which this client sends none. Statements that do + * not produce rows ignore it. Sent only when the server advertised + * {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_SKIP_ROWS = 0x20; + /** + * {@code QUERY_REQUEST.query_flags} bit: report execution statistics. + * {@code RESULT_END} and {@code EXEC_DONE} then end with + * {@code compile_ns:varint, execute_ns:varint, count_ns:varint, + * client_wait_ns:varint, stats_flags:u8}, after {@code row_count} when + * {@link #QUERY_FLAG_COUNT_ROWS} is set too. See + * {@link #STATS_FLAG_JIT_COMPILED}. Sent only when the server advertised + * {@link #CAP_QUERY_DETAILS}. + */ + public static final int QUERY_FLAG_STATS = 0x08; + /** + * {@code QUERY_REQUEST.query_flags} bit reserved for per-query timeouts, which + * this client does not send. Released clients (c-questdb-client 7.2.0 and + * later, and the Python client built on it) send it, followed by a + * {@code timeout_ms:varint}, to a server that advertises + * {@link #CAP_QUERY_TIMEOUT}, so no other flag may take this bit. A flag that + * carries a field appends it after {@code query_flags} in ascending order of + * the flag bits, so {@code timeout_ms} goes ahead of {@code skip_rows} and + * {@code max_rows}. Mirrors the server-side constant + * {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#QUERY_FLAG_TIMEOUT}. + */ + public static final int QUERY_FLAG_TIMEOUT = 0x02; public static final byte QUERY_REQUEST = 0x10; /** * Reset mask bit: clear the connection-scoped SYMBOL dict. @@ -105,6 +192,11 @@ public final class QwpEgressMsgKind { * {@link #CACHE_RESET}; SERVER_INFO lives at 0x18. */ public static final byte SERVER_INFO = 0x18; + /** + * {@code stats_flags} bit of the {@link #QUERY_FLAG_STATS} block: the query + * filter runs as JIT-compiled native code. + */ + public static final byte STATS_FLAG_JIT_COMPILED = 0x01; private QwpEgressMsgKind() { } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index dbd34fec..acc9a4a9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -100,6 +100,13 @@ * truncated / unknown frame, decoder out of sync, I/O thread interrupt) all * surface with {@link WebSocketResponse#STATUS_INTERNAL_ERROR} and the * specific cause in the message. + *

+ * Per-query options: + * {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler, QwpQueryOptions)} + * pages through a result set with a row window, and asks the server for what + * the {@code /exec} HTTP endpoint reports besides the rows: the row count of + * the whole result set, execution statistics and the error position. See + * {@link QwpQueryOptions}. */ public class QwpQueryClient implements QuietCloseable { @@ -156,6 +163,15 @@ public class QwpQueryClient implements QuietCloseable { // Warn once per process when an application-supplied token provider is used over ws:: (spec section 9). private static final AtomicBoolean CLEARTEXT_PROVIDER_WARNED = new AtomicBoolean(); private static final Logger LOG = LoggerFactory.getLogger(QwpQueryClient.class); + // query_flags bits a server honours only when it advertises CAP_QUERY_DETAILS + private static final int QUERY_DETAILS_FLAGS = QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION + | QwpEgressMsgKind.QUERY_FLAG_STATS + | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS + | QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + // query_flags bits of the row window, which change what the server streams + private static final int ROW_WINDOW_QUERY_FLAGS = QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; // Reusable typed bind-value sink. Populated on the user thread by the // {@link QwpBindSetter} passed to execute(); the pre-encoded bytes are // handed to the I/O thread via QueryRequest. Allocated once per client to @@ -172,6 +188,11 @@ public class QwpQueryClient implements QuietCloseable { private final Random failoverRandom = new Random(); // Connection health (design/qwp-token-provider-spec.md, section 8.4). private final QwpConnectionHealthTracker healthTracker = new QwpConnectionHealthTracker(); + // Deterministic barrier used by shutdown tests. Null in production. execute() + // runs it after an attempt intercepted a transport failure and before it + // checks for a stop request, the window a requestShutdown() from another + // thread can otherwise only hit by chance. + private volatile Runnable afterTransportFailureHook; private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; // Deterministic lifecycle barrier used by facade shutdown tests. Null in @@ -756,6 +777,15 @@ public void execute(CharSequence sql, QwpColumnBatchHandler handler, boolean res execute(sql, null, handler, resetSymbolDict); } + /** + * As {@link #execute(CharSequence, QwpColumnBatchHandler)}, with per-query + * {@code options}. See + * {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler, QwpQueryOptions)}. + */ + public void execute(CharSequence sql, QwpColumnBatchHandler handler, QwpQueryOptions options) { + execute(sql, null, handler, options); + } + /** * Executes {@code sql} with typed bind parameters supplied by the given * {@link QwpBindSetter}. The setter runs on the calling thread before the @@ -779,20 +809,35 @@ public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler * {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}; otherwise it is silently ignored. */ public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, boolean resetSymbolDict) { - if (!executing.compareAndSet(false, true)) { - throw new IllegalStateException( - "QwpQueryClient.execute called while another execute is in flight; one query at a time per client"); - } - // Drop any cancel latched between calls (e.g., a watchdog that fired - // while no request was in flight, or a previous pooled user that - // released the client without ever calling execute). Failover retries - // inside this execute() must still honor a fresh cancel, so the latch - // is intentionally NOT cleared inside executeOnce(). - pendingCancel = false; - try { - executeImpl(sql, binds, handler, resetSymbolDict); - } finally { - executing.set(false); + executeExclusive(sql, binds, handler, resetSymbolDict ? QwpEgressMsgKind.QUERY_FLAG_RESET_DICT : 0, 0, Long.MAX_VALUE); + } + + /** + * As {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler)}, + * with per-query {@code options}: a row window that pages through the + * result set, and what the server reports besides the rows -- the row count + * of the whole result set and execution statistics, which reach + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)} and + * {@link QwpColumnBatchHandler#onExecDone(long, short, long, QwpQueryDetails)}, + * and the error position, which reaches + * {@link QwpColumnBatchHandler#onError(long, byte, String, int)}. The client + * reads {@code options} when the call starts; null runs the query without + * options. + *

+ * The options need a server that advertises + * {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. Against any other, a row + * window fails the query through + * {@link QwpColumnBatchHandler#onError(long, byte, String, int)} with + * {@link WebSocketResponse#STATUS_INTERNAL_ERROR} before it is sent, since + * the server would stream every row; the other options are left out and the + * query runs without them. After a failover reconnect the client checks the + * new server again. + */ + public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, QwpQueryOptions options) { + if (options == null) { + executeExclusive(sql, binds, handler, 0, 0, Long.MAX_VALUE); + } else { + executeExclusive(sql, binds, handler, options.toQueryFlags(), options.getSkipRows(), options.getMaxRows()); } } @@ -976,6 +1021,11 @@ public void seedFailoverRandomForTest(long seed) { } } + @TestOnly + public void setAfterTransportFailureHookForTest(Runnable hook) { + afterTransportFailureHook = hook; + } + @TestOnly public void setBeforeCloseHookForTest(Runnable hook) { beforeCloseHook = hook; @@ -1351,6 +1401,20 @@ private static String defaultClientId() { return "questdb-java-egress/1.0.0"; } + /** + * Reports a failure that {@code execute()} detected itself, rather than one + * the server sent: a failed reconnect, an exhausted failover, a stop + * request, a bind error. Every such failure goes through this method, so it + * reaches the position-aware + * {@link QwpColumnBatchHandler#onError(long, byte, String, int)}, as the + * server's errors do, and a handler that overrides only that overload sees + * them all. The failure does not come from the SQL text, so its position is + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}. + */ + private static void deliverClientError(QwpColumnBatchHandler handler, long requestId, byte status, String message) { + handler.onError(requestId, status, message, QwpEgressMsgKind.NO_ERROR_POSITION); + } + @SuppressWarnings("BooleanMethodIsAlwaysInverted") private static boolean matchesTarget(byte role, String target) { if (TARGET_ANY.equals(target)) { @@ -1367,6 +1431,23 @@ private static boolean matchesTarget(byte role, String target) { return true; } + /** + * Returns the bits of {@code queryFlags} a server with {@code capabilities} + * honours: {@link QwpEgressMsgKind#QUERY_FLAG_RESET_DICT} needs + * {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}, and the query-details bits need + * {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. + */ + private static long resolveQueryFlags(long queryFlags, int capabilities) { + long resolved = queryFlags; + if ((capabilities & QwpEgressMsgKind.CAP_QUERY_FLAGS) == 0) { + resolved &= ~QwpEgressMsgKind.QUERY_FLAG_RESET_DICT; + } + if ((capabilities & QwpEgressMsgKind.CAP_QUERY_DETAILS) == 0) { + resolved &= ~QUERY_DETAILS_FLAGS; + } + return resolved; + } + private static TokenProviderSpec validateConfigAndResolveTokenProvider(ConfigView view, boolean tls) { view.getHostPorts("addr", DEFAULT_WS_PORT, (h, p) -> { }); @@ -1699,7 +1780,44 @@ private void connectWalk() { throw failure; } - private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, boolean resetSymbolDict) { + /** + * Runs one {@link #execute} at a time on this client. {@code queryFlags} are + * the {@code query_flags} bits the caller asks for, before + * {@link #resolveQueryFlags} drops those the server does not advertise. + */ + private void executeExclusive( + CharSequence sql, + QwpBindSetter binds, + QwpColumnBatchHandler handler, + long queryFlags, + long skipRows, + long maxRows + ) { + if (!executing.compareAndSet(false, true)) { + throw new IllegalStateException( + "QwpQueryClient.execute called while another execute is in flight; one query at a time per client"); + } + // Drop any cancel latched between calls (e.g., a watchdog that fired + // while no request was in flight, or a previous pooled user that + // released the client without ever calling execute). Failover retries + // inside this execute() must still honor a fresh cancel, so the latch + // is intentionally NOT cleared inside executeOnce(). + pendingCancel = false; + try { + executeImpl(sql, binds, handler, queryFlags, skipRows, maxRows); + } finally { + executing.set(false); + } + } + + private void executeImpl( + CharSequence sql, + QwpBindSetter binds, + QwpColumnBatchHandler handler, + long queryFlags, + long skipRows, + long maxRows + ) { if (closedFlag.get() || isShutdownRequested) { throw new IllegalStateException("QwpQueryClient is closed"); } @@ -1714,7 +1832,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa try { connect(); } catch (RuntimeException e) { - handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, + deliverClientError(handler, -1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "reconnect failed: " + e.getMessage()); return; } @@ -1735,24 +1853,28 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa int attempt = 0; while (true) { if (isShutdownRequested) { - handler.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "query client shut down"); + deliverClientError(handler, -1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "query client shut down"); return; } attempt++; FailoverProbeHandler probe = new FailoverProbeHandler(handler); - executeOnce(sql, binds, probe, resetSymbolDict); + executeOnce(sql, binds, probe, queryFlags, skipRows, maxRows); if (!probe.transportFailureIntercepted) { return; } + Runnable hook = afterTransportFailureHook; + if (hook != null) { + hook.run(); + } if (isShutdownRequested) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); return; } if (!failoverEnabled) { // With failover off the transport failure stays latched: every later execute() reports it. healthTracker.connectionLost(); healthTracker.failed(); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, probe.interceptedMessage); + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, probe.interceptedMessage); return; } if (attempt >= failoverMaxAttempts || System.nanoTime() - failoverDeadlineNanos >= 0) { @@ -1764,7 +1886,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa healthTracker.failed(); } int failovers = Math.max(0, attempt - 1); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" + (attempt == 1 ? "" : "s") + " (" + failovers + " failover reconnect" @@ -1798,7 +1920,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa long remaining = remainingNanos <= 0L ? 0L : remainingNanos / 1_000_000L; if (remainingNanos <= 0L) { int failovers = Math.max(0, attempt - 1); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" + (attempt == 1 ? "" : "s") + " (" + failovers + " failover reconnect" @@ -1814,7 +1936,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa Thread.sleep(delay); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "failover interrupted while backing off after attempt " + attempt + "; last error: " + probe.interceptedMessage); return; @@ -1835,7 +1957,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa // without recovery. Surface a distinct message so monitoring // can pull auth incidents apart from generic transport failures; // it names the failure class (design/qwp-token-provider-spec.md, 8.3). - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "auth-rejected during failover reconnect [host=" + authErr.getHost() + ':' + authErr.getPort() + ", status=" + authErr.getStatusCode() @@ -1843,12 +1965,12 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa return; } catch (QwpCredentialUnavailableException credentialErr) { // The token provider could not supply a credential; no endpoint was contacted. - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "failover reconnect failed: " + credentialErr.getMessage() + " [last error: " + probe.interceptedMessage + ']'); return; } catch (RuntimeException reconnectErr) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "failover reconnect failed after " + attempt + " attempt" + (attempt == 1 ? "" : "s") + " [last error: " + probe.interceptedMessage + ", reconnect error: " @@ -1856,7 +1978,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa return; } if (isShutdownRequested) { - handler.onError(probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); + deliverClientError(handler, probe.interceptedRequestId, probe.interceptedStatus, "query client shut down"); return; } handler.onFailoverReset(probe.interceptedRequestId, serverInfo); @@ -1868,7 +1990,14 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa * the user's handler in a {@link FailoverProbeHandler} so that the outer * loop can intercept transport failures before they reach the user. */ - private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHandler probe, boolean resetSymbolDict) { + private void executeOnce( + CharSequence sql, + QwpBindSetter binds, + FailoverProbeHandler probe, + long queryFlags, + long skipRows, + long maxRows + ) { // Cache the I/O thread reference at entry: close() may null the field while // we are inside this loop, so reading the field per-iteration would NPE // exactly when the user is mid-execute() and close() races. The queue and @@ -1876,7 +2005,7 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan // before close() returns. QwpEgressIoThread io = ioThread; if (io == null) { - probe.onError(-1L, WebSocketResponse.STATUS_INTERNAL_ERROR, "QwpQueryClient is closed"); + probe.deliverFinal(-1L, "QwpQueryClient is closed"); return; } GenerationListener listener = currentGenerationListener; @@ -1885,6 +2014,21 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan probe.markTransportFailure(-1L, tf.status, tf.message); return; } + // Resolved per attempt: a failover reconnect may land on a server that + // advertises other capabilities. Every bound connection has its + // SERVER_INFO; only a connect() racing this query from another thread + // could clear it, and then no flag goes out. + final QwpServerInfo info = serverInfo; + final int capabilities = info != null ? info.getCapabilities() : 0; + final long wireQueryFlags = resolveQueryFlags(queryFlags, capabilities); + if ((wireQueryFlags & ROW_WINDOW_QUERY_FLAGS) != (queryFlags & ROW_WINDOW_QUERY_FLAGS)) { + // A server that does not know the row window would stream every + // row of the result set. Like a bind error, this is deterministic + // on this server, not a transport failure. + probe.deliverFinal(-1L, "server does not support skipRows and maxRows [capabilities=0x" + + Integer.toHexString(capabilities) + "]; they need a server that advertises CAP_QUERY_DETAILS"); + return; + } bindValues.reset(); if (binds != null) { try { @@ -1917,7 +2061,7 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan } try { io.submitQuery(sql, requestId, initialCreditBytes, bindValues.count(), bindValues.bufferPtr(), bindValues.bufferLen(), - resolveQueryFlags(resetSymbolDict)); + wireQueryFlags, skipRows, maxRows); boolean isBatchSkipped = false; while (true) { QueryEvent ev = io.takeEvent(); @@ -1942,20 +2086,22 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan if (isBatchSkipped) { probe.deliverFinal(requestId, "query client shut down"); } else { - probe.onEnd(requestId, ev.totalRows); + probe.onEnd(requestId, ev.totalRows, ev.details); } return; case QueryEvent.KIND_EXEC_DONE: - probe.onExecDone(requestId, ev.opType, ev.rowsAffected); + probe.onExecDone(requestId, ev.opType, ev.rowsAffected, ev.details); return; case QueryEvent.KIND_ERROR: - probe.onError(requestId, ev.errorStatus, ev.errorMessage); + probe.onError(requestId, ev.errorStatus, ev.errorMessage, ev.errorPosition); return; case QueryEvent.KIND_TRANSPORT_ERROR: probe.markTransportFailure(requestId, ev.errorStatus, ev.errorMessage); return; default: - probe.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind); + // Unreachable: the I/O thread emits only the kinds above. + assert false : "unknown event kind " + ev.kind; + probe.deliverFinal(requestId, "unknown event kind " + ev.kind); return; } } finally { @@ -2199,16 +2345,6 @@ private String resolveAuthorizationHeader() { return authorizationHeader; } - private long resolveQueryFlags(boolean resetSymbolDict) { - if (!resetSymbolDict) { - return 0L; - } - QwpServerInfo info = serverInfo; - return info != null && (info.getCapabilities() & QwpEgressMsgKind.CAP_QUERY_FLAGS) != 0 - ? QwpEgressMsgKind.QUERY_FLAG_RESET_DICT - : 0L; - } - private void runUpgradeWithTimeout(Endpoint ep, String authHeader) { // Connect first, OUTSIDE the upgrade try. A connect-phase failure -- // including a connect_timeout overage flagged via flagAsTimeout() -- must @@ -2312,6 +2448,11 @@ public void onEnd(long requestId, long totalRows) { delegate.onEnd(requestId, totalRows); } + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + delegate.onEnd(requestId, totalRows, details); + } + @Override public void onError(byte status, String message) { // Server-emitted QUERY_ERROR. Pass straight through. Transport @@ -2324,6 +2465,11 @@ public void onError(long requestId, byte status, String message) { delegate.onError(requestId, status, message); } + @Override + public void onError(long requestId, byte status, String message, int position) { + delegate.onError(requestId, status, message, position); + } + @Override public void onExecDone(short opType, long rowsAffected) { delegate.onExecDone(opType, rowsAffected); @@ -2334,6 +2480,11 @@ public void onExecDone(long requestId, short opType, long rowsAffected) { delegate.onExecDone(requestId, opType, rowsAffected); } + @Override + public void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + delegate.onExecDone(requestId, opType, rowsAffected, details); + } + @Override public void onFailoverReset(QwpServerInfo newNode) { delegate.onFailoverReset(newNode); @@ -2350,7 +2501,7 @@ public void onFailoverReset(long requestId, QwpServerInfo newNode) { * errors, interrupts) so they don't trigger a spurious failover. */ void deliverFinal(long requestId, String message) { - delegate.onError(requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, message); + deliverClientError(delegate, requestId, WebSocketResponse.STATUS_INTERNAL_ERROR, message); } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java new file mode 100644 index 00000000..0b683c7a --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryDetails.java @@ -0,0 +1,165 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * What a successful query reported besides its rows, on the request of + * {@link QwpQueryOptions}: the row count of the whole result set and execution + * statistics. Passed to + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)} and + * {@link QwpColumnBatchHandler#onExecDone(long, short, long, QwpQueryDetails)}, + * never null. + *

+ * The instance is valid only for the duration of that call; the client reuses + * it afterwards. Copy any values you need to retain. + */ +public final class QwpQueryDetails { + + private long clientWaitNanos = -1; + private long compileNanos = -1; + private long countNanos = -1; + private long executeNanos = -1; + private boolean hasRowCount; + private boolean hasStats; + private boolean isJitCompiled; + private long rowCount = -1; + + /** + * Returns how long, in nanoseconds, the query waited for this client to + * read its results or grant credit, or -1 when {@link #hasStats()} is false. + * Part of {@link #getExecuteNanos()}. + */ + public long getClientWaitNanos() { + return clientWaitNanos; + } + + /** + * Returns how long, in nanoseconds, the server took to compile the SQL text, + * or -1 when {@link #hasStats()} is false. Zero when the server reused a + * compiled query from its cache. Part of {@link #getExecuteNanos()}. + */ + public long getCompileNanos() { + return compileNanos; + } + + /** + * Returns how long, in nanoseconds, the server took to count the rows the + * row window left out, or -1 when {@link #hasStats()} is false. Zero when + * the row count was not requested, or needed no counting. Part of + * {@link #getExecuteNanos()}. + */ + public long getCountNanos() { + return countNanos; + } + + /** + * Returns how long, in nanoseconds, the query ran on the server, from the + * arrival of the request to its last frame, or -1 when {@link #hasStats()} + * is false. Includes the compile, count and client-wait times. + */ + public long getExecuteNanos() { + return executeNanos; + } + + /** + * Returns the number of rows in the whole result set, including the rows + * the row window left out of the stream, or -1 when {@link #hasRowCount()} + * is false. + */ + public long getRowCount() { + return rowCount; + } + + /** + * Returns true when the server reported the row count of the whole result + * set: the query asked for it with {@link QwpQueryOptions#countRows(boolean)}, + * the server advertises {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}, and the + * statement produced rows. + */ + public boolean hasRowCount() { + return hasRowCount; + } + + /** + * Returns true when the server reported execution statistics: the query + * asked for them with {@link QwpQueryOptions#stats(boolean)} and the server + * advertises {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. + */ + public boolean hasStats() { + return hasStats; + } + + /** + * Returns true when the query filter ran as JIT-compiled native code; false + * when it did not, or when {@link #hasStats()} is false. + */ + public boolean isJitCompiled() { + return isJitCompiled; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("QwpQueryDetails{"); + if (hasRowCount) { + sb.append("rowCount=").append(rowCount); + } + if (hasStats) { + if (hasRowCount) { + sb.append(", "); + } + sb.append("compileNanos=").append(compileNanos) + .append(", executeNanos=").append(executeNanos) + .append(", countNanos=").append(countNanos) + .append(", clientWaitNanos=").append(clientWaitNanos) + .append(", jitCompiled=").append(isJitCompiled); + } + return sb.append('}').toString(); + } + + void clear() { + clientWaitNanos = -1; + compileNanos = -1; + countNanos = -1; + executeNanos = -1; + hasRowCount = false; + hasStats = false; + isJitCompiled = false; + rowCount = -1; + } + + void setRowCount(long rowCount) { + this.rowCount = rowCount; + this.hasRowCount = true; + } + + void setStats(long compileNanos, long executeNanos, long countNanos, long clientWaitNanos, boolean isJitCompiled) { + this.compileNanos = compileNanos; + this.executeNanos = executeNanos; + this.countNanos = countNanos; + this.clientWaitNanos = clientWaitNanos; + this.isJitCompiled = isJitCompiled; + this.hasStats = true; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java new file mode 100644 index 00000000..93f5034e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryOptions.java @@ -0,0 +1,258 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Per-query options of + * {@link QwpQueryClient#execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler, QwpQueryOptions)} + * and {@link io.questdb.client.Query#options(QwpQueryOptions)}: a row window + * that pages through the result set, and what the server reports besides the + * rows -- the row count of the whole result set, execution statistics and the + * position of an error in the SQL text. A new instance has every option off, + * and a query run with it is the same on the wire as one run without options. + *

+ * The row window, the row count, the statistics and the error position need a + * server that advertises {@link QwpEgressMsgKind#CAP_QUERY_DETAILS}. Against a + * server without it, the client fails a query that sets a row window, since the + * server would stream every row, and runs a query that asks only for the row + * count, the statistics or the error position without them: the handler then + * sees {@link QwpQueryDetails#hasRowCount()} and + * {@link QwpQueryDetails#hasStats()} false and the position + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}. + *

+ * The client reads the options when the query starts, so one instance can be + * reused for any number of queries, and changing it does not affect a query + * already running. Not thread-safe. + */ +public final class QwpQueryOptions { + + private boolean isErrorPositionRequested; + private boolean isRowCountRequested; + private boolean isStatsRequested; + private boolean isSymbolDictResetRequested; + private long maxRows = Long.MAX_VALUE; + private long skipRows; + + /** + * Resets every option to its default: no row window, nothing reported + * besides the rows, and the connection-scoped SYMBOL dictionary kept. + * + * @return this instance + */ + public QwpQueryOptions clear() { + isErrorPositionRequested = false; + isRowCountRequested = false; + isStatsRequested = false; + isSymbolDictResetRequested = false; + maxRows = Long.MAX_VALUE; + skipRows = 0; + return this; + } + + /** + * Copies every option of {@code other} into this instance, or resets them + * when {@code other} is null. + * + * @return this instance + */ + public QwpQueryOptions copyFrom(QwpQueryOptions other) { + if (other == null) { + return clear(); + } + isErrorPositionRequested = other.isErrorPositionRequested; + isRowCountRequested = other.isRowCountRequested; + isStatsRequested = other.isStatsRequested; + isSymbolDictResetRequested = other.isSymbolDictResetRequested; + maxRows = other.maxRows; + skipRows = other.skipRows; + return this; + } + + /** + * Asks the server to count the rows of the whole result set, whatever the + * row window leaves out of the stream, as the {@code count} of the + * {@code /exec} HTTP endpoint does. The count reaches + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)}. + * Statements that do not produce rows report no count. When the row window + * ends before the result set does, the server reads the rest of the result + * set to count it, without sending it. + * + * @return this instance + */ + public QwpQueryOptions countRows(boolean isRequested) { + isRowCountRequested = isRequested; + return this; + } + + /** + * Asks the server for the position of an error in the SQL text, as the + * {@code position} of the {@code /exec} HTTP endpoint. A failed query then + * reaches {@link QwpColumnBatchHandler#onError(long, byte, String, int)} + * with the bare error message, without the {@code [position]} prefix the + * server puts in front of it otherwise, and the position as a {@code char} + * index into the SQL text. + * + * @return this instance + */ + public QwpQueryOptions errorPosition(boolean isRequested) { + isErrorPositionRequested = isRequested; + return this; + } + + /** + * Returns the most rows the query streams, {@link Long#MAX_VALUE} when the + * stream is unbounded. + */ + public long getMaxRows() { + return maxRows; + } + + /** + * Returns how many rows the query leaves out at the start of the result set. + */ + public long getSkipRows() { + return skipRows; + } + + public boolean isErrorPositionRequested() { + return isErrorPositionRequested; + } + + public boolean isRowCountRequested() { + return isRowCountRequested; + } + + public boolean isStatsRequested() { + return isStatsRequested; + } + + public boolean isSymbolDictResetRequested() { + return isSymbolDictResetRequested; + } + + /** + * Streams at most {@code rows} rows of the result set, after the rows + * {@link #skipRows(long)} leaves out, like the {@code limit} of the + * {@code /exec} HTTP endpoint. Zero streams no rows, only the schema. + * {@link Long#MAX_VALUE}, the default, leaves the stream unbounded. + * Statements that do not produce rows ignore it. + * + * @param rows the most rows to stream, zero or more + * @return this instance + * @throws IllegalArgumentException if {@code rows} is negative + */ + public QwpQueryOptions maxRows(long rows) { + if (rows < 0) { + throw new IllegalArgumentException("maxRows must be >= 0 [maxRows=" + rows + ']'); + } + maxRows = rows; + return this; + } + + /** + * Asks the server to reset the connection-scoped SYMBOL dictionary before + * the query, scoping the dictionary to its symbols. Needs a server that + * advertises {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}; against any other the + * dictionary is kept. + * + * @return this instance + */ + public QwpQueryOptions resetSymbolDict(boolean isRequested) { + isSymbolDictResetRequested = isRequested; + return this; + } + + /** + * Leaves out the first {@code rows} rows of the result set, like the lower + * bound of the {@code limit} of the {@code /exec} HTTP endpoint. Statements + * that do not produce rows ignore it. + * + * @param rows the rows to leave out, zero or more; zero, the default, + * leaves out none + * @return this instance + * @throws IllegalArgumentException if {@code rows} is negative + */ + public QwpQueryOptions skipRows(long rows) { + if (rows < 0) { + throw new IllegalArgumentException("skipRows must be >= 0 [skipRows=" + rows + ']'); + } + skipRows = rows; + return this; + } + + /** + * Asks the server for execution statistics -- the compile, execute, count + * and client-wait times, and whether the filter is JIT-compiled -- as the + * {@code timings} and {@code explain} of the {@code /exec} HTTP endpoint + * report them. They reach + * {@link QwpColumnBatchHandler#onEnd(long, long, QwpQueryDetails)} or + * {@link QwpColumnBatchHandler#onExecDone(long, short, long, QwpQueryDetails)}; + * a failed query reports none. + * + * @return this instance + */ + public QwpQueryOptions stats(boolean isRequested) { + isStatsRequested = isRequested; + return this; + } + + @Override + public String toString() { + return "QwpQueryOptions{skipRows=" + skipRows + + ", maxRows=" + maxRows + + ", countRows=" + isRowCountRequested + + ", stats=" + isStatsRequested + + ", errorPosition=" + isErrorPositionRequested + + ", resetSymbolDict=" + isSymbolDictResetRequested + + '}'; + } + + /** + * Returns the {@code query_flags} bits these options ask for, before the + * client drops the ones the server does not advertise. + */ + long toQueryFlags() { + long flags = 0; + if (isSymbolDictResetRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_RESET_DICT; + } + if (isErrorPositionRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION; + } + if (isStatsRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_STATS; + } + if (isRowCountRequested) { + flags |= QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS; + } + if (skipRows > 0) { + flags |= QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS; + } + if (maxRows != Long.MAX_VALUE) { + flags |= QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + } + return flags; + } +} diff --git a/core/src/main/java/io/questdb/client/impl/QueryImpl.java b/core/src/main/java/io/questdb/client/impl/QueryImpl.java index 82088fa8..40a79b93 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QueryImpl.java @@ -29,7 +29,10 @@ import io.questdb.client.cutlass.qwp.client.QwpBindValues; import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; import io.questdb.client.std.str.StringSink; @@ -39,7 +42,7 @@ /** * Reusable per-{@link QueryWorker} query state: the configured SQL, optional - * binds, handler, terminal-event signalling, and a wrapping + * binds and options, handler, terminal-event signalling, and a wrapping * {@link QwpColumnBatchHandler} that forwards callbacks to the user handler and * signals completion on terminal events. One instance is pre-allocated per * worker in the constructor and reused across every borrow. @@ -68,12 +71,16 @@ final class QueryImpl { private final Condition doneCondition; private final ReentrantLock doneLock = new ReentrantLock(); + // The handle's own copy of the user's options: Query.options() copies into it, + // so the user may reuse the instance, and execute() reads it on the worker thread. + private final QwpQueryOptions options = new QwpQueryOptions(); private final StringSink sqlBuffer = new StringSink(); private final QueryWorker worker; private final QwpBindSetter wireBinds = this::applyBinds; private final WrappingHandler wrappingHandler = new WrappingHandler(); private volatile boolean done = true; private volatile String resultMessage; + private volatile int resultPosition = QwpEgressMsgKind.NO_ERROR_POSITION; private volatile byte resultStatus; private volatile Throwable unexpectedError; private QwpBindSetter userBinds; @@ -91,6 +98,7 @@ void abandon(long gen) { } userBinds = null; userHandler = null; + options.clear(); sqlBuffer.clear(); } @@ -207,6 +215,11 @@ void setHandler(long gen, QwpColumnBatchHandler handler) { this.userHandler = handler; } + void setOptions(long gen, QwpQueryOptions options) { + checkLive(gen); + this.options.copyFrom(options); + } + void setSql(long gen, CharSequence sql) { checkLive(gen); sqlBuffer.clear(); @@ -231,6 +244,7 @@ void submit(long gen) { done = false; resultStatus = 0; resultMessage = null; + resultPosition = QwpEgressMsgKind.NO_ERROR_POSITION; unexpectedError = null; } finally { doneLock.unlock(); @@ -296,6 +310,10 @@ private void rejectHandlerReentry(String op) { } private void signalDone(byte status, String message, Throwable unexpected) { + signalDone(status, message, QwpEgressMsgKind.NO_ERROR_POSITION, unexpected); + } + + private void signalDone(byte status, String message, int position, Throwable unexpected) { doneLock.lock(); try { if (done) { @@ -303,6 +321,7 @@ private void signalDone(byte status, String message, Throwable unexpected) { } this.resultStatus = status; this.resultMessage = message; + this.resultPosition = position; this.unexpectedError = unexpected; this.done = true; doneCondition.signalAll(); @@ -317,7 +336,7 @@ private void throwIfFailed() { throw new QueryException(resultStatus, resultMessage, unexpected); } if (resultStatus != 0) { - throw new QueryException(resultStatus, resultMessage); + throw new QueryException(resultStatus, resultMessage, resultPosition); } } @@ -332,9 +351,11 @@ private void throwIfFailed() { void resetForBorrow() { userBinds = null; userHandler = null; + options.clear(); sqlBuffer.clear(); resultStatus = 0; resultMessage = null; + resultPosition = QwpEgressMsgKind.NO_ERROR_POSITION; unexpectedError = null; done = true; } @@ -344,8 +365,9 @@ void runOn(QwpQueryClient client) { // reads chars and writes UTF-8 bytes straight into the send buffer. // sqlBuffer is stable for the duration of execute(): the calling // worker thread is blocked here until a terminal event arrives, and - // sql(...) cannot be invoked again until done==true. - client.execute(sqlBuffer, wireBinds, wrappingHandler); + // sql(...) cannot be invoked again until done==true. The same holds for + // options, which execute() reads once, when it starts. + client.execute(sqlBuffer, wireBinds, wrappingHandler, options); } /** @@ -381,6 +403,15 @@ public void onEnd(long requestId, long totalRows) { } } + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + try { + userHandler.onEnd(requestId, totalRows, details); + } finally { + signalDone((byte) 0, null, null); + } + } + @Override public void onError(byte status, String message) { try { @@ -399,6 +430,15 @@ public void onError(long requestId, byte status, String message) { } } + @Override + public void onError(long requestId, byte status, String message, int position) { + try { + userHandler.onError(requestId, status, message, position); + } finally { + signalDone(status, message, position, null); + } + } + @Override public void onExecDone(short opType, long rowsAffected) { try { @@ -417,6 +457,15 @@ public void onExecDone(long requestId, short opType, long rowsAffected) { } } + @Override + public void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + try { + userHandler.onExecDone(requestId, opType, rowsAffected, details); + } finally { + signalDone((byte) 0, null, null); + } + } + @Override public void onFailoverReset(QwpServerInfo newNode) { userHandler.onFailoverReset(newNode); diff --git a/core/src/main/java/io/questdb/client/impl/QueryLease.java b/core/src/main/java/io/questdb/client/impl/QueryLease.java index 4b07e965..398596c4 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryLease.java +++ b/core/src/main/java/io/questdb/client/impl/QueryLease.java @@ -28,6 +28,7 @@ import io.questdb.client.Query; import io.questdb.client.cutlass.qwp.client.QwpBindSetter; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; import java.util.concurrent.TimeUnit; @@ -97,6 +98,12 @@ public boolean isDone() { return impl.isDone(generation); } + @Override + public Query options(QwpQueryOptions options) { + impl.setOptions(generation, options); + return this; + } + @Override public Query sql(CharSequence sql) { impl.setSql(generation, sql); diff --git a/core/src/test/java/io/questdb/client/test/QueryExceptionTest.java b/core/src/test/java/io/questdb/client/test/QueryExceptionTest.java new file mode 100644 index 00000000..6ae0313c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QueryExceptionTest.java @@ -0,0 +1,54 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.QueryException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import org.junit.Assert; +import org.junit.Test; + +public class QueryExceptionTest { + + @Test + public void testPosition() { + QueryException e = new QueryException((byte) 5, "table does not exist", 14); + Assert.assertEquals(5, e.getStatus()); + Assert.assertEquals("table does not exist", e.getMessage()); + Assert.assertEquals(14, e.getPosition()); + Assert.assertNull(e.getCause()); + } + + @Test + public void testPositionAbsentByDefault() { + QueryException e = new QueryException((byte) 5, "x"); + Assert.assertEquals(5, e.getStatus()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + + IllegalStateException cause = new IllegalStateException(); + e = new QueryException((byte) 0, "x", cause); + Assert.assertSame(cause, e.getCause()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java b/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java new file mode 100644 index 00000000..9282fb2b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QuestDBQueryOptionsTest.java @@ -0,0 +1,249 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.Query; +import io.questdb.client.QueryException; +import io.questdb.client.QuestDB; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.cutlass.qwp.client.QwpEgressFrames; +import io.questdb.client.test.cutlass.qwp.client.QwpEgressMockServer; +import io.questdb.client.test.cutlass.qwp.client.RecordingQueryHandler; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +/** + * {@link Query#options(QwpQueryOptions)} on the pooled facade: the options + * reach the wire, the details the server reports reach the handler, the + * error position reaches {@link QueryException#getPosition()}, and the handle + * keeps its own copy of the options for its lifetime only. + */ +public class QuestDBQueryOptionsTest { + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; + private static final int WINDOW_FLAGS = QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + + @Test + public void testAbandonDropsOptions() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server); + Query q = db.borrowQuery()) { + q.sql("SELECT x").options(new QwpQueryOptions().maxRows(1)).handler(new RecordingQueryHandler()); + q.abandon(); + q.sql("SELECT x").handler(new RecordingQueryHandler()).submit().await(); + Assert.assertFalse(server.getRequests().get(0).hasQueryFlags); + server.assertNoFailure(); + } + }); + } + + @Test + public void testErrorPositionReachesQueryException() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server); + Query q = db.borrowQuery()) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("FAIL").options(new QwpQueryOptions().errorPosition(true)).handler(handler); + try { + q.submit().await(); + Assert.fail("the query must fail"); + } catch (QueryException e) { + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, e.getStatus()); + Assert.assertEquals("Invalid column: nope", e.getMessage()); + Assert.assertEquals(7, e.getPosition()); + } + Assert.assertEquals(7, handler.errorPosition); + + q.options(null); + try { + q.submit().await(); + Assert.fail("the query must fail"); + } catch (QueryException e) { + Assert.assertEquals("[7] Invalid column: nope", e.getMessage()); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + } + + // a failure after one with a position reports its own + q.sql("SELECT x").options(new QwpQueryOptions().maxRows(1)); + server.close(); + try { + q.submit().await(); + Assert.fail("the query must fail"); + } catch (QueryException e) { + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, e.getPosition()); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testOptionsAreCopiedAndKeptAcrossSubmits() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model)) { + try (QuestDB db = connect(server)) { + try (Query q = db.borrowQuery()) { + QwpQueryOptions options = new QwpQueryOptions().skipRows(1).maxRows(2).countRows(true).stats(true); + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("SELECT x").options(options).handler(handler); + // the handle copied the options: changing them does not reach it + options.clear().maxRows(9); + q.submit().await(); + Assert.assertEquals(Arrays.asList(1L, 2L), handler.values); + Assert.assertEquals(2, handler.totalRows); + Assert.assertEquals(10, handler.rowCount); + Assert.assertTrue(handler.hasStats); + Assert.assertEquals(123, handler.compileNanos); + + // the next submit on the handle keeps them + handler.values.clear(); + q.submit().await(); + Assert.assertEquals(Arrays.asList(1L, 2L), handler.values); + + // until they are replaced, or dropped + q.options(options); + q.submit().await(); + q.options(null); + q.submit().await(); + // the lease ends with options set + q.options(new QwpQueryOptions().stats(true).skipRows(3)); + } + // a new borrow of the same pooled client starts without options + try (Query q = db.borrowQuery()) { + q.sql("SELECT x").handler(new RecordingQueryHandler()).submit().await(); + } + } + List requests = server.getRequests(); + Assert.assertEquals(5, requests.size()); + for (int i = 0; i < 2; i++) { + QwpEgressFrames.QueryRequest r = requests.get(i); + Assert.assertEquals(WINDOW_FLAGS | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS, + r.queryFlags); + Assert.assertEquals(1, r.skipRows); + Assert.assertEquals(2, r.maxRows); + } + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, requests.get(2).queryFlags); + Assert.assertEquals(9, requests.get(2).maxRows); + Assert.assertFalse(requests.get(3).hasQueryFlags); + Assert.assertFalse(requests.get(4).hasQueryFlags); + server.assertNoFailure(); + } + }); + } + + @Test + public void testOptionsOnClosedHandleRejected() throws Exception { + // The pool holds one client, so the second borrow gets the client the + // closed handle had: its options must not reach that borrower's query. + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server)) { + Query closed = db.borrowQuery(); + closed.close(); + try (Query q = db.borrowQuery()) { + try { + closed.options(new QwpQueryOptions().maxRows(1)); + Assert.fail("a closed handle must reject options()"); + } catch (IllegalStateException e) { + Assert.assertEquals("query handle is not borrowed (closed or never leased)", e.getMessage()); + } + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("SELECT x").handler(handler).submit().await(); + Assert.assertEquals(10, handler.values.size()); + } + Assert.assertEquals(1, server.getRequests().size()); + Assert.assertFalse(server.getRequests().get(0).hasQueryFlags); + server.assertNoFailure(); + } + }); + } + + @Test + public void testStatsReachExecDone() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QuestDBQueryOptionsTest::model); + QuestDB db = connect(server); + Query q = db.borrowQuery()) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + q.sql("INSERT INTO t VALUES (1)").options(new QwpQueryOptions().stats(true)).handler(handler).submit().await(); + Assert.assertEquals(1, handler.execDoneCount); + Assert.assertEquals(4, handler.rowsAffected); + Assert.assertTrue(handler.hasStats); + Assert.assertEquals(123, handler.compileNanos); + Assert.assertFalse(handler.isJitCompiled); + server.assertNoFailure(); + } + }); + } + + private static QuestDB connect(QwpEgressMockServer server) { + return QuestDB.connect("ws::addr=" + server.getAddress() + + ";sender_pool_min=0;query_pool_min=1;query_pool_max=1;failover=off;auth_timeout_ms=2000;"); + } + + /** + * A 10-row table {@code x = 0..9} in one batch per query; an + * {@code INSERT} that reports 4 rows; anything else fails at position 7. + * Trailers follow the flags of the request, as on the server. + */ + private static byte[][] model(QwpEgressFrames.QueryRequest r) { + boolean hasStats = r.has(QwpEgressMsgKind.QUERY_FLAG_STATS); + if (r.sql.startsWith("SELECT")) { + long first = Math.max(0, Math.min(r.skipRows, 10)); + long count = Math.min(r.maxRows < 0 ? Long.MAX_VALUE : r.maxRows, 10 - first); + long[] values = new long[(int) count]; + for (int i = 0; i < count; i++) { + values[i] = first + i; + } + QwpEgressFrames.Body end = QwpEgressFrames.resultEnd(r.requestId, 0, count); + if (r.has(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS)) { + end.varint(10); + } + if (hasStats) { + end.stats(123, 456, 0, 0, 0); + } + return new byte[][]{QwpEgressFrames.resultBatchOfLongs(r.requestId, 0, values), end.toFrame()}; + } + if (r.sql.startsWith("INSERT")) { + QwpEgressFrames.Body done = QwpEgressFrames.execDone(r.requestId, 2, 4); + if (hasStats) { + done.stats(123, 456, 0, 0, 0); + } + return new byte[][]{done.toFrame()}; + } + String message = "Invalid column: nope"; + if (r.has(QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION)) { + return new byte[][]{QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, message).i32(7).toFrame()}; + } + return new byte[][]{QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, "[7] " + message).toFrame()}; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java index 9867ee95..488373c0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.QueryEvent; import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import org.junit.Assert; import org.junit.Test; @@ -50,6 +51,57 @@ public void testFreshEventDefaults() { Assert.assertEquals(0, ev.opType); Assert.assertEquals(0L, ev.rowsAffected); Assert.assertEquals(0L, ev.totalRows); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + RecordingQueryHandler.assertNoDetails(ev.details); + } + + @Test + public void testAsEndAndAsExecDoneDropStaleDetails() { + QueryEvent ev = new QueryEvent().asEnd(1).withRowCount(9).withStats(1, 2, 3, 4, true); + ev.asEnd(2); + RecordingQueryHandler.assertNoDetails(ev.details); + ev.withRowCount(9).withStats(1, 2, 3, 4, true).asExecDone((short) 1, 2); + RecordingQueryHandler.assertNoDetails(ev.details); + } + + @Test + public void testAsErrorWithPosition() { + QueryEvent ev = new QueryEvent(); + Assert.assertSame(ev, ev.asError((byte) 0x05, "Invalid column: nope", 7)); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals((byte) 0x05, ev.errorStatus); + Assert.assertEquals("Invalid column: nope", ev.errorMessage); + Assert.assertEquals(7, ev.errorPosition); + // the builders without a position drop the stale one + ev.asError((byte) 0x05, "boom"); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + ev.asError((byte) 0x05, "boom", 3).asTransportError((byte) 0x06, "socket reset"); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + } + + @Test + public void testWithRowCountAndStats() { + QueryEvent ev = new QueryEvent().asEnd(5); + Assert.assertSame(ev, ev.withRowCount(0)); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(0, ev.details.getRowCount()); + Assert.assertFalse(ev.details.hasStats()); + Assert.assertEquals("QwpQueryDetails{rowCount=0}", ev.details.toString()); + + Assert.assertSame(ev, ev.withStats(10, 20, 30, 40, true)); + Assert.assertTrue(ev.details.hasStats()); + Assert.assertEquals(10, ev.details.getCompileNanos()); + Assert.assertEquals(20, ev.details.getExecuteNanos()); + Assert.assertEquals(30, ev.details.getCountNanos()); + Assert.assertEquals(40, ev.details.getClientWaitNanos()); + Assert.assertTrue(ev.details.isJitCompiled()); + + QueryEvent statsOnly = new QueryEvent().asExecDone((short) 2, 1).withStats(0, 0, 0, 0, false); + Assert.assertFalse(statsOnly.details.hasRowCount()); + Assert.assertTrue(statsOnly.details.hasStats()); + Assert.assertFalse(statsOnly.details.isJitCompiled()); + Assert.assertEquals("QwpQueryDetails{compileNanos=0, executeNanos=0, countNanos=0, clientWaitNanos=0, " + + "jitCompiled=false}", statsOnly.details.toString()); } @Test @@ -132,17 +184,25 @@ public void testKindConstantsAreDistinct() { @Test public void testResetClearsEverythingFromError() { QueryEvent ev = new QueryEvent(); - ev.asError((byte) 0x05, "boom"); + ev.asError((byte) 0x05, "boom", 12); ev.reset(); Assert.assertEquals(-1, ev.kind); Assert.assertNull(ev.buffer); Assert.assertNull(ev.errorMessage); Assert.assertEquals(0, ev.errorStatus); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); Assert.assertEquals(0, ev.opType); Assert.assertEquals(0L, ev.rowsAffected); Assert.assertEquals(0L, ev.totalRows); } + @Test + public void testResetClearsDetails() { + QueryEvent ev = new QueryEvent().asEnd(5).withRowCount(9).withStats(1, 2, 3, 4, true); + ev.reset(); + RecordingQueryHandler.assertNoDetails(ev.details); + } + @Test public void testResetClearsEverythingFromBatch() { QueryEvent ev = new QueryEvent(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java index d4cdce6d..22a23337 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java @@ -24,11 +24,17 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.qwp.client.QueryEvent; import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * The two default methods on {@link QwpColumnBatchHandler} * ({@link QwpColumnBatchHandler#onFailoverReset(QwpServerInfo) onFailoverReset} @@ -40,6 +46,61 @@ */ public class QwpColumnBatchHandlerTest { + @Test + public void testDetailedOverloadsDelegateToSimplerOnes() { + // Each detailed overload delegates to the next simpler one, so a handler + // that overrides only the abstract methods sees every terminal event. + List calls = new ArrayList<>(); + QwpColumnBatchHandler handler = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + calls.add("onEnd " + totalRows); + } + + @Override + public void onError(byte status, String message) { + calls.add("onError " + status + ' ' + message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + calls.add("onExecDone " + opType + ' ' + rowsAffected); + } + }; + QueryEvent event = new QueryEvent().asEnd(0).withRowCount(9).withStats(1, 2, 3, 4, true); + handler.onEnd(1, 5, event.details); + handler.onError(2, (byte) 5, "Invalid column: nope", 7); + handler.onExecDone(3, (short) 2, 4, event.details); + Assert.assertEquals(Arrays.asList("onEnd 5", "onError 5 Invalid column: nope", "onExecDone 2 4"), calls); + + // and the correlation-aware overloads get the request id + calls.clear(); + QwpColumnBatchHandler correlating = new MinimalHandler() { + @Override + public void onEnd(long requestId, long totalRows) { + calls.add("onEnd " + requestId + ' ' + totalRows); + } + + @Override + public void onError(long requestId, byte status, String message) { + calls.add("onError " + requestId + ' ' + status + ' ' + message); + } + + @Override + public void onExecDone(long requestId, short opType, long rowsAffected) { + calls.add("onExecDone " + requestId + ' ' + opType + ' ' + rowsAffected); + } + }; + correlating.onEnd(1, 5, event.details); + correlating.onError(2, (byte) 5, "Invalid column: nope", 7); + correlating.onExecDone(3, (short) 2, 4, event.details); + Assert.assertEquals(Arrays.asList("onEnd 1 5", "onError 2 5 Invalid column: nope", "onExecDone 3 2 4"), calls); + } + @Test public void testDefaultMethodsAreNoOpAndDoNotThrow() { QwpColumnBatchHandler handler = new MinimalHandler(); @@ -56,7 +117,7 @@ public void testDefaultMethodsAreNoOpAndDoNotThrow() { * lost their {@code default} modifier this class would no longer compile, * so this test doubles as a compile-time guard on the interface contract. */ - private static final class MinimalHandler implements QwpColumnBatchHandler { + private static class MinimalHandler implements QwpColumnBatchHandler { @Override public void onBatch(QwpColumnBatch batch) { } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java new file mode 100644 index 00000000..5ad80e27 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressFrames.java @@ -0,0 +1,301 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import org.junit.Assert; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +/** + * Builds QWP egress server frames byte by byte, and parses the + * {@code QUERY_REQUEST} a client sends, so tests spell out the wire layout + * they exercise. Encodes the layouts of the server's + * {@code QwpEgressFrameWriter} and decodes the one of its + * {@code QwpEgressRequestDecoder}. + */ +public final class QwpEgressFrames { + + private QwpEgressFrames() { + } + + /** + * Starts a frame body with {@code msg_kind} and {@code request_id}, the + * prelude of every per-query server frame. + */ + public static Body body(byte msgKind, long requestId) { + return new Body().u8(msgKind).i64(requestId); + } + + /** + * {@code EXEC_DONE} body up to {@code rows_affected}; append the trailers + * the test needs. + */ + public static Body execDone(long requestId, int opType, long rowsAffected) { + return body(QwpEgressMsgKind.EXEC_DONE, requestId).u8(opType).varint(rowsAffected); + } + + /** + * Parses a {@code QUERY_REQUEST} payload. Supports the LONG binds tests + * send; the {@code query_flags} trailer and the row-window values follow + * the binds. + */ + public static QueryRequest parseQueryRequest(byte[] payload) { + Reader rd = new Reader(payload); + Assert.assertEquals("frame must be a QUERY_REQUEST", QwpEgressMsgKind.QUERY_REQUEST, (byte) rd.u8()); + long requestId = rd.i64(); + String sql = rd.utf8((int) rd.varint()); + rd.varint(); // initial_credit + int bindCount = (int) rd.varint(); + long[] longBinds = new long[bindCount]; + for (int i = 0; i < bindCount; i++) { + Assert.assertEquals("tests bind LONG values only", QwpConstants.TYPE_LONG, (byte) rd.u8()); + Assert.assertEquals("tests bind non-NULL values only", 0, rd.u8()); + longBinds[i] = rd.i64(); + } + boolean hasQueryFlags = rd.remaining() > 0; + long queryFlags = hasQueryFlags ? rd.varint() : 0; + long skipRows = (queryFlags & QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS) != 0 ? rd.varint() : -1; + long maxRows = (queryFlags & QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS) != 0 ? rd.varint() : -1; + return new QueryRequest(payload, requestId, sql, longBinds, hasQueryFlags, queryFlags, skipRows, maxRows, rd.remaining()); + } + + /** + * {@code QUERY_ERROR} body up to the message; append the position trailer + * when the test needs it. + */ + public static Body queryError(long requestId, byte status, String message) { + byte[] msg = message.getBytes(StandardCharsets.UTF_8); + return body(QwpEgressMsgKind.QUERY_ERROR, requestId).u8(status).u16(msg.length).bytes(msg); + } + + /** + * {@code RESULT_BATCH} carrying one LONG column {@code x}; the schema rides + * the first batch of a query only. + */ + public static byte[] resultBatchOfLongs(long requestId, long batchSeq, long... values) { + Body b = body(QwpEgressMsgKind.RESULT_BATCH, requestId) + .varint(batchSeq) + .varint(0) // table name length + .varint(values.length); + if (batchSeq == 0) { + b.varint(1).varint(1).u8('x').u8(QwpConstants.TYPE_LONG); + } + b.u8(0); // no NULLs + for (long v : values) { + b.i64(v); + } + return b.toFrame(); + } + + /** + * {@code RESULT_END} body up to {@code total_rows}; append the trailers the + * test needs. + */ + public static Body resultEnd(long requestId, long finalSeq, long totalRows) { + return body(QwpEgressMsgKind.RESULT_END, requestId).varint(finalSeq).varint(totalRows); + } + + /** + * Growable little-endian frame body. {@link #toFrame()} prepends the + * 12-byte QWP message header. + */ + public static final class Body { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + public Body bytes(byte[] bytes) { + out.write(bytes, 0, bytes.length); + return this; + } + + public Body i32(int v) { + for (int i = 0; i < 4; i++) { + out.write(v >>> (8 * i)); + } + return this; + } + + public Body i64(long v) { + for (int i = 0; i < 8; i++) { + out.write((int) (v >>> (8 * i))); + } + return this; + } + + public int size() { + return out.size(); + } + + /** + * Appends the {@code QUERY_FLAG_STATS} block: four varint durations and + * the {@code stats_flags} byte. + */ + public Body stats(long compileNanos, long executeNanos, long countNanos, long clientWaitNanos, int statsFlags) { + return varint(compileNanos).varint(executeNanos).varint(countNanos).varint(clientWaitNanos).u8(statsFlags); + } + + public byte[] toBody() { + return out.toByteArray(); + } + + public byte[] toFrame() { + byte[] body = out.toByteArray(); + return new Body() + .i32(QwpConstants.MAGIC_MESSAGE) + .u8(QwpConstants.VERSION) + .u8(0) // flags + .u16(0) // table_count + .i32(body.length) + .bytes(body) + .toBody(); + } + + public Body u16(int v) { + out.write(v); + out.write(v >>> 8); + return this; + } + + public Body u8(int v) { + out.write(v); + return this; + } + + /** + * Unsigned LEB128, the encoding of the server's {@code QwpVarint}. + */ + public Body varint(long v) { + while ((v & ~0x7FL) != 0) { + out.write((int) ((v & 0x7F) | 0x80)); + v >>>= 7; + } + out.write((int) v); + return this; + } + } + + /** + * A {@code QUERY_REQUEST} as the server decodes it. {@link #skipRows} and + * {@link #maxRows} are -1 when the request does not carry them. + */ + public static final class QueryRequest { + public final boolean hasQueryFlags; + public final long[] longBinds; + public final long maxRows; + // the raw payload, as the client sent it + public final byte[] payload; + public final long queryFlags; + public final long requestId; + public final long skipRows; + public final String sql; + public final int trailingBytes; + + QueryRequest( + byte[] payload, + long requestId, + String sql, + long[] longBinds, + boolean hasQueryFlags, + long queryFlags, + long skipRows, + long maxRows, + int trailingBytes + ) { + this.payload = payload; + this.requestId = requestId; + this.sql = sql; + this.longBinds = longBinds; + this.hasQueryFlags = hasQueryFlags; + this.queryFlags = queryFlags; + this.skipRows = skipRows; + this.maxRows = maxRows; + this.trailingBytes = trailingBytes; + } + + public boolean has(int queryFlag) { + return (queryFlags & queryFlag) != 0; + } + + @Override + public String toString() { + return "QueryRequest{requestId=" + requestId + + ", sql='" + sql + '\'' + + ", hasQueryFlags=" + hasQueryFlags + + ", queryFlags=0x" + Long.toHexString(queryFlags) + + ", skipRows=" + skipRows + + ", maxRows=" + maxRows + + ", trailingBytes=" + trailingBytes + + '}'; + } + } + + private static final class Reader { + private final byte[] buf; + private int pos; + + Reader(byte[] buf) { + this.buf = buf; + } + + long i64() { + Assert.assertTrue("request ends inside an i64", remaining() >= 8); + long v = 0; + for (int i = 0; i < 8; i++) { + v |= (long) (buf[pos++] & 0xFF) << (8 * i); + } + return v; + } + + int remaining() { + return buf.length - pos; + } + + int u8() { + Assert.assertTrue("request ends before the next byte", remaining() > 0); + return buf[pos++] & 0xFF; + } + + String utf8(int len) { + Assert.assertTrue("request ends inside the SQL text", remaining() >= len); + String s = new String(buf, pos, len, StandardCharsets.UTF_8); + pos += len; + return s; + } + + long varint() { + long v = 0; + for (int shift = 0; ; shift += 7) { + int b = u8(); + v |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return v; + } + Assert.assertTrue("varint longer than 10 bytes", shift < 63); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java new file mode 100644 index 00000000..f4558232 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressMockServer.java @@ -0,0 +1,150 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.ToIntFunction; + +/** + * Egress endpoint that advertises the given {@code SERVER_INFO} capabilities, + * records every {@code QUERY_REQUEST} it receives, and answers each with the + * frames a test-supplied {@link Responder} builds from it. A responder that + * returns null drops the connection instead, the way a node that dies + * mid-query does. + */ +public final class QwpEgressMockServer implements AutoCloseable { + private final AtomicReference failure = new AtomicReference<>(); + private final List requests = new CopyOnWriteArrayList<>(); + private final TestWebSocketServer server; + + public QwpEgressMockServer(int capabilities, Responder responder) throws IOException, InterruptedException { + server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length == 0 || data[0] != QwpEgressMsgKind.QUERY_REQUEST) { + return; + } + byte[][] frames; + try { + QwpEgressFrames.QueryRequest request = QwpEgressFrames.parseQueryRequest(data); + requests.add(request); + frames = responder.respond(request); + } catch (Throwable th) { + // A broken request or responder must fail the test, not hang it. + failure.compareAndSet(null, th); + frames = null; + } + if (frames == null) { + client.close(); + return; + } + try { + for (byte[] frame : frames) { + client.sendBinary(frame); + } + } catch (IOException e) { + // surfaces to the client as a transport error + } + } + }); + server.setSendServerInfo(true); + server.setCapabilities(capabilities); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + } + + /** + * Fails when parsing a request or building an answer threw on the server. + */ + public void assertNoFailure() { + Throwable th = failure.get(); + if (th != null) { + throw new AssertionError("mock egress server failed", th); + } + } + + @Override + public void close() { + server.close(); + } + + public String getAddress() { + return "localhost:" + server.getPort(); + } + + public List getRequests() { + return requests; + } + + /** + * Number of WebSocket upgrades the server completed: the initial connect + * and each reconnect. + */ + public int handshakeCount() { + return server.handshakeCount(); + } + + /** + * Decides every new egress upgrade from its {@code Authorization} header: a + * positive return is the HTTP status the server rejects the upgrade with; + * see {@link TestWebSocketServer#setAuthorizationValidator}. Null removes it. + */ + public void setAuthorizationValidator(ToIntFunction validator) { + server.setAuthorizationValidator(validator); + } + + /** + * Runs {@code hook} on each connection right before the server sends + * {@code SERVER_INFO}; see {@link TestWebSocketServer#setBeforeServerInfoHook}. + */ + public void setBeforeServerInfoHook(Runnable hook) { + server.setBeforeServerInfoHook(hook); + } + + /** + * While set, the server drops every new egress connection before the + * upgrade completes, so connects and failover reconnects fail. + */ + public void setRejectUpgrades(boolean isRejecting) { + server.setRejectReadUpgrade(isRejecting); + } + + @FunctionalInterface + public interface Responder { + /** + * Returns the frames that answer {@code request}, or null to drop the + * connection. + */ + byte[][] respond(QwpEgressFrames.QueryRequest request); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java new file mode 100644 index 00000000..e337517d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressTerminatorDecodeTest.java @@ -0,0 +1,555 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QueryEvent; +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Decoding of the trailers the query-details flags add to the terminator + * frames: {@code row_count} and the statistics block on {@code RESULT_END}, + * the statistics block on {@code EXEC_DONE}, and {@code position} on + * {@code QUERY_ERROR}. The frames are built byte by byte in the layout of the + * server's {@code QwpEgressFrameWriter}, and decoded through the same static + * decoders the I/O thread runs, with the flags the request carried. + */ +public class QwpEgressTerminatorDecodeTest { + private static final int ALL_FLAGS = QwpEgressMsgKind.QUERY_FLAG_RESET_DICT + | QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION + | QwpEgressMsgKind.QUERY_FLAG_STATS + | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS + | QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS; + private static final int PRELUDE = QwpConstants.HEADER_SIZE + 1 + 8; + private static final long[] VARINT_VALUES = { + 0, 1, 127, 128, 16_383, 16_384, 1_000_000_000L, Long.MAX_VALUE - 1, Long.MAX_VALUE + }; + + @Test + public void testExecDoneCarriesNoRowCount() throws Exception { + // COUNT_ROWS does not apply to statements without rows: the statistics + // follow rows_affected directly, and a frame without trailers is whole. + TestUtils.assertMemoryLeak(() -> { + byte[] withStats = QwpEgressFrames.execDone(3, 2, 30).stats(11, 22, 0, 44, 1).toFrame(); + QueryEvent ev = decodeExecDone(withStats, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(30, ev.rowsAffected); + assertStats(ev.details, 11, 22, 0, 44, true); + Assert.assertFalse(ev.details.hasRowCount()); + Assert.assertEquals(-1, ev.details.getRowCount()); + + ev = decodeExecDone(QwpEgressFrames.execDone(3, 2, 30).toFrame(), QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertEquals(30, ev.rowsAffected); + RecordingQueryHandler.assertNoDetails(ev.details); + }); + } + + @Test + public void testExecDoneStats() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (long v : VARINT_VALUES) { + for (int jit = 0; jit < 2; jit++) { + byte[] frame = QwpEgressFrames.execDone(3, 7, v).stats(v, Long.MAX_VALUE - v, v / 2, v / 3, jit).toFrame(); + QueryEvent ev = decodeExecDone(frame, QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(QueryEvent.KIND_EXEC_DONE, ev.kind); + Assert.assertEquals(7, ev.opType); + Assert.assertEquals(v, ev.rowsAffected); + assertStats(ev.details, v, Long.MAX_VALUE - v, v / 2, v / 3, jit == 1); + Assert.assertFalse(ev.details.hasRowCount()); + } + } + }); + } + + @Test + public void testExecDoneTruncatedAtEveryByte() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List fields = new ArrayList<>(); + QwpEgressFrames.Body body = new QwpEgressFrames.Body(); + body.u8(QwpEgressMsgKind.EXEC_DONE).i64(3); + appendField(body, fields, "op_type", b -> b.u8(2)); + appendField(body, fields, "rows_affected", b -> b.varint(16_384)); + appendStatsFields(body, fields); + assertTruncatedAtEveryByte(body.toFrame(), fields, "EXEC_DONE", QwpEgressMsgKind.QUERY_FLAG_STATS, true); + }); + } + + @Test + public void testExecDoneVarintOverflow() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (byte[] bad : overflowingVarints()) { + byte[] frame = new QwpEgressFrames.Body().u8(QwpEgressMsgKind.EXEC_DONE).i64(3).u8(2).bytes(bad).toFrame(); + assertDecodeFails("EXEC_DONE rows_affected varint overflow", frame, 0, true); + } + for (String field : new String[]{"compile_ns", "execute_ns", "count_ns", "client_wait_ns"}) { + for (byte[] bad : overflowingVarints()) { + byte[] frame = statsWithBadField(QwpEgressFrames.execDone(3, 2, 30), field, bad).toFrame(); + assertDecodeFails("EXEC_DONE " + field + " varint overflow", frame, QwpEgressMsgKind.QUERY_FLAG_STATS, true); + } + } + }); + } + + @Test + public void testExecDoneWithoutFlagsIgnoresTrailers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.execDone(3, 2, 30).stats(11, 22, 33, 44, 1).toFrame(); + QueryEvent ev = decodeExecDone(frame, 0); + Assert.assertEquals(2, ev.opType); + Assert.assertEquals(30, ev.rowsAffected); + RecordingQueryHandler.assertNoDetails(ev.details); + }); + } + + @Test + public void testQueryErrorEmptyMessageWithPosition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, "").i32(3).toFrame(); + QueryEvent ev = decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals("", ev.errorMessage); + Assert.assertEquals(3, ev.errorPosition); + }); + } + + @Test + public void testQueryErrorMalformedWithPositionRequested() throws Exception { + // the guards that predate the position trailer hold when it is requested + TestUtils.assertMemoryLeak(() -> { + byte[] truncated = new QwpEgressFrames.Body().u8(QwpEgressMsgKind.QUERY_ERROR).i64(5).u8(5).u8(4).toFrame(); + QueryEvent ev = decodeError(truncated, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, ev.errorStatus); + Assert.assertEquals("QUERY_ERROR frame truncated before msg_len", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + + // msg_len claims the message and the position, and one byte more + byte[] overrun = new QwpEgressFrames.Body() + .u8(QwpEgressMsgKind.QUERY_ERROR).i64(5).u8(5).u16(9).bytes("boom".getBytes()).i32(7) + .toFrame(); + ev = decodeError(overrun, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, ev.errorStatus); + Assert.assertEquals("QUERY_ERROR msg_len 9 exceeds frame remainder 8", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + }); + } + + @Test + public void testQueryErrorPosition() throws Exception { + // 'é' and '€' are one char each but 2 and 3 UTF-8 bytes, and the emoji is + // two chars and 4 bytes: the message survives the trailer whole + final String message = "table does not exist [table=é€\uD83D\uDE00]"; + TestUtils.assertMemoryLeak(() -> { + for (int position : new int[]{QwpEgressMsgKind.NO_ERROR_POSITION, 0, 1, 14, 65_535, Integer.MAX_VALUE}) { + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, message).i32(position).toFrame(); + for (long flags : new long[]{QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION, ALL_FLAGS}) { + QueryEvent ev = decodeError(frame, flags); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals(message, ev.errorMessage); + Assert.assertEquals(position, ev.errorPosition); + } + } + }); + } + + @Test + public void testQueryErrorPositionFollowedByMoreBytes() throws Exception { + // bytes after the known fields are ignored, as a newer server may add some + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.queryError(5, QwpConstants.STATUS_CANCELLED, "cancelled by client") + .i32(QwpEgressMsgKind.NO_ERROR_POSITION).u8(1).u8(2).u8(3).toFrame(); + QueryEvent ev = decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QwpConstants.STATUS_CANCELLED, ev.errorStatus); + Assert.assertEquals("cancelled by client", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + + frame = QwpEgressFrames.queryError(5, QwpConstants.STATUS_CANCELLED, "cancelled by client") + .i32(42).u8(1).toFrame(); + Assert.assertEquals(42, decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION).errorPosition); + }); + } + + @Test + public void testQueryErrorPositionTrailerCutShort() throws Exception { + // 1 to 3 bytes after the message: neither a baseline frame nor a whole + // position, so the event names the defect and keeps what the server said + TestUtils.assertMemoryLeak(() -> { + for (int extra = 1; extra < Integer.BYTES; extra++) { + QwpEgressFrames.Body body = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, "Invalid column: nope"); + for (int i = 0; i < extra; i++) { + body.u8(7); + } + QueryEvent ev = decodeError(body.toFrame(), QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, ev.errorStatus); + Assert.assertEquals("QUERY_ERROR frame truncated mid position [status=5, message=Invalid column: nope]", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + } + }); + } + + @Test + public void testQueryErrorTrailerIgnoredWhenNotRequested() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, "[14] table does not exist") + .i32(14).toFrame(); + for (long flags : new long[]{0, ALL_FLAGS & ~QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION}) { + QueryEvent ev = decodeError(frame, flags); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals("[14] table does not exist", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + } + // the decoder of a request without flags + QueryEvent ev = decodeBaselineError(frame); + Assert.assertEquals("[14] table does not exist", ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + }); + } + + @Test + public void testQueryErrorWithoutTrailerWhenPositionRequested() throws Exception { + // the server answers a request it could not decode, or one it rejects + // while another query streams, with the baseline frame + TestUtils.assertMemoryLeak(() -> { + String message = "Phase 1 egress supports a single in-flight query per connection"; + byte[] frame = QwpEgressFrames.queryError(5, WebSocketResponse.STATUS_PARSE_ERROR, message).toFrame(); + QueryEvent ev = decodeError(frame, QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, ev.errorStatus); + Assert.assertEquals(message, ev.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, ev.errorPosition); + }); + } + + @Test + public void testResultEndRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (long rowCount : VARINT_VALUES) { + byte[] frame = QwpEgressFrames.resultEnd(7, 0, 5).varint(rowCount).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertEquals(QueryEvent.KIND_END, ev.kind); + Assert.assertEquals(5, ev.totalRows); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(rowCount, ev.details.getRowCount()); + Assert.assertFalse(ev.details.hasStats()); + } + }); + } + + @Test + public void testResultEndRowCountThenStats() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.resultEnd(7, 2, 50).varint(100).stats(1_500, 900_000, 300, 0, 0).toFrame(); + for (long flags : new long[]{QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS, ALL_FLAGS}) { + QueryEvent ev = decodeResultEnd(frame, flags); + Assert.assertEquals(50, ev.totalRows); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(100, ev.details.getRowCount()); + assertStats(ev.details, 1_500, 900_000, 300, 0, false); + Assert.assertEquals("QwpQueryDetails{rowCount=100, compileNanos=1500, executeNanos=900000, " + + "countNanos=300, clientWaitNanos=0, jitCompiled=false}", ev.details.toString()); + } + }); + } + + @Test + public void testResultEndStats() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (long v : VARINT_VALUES) { + for (int jit = 0; jit < 2; jit++) { + byte[] frame = QwpEgressFrames.resultEnd(7, 3, 40).stats(v, Long.MAX_VALUE - v, v / 2, v / 3, jit).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(40, ev.totalRows); + assertStats(ev.details, v, Long.MAX_VALUE - v, v / 2, v / 3, jit == 1); + Assert.assertFalse(ev.details.hasRowCount()); + Assert.assertEquals(-1, ev.details.getRowCount()); + } + } + }); + } + + @Test + public void testResultEndTrailersFollowedByMoreBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] frame = QwpEgressFrames.resultEnd(7, 2, 50).varint(100).stats(1, 2, 3, 4, 1).u8(0xFF).u8(0x80).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertEquals(100, ev.details.getRowCount()); + assertStats(ev.details, 1, 2, 3, 4, true); + }); + } + + @Test + public void testResultEndTruncatedAtEveryByte() throws Exception { + // multi-byte varints in every field, so cuts land inside each of them + TestUtils.assertMemoryLeak(() -> { + List fields = new ArrayList<>(); + QwpEgressFrames.Body body = new QwpEgressFrames.Body(); + body.u8(QwpEgressMsgKind.RESULT_END).i64(7); + appendField(body, fields, "final_seq", b -> b.varint(300)); + appendField(body, fields, "total_rows", b -> b.varint(16_384)); + appendField(body, fields, "row_count", b -> b.varint(1_000_000)); + appendStatsFields(body, fields); + assertTruncatedAtEveryByte(body.toFrame(), fields, "RESULT_END", + QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS, false); + }); + } + + @Test + public void testResultEndVarintOverflow() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (byte[] bad : overflowingVarints()) { + byte[] frame = QwpEgressFrames.body(QwpEgressMsgKind.RESULT_END, 7).bytes(bad).varint(5).toFrame(); + assertDecodeFails("RESULT_END final_seq varint overflow", frame, 0, false); + frame = QwpEgressFrames.body(QwpEgressMsgKind.RESULT_END, 7).varint(0).bytes(bad).toFrame(); + assertDecodeFails("RESULT_END total_rows varint overflow", frame, 0, false); + frame = QwpEgressFrames.resultEnd(7, 0, 5).bytes(bad).toFrame(); + assertDecodeFails("RESULT_END row_count varint overflow", frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS, false); + } + for (String field : new String[]{"compile_ns", "execute_ns", "count_ns", "client_wait_ns"}) { + for (byte[] bad : overflowingVarints()) { + byte[] frame = statsWithBadField(QwpEgressFrames.resultEnd(7, 0, 5), field, bad).toFrame(); + assertDecodeFails("RESULT_END " + field + " varint overflow", frame, QwpEgressMsgKind.QUERY_FLAG_STATS, false); + } + } + }); + } + + @Test + public void testResultEndVarintUsesAll64Bits() throws Exception { + // a tenth byte of 0x01 sets bit 63, the last bit a long holds + TestUtils.assertMemoryLeak(() -> { + byte[] bit63 = {(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}; + byte[] frame = QwpEgressFrames.resultEnd(7, 0, 5).bytes(bit63).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertTrue(ev.details.hasRowCount()); + Assert.assertEquals(Long.MIN_VALUE, ev.details.getRowCount()); + }); + } + + @Test + public void testResultEndWithoutFlagsIgnoresTrailers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] baseline = QwpEgressFrames.resultEnd(7, 2, 30).toFrame(); + byte[] withTrailers = QwpEgressFrames.resultEnd(7, 2, 30).varint(100).stats(1, 2, 3, 4, 1).toFrame(); + for (byte[] frame : new byte[][]{baseline, withTrailers}) { + for (long flags : new long[]{0, ALL_FLAGS & ~(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_STATS)}) { + QueryEvent ev = decodeResultEnd(frame, flags); + Assert.assertEquals(QueryEvent.KIND_END, ev.kind); + Assert.assertEquals(30, ev.totalRows); + RecordingQueryHandler.assertNoDetails(ev.details); + } + } + // COUNT_ROWS alone reads the row count and leaves what follows alone + QueryEvent ev = decodeResultEnd(withTrailers, QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS); + Assert.assertEquals(100, ev.details.getRowCount()); + Assert.assertFalse(ev.details.hasStats()); + }); + } + + @Test + public void testStatsFlagsReservedBitsIgnored() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int[] statsFlags = {0x00, 0x01, 0x02, 0x80, 0x81, 0xFE, 0xFF}; + for (int flags : statsFlags) { + byte[] frame = QwpEgressFrames.resultEnd(7, 0, 1).stats(1, 2, 3, 4, flags).toFrame(); + QueryEvent ev = decodeResultEnd(frame, QwpEgressMsgKind.QUERY_FLAG_STATS); + Assert.assertTrue(ev.details.hasStats()); + Assert.assertEquals("stats_flags 0x" + Integer.toHexString(flags), + (flags & QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED) != 0, ev.details.isJitCompiled()); + } + }); + } + + private static void appendField(QwpEgressFrames.Body body, List fields, String field, FieldWriter writer) { + int before = body.size(); + writer.write(body); + for (int i = before; i < body.size(); i++) { + fields.add(field); + } + } + + private static void appendStatsFields(QwpEgressFrames.Body body, List fields) { + appendField(body, fields, "compile_ns", b -> b.varint(128)); + appendField(body, fields, "execute_ns", b -> b.varint(Long.MAX_VALUE)); + appendField(body, fields, "count_ns", b -> b.varint(0)); + appendField(body, fields, "client_wait_ns", b -> b.varint(200)); + appendField(body, fields, "stats_flags", b -> b.u8(QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED)); + } + + private static void assertDecodeFails(String expectedMessage, byte[] frame, long flags, boolean isExecDone) throws Exception { + try { + if (isExecDone) { + decodeExecDone(frame, flags); + } else { + decodeResultEnd(frame, flags); + } + Assert.fail("decode must fail: " + expectedMessage); + } catch (QwpDecodeException e) { + Assert.assertEquals(expectedMessage, e.getMessage()); + } + } + + private static void assertStats( + QwpQueryDetails details, + long compileNanos, + long executeNanos, + long countNanos, + long clientWaitNanos, + boolean isJitCompiled + ) { + Assert.assertTrue(details.hasStats()); + Assert.assertEquals(compileNanos, details.getCompileNanos()); + Assert.assertEquals(executeNanos, details.getExecuteNanos()); + Assert.assertEquals(countNanos, details.getCountNanos()); + Assert.assertEquals(clientWaitNanos, details.getClientWaitNanos()); + Assert.assertEquals(isJitCompiled, details.isJitCompiled()); + } + + /** + * Cuts {@code frame} after every byte from the end of the prelude to the + * last but one: each cut lands in the field {@code fields} names for the + * first missing byte, and the decode fails naming that field. + */ + private static void assertTruncatedAtEveryByte( + byte[] frame, + List fields, + String frameName, + long flags, + boolean isExecDone + ) throws Exception { + Assert.assertEquals(PRELUDE + fields.size(), frame.length); + for (int len = PRELUDE; len < frame.length; len++) { + String field = fields.get(len - PRELUDE); + boolean isByteField = "op_type".equals(field) || "stats_flags".equals(field); + String expected = frameName + " frame truncated " + (isByteField ? "before " + field : "mid " + field + " varint"); + assertDecodeFails(expected, Arrays.copyOf(frame, len), flags, isExecDone); + } + // the whole frame decodes + if (isExecDone) { + decodeExecDone(frame, flags); + } else { + decodeResultEnd(frame, flags); + } + } + + private static QueryEvent decodeBaselineError(byte[] frame) { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeError(buf, frame.length); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static QueryEvent decodeError(byte[] frame, long flags) { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeError(buf, frame.length, flags); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static QueryEvent decodeExecDone(byte[] frame, long flags) throws QwpDecodeException { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeExecDone(buf, frame.length, flags); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static QueryEvent decodeResultEnd(byte[] frame, long flags) throws QwpDecodeException { + long buf = toNative(frame); + try { + return QwpEgressIoThread.decodeResultEnd(buf, frame.length, flags); + } finally { + Unsafe.free(buf, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + /** + * Varints that do not fit in 64 bits: a continuation bit on the tenth + * byte, whose data bit still fits, and data bits above bit 63 on the + * tenth byte. + */ + private static byte[][] overflowingVarints() { + byte[] continued = new byte[11]; + Arrays.fill(continued, (byte) 0x80); + continued[9] = (byte) 0x81; + continued[10] = 0x00; + byte[] highBits = new byte[10]; + Arrays.fill(highBits, (byte) 0x80); + highBits[9] = 0x02; + byte[] allHighBits = new byte[10]; + Arrays.fill(allHighBits, (byte) 0xFF); + allHighBits[9] = 0x7F; + return new byte[][]{continued, highBits, allHighBits}; + } + + /** + * Appends a statistics block whose {@code field} carries {@code bad} and + * whose other fields are well-formed. + */ + private static QwpEgressFrames.Body statsWithBadField(QwpEgressFrames.Body body, String field, byte[] bad) { + String[] names = {"compile_ns", "execute_ns", "count_ns", "client_wait_ns"}; + for (String name : names) { + if (name.equals(field)) { + body.bytes(bad); + } else { + body.varint(1); + } + } + return body.u8(0); + } + + private static long toNative(byte[] frame) { + long buf = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(buf + i, frame[i]); + } + return buf; + } + + @FunctionalInterface + private interface FieldWriter { + void write(QwpEgressFrames.Body body); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java new file mode 100644 index 00000000..e42f0870 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFailurePathsTest.java @@ -0,0 +1,335 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.auth.TokenUnavailableException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * The failures {@link QwpQueryClient#execute} detects itself -- a failover + * reconnect that fails or meets a rejected or unavailable credential, a + * failover that runs out of attempts, a stop request or a {@code close()} + * between attempts -- reach the handler through the position-aware + * {@code onError(requestId, status, message, position)} with + * {@link QwpEgressMsgKind#NO_ERROR_POSITION}, like the server's errors, so a + * handler that overrides only that overload sees them all. Each query asks for + * the details; the first one dies with its connection, which sends + * {@code execute()} down its failover path. + */ +public class QwpQueryClientFailurePathsTest { + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; + private static final QwpQueryOptions OPTIONS = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true); + private static final String SQL = "SELECT x FROM t"; + + @Test(timeout = 30_000) + public void testAuthRejectedFailoverReportsNoPosition() throws Exception { + // the server dies with the first query and rejects the credential of + // the failover reconnect + TestUtils.assertMemoryLeak(() -> { + AtomicReference serverRef = new AtomicReference<>(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> { + serverRef.get().setAuthorizationValidator(authorization -> 401); + return null; + }); + QwpQueryClient client = newClient(server)) { + serverRef.set(server); + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertTrue(handler.errorMessage, + handler.errorMessage.startsWith("auth-rejected during failover reconnect [host=localhost:")); + TestUtils.assertContains(handler.errorMessage, ", status=401, last error: "); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("the failover reconnect must not complete", 1, server.handshakeCount()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testCloseBetweenAttemptsReportsClosed() throws Exception { + // close() from the failover callback: the next attempt finds no I/O + // thread and ends the query instead of touching the freed client + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler() { + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + super.onFailoverReset(requestId, newNode); + client.close(); + } + }; + client.execute(SQL, handler, OPTIONS); + Assert.assertEquals(1, handler.failoverResetCount); + handler.assertOnlyError(); + Assert.assertEquals(-1, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("QwpQueryClient is closed", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertFalse(client.wasLastCloseTimedOut()); + Assert.assertEquals("the closed client must not replay the query", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testCredentialUnavailableFailoverReportsNoPosition() throws Exception { + // the token provider fails when the failover reconnect asks it for a + // credential, so the reconnect contacts no endpoint + TestUtils.assertMemoryLeak(() -> { + AtomicInteger tokenPulls = new AtomicInteger(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server).withBearerTokenProvider(() -> { + if (tokenPulls.incrementAndGet() > 1) { + throw TokenUnavailableException.retryable("IdP unreachable"); + } + return "TOKEN"; + })) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertTrue(handler.errorMessage, handler.errorMessage.startsWith("failover reconnect failed: ")); + TestUtils.assertContains(handler.errorMessage, "IdP unreachable"); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("the failover reconnect must contact no endpoint", 1, server.handshakeCount()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testExhaustedFailoverReportsNoPosition() throws Exception { + // failover_max_attempts=1 leaves no attempt for a failover: the query + // dies with its connection, and the next query hits the latched failure + TestUtils.assertMemoryLeak(() -> { + final String failure = "transport failure after 1 execute attempt (0 failover reconnects); last error: "; + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpQueryClient client = newClient(server, "failover_max_attempts=1;")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertTrue(handler.errorMessage, handler.errorMessage.startsWith(failure)); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + + RecordingQueryHandler latched = new RecordingQueryHandler(); + client.execute(SQL, latched, OPTIONS); + latched.assertOnlyError(); + Assert.assertEquals(-1, latched.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, latched.errorStatus); + Assert.assertTrue(latched.errorMessage, latched.errorMessage.startsWith(failure)); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, latched.errorPosition); + Assert.assertEquals("no failover reconnect", 1, server.handshakeCount()); + Assert.assertEquals("the latched failure sends nothing", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testFailedReconnectsReportNoPosition() throws Exception { + // The server dies with the first query and then refuses connections: the + // failover reconnect fails, and so does the reconnect of the next query. + // Once it accepts connections again, the client recovers. + TestUtils.assertMemoryLeak(() -> { + AtomicReference serverRef = new AtomicReference<>(); + AtomicInteger queries = new AtomicInteger(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> { + if (queries.getAndIncrement() == 0) { + serverRef.get().setRejectUpgrades(true); + return null; + } + return answer(r); + }); + QwpQueryClient client = newClient(server)) { + serverRef.set(server); + client.connect(); + + RecordingQueryHandler failover = new RecordingQueryHandler(); + client.execute(SQL, failover, OPTIONS); + failover.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, failover.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, failover.errorStatus); + TestUtils.assertContains(failover.errorMessage, "failover reconnect failed after 1 attempt [last error: "); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, failover.errorPosition); + Assert.assertFalse(client.isConnected()); + + RecordingQueryHandler reconnect = new RecordingQueryHandler(); + client.execute(SQL, reconnect, OPTIONS); + reconnect.assertOnlyError(); + Assert.assertEquals(-1, reconnect.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, reconnect.errorStatus); + Assert.assertTrue(reconnect.errorMessage, reconnect.errorMessage.startsWith("reconnect failed: ")); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, reconnect.errorPosition); + + server.setRejectUpgrades(false); + RecordingQueryHandler recovered = new RecordingQueryHandler(); + client.execute(SQL, recovered, OPTIONS); + Assert.assertEquals(1, recovered.endCount); + Assert.assertEquals(Collections.singletonList(42L), recovered.values); + Assert.assertEquals(1, recovered.rowCount); + Assert.assertTrue(recovered.hasStats); + Assert.assertEquals(2, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testShutdownAfterTransportFailureReportsNoPosition() throws Exception { + // a stop request that lands right after the connection dies ends the + // query instead of failing over + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpQueryClient client = newClient(server)) { + client.connect(); + client.setAfterTransportFailureHookForTest(client::requestShutdown); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(0, handler.failoverResetCount); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("query client shut down", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("no failover reconnect after the stop request", 1, server.handshakeCount()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testShutdownBetweenAttemptsReportsNoPosition() throws Exception { + // a stop request from the failover callback ends the query before the + // replay + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler() { + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + super.onFailoverReset(requestId, newNode); + client.requestShutdown(); + } + }; + client.execute(SQL, handler, OPTIONS); + Assert.assertEquals(1, handler.failoverResetCount); + handler.assertOnlyError(); + Assert.assertEquals(-1, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("query client shut down", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals("the query must not be replayed", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + @Test(timeout = 30_000) + public void testShutdownDuringReconnectReportsNoPosition() throws Exception { + // the stop request arrives while the failover reconnect waits for + // SERVER_INFO: the reconnect completes, then the query ends + TestUtils.assertMemoryLeak(() -> { + AtomicReference clientRef = new AtomicReference<>(); + AtomicInteger serverInfos = new AtomicInteger(); + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, dropFirstQuery()); + QwpQueryClient client = newClient(server)) { + clientRef.set(client); + server.setBeforeServerInfoHook(() -> { + if (serverInfos.incrementAndGet() == 2) { + clientRef.get().requestShutdown(); + } + }); + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, OPTIONS); + handler.assertOnlyError(); + Assert.assertEquals(0, handler.failoverResetCount); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("query client shut down", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertEquals(2, server.handshakeCount()); + Assert.assertEquals("the query must not be replayed", 1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + + /** + * One LONG row, x = 42, and the trailers the request asks for. + */ + private static byte[][] answer(QwpEgressFrames.QueryRequest r) { + QwpEgressFrames.Body end = QwpEgressFrames.resultEnd(r.requestId, 0, 1); + if (r.has(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS)) { + end.varint(1); + } + if (r.has(QwpEgressMsgKind.QUERY_FLAG_STATS)) { + end.stats(1, 2, 3, 4, 0); + } + return new byte[][]{QwpEgressFrames.resultBatchOfLongs(r.requestId, 0, 42), end.toFrame()}; + } + + /** + * Drops the connection on the first query, then answers like + * {@link #answer}. + */ + private static QwpEgressMockServer.Responder dropFirstQuery() { + AtomicInteger queries = new AtomicInteger(); + return r -> queries.getAndIncrement() == 0 ? null : answer(r); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server) { + return newClient(server, ""); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server, String extraConfig) { + return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;" + extraConfig); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java new file mode 100644 index 00000000..dc010902 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryDetailsTest.java @@ -0,0 +1,518 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * End-to-end coverage of {@link QwpQueryOptions} through + * {@link QwpQueryClient#execute}: the I/O thread decodes the trailers the + * request asked for and the handler receives them. The mock server models the + * server's {@code QwpEgressUpgradeProcessor}: a 10-row table {@code x = 0..9} + * streamed in batches of up to 3 rows, cut by the row window; trailers only + * for the flags the request carries; the bare message and the position of an + * error only under {@code QUERY_FLAG_ERROR_POSITION}. The questdb repo checks + * the same against a live server. + */ +public class QwpQueryClientQueryDetailsTest { + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; + private static final long CLIENT_WAIT_NANOS = 44; + private static final long COMPILE_NANOS = 11; + private static final long COUNT_NANOS = 33; + private static final String ERROR_MESSAGE = "Invalid column: nope"; + private static final int ERROR_POSITION = 7; + private static final long EXECUTE_NANOS = 22; + private static final int OP_INSERT = 2; + private static final long ROWS_AFFECTED = 5; + private static final int TABLE_ROWS = 10; + + @Test + public void testDetailsAbsentWithoutCapability() throws Exception { + // the client leaves out what the server cannot report, and the server + // answers with the baseline frames + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(QwpEgressMsgKind.CAP_QUERY_FLAGS, + QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true); + + RecordingQueryHandler select = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", select, options); + Assert.assertEquals(1, select.endCount); + Assert.assertEquals(TABLE_ROWS, select.totalRows); + select.assertNoDetails(); + + RecordingQueryHandler insert = new RecordingQueryHandler(); + client.execute("INSERT INTO t VALUES (1)", insert, options); + Assert.assertEquals(1, insert.execDoneCount); + insert.assertNoDetails(); + + RecordingQueryHandler failed = new RecordingQueryHandler(); + client.execute("FAIL", failed, options); + failed.assertOnlyError(); + Assert.assertEquals("[7] " + ERROR_MESSAGE, failed.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, failed.errorPosition); + + for (QwpEgressFrames.QueryRequest r : server.getRequests()) { + Assert.assertFalse(r.toString(), r.hasQueryFlags); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testDetailsDoNotLeakToTheNextQuery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true).maxRows(2); + for (int i = 0; i < 2; i++) { + RecordingQueryHandler detailed = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", detailed, options); + Assert.assertEquals(Arrays.asList(0L, 1L), detailed.values); + Assert.assertEquals(TABLE_ROWS, detailed.rowCount); + Assert.assertTrue(detailed.hasStats); + + RecordingQueryHandler plain = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", plain); + Assert.assertEquals(TABLE_ROWS, plain.values.size()); + Assert.assertEquals(TABLE_ROWS, plain.totalRows); + plain.assertNoDetails(); + + RecordingQueryHandler failed = new RecordingQueryHandler(); + client.execute("FAIL", failed, options); + Assert.assertEquals(ERROR_POSITION, failed.errorPosition); + + RecordingQueryHandler plainFailed = new RecordingQueryHandler(); + client.execute("FAIL", plainFailed); + Assert.assertEquals("[7] " + ERROR_MESSAGE, plainFailed.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, plainFailed.errorPosition); + + RecordingQueryHandler plainInsert = new RecordingQueryHandler(); + client.execute("INSERT INTO t VALUES (1)", plainInsert); + plainInsert.assertNoDetails(); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testErrorPositionReachesOnError() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("FAIL", handler, new QwpQueryOptions().errorPosition(true)); + handler.assertOnlyError(); + Assert.assertEquals(server.getRequests().get(0).requestId, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, handler.errorStatus); + Assert.assertEquals(ERROR_MESSAGE, handler.errorMessage); + Assert.assertEquals(ERROR_POSITION, handler.errorPosition); + server.assertNoFailure(); + } + }); + } + + @Test + public void testFailoverChecksTheNewServerAgain() throws Exception { + // the first node dies mid-query; the replay goes to a node without + // CAP_QUERY_DETAILS, which gets no detail flags and fails a row window + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer dying = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpEgressMockServer old = new QwpEgressMockServer(QwpEgressMsgKind.CAP_QUERY_FLAGS, + QwpQueryClientQueryDetailsTest::model)) { + String config = "ws::addr=" + dying.getAddress() + ',' + old.getAddress() + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(config)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, new QwpQueryOptions().stats(true).countRows(true)); + Assert.assertEquals(1, handler.failoverResetCount); + Assert.assertEquals(1, handler.endCount); + Assert.assertEquals(TABLE_ROWS, handler.values.size()); + handler.assertNoDetails(); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_STATS | QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS, + dying.getRequests().get(0).queryFlags); + Assert.assertFalse(old.getRequests().get(0).hasQueryFlags); + } + try (QwpQueryClient client = QwpQueryClient.fromConfig(config)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, new QwpQueryOptions().maxRows(2)); + Assert.assertEquals(1, handler.failoverResetCount); + handler.assertOnlyError(); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("server does not support skipRows and maxRows [capabilities=0x2]; " + + "they need a server that advertises CAP_QUERY_DETAILS", handler.errorMessage); + Assert.assertEquals(2, dying.getRequests().size()); + Assert.assertEquals(2, dying.getRequests().get(1).maxRows); + Assert.assertEquals("the replay must not reach the old node", 1, old.getRequests().size()); + } + dying.assertNoFailure(); + old.assertNoFailure(); + } + }); + } + + @Test + public void testFailoverReplaysTheOptionsTheQueryStartedWith() throws Exception { + // the client copies the options when execute() starts: a change during + // the query does not reach the replay after a failover + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer dying = new QwpEgressMockServer(ALL_CAPABILITIES, r -> null); + QwpEgressMockServer healthy = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model)) { + String config = "ws::addr=" + dying.getAddress() + ',' + healthy.getAddress() + + ";auth_timeout_ms=2000;failover_backoff_initial_ms=0;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(config)) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().skipRows(4).maxRows(3).countRows(true).stats(true); + RecordingQueryHandler handler = new RecordingQueryHandler() { + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + super.onFailoverReset(requestId, newNode); + options.clear(); + } + }; + client.execute("SELECT x FROM t", handler, options); + Assert.assertEquals(1, handler.failoverResetCount); + Assert.assertEquals(Arrays.asList(4L, 5L, 6L), handler.values); + Assert.assertEquals(3, handler.totalRows); + Assert.assertEquals(TABLE_ROWS, handler.rowCount); + Assert.assertTrue(handler.hasStats); + QwpEgressFrames.QueryRequest replay = healthy.getRequests().get(0); + Assert.assertEquals(4, replay.skipRows); + Assert.assertEquals(3, replay.maxRows); + } + dying.assertNoFailure(); + healthy.assertNoFailure(); + } + }); + } + + @Test + public void testFlaggedRequestAnsweredWithBaselineError() throws Exception { + // the server answers a request it could not decode with the baseline + // frame, although the request asked for the position + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, r -> new byte[][]{ + QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, + "incomplete varint: buffer underflow").toFrame() + }); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, new QwpQueryOptions().errorPosition(true).maxRows(1)); + handler.assertOnlyError(); + Assert.assertEquals(WebSocketResponse.STATUS_PARSE_ERROR, handler.errorStatus); + Assert.assertEquals("incomplete varint: buffer underflow", handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + Assert.assertTrue("the connection stays usable", client.isConnected()); + + RecordingQueryHandler next = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", next, new QwpQueryOptions().errorPosition(true)); + Assert.assertEquals(1, next.errorCount); + server.assertNoFailure(); + } + }); + } + + @Test + public void testLegacyHandlerOverloadsStillFire() throws Exception { + // a handler written before the details overloads gets every callback + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().countRows(true).stats(true).errorPosition(true).skipRows(1); + List calls = new ArrayList<>(); + QwpColumnBatchHandler legacy = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + calls.add("onEnd " + totalRows); + } + + @Override + public void onError(byte status, String message) { + calls.add("onError " + status + ' ' + message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + calls.add("onExecDone " + opType + ' ' + rowsAffected); + } + }; + client.execute("SELECT x FROM t", legacy, options); + client.execute("INSERT INTO t VALUES (1)", legacy, options); + client.execute("FAIL", legacy, options); + Assert.assertEquals(Arrays.asList( + "onEnd 9", + "onExecDone 2 5", + "onError 5 " + ERROR_MESSAGE + ), calls); + + List requestIds = new ArrayList<>(); + QwpColumnBatchHandler correlating = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + Assert.fail(); + } + + @Override + public void onEnd(long requestId, long totalRows) { + requestIds.add("onEnd " + requestId); + } + + @Override + public void onError(byte status, String message) { + Assert.fail(); + } + + @Override + public void onError(long requestId, byte status, String message) { + requestIds.add("onError " + requestId); + } + + @Override + public void onExecDone(long requestId, short opType, long rowsAffected) { + requestIds.add("onExecDone " + requestId); + } + }; + client.execute("SELECT x FROM t", correlating, options); + client.execute("INSERT INTO t VALUES (1)", correlating, options); + client.execute("FAIL", correlating, options); + List requests = server.getRequests(); + Assert.assertEquals(Arrays.asList( + "onEnd " + requests.get(3).requestId, + "onExecDone " + requests.get(4).requestId, + "onError " + requests.get(5).requestId + ), requestIds); + server.assertNoFailure(); + } + }); + } + + @Test + public void testMissingTrailerIsTransportFailure() throws Exception { + // A server that advertises CAP_QUERY_DETAILS must send the trailers the + // request asks for; a terminator without them is a broken stream. + Object[][] cases = { + {"SELECT x FROM t", new QwpQueryOptions().countRows(true), "RESULT_END frame truncated mid row_count varint"}, + {"SELECT x FROM t", new QwpQueryOptions().stats(true), "RESULT_END frame truncated mid compile_ns varint"}, + {"INSERT INTO t VALUES (1)", new QwpQueryOptions().stats(true), "EXEC_DONE frame truncated mid compile_ns varint"}, + }; + for (Object[] c : cases) { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, + r -> model(r, 0)); + QwpQueryClient client = newClient(server, "failover=off;")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute((String) c[0], handler, (QwpQueryOptions) c[1]); + handler.assertOnlyError(); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals(c[2], handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + server.assertNoFailure(); + } + }); + } + } + + @Test + public void testReentrantExecuteWithOptionsRejected() throws Exception { + // one query at a time: an execute() from inside a handler fails, its + // exception leaves the outer execute(), and the client stays usable + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + QwpQueryOptions options = new QwpQueryOptions().maxRows(1).stats(true); + RecordingQueryHandler reentrant = new RecordingQueryHandler() { + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + super.onEnd(requestId, totalRows, details); + client.execute("SELECT x FROM t", new RecordingQueryHandler(), options); + } + }; + try { + client.execute("SELECT x FROM t", reentrant, options); + Assert.fail("a nested execute() must throw"); + } catch (IllegalStateException e) { + Assert.assertEquals("QwpQueryClient.execute called while another execute is in flight; " + + "one query at a time per client", e.getMessage()); + } + Assert.assertEquals(1, reentrant.endCount); + Assert.assertEquals("the nested query must not reach the server", 1, server.getRequests().size()); + + RecordingQueryHandler next = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", next, options); + Assert.assertEquals(Arrays.asList(0L), next.values); + Assert.assertTrue(next.hasStats); + server.assertNoFailure(); + } + }); + } + + @Test + public void testRowWindowCountAndStatsReachOnEnd() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + // {skip, max, expected first value, expected rows} + long[][] windows = {{3, 4, 3, 4}, {0, 10, 0, 10}, {8, 5, 8, 2}, {10, 1, 0, 0}, {0, 0, 0, 0}, {1, Long.MAX_VALUE, 1, 9}}; + for (long[] w : windows) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t", handler, + new QwpQueryOptions().skipRows(w[0]).maxRows(w[1]).countRows(true).stats(true)); + String msg = "skip=" + w[0] + ", max=" + w[1]; + Assert.assertEquals(msg, 1, handler.endCount); + Assert.assertEquals(msg, w[3], handler.values.size()); + for (int i = 0; i < w[3]; i++) { + Assert.assertEquals(msg, w[2] + i, (long) handler.values.get(i)); + } + Assert.assertEquals(msg, w[3], handler.totalRows); + Assert.assertTrue(msg, handler.hasRowCount); + Assert.assertEquals(msg, TABLE_ROWS, handler.rowCount); + Assert.assertTrue(msg, handler.hasStats); + Assert.assertEquals(COMPILE_NANOS, handler.compileNanos); + Assert.assertEquals(EXECUTE_NANOS, handler.executeNanos); + Assert.assertEquals(COUNT_NANOS, handler.countNanos); + Assert.assertEquals(CLIENT_WAIT_NANOS, handler.clientWaitNanos); + Assert.assertTrue(handler.isJitCompiled); + Assert.assertEquals(server.getRequests().get(server.getRequests().size() - 1).requestId, handler.endRequestId); + } + server.assertNoFailure(); + } + }); + } + + @Test + public void testStatsReachOnExecDone() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryDetailsTest::model); + QwpQueryClient client = newClient(server, "")) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + // the row options do not apply to a statement without rows + client.execute("INSERT INTO t VALUES (1)", handler, + new QwpQueryOptions().stats(true).countRows(true).skipRows(1).maxRows(1)); + Assert.assertEquals(1, handler.execDoneCount); + Assert.assertEquals(OP_INSERT, handler.opType); + Assert.assertEquals(ROWS_AFFECTED, handler.rowsAffected); + Assert.assertFalse(handler.hasRowCount); + Assert.assertTrue(handler.hasStats); + Assert.assertEquals(COMPILE_NANOS, handler.compileNanos); + Assert.assertEquals(EXECUTE_NANOS, handler.executeNanos); + Assert.assertEquals(COUNT_NANOS, handler.countNanos); + Assert.assertEquals(CLIENT_WAIT_NANOS, handler.clientWaitNanos); + Assert.assertTrue(handler.isJitCompiled); + server.assertNoFailure(); + } + }); + } + + private static byte[][] model(QwpEgressFrames.QueryRequest r) { + return model(r, r.queryFlags); + } + + /** + * Answers {@code r} the way the server does, with the trailers of + * {@code trailerFlags}: a {@code SELECT} streams the row window of the + * 10-row table, an {@code INSERT} reports {@value #ROWS_AFFECTED} rows, and + * anything else fails at position {@value #ERROR_POSITION}. + */ + private static byte[][] model(QwpEgressFrames.QueryRequest r, long trailerFlags) { + List frames = new ArrayList<>(); + if (r.sql.startsWith("SELECT")) { + long skip = r.skipRows < 0 ? 0 : r.skipRows; + long max = r.maxRows < 0 ? Long.MAX_VALUE : r.maxRows; + long first = Math.min(skip, TABLE_ROWS); + long end = first + Math.min(max, TABLE_ROWS - first); + long seq = 0; + long x = first; + do { + long n = Math.min(3, end - x); + long[] values = new long[(int) n]; + for (int i = 0; i < n; i++) { + values[i] = x++; + } + frames.add(QwpEgressFrames.resultBatchOfLongs(r.requestId, seq++, values)); + } while (x < end); + QwpEgressFrames.Body end0 = QwpEgressFrames.resultEnd(r.requestId, seq - 1, end - first); + if ((trailerFlags & QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS) != 0) { + end0.varint(TABLE_ROWS); + } + if ((trailerFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + end0.stats(COMPILE_NANOS, EXECUTE_NANOS, COUNT_NANOS, CLIENT_WAIT_NANOS, QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED); + } + frames.add(end0.toFrame()); + } else if (r.sql.startsWith("INSERT")) { + QwpEgressFrames.Body done = QwpEgressFrames.execDone(r.requestId, OP_INSERT, ROWS_AFFECTED); + if ((trailerFlags & QwpEgressMsgKind.QUERY_FLAG_STATS) != 0) { + done.stats(COMPILE_NANOS, EXECUTE_NANOS, COUNT_NANOS, CLIENT_WAIT_NANOS, QwpEgressMsgKind.STATS_FLAG_JIT_COMPILED); + } + frames.add(done.toFrame()); + } else if ((r.queryFlags & QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION) != 0) { + frames.add(QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, ERROR_MESSAGE) + .i32(ERROR_POSITION).toFrame()); + } else { + frames.add(QwpEgressFrames.queryError(r.requestId, WebSocketResponse.STATUS_PARSE_ERROR, + "[" + ERROR_POSITION + "] " + ERROR_MESSAGE).toFrame()); + } + return frames.toArray(new byte[0][]); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server, String extraConfig) { + return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() + ";auth_timeout_ms=2000;" + extraConfig); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java index 07a1643c..41e7ad9f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientQueryFlagsTest.java @@ -28,8 +28,11 @@ import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -37,6 +40,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -58,9 +62,16 @@ * The decode -> dict-reset effect on the server, and the end-to-end honouring * of the flag, are covered against a live server in the questdb repo * (QwpEgressQueryFlagsResetTest, QwpEgressCacheResetWireTest). + *

+ * The {@link QwpQueryOptions} tests pin the query-details bits and the + * row-window values the server's {@code QwpEgressRequestDecoder} reads after + * the flags: each option sets its own bit, {@code skip_rows} and + * {@code max_rows} follow the flags in that order, a server without + * {@link QwpEgressMsgKind#CAP_QUERY_DETAILS} gets none of them, and a row + * window it would ignore fails the query before it is sent. */ public class QwpQueryClientQueryFlagsTest { - + private static final int ALL_CAPABILITIES = QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_DETAILS; private static final QwpColumnBatchHandler NOOP_HANDLER = new QwpColumnBatchHandler() { @Override public void onBatch(QwpColumnBatch batch) { @@ -74,6 +85,58 @@ public void onEnd(long totalRows) { public void onError(byte status, String message) { } }; + private static final String SQL = "SELECT 1"; + + @Test + public void testDetailOptionsDroppedWithoutCapability() throws Exception { + // the query runs without what the server cannot report + QwpQueryOptions options = new QwpQueryOptions().errorPosition(true).stats(true).countRows(true).resetSymbolDict(true); + QwpEgressFrames.QueryRequest r = executeOnce(QwpEgressMsgKind.CAP_QUERY_FLAGS, options); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, r.queryFlags); + Assert.assertEquals(0, r.trailingBytes); + + // a server with per-query timeouts, but without the query details + r = executeOnce(QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_TIMEOUT, options); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, r.queryFlags); + Assert.assertEquals(0, r.trailingBytes); + + r = executeOnce(0, options); + Assert.assertFalse("a server without CAP_QUERY_FLAGS gets no trailer", r.hasQueryFlags); + Assert.assertEquals(0, r.trailingBytes); + } + + @Test + public void testEachBooleanOptionSendsItsFlag() throws Exception { + Object[][] cases = { + {new QwpQueryOptions().errorPosition(true), QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION}, + {new QwpQueryOptions().stats(true), QwpEgressMsgKind.QUERY_FLAG_STATS}, + {new QwpQueryOptions().countRows(true), QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS}, + {new QwpQueryOptions().resetSymbolDict(true), QwpEgressMsgKind.QUERY_FLAG_RESET_DICT}, + }; + for (Object[] c : cases) { + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, (QwpQueryOptions) c[0]); + Assert.assertEquals(c[0].toString(), (long) (int) c[1], r.queryFlags); + Assert.assertEquals(-1, r.skipRows); + Assert.assertEquals(-1, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + } + + @Test + public void testEveryOptionOnTheWire() throws Exception { + QwpQueryOptions options = new QwpQueryOptions() + .resetSymbolDict(true).errorPosition(true).stats(true).countRows(true) + .skipRows(300).maxRows(16_384); + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, options); + // every bit but 0x02, which per-query timeouts own + Assert.assertEquals(0x7D, r.queryFlags); + Assert.assertEquals(300, r.skipRows); + Assert.assertEquals(16_384, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + // flags, skip_rows, max_rows: the last bytes of the request, in that order + byte[] tail = new QwpEgressFrames.Body().varint(0x7D).varint(300).varint(16_384).toBody(); + Assert.assertArrayEquals(tail, Arrays.copyOfRange(r.payload, r.payload.length - tail.length, r.payload.length)); + } @Test public void testFlagDroppedWhenServerLacksCapability() throws Exception { @@ -94,6 +157,79 @@ public void testLegacyTwoArgOverloadSendsNoTrailer() throws Exception { Assert.assertEquals(-1L, queryFlagsTrailer(frame)); } + @Test + public void testMaxRowsOnTheWire() throws Exception { + // zero streams only the schema, so it goes on the wire like any bound + for (long maxRows : new long[]{0, 1, 127, 128, 16_383, 16_384, Long.MAX_VALUE - 1}) { + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().maxRows(maxRows)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(-1, r.skipRows); + Assert.assertEquals(maxRows, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + // the unbounded default sends nothing + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().maxRows(5).maxRows(Long.MAX_VALUE)); + Assert.assertFalse(r.hasQueryFlags); + Assert.assertEquals(0, r.trailingBytes); + } + + @Test + public void testOptionsFollowBinds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute("SELECT x FROM t WHERE x > $1 AND x < $2", binds -> binds.setLong(0, -7).setLong(1, Long.MAX_VALUE), + handler, new QwpQueryOptions().countRows(true).skipRows(2).maxRows(3)); + Assert.assertEquals(1, handler.execDoneCount); + server.assertNoFailure(); + QwpEgressFrames.QueryRequest r = server.getRequests().get(0); + Assert.assertArrayEquals(new long[]{-7, Long.MAX_VALUE}, r.longBinds); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS | QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS + | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(2, r.skipRows); + Assert.assertEquals(3, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + }); + } + + @Test + public void testOptionsWithNothingSetAreTheBaseline() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(ALL_CAPABILITIES, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler); + client.execute(SQL, handler, new QwpQueryOptions()); + client.execute(SQL, handler, (QwpQueryOptions) null); + client.execute(SQL, null, handler, new QwpQueryOptions().skipRows(5).maxRows(5).clear()); + Assert.assertEquals(4, handler.execDoneCount); + server.assertNoFailure(); + List requests = server.getRequests(); + Assert.assertEquals(4, requests.size()); + byte[] baseline = requests.get(0).payload; + zeroRequestId(baseline); + for (int i = 1; i < requests.size(); i++) { + byte[] payload = requests.get(i).payload; + zeroRequestId(payload); + Assert.assertArrayEquals("request " + i, baseline, payload); + } + } + }); + } + + @Test + public void testResetDictDroppedWithoutQueryFlagsCapability() throws Exception { + QwpEgressFrames.QueryRequest r = executeOnce(QwpEgressMsgKind.CAP_QUERY_DETAILS, + new QwpQueryOptions().resetSymbolDict(true).stats(true).maxRows(9)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_STATS | QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(9, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + @Test public void testResetFlagAppendsTrailerWhenCapabilityAdvertised() throws Exception { byte[] frame = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", NOOP_HANDLER, true)); @@ -113,6 +249,101 @@ public void testResetTrailerIsTheOnlyDifferenceFromBaseline() throws Exception { Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, on[on.length - 1]); } + @Test + public void testRowWindowFailsWithoutCapability() throws Exception { + // A server without CAP_QUERY_DETAILS would stream every row. The query + // fails before it is sent, and the connection stays usable. + QwpQueryOptions[] windows = { + new QwpQueryOptions().skipRows(1), + new QwpQueryOptions().maxRows(0), + new QwpQueryOptions().skipRows(10).maxRows(10).stats(true), + }; + final int[] capabilitiesWithoutDetails = { + 0, + QwpEgressMsgKind.CAP_QUERY_FLAGS, + QwpEgressMsgKind.CAP_QUERY_FLAGS | QwpEgressMsgKind.CAP_QUERY_TIMEOUT + }; + for (int capabilities : capabilitiesWithoutDetails) { + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(capabilities, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + for (QwpQueryOptions window : windows) { + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, binds -> Assert.fail("binds of a refused query must not be encoded"), handler, window); + handler.assertOnlyError(); + Assert.assertEquals(-1, handler.errorRequestId); + Assert.assertEquals(WebSocketResponse.STATUS_INTERNAL_ERROR, handler.errorStatus); + Assert.assertEquals("server does not support skipRows and maxRows [capabilities=0x" + + Integer.toHexString(capabilities) + "]; they need a server that advertises CAP_QUERY_DETAILS", + handler.errorMessage); + Assert.assertEquals(QwpEgressMsgKind.NO_ERROR_POSITION, handler.errorPosition); + } + Assert.assertEquals("a refused query must not reach the server", 0, server.getRequests().size()); + + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, new QwpQueryOptions().stats(true)); + Assert.assertEquals(1, handler.execDoneCount); + Assert.assertEquals(1, server.getRequests().size()); + server.assertNoFailure(); + } + }); + } + } + + @Test + public void testSkipRowsOnTheWire() throws Exception { + for (long skipRows : new long[]{1, 127, 128, 16_383, 16_384, Long.MAX_VALUE}) { + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().skipRows(skipRows)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS, r.queryFlags); + Assert.assertEquals(skipRows, r.skipRows); + Assert.assertEquals(-1, r.maxRows); + Assert.assertEquals(0, r.trailingBytes); + } + // skipping no rows sends nothing + QwpEgressFrames.QueryRequest r = executeOnce(ALL_CAPABILITIES, new QwpQueryOptions().skipRows(5).skipRows(0).maxRows(10)); + Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS, r.queryFlags); + Assert.assertEquals(-1, r.skipRows); + Assert.assertEquals(10, r.maxRows); + } + + /** + * Each capability and each query flag is a bit of its own, and the query + * details leave alone the per-query timeout bits that released clients + * already use: capability 0x08 and flag 0x02, followed by {@code timeout_ms} + * (c-questdb-client 7.2.0 and py-questdb-client 6.0.0). + */ + @Test + public void testWireBitsAreDistinctAndLeaveTimeoutBitsFree() { + Assert.assertEquals(0x08, QwpEgressMsgKind.CAP_QUERY_TIMEOUT); + Assert.assertEquals(0x02, QwpEgressMsgKind.QUERY_FLAG_TIMEOUT); + assertDistinctBits( + QwpEgressMsgKind.CAP_ZONE, + QwpEgressMsgKind.CAP_QUERY_FLAGS, + 0x04, // the server's CAP_COMPRESSION, which this client does not read + QwpEgressMsgKind.CAP_QUERY_TIMEOUT, + QwpEgressMsgKind.CAP_QUERY_DETAILS + ); + assertDistinctBits( + QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, + QwpEgressMsgKind.QUERY_FLAG_TIMEOUT, + QwpEgressMsgKind.QUERY_FLAG_ERROR_POSITION, + QwpEgressMsgKind.QUERY_FLAG_STATS, + QwpEgressMsgKind.QUERY_FLAG_COUNT_ROWS, + QwpEgressMsgKind.QUERY_FLAG_SKIP_ROWS, + QwpEgressMsgKind.QUERY_FLAG_MAX_ROWS + ); + } + + private static void assertDistinctBits(int... bits) { + int union = 0; + for (int bit : bits) { + Assert.assertEquals("0x" + Integer.toHexString(bit) + " must be a single bit", 1, Integer.bitCount(bit)); + Assert.assertEquals("0x" + Integer.toHexString(bit) + " is taken twice", 0, union & bit); + union |= bit; + } + } + private static byte[] buildExecDone(byte[] queryRequest) { int bodyLen = 1 + 8 + 1 + 1; // msg_kind + request_id + op_type + rows_affected varint byte[] frame = new byte[QwpConstants.HEADER_SIZE + bodyLen]; @@ -129,6 +360,43 @@ private static byte[] buildExecDone(byte[] queryRequest) { return frame; } + /** + * Answers a request with an {@code EXEC_DONE}, carrying the statistics + * block when the request asked for it, as the server does. + */ + private static byte[][] execDone(QwpEgressFrames.QueryRequest r) { + QwpEgressFrames.Body body = QwpEgressFrames.execDone(r.requestId, 0, 0); + if (r.has(QwpEgressMsgKind.QUERY_FLAG_STATS)) { + body.stats(0, 0, 0, 0, 0); + } + return new byte[][]{body.toFrame()}; + } + + /** + * Runs {@link #SQL} once with {@code options} against a server that + * advertises {@code capabilities}, and returns the request it received. + */ + private static QwpEgressFrames.QueryRequest executeOnce(int capabilities, QwpQueryOptions options) throws Exception { + AtomicReference request = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + try (QwpEgressMockServer server = new QwpEgressMockServer(capabilities, QwpQueryClientQueryFlagsTest::execDone); + QwpQueryClient client = newClient(server)) { + client.connect(); + RecordingQueryHandler handler = new RecordingQueryHandler(); + client.execute(SQL, handler, options); + Assert.assertEquals("the query must complete", 1, handler.execDoneCount); + server.assertNoFailure(); + Assert.assertEquals(1, server.getRequests().size()); + request.set(server.getRequests().get(0)); + } + }); + return request.get(); + } + + private static QwpQueryClient newClient(QwpEgressMockServer server) { + return QwpQueryClient.fromConfig("ws::addr=" + server.getAddress() + ";auth_timeout_ms=2000;failover=off;"); + } + /** * Parses the {@code QUERY_REQUEST} the client emitted and returns the * decoded {@code query_flags} trailer, or {@code -1} when no trailer was diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java new file mode 100644 index 00000000..9d50134c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryOptionsTest.java @@ -0,0 +1,151 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpQueryOptions; +import org.junit.Assert; +import org.junit.Test; + +/** + * State of {@link QwpQueryOptions}: defaults, setters, validation, reset and + * copy. What the options put on the wire is covered by + * {@link QwpQueryClientQueryFlagsTest}. + */ +public class QwpQueryOptionsTest { + + @Test + public void testClearRestoresDefaults() { + QwpQueryOptions options = allSet(); + Assert.assertSame(options, options.clear()); + assertDefaults(options); + } + + @Test + public void testCopyFrom() { + QwpQueryOptions source = allSet(); + QwpQueryOptions target = new QwpQueryOptions(); + Assert.assertSame(target, target.copyFrom(source)); + Assert.assertEquals(source.toString(), target.toString()); + assertAllSet(target); + + // the copy is independent of its source + source.clear(); + assertAllSet(target); + + Assert.assertSame(target, target.copyFrom(target)); + assertAllSet(target); + + // null copies the defaults + Assert.assertSame(target, target.copyFrom(null)); + assertDefaults(target); + } + + @Test + public void testDefaults() { + QwpQueryOptions options = new QwpQueryOptions(); + assertDefaults(options); + Assert.assertEquals("QwpQueryOptions{skipRows=0, maxRows=9223372036854775807, countRows=false, " + + "stats=false, errorPosition=false, resetSymbolDict=false}", options.toString()); + } + + @Test + public void testNegativeRowsRejected() { + QwpQueryOptions options = new QwpQueryOptions().skipRows(3).maxRows(4); + for (long rows : new long[]{-1, -128, Long.MIN_VALUE}) { + try { + options.skipRows(rows); + Assert.fail("skipRows(" + rows + ") must throw"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("skipRows must be >= 0 [skipRows=" + rows + ']', e.getMessage()); + } + try { + options.maxRows(rows); + Assert.fail("maxRows(" + rows + ") must throw"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("maxRows must be >= 0 [maxRows=" + rows + ']', e.getMessage()); + } + // a rejected value leaves the previous one + Assert.assertEquals(3, options.getSkipRows()); + Assert.assertEquals(4, options.getMaxRows()); + } + } + + @Test + public void testRowBounds() { + QwpQueryOptions options = new QwpQueryOptions(); + for (long rows : new long[]{0, 1, 127, 128, Long.MAX_VALUE - 1, Long.MAX_VALUE}) { + Assert.assertSame(options, options.skipRows(rows)); + Assert.assertEquals(rows, options.getSkipRows()); + Assert.assertSame(options, options.maxRows(rows)); + Assert.assertEquals(rows, options.getMaxRows()); + } + } + + @Test + public void testSetters() { + QwpQueryOptions options = new QwpQueryOptions(); + Assert.assertSame(options, options.countRows(true)); + Assert.assertTrue(options.isRowCountRequested()); + Assert.assertSame(options, options.stats(true)); + Assert.assertTrue(options.isStatsRequested()); + Assert.assertSame(options, options.errorPosition(true)); + Assert.assertTrue(options.isErrorPositionRequested()); + Assert.assertSame(options, options.resetSymbolDict(true)); + Assert.assertTrue(options.isSymbolDictResetRequested()); + Assert.assertEquals("QwpQueryOptions{skipRows=0, maxRows=9223372036854775807, countRows=true, " + + "stats=true, errorPosition=true, resetSymbolDict=true}", options.toString()); + + options.countRows(false).stats(false).errorPosition(false).resetSymbolDict(false); + assertDefaults(options); + } + + private static QwpQueryOptions allSet() { + return new QwpQueryOptions() + .skipRows(100) + .maxRows(50) + .countRows(true) + .stats(true) + .errorPosition(true) + .resetSymbolDict(true); + } + + private static void assertAllSet(QwpQueryOptions options) { + Assert.assertEquals(100, options.getSkipRows()); + Assert.assertEquals(50, options.getMaxRows()); + Assert.assertTrue(options.isRowCountRequested()); + Assert.assertTrue(options.isStatsRequested()); + Assert.assertTrue(options.isErrorPositionRequested()); + Assert.assertTrue(options.isSymbolDictResetRequested()); + } + + private static void assertDefaults(QwpQueryOptions options) { + Assert.assertEquals(0, options.getSkipRows()); + Assert.assertEquals(Long.MAX_VALUE, options.getMaxRows()); + Assert.assertFalse(options.isRowCountRequested()); + Assert.assertFalse(options.isStatsRequested()); + Assert.assertFalse(options.isErrorPositionRequested()); + Assert.assertFalse(options.isSymbolDictResetRequested()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java new file mode 100644 index 00000000..2a17cb52 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecordingQueryHandler.java @@ -0,0 +1,163 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryDetails; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.List; + +/** + * Records what the most detailed callbacks of {@link QwpColumnBatchHandler} + * receive, copying the {@link QwpQueryDetails} values out while they are + * valid. The simpler overloads fail: the client must call the detailed ones. + * Batches must carry a single LONG column. Also hosts the assertion on a + * {@link QwpQueryDetails} that carries nothing, which the decoder tests share. + */ +public class RecordingQueryHandler implements QwpColumnBatchHandler { + public final List values = new ArrayList<>(); + public long clientWaitNanos; + public long compileNanos; + public long countNanos; + public int endCount; + public long endRequestId; + public int errorCount; + public String errorMessage; + public int errorPosition; + public long errorRequestId; + public byte errorStatus; + public int execDoneCount; + public long executeNanos; + public int failoverResetCount; + public boolean hasRowCount; + public boolean hasStats; + public boolean isJitCompiled; + public short opType; + public long rowCount; + public long rowsAffected; + public long totalRows; + + /** + * Asserts that {@code details} carries neither a row count nor statistics: + * every getter returns its absent value. + */ + public static void assertNoDetails(QwpQueryDetails details) { + Assert.assertFalse(details.hasRowCount()); + Assert.assertEquals(-1, details.getRowCount()); + Assert.assertFalse(details.hasStats()); + Assert.assertEquals(-1, details.getCompileNanos()); + Assert.assertEquals(-1, details.getExecuteNanos()); + Assert.assertEquals(-1, details.getCountNanos()); + Assert.assertEquals(-1, details.getClientWaitNanos()); + Assert.assertFalse(details.isJitCompiled()); + Assert.assertEquals("QwpQueryDetails{}", details.toString()); + } + + public void assertNoDetails() { + Assert.assertFalse("row count", hasRowCount); + Assert.assertEquals(-1, rowCount); + Assert.assertFalse("stats", hasStats); + Assert.assertEquals(-1, compileNanos); + Assert.assertEquals(-1, executeNanos); + Assert.assertEquals(-1, countNanos); + Assert.assertEquals(-1, clientWaitNanos); + Assert.assertFalse(isJitCompiled); + } + + public void assertOnlyError() { + Assert.assertEquals("errors", 1, errorCount); + Assert.assertEquals("ends", 0, endCount); + Assert.assertEquals("exec dones", 0, execDoneCount); + } + + @Override + public void onBatch(QwpColumnBatch batch) { + for (int row = 0; row < batch.getRowCount(); row++) { + values.add(batch.getLongValue(0, row)); + } + } + + @Override + public void onEnd(long totalRows) { + Assert.fail("the client must call onEnd(requestId, totalRows, details)"); + } + + @Override + public void onEnd(long requestId, long totalRows, QwpQueryDetails details) { + endCount++; + endRequestId = requestId; + this.totalRows = totalRows; + copy(details); + } + + @Override + public void onError(byte status, String message) { + Assert.fail("the client must call onError(requestId, status, message, position)"); + } + + @Override + public void onError(long requestId, byte status, String message, int position) { + errorCount++; + errorRequestId = requestId; + errorStatus = status; + errorMessage = message; + errorPosition = position; + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + Assert.fail("the client must call onExecDone(requestId, opType, rowsAffected, details)"); + } + + @Override + public void onExecDone(long requestId, short opType, long rowsAffected, QwpQueryDetails details) { + execDoneCount++; + this.opType = opType; + this.rowsAffected = rowsAffected; + copy(details); + } + + @Override + public void onFailoverReset(long requestId, QwpServerInfo newNode) { + failoverResetCount++; + values.clear(); + } + + private void copy(QwpQueryDetails details) { + Assert.assertNotNull("details must never be null", details); + hasRowCount = details.hasRowCount(); + rowCount = details.getRowCount(); + hasStats = details.hasStats(); + compileNanos = details.getCompileNanos(); + executeNanos = details.getExecuteNanos(); + countNanos = details.getCountNanos(); + clientWaitNanos = details.getClientWaitNanos(); + isJitCompiled = details.isJitCompiled(); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 102922d6..e63637b1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -112,6 +112,10 @@ public class TestWebSocketServer implements Closeable { // connections carry only ACK frames. private volatile boolean sendServerInfo; private volatile int capabilities; + // Runs on the connection's thread right before it sends SERVER_INFO, while + // the client waits for the frame inside connect() or a failover reconnect. + // Null for none. Set via setBeforeServerInfoHook(). + private volatile Runnable beforeServerInfoHook; // When true, the server fails the WebSocket upgrade on the egress read path // (/read...) by dropping the connection before the 101, while still serving // the ingest write path (/write...) normally. Lets one server + one cluster @@ -434,6 +438,16 @@ public void setSendServerInfo(boolean sendServerInfo) { this.sendServerInfo = sendServerInfo; } + /** + * Runs {@code hook} on each egress connection right before the server sends + * its {@code SERVER_INFO} frame, which the client awaits inside + * {@code connect()} or a failover reconnect. Lets a test act at a fixed + * point of the client's connect sequence. Null removes the hook. + */ + public void setBeforeServerInfoHook(Runnable hook) { + this.beforeServerInfoHook = hook; + } + // Advertised SERVER_INFO capabilities. CAP_ZONE is unsupported here: the // frame builder emits no zone_id trailer. public void setCapabilities(int capabilities) { @@ -792,6 +806,9 @@ private boolean performHandshake() throws IOException { // query pool's connect fails fast, while ingest write-path upgrades // still complete on this same server. if (rejectReadUpgrade && isReadPath(requestPath)) { + // Drop the connection before the 101, so the client fails its + // upgrade at once instead of waiting out its auth timeout. + close(); return false; } @@ -941,6 +958,10 @@ void start() { // connection parses every inbound frame as an ACK and // would fail on it. if (sendServerInfo && isReadPath(requestPath)) { + Runnable hook = beforeServerInfoHook; + if (hook != null) { + hook.run(); + } sendBinary(buildServerInfoFrame(roleByte(advertisedRole), capabilities)); }