diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 8cfc437b..02a0cb6d 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -139,7 +139,11 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token * {@link Sender#close()} on the returned instance to release it back to * the pool. {@code close()} on a pooled Sender flushes pending rows * before returning to the pool; a real disconnect only happens at - * {@link #close()} on this {@code QuestDB} handle. + * {@link #close()} on this {@code QuestDB} handle. A Sender in strict + * transactional mode ({@code transaction=strict}, or a WebSocket config + * with no auto-flush trigger, such as {@code auto_flush=off}) commits only + * on {@link Sender#flush()}, so its {@code close()} discards the rows it + * did not flush instead. *

* Allocation: zero at steady state -- the returned instance is a * pre-allocated decorator backed by a pre-allocated underlying Sender. diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index d16e9780..fb21442f 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -773,13 +773,46 @@ default void resetSymbolDictionary() { * data format) and want to continue sending new data without retrying the problematic data. * After calling this method, you can start building a new row by calling {@link #table(CharSequence)}. *
- * Note: This method is only available for HTTP transport. TCP transport doesn't support - * this operation. + * In WebSocket transactional mode ({@code transaction=on} or {@code strict}) it also + * discards the rows of the open transaction that auto-flush already sent, as + * {@link #rollback()} does, when the server supports QWP rollback. + *
+ * Note: the ILP TCP and UDP transports don't support this operation. * * @see #flush() */ void reset(); + /** + * Discard the open transaction: the rows buffered since the last flush and, in + * WebSocket transactional mode ({@code transaction=on} or {@code strict}), the + * rows of the transaction that auto-flush already sent to the server, which + * the server discards on request. + *
+ * Like {@link #flush()}, this returns without waiting for the server, so a + * server outage makes it fail only when the store-and-forward buffer is full, + * as it would a flush. The request goes out ahead of anything sent later, so + * later rows never commit the discarded ones, even across reconnects. Call + * {@link #drain(long)} to wait until the server confirms. + *
+ * A {@code transaction=on} transaction that already committed in parts (see + * {@link LineSenderBuilder#transactional(boolean)}) keeps the parts it committed; + * this discards only the rest. + *
+ * Discarding rows already sent needs a server that supports QWP rollback. When + * the server does not, this method throws and aborts the transaction: every + * later flush throws too, and closing the sender discards the rows, because the + * server rolls back uncommitted rows when the connection closes. + *
+ * Only the WebSocket transport supports this method. + * + * @throws LineSenderException if the transport does not support rollback, or the + * server cannot discard rows already sent + */ + default void rollback() { + throw new LineSenderException("rollback is only supported for WebSocket transport"); + } + /** * Add a column with a 16-bit signed integer value. * @@ -1075,7 +1108,10 @@ final class LineSenderBuilder { private static final int DEFAULT_TCP_PORT = 9009; private static final int DEFAULT_UDP_PORT = 9007; private static final int DEFAULT_WEBSOCKET_PORT = 9000; - private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 0; + // Unset auto_flush_bytes on WebSocket sizes the byte trigger from what one + // frame can carry, so no batch outgrows a frame; see + // QwpWebSocketSender.AUTO_FLUSH_BYTES_AUTO. + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = QwpWebSocketSender.AUTO_FLUSH_BYTES_AUTO; private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 1_000; // Cadence at which the SF cursor I/O loop sends a keepalive PING @@ -1118,10 +1154,12 @@ final class LineSenderBuilder { private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY; - // Set by disableAutoFlush(). auto_flush_interval=off also stores the interval as - // Integer.MAX_VALUE, so the WebSocket validation keys on this flag to reject only the - // full disable and accept the interval-only opt-out. - private boolean autoFlushDisabled; + // disableAutoFlush() (auto_flush=off) sets it. That call turns off every + // auto-flush trigger, so a positive auto_flush_bytes contradicts it, and only + // that conflict check reads this flag. auto_flush_rows=off with + // auto_flush_interval=off reaches the same trigger state, which is what + // hasAutoFlushTrigger() reads to pick the sender's mode. + private boolean isAutoFlushDisabled; private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; private int autoFlushRows = PARAMETER_NOT_SET_EXPLICITLY; private int bufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; @@ -1234,7 +1272,12 @@ public int getConnectTimeout() { private long reconnectMaxDurationMillis = PARAMETER_NOT_SET_EXPLICITLY; private int durableAckTiers = DurableAckTiers.NONE; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; + private boolean strictTransactional; private boolean transactional; + // True once transaction=... (or transactional()/strictTransactional()) + // set the mode explicitly, so auto_flush=off can refuse to override an + // explicit transaction=off. + private boolean transactionModeSet; private String senderId = DEFAULT_SENDER_ID; // Per-append deadline for SF appendBlocking spin-then-throw. Used to // be a hardcoded 30s constant; expose so tight-SLA users can lower @@ -1385,7 +1428,13 @@ public LineSenderBuilder authTimeoutMillis(long millis) { *
* This is only used when communicating over WebSocket transport. *
- * Default value is 0, which disables byte-based auto-flush. + * By default the byte trigger fires at 90% of the largest frame the connection can carry: the server's + * advertised maximum batch size, or the store-and-forward segment when that is smaller. A positive value sets + * the trigger, capped at that size, and counts as an auto-flush trigger: unless the sender is transactional, + * every batch it sends commits. 0 turns the byte trigger off, and then every batch must fit one frame per + * table. + *
+ * You cannot set a positive value when auto-flush is disabled. See {@link #disableAutoFlush()}. * * @param bytes maximum bytes per batch * @return this instance for method chaining @@ -1397,6 +1446,8 @@ public LineSenderBuilder autoFlushBytes(int bytes) { if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("auto flush bytes was already configured") .put("[bytes=").put(this.autoFlushBytes).put("]"); + } else if (isAutoFlushDisabled && bytes > 0) { + throw new LineSenderException("cannot set auto flush bytes when auto-flush is already disabled"); } if (bytes < 0) { throw new LineSenderException("auto flush bytes cannot be negative") @@ -1455,7 +1506,9 @@ public LineSenderBuilder autoFlushIntervalMillis(int autoFlushIntervalMillis) { * Setting this to 1 means that the Sender will send each row to a server immediately after it is added. This * effectively disables batching and may lead to a significant performance degradation. *
- * Setting this to 0 disables row-based auto-flush. Interval-based auto-flush remains enabled. + * Setting this to 0 disables row-based auto-flush. Interval-based auto-flush remains enabled. On WebSocket, a + * sender whose interval trigger is off too, and that has no positive {@link #autoFlushBytes(int)}, has no + * auto-flush trigger left and works like one built with {@link #disableAutoFlush()}. *

* You cannot set this value when auto-flush is disabled. See {@link #disableAutoFlush()}. * @@ -1868,9 +1921,17 @@ public LineSenderBuilder connectionListenerInboxCapacity(int capacity) { *

* It is essential to ensure the maximum buffer capacity is sufficient to accommodate all locally buffered data. *

- * This method should only be used when communicating via the HTTP protocol. Calling this method is illegal when - * communicating over the TCP or WebSocket protocol. On WebSocket, {@code auto_flush_interval=off} turns off - * only the time-based trigger. + * This method can be used with the HTTP and WebSocket protocols; calling it is illegal when communicating over + * the TCP protocol. Turning off both the row and the interval trigger ({@code auto_flush_rows=off} and + * {@code auto_flush_interval=off}) has the same effect. + *

+ * On WebSocket, a sender with no auto-flush trigger is in strict transactional mode (see + * {@link #strictTransactional(boolean)}) unless {@link #transactional(boolean)} selects the mode: rows still go + * to the server in frames sized for the connection, so a batch can be any size, but only {@code flush()} + * commits them, and closing the sender or returning a pooled sender without a flush discards them. + * {@code transactional(true)} keeps such a sender non-strict, so closing it commits, and + * {@code transactional(false)} cannot be combined with it. Nor can a positive {@link #autoFlushBytes(int)}, + * which is an auto-flush trigger. * * @return this instance for method chaining * @see #autoFlushRows(int) @@ -1885,10 +1946,14 @@ public LineSenderBuilder disableAutoFlush() { throw new LineSenderException("auto flush interval was already configured ") .put("[autoFlushIntervalMillis=").put(this.autoFlushIntervalMillis).put("]"); } + if (this.autoFlushBytes > 0) { + throw new LineSenderException("auto flush bytes was already configured ") + .put("[autoFlushBytes=").put(this.autoFlushBytes).put("]"); + } this.autoFlushRows = AUTO_FLUSH_DISABLED; this.autoFlushIntervalMillis = Integer.MAX_VALUE; - this.autoFlushDisabled = true; + this.isAutoFlushDisabled = true; return this; } @@ -2757,12 +2822,54 @@ private LineSenderBuilder requestDurableAckTiers(int tiers) { return this; } + /** + * Enables strict transactional mode, the builder form of + * {@code transaction=strict}. Like {@link #transactional(boolean)}, auto-flush + * sends rows to the server without committing them and {@code flush()} + * commits, so a transaction can be any size. In addition the transaction + * never commits in parts: a row that would take a table past the server's + * {@code qwp.max.uncommitted.rows}, or the transaction past + * {@code sf_max_total_bytes}, throws + * {@link io.questdb.client.cutlass.qwp.client.TransactionTooLargeException} + * and is not added, and {@code close()} without {@code flush()} discards the + * transaction instead of committing it. + * A sender with no auto-flush trigger (see {@link #disableAutoFlush()}) is + * in this mode unless {@link #transactional(boolean)} selects the mode. + * WebSocket transport only. + * + * @param enabled true to enable strict transactional mode; false leaves + * {@link #transactional(boolean) transactional} mode as set + * @return this instance for method chaining + */ + public LineSenderBuilder strictTransactional(boolean enabled) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("transactional is only supported for WebSocket transport"); + } + this.strictTransactional = enabled; + if (enabled) { + this.transactional = true; + } + this.transactionModeSet = true; + return this; + } + /** * Enables transactional mode. Auto-flush sends data to the server * with {@code FLAG_DEFER_COMMIT}; only an explicit {@code flush()} - * triggers the server-side WAL commit. This allows accumulating - * datasets larger than the server's recv buffer while committing - * atomically per table. + * (or {@code close()}) triggers the server-side WAL commit. This allows + * accumulating datasets larger than the server's recv buffer while + * committing atomically per table. A transaction too large for either side + * commits in parts: the server commits a table on its own once + * {@code qwp.max.uncommitted.rows} of its rows are uncommitted, and the + * sender, which holds the transaction in {@code sf_max_total_bytes} until + * the server commits it, commits the rows it already sent once the + * transaction outgrows that buffer, and logs a warning. Use + * {@link #strictTransactional(boolean)} to fail instead. + *

+ * A sender with no auto-flush trigger (see {@link #disableAutoFlush()}) is + * in strict transactional mode unless this method selects the mode: + * {@code true} keeps it in this non-strict mode, and {@code false} cannot + * be combined with it. * * @param enabled true to enable transactional mode * @return this instance for method chaining @@ -2772,6 +2879,10 @@ public LineSenderBuilder transactional(boolean enabled) { throw new LineSenderException("transactional is only supported for WebSocket transport"); } this.transactional = enabled; + if (!enabled) { + this.strictTransactional = false; + } + this.transactionModeSet = true; return this; } @@ -3822,6 +3933,11 @@ private QwpWebSocketSender buildWebSocket(HttpTokenProvider tokenProvider) { // dispatcher daemon, drainer pool, microbatch buffers and // WebSocketClient inside the abandoned `connected`. connected.setTransactional(transactional); + // With no auto-flush trigger only flush() commits: a strict transaction of + // any size, unless transaction= chose the mode. + if (strictTransactional || (!transactionModeSet && !hasAutoFlushTrigger())) { + connected.setStrictTransactional(true); + } try { final String rebuildSfDir = sfDir; final String rebuildSenderId = senderId; @@ -4230,8 +4346,10 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { transactional(true); } else if (Chars.equalsIgnoreCase("off", sink)) { transactional(false); + } else if (Chars.equalsIgnoreCase("strict", sink)) { + strictTransactional(true); } else { - throw new LineSenderException("invalid transaction [value=").put(sink).put(", allowed-values=[on, off]]"); + throw new LineSenderException("invalid transaction [value=").put(sink).put(", allowed-values=[on, off, strict]]"); } } else if (Chars.equals("sf_dir", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { @@ -4682,8 +4800,10 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) transactional(true); } else if (s.equalsIgnoreCase("off")) { transactional(false); + } else if (s.equalsIgnoreCase("strict")) { + strictTransactional(true); } else { - throw new LineSenderException("invalid transaction [value=").put(s).put(", allowed-values=[on, off]]"); + throw new LineSenderException("invalid transaction [value=").put(s).put(", allowed-values=[on, off, strict]]"); } } s = view.getStr("request_durable_ack"); @@ -4821,7 +4941,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("auto_flush_bytes", autoFlushBytes); m.put("auto_flush_interval", autoFlushIntervalMillis); m.put("max_name_len", maxNameLength); - m.put("transaction", transactional); + m.put("transaction", strictTransactional ? "strict" : transactional ? "on" : "off"); m.put("request_durable_ack", DurableAckTiers.configValue(durableAckTiers)); m.put("sender_id", senderId); m.put("sf_dir", sfDir); @@ -4865,6 +4985,16 @@ public java.util.Map wsConfigSnapshotForTest() { return m; } + // True when a configured trigger flushes rows on its own: the row or interval + // trigger, or a positive auto_flush_bytes. auto_flush=off, and + // auto_flush_rows=off with auto_flush_interval=off, leave none. The WebSocket + // default byte trigger does not count: it only sizes frames. + private boolean hasAutoFlushTrigger() { + return autoFlushRows != AUTO_FLUSH_DISABLED + || autoFlushIntervalMillis != Integer.MAX_VALUE + || autoFlushBytes > 0; + } + /** * Use HTTP protocol as transport. *
@@ -5068,8 +5198,13 @@ private void validateParameters() { if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol version is not supported for WebSocket protocol"); } - if (autoFlushDisabled) { - throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol"); + if (transactionModeSet && !transactional && !hasAutoFlushTrigger()) { + // Only flush() can commit this sender's rows, and a non-transactional + // flush has to fit one frame per table. + throw new LineSenderException("auto-flush is off (auto_flush=off, or auto_flush_rows=off and " + + "auto_flush_interval=off), which on WebSocket selects strict transactional mode and " + + "cannot be combined with transaction=off; set transaction=on or turn on an auto-flush " + + "trigger"); } if ((sfDurability == SfDurability.FLUSH || sfDurability == SfDurability.APPEND)) { throw new LineSenderException( diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 938d5175..ab479d60 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -84,6 +84,11 @@ public abstract class WebSocketClient implements QuietCloseable { private static final String QWP_CONTENT_ENCODING_HEADER_NAME = "X-QWP-Content-Encoding:"; private static final String QWP_DURABLE_ACK_HEADER_NAME = "X-QWP-Durable-Ack:"; private static final String QWP_MAX_BATCH_SIZE_HEADER_NAME = "X-QWP-Max-Batch-Size:"; + private static final String QWP_MAX_UNCOMMITTED_ROWS_HEADER_NAME = "X-QWP-Max-Uncommitted-Rows:"; + private static final String QWP_ROLLBACK_HEADER_NAME = "X-QWP-Rollback:"; + // Token the server echoes in X-QWP-Rollback when it accepts the rollback + // control frame on this connection. + private static final String QWP_ROLLBACK_TOKEN_ENABLED = "enabled"; private static final String QWP_VERSION_HEADER_NAME = "X-QWP-Version:"; private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { try { @@ -148,6 +153,9 @@ public abstract class WebSocketClient implements QuietCloseable { // Requested durable-ack tier set (DurableAckTiers bitmask), sent as // X-QWP-Request-Durable-Ack: private int qwpDurableAckTiers = DurableAckTiers.NONE; + // When true, the upgrade request carries X-QWP-Request-Rollback: true, asking + // the server to confirm that it accepts the rollback control frame. + private boolean qwpRequestRollback; // Receive buffer (native memory) private long recvBufPtr; private int recvBufSize; @@ -170,6 +178,11 @@ public abstract class WebSocketClient implements QuietCloseable { // not advertise the header (older builds), in which case the sender falls // back to its locally configured budget. private int serverMaxBatchSize; + // Server's per-table cap on rows a deferred-commit group may hold before the + // server commits them on its own (qwp.max.uncommitted.rows), extracted from + // X-QWP-Max-Uncommitted-Rows on the 101 upgrade response. 0 when the server + // did not advertise it (older builds). + private long serverMaxUncommittedRows; // Zstd compression level the server actually applied for this connection, // extracted from the echoed X-QWP-Content-Encoding response header. // 0 means "no zstd" -- either the server picked raw, the header was absent @@ -178,6 +191,11 @@ public abstract class WebSocketClient implements QuietCloseable { // the wire said without re-clamping so a misconfigured server is observable. private int serverNegotiatedZstdLevel; private int serverQwpVersion = 1; + // Set during upgrade response validation when the client asked for rollback + // (qwpRequestRollback) and the server confirmed it with X-QWP-Rollback: + // enabled. A server that does not confirm treats the rollback control frame + // as an empty deferred frame, so the client must never send it there. + private boolean serverRollbackEnabled; private String upgradeRejectRole; // WWW-Authenticate value(s) from the most recent rejected upgrade, joined with ", " when the server sent // several, or null when absent. Lets a 401 carrying "Bearer error=insufficient_scope" be told apart from an @@ -412,6 +430,16 @@ public int getServerMaxBatchSize() { return serverMaxBatchSize; } + /** + * Server's per-table limit on rows a deferred-commit group may hold before + * the server commits them on its own ({@code qwp.max.uncommitted.rows}), taken + * from the {@code X-QWP-Max-Uncommitted-Rows} response header on the 101 + * upgrade. Returns {@code 0} when the server did not advertise the header. + */ + public long getServerMaxUncommittedRows() { + return serverMaxUncommittedRows; + } + /** * Zstd level the server actually applied for this connection, parsed * from the echoed {@code X-QWP-Content-Encoding} response header. @@ -494,6 +522,16 @@ public boolean isServerDurableAckEnabled() { return serverDurableAckEnabled; } + /** + * Returns true when the client asked for rollback support via + * {@link #setQwpRequestRollback} and the 101 upgrade response confirmed it + * with {@code X-QWP-Rollback: enabled}. Meaningful only after + * {@link #upgrade} returns. + */ + public boolean isServerRollbackEnabled() { + return serverRollbackEnabled; + } + /** * Receives and processes WebSocket frames. * @@ -670,6 +708,16 @@ public void setQwpMaxVersion(int maxVersion) { this.qwpMaxVersion = maxVersion; } + /** + * When {@code true}, the upgrade request carries + * {@code X-QWP-Request-Rollback: true}. A server that accepts the rollback + * control frame confirms it with {@code X-QWP-Rollback: enabled}; see + * {@link #isServerRollbackEnabled()}. + */ + public void setQwpRequestRollback(boolean requestRollback) { + this.qwpRequestRollback = requestRollback; + } + /** * Sets the requested durable-ack tier set ({@link DurableAckTiers} * bitmask) for the opt-in X-QWP-Request-Durable-Ack upgrade header. When @@ -795,6 +843,9 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe sendBuffer.putAscii(DurableAckTiers.requestHeaderValue(qwpDurableAckTiers)); sendBuffer.putAscii("\r\n"); } + if (qwpRequestRollback) { + sendBuffer.putAscii("X-QWP-Request-Rollback: true\r\n"); + } if (authorizationHeader != null) { sendBuffer.putAscii("Authorization: "); sendBuffer.putAscii(authorizationHeader); @@ -912,6 +963,22 @@ private static String extractDurableAckEcho(String response) { return null; } + private static String extractHeaderValue(String response, String headerName) { + int headerLen = headerName.length(); + int responseLen = response.length(); + for (int i = 0; i <= responseLen - headerLen; i++) { + if (response.regionMatches(true, i, headerName, 0, headerLen)) { + int valueStart = i + headerLen; + int lineEnd = response.indexOf('\r', valueStart); + if (lineEnd < 0) { + lineEnd = responseLen; + } + return response.substring(valueStart, lineEnd).trim(); + } + } + return null; + } + private static int extractMaxBatchSize(String response) { int headerLen = QWP_MAX_BATCH_SIZE_HEADER_NAME.length(); int responseLen = response.length(); @@ -934,6 +1001,18 @@ private static int extractMaxBatchSize(String response) { return 0; } + private static long extractMaxUncommittedRows(String response) { + String value = extractHeaderValue(response, QWP_MAX_UNCOMMITTED_ROWS_HEADER_NAME); + if (value == null) { + return 0; + } + try { + return Math.max(Long.parseLong(value), 0); + } catch (NumberFormatException e) { + return 0; + } + } + private static int extractQwpVersion(String response) { int headerLen = QWP_VERSION_HEADER_NAME.length(); int responseLen = response.length(); @@ -1541,6 +1620,16 @@ private void validateUpgradeResponse(int headerEnd) { // sender falls back to its locally configured byte budget in that case. serverMaxBatchSize = extractMaxBatchSize(response); + // Extract X-QWP-Max-Uncommitted-Rows (optional). Older servers omit it; + // a strict transaction then assumes the server's default. + serverMaxUncommittedRows = extractMaxUncommittedRows(response); + + // Extract the X-QWP-Rollback confirmation (optional). Only a client that + // asked for rollback may treat it as granted: the server echoes it only + // on request, and an older server never sends it. + serverRollbackEnabled = qwpRequestRollback + && QWP_ROLLBACK_TOKEN_ENABLED.equalsIgnoreCase(extractHeaderValue(response, QWP_ROLLBACK_HEADER_NAME)); + // Extract X-QWP-Content-Encoding (optional). Surfaces what level the // server actually applied -- which may differ from what this client // asked for if the server has qwp.egress.compression.force.level set diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index f38d9e93..68c33866 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -184,6 +184,26 @@ public int encodeWithDeltaDict( return finishMessage(); } + /** + * Encodes the rollback control frame into the buffer: a bare header with + * {@code ROLLBACK_FRAME_FLAGS}, no table blocks and an empty payload. The + * server matches it on exactly that shape. + * + * @return the frame size in bytes + */ + public int encodeRollbackMessage() { + buffer.reset(); + buffer.putByte((byte) 'Q'); + buffer.putByte((byte) 'W'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '1'); + buffer.putByte(version); + buffer.putByte(ROLLBACK_FRAME_FLAGS); + buffer.putShort((short) 0); + buffer.putInt(0); + return buffer.getPosition(); + } + public int finishMessage() { int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1fc43484..7e6f01a6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -72,6 +72,7 @@ import io.questdb.client.std.NumericException; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.TestOnly; @@ -135,6 +136,13 @@ */ public class QwpWebSocketSender implements Sender { + /** + * {@code autoFlushBytes} value that sizes the byte trigger from the frames the + * connection can carry: 90% of the smaller of the server's advertised + * {@code X-QWP-Max-Batch-Size} and the store-and-forward segment. It is the + * builder default, so a batch never grows past what one frame can hold. + */ + public static final int AUTO_FLUSH_BYTES_AUTO = -1; public static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; // Soft per-batch byte budget. Trips a flush when raw column-buffer bytes // cross the threshold, well before the server's 16 MB wire cap. Wide-row @@ -174,8 +182,23 @@ public class QwpWebSocketSender implements Sender { // Default for symbol_dict_reset_threshold: distinct-symbol count that // triggers a recycle once symbol_dict_reset is on. public static final int DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS = 100_000; + // Largest bare commit frame: the header and an empty symbol-dictionary delta, + // whose start id takes at most five varint bytes and whose count one. The + // rollback control frame is the header alone. A transaction=on transaction keeps + // room for one such frame in the client buffer, so the frame that ends it never + // waits for the buffer to drain. + private static final int COMMIT_FRAME_MAX_BYTES = QwpConstants.HEADER_SIZE + 6; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + // Frames of at most the frame cap that a strict transaction keeps room for in + // the client buffer, so the commit that ends it can always be published. + private static final int STRICT_COMMIT_RESERVE_FRAMES = 3; + // Frame cap the byte trigger assumes when the server does not advertise + // X-QWP-Max-Batch-Size: the server's default 2 MiB recv buffer minus the + // worst-case WebSocket frame header, the same value such a server would have + // advertised. Sizes only the soft trigger, never the hard per-frame guard, so an + // older server with a larger buffer still accepts every frame it used to. + private static final int FALLBACK_SERVER_MAX_BATCH_SIZE = 2 * 1024 * 1024 - 14; private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final int MAX_TABLE_NAME_LENGTH = 127; // sf-client.md section 4.4 floor: drop-oldest under bursts needs a wide @@ -678,6 +701,58 @@ public class QwpWebSocketSender implements Sender { // refresh this from the cursor I/O thread on a mid-stream reconnect while // sendRow reads it on the producer thread with no synchronization. private volatile int serverMaxBatchSize; + // Byte length of the symbol-dictionary entries the pending batch introduced: + // the rows reference these ids, so the batch's next frame must carry the + // entries too. Counted toward the byte trigger alongside pendingBytes. Reset + // with pendingBytes. + private long pendingSymbolDictBytes; + // Server's per-table cap on rows a deferred-commit group may hold before it + // commits them on its own (X-QWP-Max-Uncommitted-Rows), 0 when not advertised. + // Captured on each FOREGROUND handshake, like serverMaxBatchSize. + private volatile long serverMaxUncommittedRows; + // True when the current foreground connection confirmed X-QWP-Rollback, so the + // rollback control frame may go out. Captured on each FOREGROUND handshake. + private volatile boolean serverRollbackEnabled; + // Strict transactional mode (transaction=strict, or a sender with no auto-flush + // trigger): implies transactional, never lets an open transaction commit in + // parts, and close() discards instead of committing. See setStrictTransactional. + private boolean strictTransactional; + // Set once the open transaction can no longer be rolled back or committed as + // one: rollback()/reset() could not undo frames already sent because the + // server does not support rollback. Every later flush throws, and close() + // discards; only closing the sender ends this state. + private boolean txnAborted; + // Strict mode only: set when the client buffer cannot take another deferred + // frame of the open transaction and still commit it. Rejects further rows + // until flush() or rollback() ends the transaction. The rows accepted before + // it stay in the transaction, including the one whose auto-flush found no + // room: the room kept for the commit holds them. + private boolean txnBufferFull; + // Transactional mode: simulation of how the ring packs the open transaction's + // frames, mirroring MmapSegment's append rule. txnSimSegments counts the + // segments the transaction's frames occupy (0 = no frame yet) and + // txnSimRemaining is the raw space left in the last of them. + private long txnSimRemaining; + private int txnSimSegments; + // transaction=on: rows of the open transaction the sender already committed + // because the transaction outgrew the client buffer, 0 when it did not. Reset + // with the transaction; committing a part keeps it. + private long txnRowsCommittedInParts; + // Tables with rows in deferred frames of the open transaction, so a commit + // or rollback resets only those tables' counters. Producer thread only. + private final ObjList txnTables = new ObjList<>(); + // Logs the reset()-cannot-roll-back warning once per sender. + private boolean isResetRollbackWarned; + // Transactional mode: payload bytes of the open transaction's frames on the + // ring. Reset with the transaction. + private long txnBytes; + // Strict mode only: the size the open transaction would have reached with the + // flush the client buffer last refused, for the error messages. Reset with the + // transaction. + private long txnRefusedBytes; + // Scratch for hasTransactionRoom: the frame sizes a flush will publish. + // Producer thread only. + private final IntList transactionFrameSizes = new IntList(); private QwpWebSocketSender( List endpoints, @@ -703,11 +778,12 @@ private QwpWebSocketSender( this.closed = false; this.autoFlushRows = autoFlushRows; this.autoFlushBytes = autoFlushBytes; - // Until the handshake completes, honor the configured budget verbatim. - // applyServerBatchSizeLimit() clamps this on connect once the server's - // X-QWP-Max-Batch-Size is known. - this.effectiveAutoFlushBytes = autoFlushBytes; this.autoFlushIntervalNanos = autoFlushIntervalNanos; + // Until the handshake completes, size the byte trigger without a server + // cap. applyServerBatchSizeLimit() re-sizes it on connect once the + // server's X-QWP-Max-Batch-Size is known, and setCursorEngine() once the + // segment size is. + recomputeFrameBudget(); this.globalSymbolDictionary = new GlobalSymbolDictionary(); int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); @@ -1795,29 +1871,54 @@ private void close0(boolean[] restoreInterrupt) { // sender to discard the batch, so honour that: // discard it, remember the error, and let the rest of // close() run. rethrowTerminal below still surfaces it. - try { - flushPendingRows(deferCommit); - } catch (BatchTooLargeForCapException e) { - resetTableBuffersAfterFlush(); - terminalError = captureCloseError(terminalError, e); - } catch (Throwable t) { - // Same reasoning as the pre-flight rejection above, for the - // failures a size check cannot see: sealAndSwapBuffer's - // buffer-recycle timeout and appendBlocking's backpressure - // deadline. Letting those escape to the outer catch skipped - // sendCommitMessage, sealAndSwapBuffer and drainOnClose -- so a - // flush that had already published deferred dictionary chunks - // left their group open forever, and every row an EARLIER - // successful flush published was abandoned unacked. The batch is - // NOT discarded here (unlike the over-cap case, this failure is - // not a verdict on the batch's contents), but the rest of close() - // must still run. rethrowTerminal below surfaces it. - terminalError = captureCloseError(terminalError, t); - } - if (!deferCommit && hasDeferredMessages) { - sendCommitMessage(); + if (strictTransactional || txnAborted) { + // A strict transaction commits only on an explicit flush(), so + // closing without one discards it, and an aborted one must never + // commit. The pending rows never left the client. The frames of + // the open transaction already sent stay uncommitted: a server + // with rollback support discards them on the rollback frame, whose + // ack the drain below waits for; any other server rolls them back + // when the connection closes, and a recovered slot retires them + // as an aborted transaction. + if (!txnAborted) { + warnDiscardedTransaction("close()"); + } + discardPendingRows(); + if (hasDeferredMessages && serverRollbackEnabled && !txnAborted) { + try { + publishRollbackFrame(); + hasDeferredMessages = false; + lastCommitBoundaryFsn = cursorEngine.publishedFsn(); + clearTransactionState(); + } catch (Throwable t) { + terminalError = captureCloseError(terminalError, t); + } + } + } else { + try { + flushPendingRows(deferCommit, false); + } catch (BatchTooLargeForCapException e) { + resetTableBuffersAfterFlush(); + terminalError = captureCloseError(terminalError, e); + } catch (Throwable t) { + // Same reasoning as the pre-flight rejection above, for the + // failures a size check cannot see: sealAndSwapBuffer's + // buffer-recycle timeout and appendBlocking's backpressure + // deadline. Letting those escape to the outer catch skipped + // sendCommitMessage, sealAndSwapBuffer and drainOnClose -- so a + // flush that had already published deferred dictionary chunks + // left their group open forever, and every row an EARLIER + // successful flush published was abandoned unacked. The batch is + // NOT discarded here (unlike the over-cap case, this failure is + // not a verdict on the batch's contents), but the rest of close() + // must still run. rethrowTerminal below surfaces it. + terminalError = captureCloseError(terminalError, t); + } + if (!deferCommit && hasDeferredMessages) { + sendCommitMessage(); + } } - if (activeBuffer != null && activeBuffer.hasData()) { + if (!strictTransactional && !txnAborted && activeBuffer != null && activeBuffer.hasData()) { sealAndSwapBuffer(); if (!deferCommit) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); @@ -2272,6 +2373,11 @@ public void flush() { public long flushAndGetSequence() { checkNotClosed(); checkRecycleFailure(); + if (txnAborted) { + // Committing now would commit the frames a failed rollback left on + // the server together with whatever came after them. + throw transactionAbortedError(); + } if (cursorEngine != null) { cursorEngine.checkDurability(); } @@ -2285,7 +2391,7 @@ public long flushAndGetSequence() { // is published in its mmap'd segment. PERIODIC stable-storage barriers // run independently in the manager. No processingCount to drain and no // awaitPendingAcks here; just surface any I/O thread error. - flushPendingRows(deferCommit); + flushPendingRows(deferCommit, false); if (!deferCommit && hasDeferredMessages) { sendCommitMessage(); } @@ -2618,7 +2724,16 @@ public SenderError getLastTerminalError() { * @return the global symbol ID */ public int getOrAddGlobalSymbol(CharSequence symbol) { + int sizeBefore = globalSymbolDictionary.size(); int globalId = globalSymbolDictionary.getOrAddSymbol(symbol); + if (globalId >= sizeBefore) { + // A new entry: the batch's next frame carries it in the dictionary + // section ([len varint][utf8]), which the column bytes behind + // pendingBytes do not count. Without this a batch of new long symbols + // could pass the byte trigger and still outgrow the frame cap. + int len = NativeBufferWriter.utf8Length(symbol); + pendingSymbolDictBytes += NativeBufferWriter.varintSize(len) + len; + } if (globalId > currentBatchMaxSymbolId) { currentBatchMaxSymbolId = globalId; } @@ -2687,6 +2802,37 @@ public int getServerMaxBatchSize() { return serverMaxBatchSize; } + /** + * The server's per-table {@code qwp.max.uncommitted.rows} from the last + * foreground handshake ({@code X-QWP-Max-Uncommitted-Rows}), or 0 when the + * server did not advertise it. + */ + public long getServerMaxUncommittedRows() { + return serverMaxUncommittedRows; + } + + /** + * True when the server of the last foreground handshake confirmed QWP + * rollback support, so {@link #rollback()} can discard frames already sent. + */ + public boolean isServerRollbackEnabled() { + return serverRollbackEnabled; + } + + /** + * True in strict transactional mode; see {@link #setStrictTransactional(boolean)}. + */ + public boolean isStrictTransactional() { + return strictTransactional; + } + + /** + * True in transactional mode (strict or not); see {@link #setTransactional(boolean)}. + */ + public boolean isTransactional() { + return transactional; + } + /** Resolved value of {@code symbol_dict_reset_max_wait_millis}. */ @TestOnly public long getSymbolDictResetMaxWaitMillis() { @@ -2976,6 +3122,16 @@ public long getTotalServerErrors() { return counters.serverErrors.get(); } + /** + * Rows of the open transaction the sender already committed because the + * transaction outgrew the client buffer (see {@link #setTransactional}); 0 + * when it did not. + */ + @TestOnly + public long getTxnRowsCommittedInPartsForTesting() { + return txnRowsCommittedInParts; + } + /** * Adds an INT column value to the current row. * @@ -3250,9 +3406,108 @@ public void setCloseStartedHook(Runnable hook) { this.closeStartedHook = hook; } + /** + * Discards the rows buffered since the last flush, like before. In + * transactional mode it also discards the open transaction's frames already + * sent, when the server supports QWP rollback (see {@link #rollback()}), and + * like {@link #rollback()} it does not wait for the server. Without that + * support a strict transaction is aborted -- the next flush throws and only + * closing the sender ends it -- while a non-strict one keeps the frames + * already sent, and the next flush commits them, as before. + */ @Override public void reset() { checkNotClosed(); + discardPendingRows(); + if (transactional && hasDeferredMessages && !txnAborted) { + if (serverRollbackEnabled) { + rollbackOpenTransaction("reset()"); + } else if (strictTransactional) { + txnAborted = true; + } else if (!isResetRollbackWarned) { + isResetRollbackWarned = true; + LOG.warn("reset() discarded the rows not yet sent, but the server does not support QWP " + + "rollback, so the open transaction's rows already sent stay on the server and the " + + "next flush() commits them. Close the sender to discard them instead"); + } + } + } + + /** + * Discards the open transaction: the rows buffered since the last flush, + * and the transaction's frames already sent, which the server rolls back on + * the rollback control frame. Without an open transaction this only discards + * the buffered rows. + *

+ * Like {@link #flush()}, it hands the rollback frame to store-and-forward and + * returns without waiting for the server, so a server outage makes it fail + * only when the store-and-forward buffer is full, as it would a flush. The + * frame goes out behind the transaction's frames and ahead of anything sent + * later, on every connection and after a restart, so nothing sent later can + * commit the discarded rows. Call {@link #drain(long)} to wait for the server + * to confirm. + *

+ * A transaction=on transaction that already committed in parts, at the + * client buffer or the server's {@code qwp.max.uncommitted.rows}, keeps the + * parts it committed; this discards only the rest. + * + * @throws LineSenderException when frames of the open transaction were already + * sent and the server does not support QWP rollback + * (the transaction is then aborted: the next flush + * throws, and only closing the sender ends it) + */ + @Override + public void rollback() { + checkNotClosed(); + checkRecycleFailure(); + discardPendingRows(); + if (txnAborted) { + throw transactionAbortedError(); + } + if (!hasDeferredMessages) { + clearTransactionState(); + return; + } + if (!serverRollbackEnabled) { + txnAborted = true; + throw transactionAbortedError(); + } + rollbackOpenTransaction("rollback()"); + } + + /** + * Pool hook for a strict-mode lease being returned: discards what the + * borrower left uncommitted, so the next borrower starts with no open + * transaction. Returns {@code false} when frames already sent could not be + * rolled back -- the server does not support QWP rollback, or the transaction + * was already aborted -- and the pool must retire the connection instead; the + * server rolls those frames back when the connection closes. + */ + public boolean discardTransactionForReuse() { + checkNotClosed(); + if (!txnAborted) { + warnDiscardedTransaction("returning the sender to the pool"); + } + discardPendingRows(); + if (txnAborted) { + return false; + } + if (!hasDeferredMessages) { + clearTransactionState(); + return true; + } + if (!serverRollbackEnabled) { + // Only closing the connection discards the frames already sent, so the + // transaction is over. Aborting it also keeps the close() that retires + // the connection from warning about the same rows again. + txnAborted = true; + return false; + } + rollbackOpenTransaction("returning the sender to the pool"); + return true; + } + + private void discardPendingRows() { // Reset ALL table buffers, not just the current one ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { @@ -3273,6 +3528,7 @@ public void reset() { currentBatchMaxSymbolId = -1; reclaimUnsentSymbolIds(); pendingBytes = 0; + pendingSymbolDictBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; currentTableBuffer = null; @@ -3282,6 +3538,57 @@ public void reset() { cachedTimestampNanosColumn = null; } + /** + * Publishes the rollback control frame behind the open transaction's + * frames: a bare header the server answers by rolling back every row this + * connection holds uncommitted. Sent only to a server that confirmed + * X-QWP-Rollback. + */ + private void publishRollbackFrame() { + int messageSize = encoder.encodeRollbackMessage(); + QwpBufferWriter buffer = encoder.getBuffer(); + ensureActiveBufferReady(); + activeBuffer.ensureCapacity(messageSize); + activeBuffer.write(buffer.getBufferPtr(), messageSize); + activeBuffer.incrementRowCount(); + sealAndSwapBuffer(); + } + + /** + * Rolls back the open transaction: publishes the rollback frame behind its + * frames and ends it, without waiting for the server. + *

+ * No wait is needed to keep the rolled-back frames from ever committing. The + * store-and-forward log orders the rollback frame ahead of every frame + * published later, and every connection replays the log in that order, so a + * server that receives a later commit has rolled the transaction back first. + * Only a server that does not support rollback, reached by a reconnect + * before the frame was acked, could commit them: it would read the frame as + * an empty deferred frame. The I/O loop never sends it there; it discards + * the transaction by closing that connection instead (see + * {@code CursorWebSocketSendLoop.retireRollbackFrame}). The same holds for + * frames replayed after a restart or by a background drainer, which use the + * same loop. + *

+ * The rows a transaction=on transaction committed in parts stay committed; + * {@code action} names the call in the warning that says so. + */ + private void rollbackOpenTransaction(String action) { + long committedRows = txnRowsCommittedInParts; + publishRollbackFrame(); + hasDeferredMessages = false; + // The rollback frame gets an OK ack covering the transaction's frames, or the + // I/O loop acknowledges it when it discards the transaction itself, so + // close-time drains may wait for it like for a commit. + lastCommitBoundaryFsn = cursorEngine.publishedFsn(); + clearTransactionState(); + if (committedRows > 0) { + LOG.warn("{} discarded the rows of the open transaction sent since the sender last committed part of " + + "it, but the {} rows it committed when the transaction outgrew the client buffer stay " + + "committed", action, committedRows); + } + } + /** * Advisory request to start a fresh symbol-dictionary epoch. Sets * {@link #manualResetRequested}; if no flush is in flight @@ -3400,6 +3707,8 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { engine.adoptCounters(counters); engine.setSlotLockReleaseListener(this::onSlotLockReleased); } + // The segment size bounds every frame; re-size the byte trigger for it. + recomputeFrameBudget(); } /** @@ -3549,8 +3858,37 @@ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) { this.recycleDeferredCloseMaxWaitMillis = millis; } + /** + * Strict transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames + * like {@link #setTransactional(boolean) transactional} mode, only an explicit + * flush commits, and the open transaction never commits in parts. A row that + * would push a table past the server's {@code qwp.max.uncommitted.rows}, or the + * open transaction past what the client buffer can hold alongside the commit, + * throws {@link TransactionTooLargeException} instead and is not added. + * {@link #close()} discards an uncommitted transaction rather than committing + * it. Enabling it enables transactional mode; disabling it leaves transactional + * mode on. + */ + public void setStrictTransactional(boolean strict) { + this.strictTransactional = strict; + if (strict) { + this.transactional = true; + } + } + + /** + * Transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames and + * only an explicit flush (or close) commits. A transaction that outgrows the + * client buffer commits in parts: before a flush that does not fit beside the + * frames already sent, the sender commits those frames and logs a warning. + * Disabling it also disables {@link #setStrictTransactional(boolean) strict} + * mode, which refuses rows instead. + */ public void setTransactional(boolean transactional) { this.transactional = transactional; + if (!transactional) { + this.strictTransactional = false; + } } /** @@ -3886,44 +4224,30 @@ private static List singleEndpoint(String host, int port) { } /** - * Clamps the soft-flush byte budget to fit under the server's advertised - * X-QWP-Max-Batch-Size minus a safety margin for encoding overhead - * (schema, dict deltas, framing). A 0 advertisement means the server did - * not send the header (older build) and the configured budget is kept - * verbatim. Called on every successful connect because a rolling upgrade - * can leave neighbouring endpoints with different caps. + * Records the server's advertised X-QWP-Max-Batch-Size and re-sizes the + * byte trigger from it. A 0 advertisement means the server did not send + * the header (older build). Called on every successful connect because a + * rolling upgrade can leave neighbouring endpoints with different caps. *

- * Always updates {@link #serverMaxBatchSize} so the single-row hard guard - * in {@link #sendRow} fires against the freshly advertised value. The - * byte trigger, however, is only adjusted when the user left it enabled: - * an explicit {@code auto_flush_bytes=off} (autoFlushBytes == 0) is - * preserved even when the server advertises a cap, so applications that - * opted out keep the contract they asked for. + * Always updates {@link #serverMaxBatchSize}, so the single-row guard in + * {@link #sendRow} fires against the freshly advertised value. See + * {@link #recomputeFrameBudget()} for the byte trigger. */ @TestOnly public void applyServerBatchSizeLimit(int advertisedMaxBatchSize) { serverMaxBatchSize = advertisedMaxBatchSize; - if (autoFlushBytes <= 0) { - // User opted out of byte-based auto-flush; respect that even when - // the server advertises a cap. The single-row guard still protects - // against oversize individual rows via serverMaxBatchSize. - effectiveAutoFlushBytes = 0; - return; - } - if (advertisedMaxBatchSize <= 0) { - effectiveAutoFlushBytes = autoFlushBytes; - return; - } - // Cap at 90% of the server's hard limit. Raw column-buffer bytes are - // a conservative proxy for wire size (compression usually shrinks the - // payload), but schema and dict-delta overhead can push the wire size - // above the raw total in pathological cases. - long safeBudget = (long) advertisedMaxBatchSize * 9 / 10; - if (autoFlushBytes < safeBudget) { - effectiveAutoFlushBytes = autoFlushBytes; - } else { - effectiveAutoFlushBytes = (int) safeBudget; - } + recomputeFrameBudget(); + } + + /** + * Records the transaction limits the server advertised on the last + * foreground handshake: its per-table {@code qwp.max.uncommitted.rows} + * ({@code 0} when not advertised) and whether it confirmed rollback support. + */ + @TestOnly + public void applyServerTransactionLimits(long maxUncommittedRows, boolean rollbackEnabled) { + serverMaxUncommittedRows = maxUncommittedRows; + serverRollbackEnabled = rollbackEnabled; } private void atMicros(long timestampMicros) { @@ -4256,6 +4580,9 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpDurableAckTiers(durableAckTiers); + // Ask every server whether it accepts the rollback control frame; + // only one that confirms ever receives it. + newClient.setQwpRequestRollback(true); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); if (cancellation != null) { // Publish the client we are about to block on so a @@ -4460,6 +4787,8 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo // endpoint's cap; an oversize row then escapes the producer-side // guard and trips a wire-level ws-close[1009] downstream. applyServerBatchSizeLimit(newClient.getServerMaxBatchSize()); + applyServerTransactionLimits( + newClient.getServerMaxUncommittedRows(), newClient.isServerRollbackEnabled()); return newClient; } catch (Error e) { closeQuietlyOnError(newClient); @@ -5189,8 +5518,12 @@ private void ensureNoInProgressRow() { * @param deferCommit when true, the message carries FLAG_DEFER_COMMIT * so the server appends rows but does not commit. * Used by auto-flush in transactional mode. + * @param isAutoFlush true when a row triggered the flush: when the client + * buffer cannot take it, a strict transaction keeps the + * rows pending instead of throwing, and refuses the next + * row. */ - private void flushPendingRows(boolean deferCommit) { + private void flushPendingRows(boolean deferCommit, boolean isAutoFlush) { if (pendingRowCount <= 0) { return; } @@ -5202,6 +5535,7 @@ private void flushPendingRows(boolean deferCommit) { if (tableCount == 0) { clearPendingTables(); pendingBytes = 0; + pendingSymbolDictBytes = 0; currentTableBufferSnapshotBytes = currentTableBuffer == null ? 0 : currentTableBuffer.getBufferedBytes(); pendingRowCount = 0; @@ -5233,6 +5567,39 @@ private void flushPendingRows(boolean deferCommit) { int combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); int messageSize = encoder.finishMessage(); + // The client buffer holds a transaction's frames until the server commits it, + // so it cannot trim them to make room. Before any frame of this flush reaches + // the ring, check that the buffer holds the flush beside them. + if (transactional && cursorEngine != null) { + if (strictTransactional) { + // A strict transaction must be able to commit as one: refuse a flush + // the client buffer could not hold together with the commit that has + // to follow it. + if (!hasTransactionRoom(cap, deltaBaseline, messageSize, deferCommit)) { + if (deferCommit) { + txnBufferFull = true; + if (isAutoFlush) { + // The rows stay pending, the one that triggered this flush + // included, and the room kept for the commit holds them. The + // next row is refused, so a refused row is never part of the + // transaction, as at the server's row limit. + return; + } + } + throw transactionTooLargeForClientBuffer("so the flush was not sent"); + } + } else if (txnSimSegments > 0 && !hasTransactionRoom(cap, deltaBaseline, messageSize, deferCommit)) { + // transaction=on commits in parts instead: commit the frames already + // sent, which lets the buffer trim them, and send this flush as the + // start of the next part. A flush that does not fit even alone goes + // out as it is. + commitTransactionPart(); + // The commit frame went through the encoder: encode the batch again. + combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); + messageSize = encoder.finishMessage(); + } + } + // Full-dict over-cap fallback. A full-dictionary frame carries the whole // dictionary from id 0, so its fixed overhead grows with lifetime symbol // cardinality: eventually the section alone fills the cap, and before that @@ -5302,6 +5669,7 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { if (!deferCommit) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + accountTransactionRows(deferCommit); resetTableBuffersAfterFlush(); } catch (Throwable t) { @@ -5531,6 +5899,7 @@ private void flushPendingRowsSplit( // committed the whole group. lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + accountTransactionRows(deferCommit); resetTableBuffersAfterFlush(); } @@ -5554,6 +5923,7 @@ private void resetTableBuffersAfterFlush() { flushTableBuffers.setPos(0); currentBatchMaxSymbolId = -1; pendingBytes = 0; + pendingSymbolDictBytes = 0; currentTableBufferSnapshotBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; @@ -6461,6 +6831,7 @@ private void sendCommitMessage() { sealAndSwapBuffer(); hasDeferredMessages = false; lastCommitBoundaryFsn = cursorEngine.publishedFsn(); + clearTransactionState(); } private SenderErrorHandler userErrorHandler() { @@ -7110,7 +7481,17 @@ private void sealAndSwapBuffer() { // (durable mmap append, returns once published). try { toSend.markSending(); + // Transactional mode tracks how the open transaction's frames pack + // into segments; the first frame's placement depends on the active + // segment's room before the append. + boolean isTxnFrame = transactional + && isTransactionFrame(toSend.getBufferPtr(), toSend.getBufferPos()); + long activeRemaining = isTxnFrame && txnSimSegments == 0 + ? cursorEngine.activeSegment().capacityRemaining() : 0; cursorEngine.appendBlocking(toSend.getBufferPtr(), toSend.getBufferPos()); + if (isTxnFrame) { + recordTransactionFrame(toSend.getBufferPos(), activeRemaining); + } toSend.markRecycled(); } catch (Throwable t) { // appendBlocking failed synchronously on the user thread — the @@ -7135,6 +7516,315 @@ private void sealAndSwapBuffer() { } } + /** + * Adds the rows a just-published FLAG_DEFER_COMMIT flush shipped to each + * flushed table's open-transaction count, or forgets the open transaction + * when the flush committed it. + */ + private void accountTransactionRows(boolean deferred) { + if (!deferred) { + clearTransactionState(); + return; + } + for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { + QwpTableBuffer table = flushTableBuffers.getQuick(i); + long sentRows = table.getTxnSentRows(); + if (sentRows == 0) { + txnTables.add(table); + } + table.setTxnSentRows(sentRows + table.getRowCount()); + } + } + + private void checkTransactionAcceptsRow() { + if (txnAborted) { + throw transactionAbortedError(); + } + if (!strictTransactional) { + return; + } + if (txnBufferFull) { + throw transactionTooLargeForClientBuffer("so the row was not added"); + } + long maxRows = serverMaxUncommittedRows; + if (maxRows <= 0) { + maxRows = QwpConstants.DEFAULT_SERVER_MAX_UNCOMMITTED_ROWS; + } + // The server commits a table on its own once its uncommitted rows reach + // maxRows, so a strict transaction holds at most maxRows - 1 rows per table. + long rows = currentTableBuffer.getTxnSentRows() + currentTableBuffer.getRowCount() + 1; + if (rows >= maxRows) { + throw new TransactionTooLargeException("transaction too large for the server") + .put(" [table=").put(currentTableName) + .put(", rows=").put(rows) + .put(", maxUncommittedRows=").put(maxRows) + .put("]; the server commits a table's rows on its own once qwp.max.uncommitted.rows " + + "of them are uncommitted, which would split the transaction, so the row was " + + "not added. Raise qwp.max.uncommitted.rows on the server"); + } + } + + /** + * Forgets the open transaction: its tables' row counts, the client-buffer + * simulation and the buffer-full latch. Called when a frame commits it or + * the rollback frame discards it; O(tables in the transaction). + */ + private void clearTransactionState() { + for (int i = 0, n = txnTables.size(); i < n; i++) { + txnTables.getQuick(i).setTxnSentRows(0); + } + txnTables.setPos(0); + txnBufferFull = false; + txnBytes = 0; + txnRefusedBytes = 0; + txnRowsCommittedInParts = 0; + txnSimSegments = 0; + txnSimRemaining = 0; + } + + /** + * transaction=on: commits the frames of the open transaction already sent, + * because the next flush does not fit beside them in the client buffer, which + * cannot trim them before the server commits them. The transaction goes on + * with the rows sent from here on, so it commits in parts, as it does when the + * server commits a table at {@code qwp.max.uncommitted.rows}. Logs the first + * part each transaction commits this way. + */ + private void commitTransactionPart() { + long rows = sentTransactionRows(); + long bytes = txnBytes; + long committedRows = txnRowsCommittedInParts; + sendCommitMessage(); + // sendCommitMessage() forgot the transaction, but it goes on. + txnRowsCommittedInParts = committedRows + rows; + if (committedRows == 0) { + LOG.warn("the open transaction outgrew the client buffer, so the sender committed the {} rows it had " + + "already sent, and the rest of the transaction commits separately [transactionBytes={}, " + + "ringCapacityBytes={}]. A transaction=on transaction commits in parts when " + + "sf_max_total_bytes cannot hold it: raise sf_max_total_bytes or set sf_dir to keep such " + + "transactions whole, or use transaction=strict to refuse their rows instead. Logged once " + + "per transaction", + rows, bytes, cursorEngine.ringCapacityBytes()); + } + } + + /** + * Transactional mode: whether the ring under {@code sf_max_total_bytes} can hold + * a flush's frames together with the open transaction's frames already on it + * and, for a deferred flush, room for the frame that ends the transaction. The + * server acks none of a transaction's frames before its commit, so the ring + * cannot trim them; a transaction that does not fit would block its own commit. + * A strict transaction keeps room for a commit that carries the rows still + * pending when it ends, a transaction=on transaction for a bare commit frame: + * it commits its frames with one when the next flush does not fit. + *

+ * Mirrors how the ring packs frames: frames never span segments, and a frame + * that does not fit the current segment opens the next one. The flush's + * frame sizes are derived the way flushPendingRows and the split publish + * them; dictionary chunks of the full-dictionary fallback count as full + * frames. Runs before anything of the flush is published, so a refusal + * leaves the batch pending and the ring untouched. On a strict refusal it + * records the size the transaction would have reached in + * {@link #txnRefusedBytes}. + */ + private boolean hasTransactionRoom(int cap, int deltaBaseline, int messageSize, boolean deferCommit) { + CursorSendEngine engine = cursorEngine; + long segmentPayload = segmentPayloadCapacity(); + transactionFrameSizes.clear(); + if (cap <= 0 || messageSize <= cap) { + transactionFrameSizes.add(messageSize); + } else { + int baseline = deltaBaseline; + if (!deltaDictEnabled && currentBatchMaxSymbolId > deltaBaseline + && !splitFramesFit(cap, deltaBaseline) + && splitFramesFit(cap, currentBatchMaxSymbolId)) { + // The full-dictionary fallback ships the section as chunks of at + // most cap bytes ahead of the table frames. + long dictBytes = encoder.getSplitMessageSize(0, deltaBaseline, currentBatchMaxSymbolId) + - QwpConstants.HEADER_SIZE; + long chunkPayload = Math.max(1, cap - QwpConstants.HEADER_SIZE - 16); + for (long chunks = dictBytes / chunkPayload + 1; chunks > 0; chunks--) { + transactionFrameSizes.add(cap); + } + baseline = currentBatchMaxSymbolId; + } + for (int i = 0, n = splitFrameBodyBytes.size(); i < n; i++) { + transactionFrameSizes.add(encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(i), baseline, currentBatchMaxSymbolId)); + if (deltaDictEnabled && currentBatchMaxSymbolId > baseline) { + baseline = currentBatchMaxSymbolId; + } + } + } + if (deferCommit && !strictTransactional) { + // Keep room for the bare commit or rollback frame that ends the + // transaction: the next flush that does not fit commits it with one. + transactionFrameSizes.add(COMMIT_FRAME_MAX_BYTES); + } + int segments = txnSimSegments; + long remaining = segments == 0 ? engine.activeSegment().capacityRemaining() : txnSimRemaining; + long bytes = txnBytes; + for (int i = 0, n = transactionFrameSizes.size(); i < n; i++) { + long size = transactionFrameSizes.getQuick(i); + bytes += size; + if (segments == 0 || size > remaining) { + if (segments > 0 || size > remaining) { + remaining = segmentPayload; + } + segments++; + } + remaining = Math.max(0, remaining - size - MmapSegment.FRAME_HEADER_SIZE); + } + if (deferCommit && strictTransactional) { + // Keep room for the commit: the rows pending when this transaction + // ends are at most a frame's worth plus one row, which a split + // ships as a few frames of at most cap bytes. + long reserve = cap > 0 ? Math.min(cap, segmentPayload) : segmentPayload; + for (int i = 0; i < STRICT_COMMIT_RESERVE_FRAMES; i++) { + if (reserve > remaining) { + remaining = segmentPayload; + segments++; + } + remaining = Math.max(0, remaining - reserve - MmapSegment.FRAME_HEADER_SIZE); + } + } + long maxSegments = Math.max(2, engine.ringCapacityBytes() / engine.segmentSizeBytes()); + if (segments > maxSegments) { + if (strictTransactional) { + txnRefusedBytes = bytes; + } + return false; + } + return true; + } + + /** + * True for a QWP frame that joins the open transaction: FLAG_DEFER_COMMIT + * set, and not the rollback control frame (which also carries the flag). + */ + private static boolean isTransactionFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DEFER_COMMIT) != 0 && (flags & QwpConstants.FLAG_ROLLBACK) == 0; + } + + /** + * Re-sizes the byte trigger {@link #effectiveAutoFlushBytes} from the frame + * size the connection can carry: the smaller of the advertised + * X-QWP-Max-Batch-Size and the payload a store-and-forward segment can hold. + *

+ */ + private void recomputeFrameBudget() { + long segmentPayload = segmentPayloadCapacity(); + int advertised = serverMaxBatchSize; + long frameSize = Math.min(advertised > 0 ? advertised : FALLBACK_SERVER_MAX_BATCH_SIZE, segmentPayload); + long budget = frameSize * 9 / 10; + if (autoFlushBytes > 0) { + long limit = advertised > 0 ? budget + : (segmentPayload == Long.MAX_VALUE ? Long.MAX_VALUE : segmentPayload * 9 / 10); + effectiveAutoFlushBytes = (int) Math.min(autoFlushBytes, limit); + } else if (autoFlushBytes == AUTO_FLUSH_BYTES_AUTO) { + effectiveAutoFlushBytes = (int) budget; + } else { + effectiveAutoFlushBytes = 0; + } + } + + /** + * Transactional mode: advances the client-buffer simulation by one + * open-transaction frame that just reached the ring. {@code activeRemaining} + * is the active segment's room before the append, needed for the + * transaction's first frame. + */ + private void recordTransactionFrame(int payloadLen, long activeRemaining) { + txnBytes += payloadLen; + if (txnSimSegments == 0) { + txnSimRemaining = payloadLen > activeRemaining ? segmentPayloadCapacity() : activeRemaining; + txnSimSegments = 1; + } else if (payloadLen > txnSimRemaining) { + txnSimRemaining = segmentPayloadCapacity(); + txnSimSegments++; + } + txnSimRemaining = Math.max(0, txnSimRemaining - payloadLen - MmapSegment.FRAME_HEADER_SIZE); + } + + /** + * Largest frame payload a store-and-forward segment can hold, or + * {@link Long#MAX_VALUE} before an engine is attached. + */ + private long segmentPayloadCapacity() { + CursorSendEngine engine = cursorEngine; + return engine == null + ? Long.MAX_VALUE + : engine.segmentSizeBytes() - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE; + } + + /** + * Rows the open transaction's deferred frames already sent, over all its tables. + */ + private long sentTransactionRows() { + long rows = 0; + for (int i = 0, n = txnTables.size(); i < n; i++) { + rows += txnTables.getQuick(i).getTxnSentRows(); + } + return rows; + } + + private LineSenderException transactionAbortedError() { + return new LineSenderException("transaction aborted: rollback could not undo the frames of the " + + "open transaction already sent, because the server does not support QWP rollback. Close " + + "the sender to discard them; the server rolls them back when the connection closes"); + } + + /** + * The error for a strict transaction the client buffer cannot hold: the size + * the transaction would have reached, the buffer, what was refused, and the + * settings that make room. + */ + private LineSenderException transactionTooLargeForClientBuffer(String outcome) { + CursorSendEngine engine = cursorEngine; + return new TransactionTooLargeException("transaction too large for the client buffer") + .put(" [transactionBytes=").put(txnRefusedBytes) + .put(", ringCapacityBytes=").put(engine.ringCapacityBytes()) + .put(", segmentBytes=").put(engine.segmentSizeBytes()) + .put("]; the client holds a strict transaction until the server commits it, and " + + "sf_max_total_bytes cannot hold this one, ").put(outcome) + .put(". Raise sf_max_total_bytes, or set sf_dir to keep the transaction on disk"); + } + + /** + * Strict mode commits only on flush(), so closing the sender or returning a + * pooled lease discards the open transaction. Logs it when the transaction holds + * rows: a caller that relied on close() to commit, as a sender with auto-flush + * triggers does, would otherwise lose them without a trace. Counts the rows + * already sent in deferred frames and those still buffered; call it before + * discardPendingRows() clears the latter. + */ + private void warnDiscardedTransaction(String action) { + long rows = pendingRowCount + sentTransactionRows(); + if (rows > 0) { + LOG.warn("{} discarded an uncommitted transaction of {} rows: a strict transactional sender " + + "(transaction=strict, or one with no auto-flush trigger) commits only on flush(). Call " + + "flush() before close() to keep the rows, or ignore this if the discard is intentional", + action, rows); + } + } + /** * Accumulates the current row. * Rows buffer until flush (explicit or auto-flush). @@ -7142,6 +7832,12 @@ private void sealAndSwapBuffer() { private void sendRow() { checkRecycleFailure(); ensureConnected(); + if (transactional) { + // Before nextRow(): a refusal leaves the row uncommitted, so the + // at()/atNow() error path discards it and the transaction keeps the + // rows accepted before it. + checkTransactionAcceptsRow(); + } // Hard guard: a single row whose bytes exceed the server's wire cap // would flush as an oversize WS frame the server closes with @@ -7179,7 +7875,7 @@ private void sendRow() { currentTableBufferSnapshotBytes = bufferedNow; if (shouldAutoFlush()) { - flushPendingRows(transactional); + flushPendingRows(transactional, true); } } @@ -7193,7 +7889,7 @@ private boolean shouldAutoFlush() { if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { return true; } - if (effectiveAutoFlushBytes > 0 && getPendingBytes() >= effectiveAutoFlushBytes) { + if (effectiveAutoFlushBytes > 0 && pendingBytes + pendingSymbolDictBytes >= effectiveAutoFlushBytes) { return true; } // pendingRowCount == 1: sendRow() just timestamped this very row, so its diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java new file mode 100644 index 00000000..c7cc37f8 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java @@ -0,0 +1,51 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** + * A strict transaction ({@code transaction=strict}, or a sender with no auto-flush + * trigger such as {@code auto_flush=off}, on the WebSocket transport) outgrew a + * limit that would stop it from committing as one transaction: the server's + * per-table {@code qwp.max.uncommitted.rows}, past which the server commits the + * rows on its own, or the client's {@code sf_max_total_bytes}, which must hold the + * whole uncommitted transaction until the server commits it. To send transactions + * of that size, raise the limit the message names: {@code qwp.max.uncommitted.rows} + * on the server, or {@code sf_max_total_bytes} on the client, whose default is + * larger with {@code sf_dir} set. + *

+ * {@code at()} and {@code atNow()} throw it for the row that would take the + * transaction past either limit, and that row is not part of the transaction. + * {@code flush()} throws it when the client buffer cannot hold the flush, and + * sends nothing. Either way nothing past the limit reaches the server, and the + * transaction keeps the rows accepted before the error. + */ +public class TransactionTooLargeException extends LineSenderException { + + public TransactionTooLargeException(CharSequence message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index ba46365a..16d9296d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -1900,6 +1900,17 @@ public int sealedSegmentsSnapshot(MmapSegment[] target) { return ring.snapshotSealedSegments(target); } + /** + * Bytes of segments the ring may hold under {@code sf_max_total_bytes}: the + * budget's capacity minus the side-file bytes it also charges (disk mode). + * In a sender pool the budget is shared, so this bounds the rings of all + * pooled senders together, not this ring alone. + */ + public long ringCapacityBytes() { + SegmentBudget budget = manager.getBudget(); + return budget.getCapacityBytes() - budget.getSideFileBytes(); + } + /** * Configured per-segment size in bytes. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 8a76edcc..e45e53c8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -591,6 +591,16 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // and by the I/O thread afterwards -- never concurrently. private long orphanSkipStartFsn = -1L; private long orphanSkipTipFsn = -1L; + // Whether the current connection's server confirmed X-QWP-Rollback, so a rollback + // control frame may go out on it. setWireBaselineWithCatchUp sets it for every + // connection from the client it runs on; false until then, the safe side (see + // retireRollbackFrame). Written by start() on the user thread before the I/O thread + // exists, and by the I/O thread afterwards -- never concurrently. + private boolean isRollbackConfirmed; + // FSN of the last commit-bearing frame sent on the current connection, -1 when none. + // Tracked only while !isRollbackConfirmed, for retireRollbackFrame. Reset with + // isRollbackConfirmed; same threading. + private long lastSentCommitBearingFsn = -1L; // Poison-frame detector state (I/O thread only). poisonFsn is the FSN of the // frame implicated by the most recent server-active rejection: the NACK-named // frame, or the OK-level head-of-line frame (highestOkFsn+1) for a @@ -3198,6 +3208,10 @@ private void setWireBaselineWithCatchUp(long replayStart) { // handleServerRejection can tell "only the catch-up went out" from "the // head data frame went out". dataFrameSentThisConnection = false; + // The server this connection reached decides whether a rollback frame may go + // out on it; until a client exists no frame goes out at all. + isRollbackConfirmed = client != null && client.isServerRollbackEnabled(); + lastSentCommitBearingFsn = -1L; // Do not gate solely on isDeltaDictEnabled(): a recovered delta slot can // lose access to its side-file while its on-disk frames still start above // zero. hasReplayDictionaryDependency preserves that distinction while @@ -3230,6 +3244,26 @@ private int frameDeltaStart(long payloadAddr, int payloadLen) { return encoded < 0L ? -1 : (int) (encoded >>> 3); } + // True for a QWP frame carrying FLAG_DEFER_COMMIT. A short or foreign payload counts + // as commit-bearing, so retireRollbackFrame waits for its ack: the safe side, and the + // rule RecoveredFrameAnalysis applies to the orphaned deferred tail. + private static boolean isDeferredFrame(long payloadAddr, int payloadLen) { + return payloadLen >= QwpConstants.HEADER_SIZE + && Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE + && (Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) + & QwpConstants.FLAG_DEFER_COMMIT) != 0; + } + + // True only for the rollback control frame (QwpWebSocketEncoder.encodeRollbackMessage): + // a bare QWP header whose flags byte equals ROLLBACK_FRAME_FLAGS, compared whole as the + // server matches it. + private static boolean isRollbackFrame(long payloadAddr, int payloadLen) { + return payloadLen == QwpConstants.HEADER_SIZE + && Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE + && Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) + == QwpConstants.ROLLBACK_FRAME_FLAGS; + } + // True only for a well-formed QWP frame this encoder produced that carries a // delta symbol-dict section. The magic check keeps the dict logic from // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose @@ -4001,6 +4035,16 @@ private boolean trySendBatch() { break; // fixed byte cap: the next call carries on from this frame } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + if (!isRollbackConfirmed && isRollbackFrame(frameAddr, payloadLen)) { + if (batchFrames > 0) { + break; + } + // This server cannot roll back, so the frame never goes out on this + // connection: the loop waits for acks, or it discarded the rolled-back + // transaction and recycled. + retireRollbackFrame(); + return false; + } // Torn-dictionary guard. sentDictCount is this loop's model of how many ids the // CURRENT server has been told about. A frame whose delta starts ABOVE that // coverage references ids the server was never given, and the server now rejects @@ -4063,6 +4107,9 @@ private boolean trySendBatch() { sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; nextWireSeq++; + if (!isRollbackConfirmed && !isDeferredFrame(frameAddr, payloadLen)) { + lastSentCommitBearingFsn = fsnSent; + } batchFrames++; batchBytes += wireLen; if (replayTargetFsn >= 0) { @@ -4114,6 +4161,45 @@ void positionCursorForStart() { positionCursorAt(replayStart); } + /** + * Retires the rollback control frame at the send cursor of a connection whose + * server did not confirm X-QWP-Rollback. The producer sends that frame only after + * a server confirmed rollback, so this connection is a reconnect that reached a + * server without rollback support, such as an older node after a failover, with + * the frame not yet acked. That server would read the frame as an empty deferred + * frame and commit the rolled-back transaction, whose frames it holds uncommitted + * on this connection, with the next commit frame. So the frame never goes out: + * closing the connection discards those frames instead, as every server does with + * the rows a closed connection leaves uncommitted. + *

+ * The close waits until the server acks every commit-bearing frame sent on this + * connection before the rollback frame: the self-acknowledge below covers those + * frames too, so a commit the server had not yet processed would be lost with the + * connection. Until then this returns without doing anything, and tryReceiveAcks + * runs every loop iteration. Once they are acked, the frames this connection holds + * unacked are exactly the rolled-back transaction's deferred frames. The cumulative + * self-acknowledge of the rollback frame retires them, and the recycle discards + * them server-side and re-anchors the next connection past the rollback frame, the + * way {@link #tryRetireOrphanTail} handles an orphaned deferred tail. + */ + private void retireRollbackFrame() { + long rollbackFsn = fsnAtZero + nextWireSeq; + long ackedFsn = engine.ackedFsn(); + if (ackedFsn < lastSentCommitBearingFsn) { + return; + } + LOG.info("server does not support QWP rollback: discarding the rolled-back transaction " + + "[fsn {}..{}] by closing the connection that holds it uncommitted", + externalFsnBase + ackedFsn + 1L, externalFsnBase + rollbackFsn); + boolean isAdvanced = engine.acknowledge(rollbackFsn); + // Every ack is clamped to the frames this loop sent, and the cursor stands on + // the rollback frame, so the self-acknowledge always advances the watermark. + assert isAdvanced : "rollback frame already acknowledged [fsn=" + rollbackFsn + ']'; + dispatchProgress(rollbackFsn); + fail(new LineSenderException( + "recycling connection to discard a rolled-back transaction the server cannot roll back")); + } + /** * Retires the recovered orphaned deferred tail once retirable. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 7d3f5f56..ba400016 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -472,6 +472,14 @@ public synchronized boolean isWorkerReaped() { return workerThread == null; } + /** + * The {@code sf_max_total_bytes} budget this manager charges its rings' + * segments to; shared with other managers in a sender pool. + */ + public SegmentBudget getBudget() { + return budget; + } + /** * Stop tracking {@code ring}. Idempotent; safe to call from any thread. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 0fa7a645..ba1ee0e5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -36,6 +36,13 @@ public final class QwpConstants { * Client identifier sent in the X-QWP-Client-Id upgrade header. */ public static final String CLIENT_ID = "java/1.0.2"; + /** + * Default of the server's {@code qwp.max.uncommitted.rows}: the per-table + * count of rows a deferred-commit group may hold before the server commits + * them on its own. A strict transaction assumes this value when the server + * does not advertise {@code X-QWP-Max-Uncommitted-Rows}. + */ + public static final long DEFAULT_SERVER_MAX_UNCOMMITTED_ROWS = 1_000_000L; /** * Flag bit: defer WAL commit. The server appends rows to WAL writers * but skips the commit until a subsequent message without this flag. @@ -50,6 +57,12 @@ public final class QwpConstants { * Flag bit: Gorilla timestamp encoding enabled. */ public static final byte FLAG_GORILLA = 0x04; + /** + * Flag bit: roll back the connection's open deferred-commit group. Only ever + * set on the rollback control frame (see {@link #ROLLBACK_FRAME_FLAGS}), which + * the client sends only after the server confirmed {@code X-QWP-Rollback}. + */ + public static final byte FLAG_ROLLBACK = 0x20; /** * Flag bit: payload region after the prelude is zstd-compressed. Set only * when the handshake negotiated zstd compression. Mirror of the server-side @@ -108,6 +121,13 @@ public final class QwpConstants { * constant; used by the decoder to reject malformed wire bytes. */ public static final int MAX_TABLE_NAME_LENGTH = 127; + /** + * Flags byte of the rollback control frame: a bare 12-byte header with no + * table blocks and no payload. It carries {@link #FLAG_DEFER_COMMIT} too, so a + * server that does not know {@link #FLAG_ROLLBACK} treats it as an empty + * deferred frame instead of a commit. + */ + public static final byte ROLLBACK_FRAME_FLAGS = FLAG_ROLLBACK | FLAG_DEFER_COMMIT; /** * Status byte on a {@code QUERY_ERROR} frame: the query was cancelled, * either by a client {@code CANCEL} frame or by explicit server-side diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 02f95b91..7e7ce99b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -78,6 +78,9 @@ public class QwpTableBuffer implements QuietCloseable { // buffer is listed once. Producer thread only. private boolean listedPending; private int rowCount; + // Rows of this table the sender already shipped in FLAG_DEFER_COMMIT frames + // of the open transaction. Producer thread only. + private long txnSentRows; public QwpTableBuffer(String tableName) { this(tableName, null); @@ -282,6 +285,14 @@ public String getTableName() { return tableName; } + /** + * Rows of this table already shipped in FLAG_DEFER_COMMIT frames of the + * sender's open transaction; 0 outside a transaction. + */ + public long getTxnSentRows() { + return txnSentRows; + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -426,6 +437,13 @@ public void setListedPending(boolean listedPending) { this.listedPending = listedPending; } + /** + * Sets the count reported by {@link #getTxnSentRows()}. + */ + public void setTxnSentRows(long txnSentRows) { + this.txnSentRows = txnSentRows; + } + private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { if (column.type != type) { throw new LineSenderException( diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index a9ef418a..e25bad14 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -28,6 +28,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -47,7 +48,9 @@ *

* Behaviour difference from a raw Sender: {@link #close()} flushes the buffer * and returns the slot to the pool. The underlying Sender is only truly closed - * when {@link io.questdb.client.QuestDB#close()} shuts the pool down. + * when {@link io.questdb.client.QuestDB#close()} shuts the pool down. A + * WebSocket Sender in strict transactional mode commits only on flush(), so + * there close() discards the open transaction instead of flushing it. *

* Because the slot is reused across borrows, this wrapper -- not the slot -- * carries the lease identity. A stale handle (held after {@link #close()}, with @@ -167,8 +170,18 @@ public void close() { // above. A stale duplicate throws here, and the finally routes it // to discardBroken, which drops it on the same stale-generation // check under the pool lock. - slot.live(generation).flush(); - flushed = true; + Sender live = slot.live(generation); + if (live instanceof QwpWebSocketSender && ((QwpWebSocketSender) live).isStrictTransactional()) { + // A strict transaction commits only on an explicit flush(), so + // returning the lease discards whatever the borrower left open. + // The next borrower must not inherit it, so a transaction the + // server cannot roll back retires the connection instead (the + // server rolls it back when the connection closes). + flushed = ((QwpWebSocketSender) live).discardTransactionForReuse(); + } else { + live.flush(); + flushed = true; + } } finally { if (flushed) { slot.pool().giveBack(this); @@ -341,6 +354,11 @@ public void reset() { slot.live(generation).reset(); } + @Override + public void rollback() { + slot.live(generation).rollback(); + } + @Override public void resetSymbolDictionary() { slot.live(generation).resetSymbolDictionary(); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index e304df95..d1571390 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -215,12 +215,20 @@ public void testMalformedEgressConfigRejectedAtBuildWithMinZero() { @Test public void testAutoFlushIntervalOffAcceptedAtBuildWithMinZero() { // auto_flush_interval=off turns off only the time-based trigger and is legal on - // WebSocket, alone or with the row trigger off too (only auto_flush=off is rejected). + // WebSocket: alone, with the row trigger off too (a sender with no auto-flush + // trigger, in strict transactional mode), and with a byte trigger left as the + // only one. QuestDB.builder().fromConfig( "ws::addr=127.0.0.1:1;auto_flush_interval=off;sender_pool_min=0;query_pool_min=0;").build().close(); + QuestDB.builder().fromConfig( + "ws::addr=127.0.0.1:1;auto_flush_rows=off;auto_flush_interval=off;" + + "sender_pool_min=0;query_pool_min=0;").build().close(); QuestDB.builder().fromConfig( "ws::addr=127.0.0.1:1;auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=65536;" + "sender_pool_min=0;query_pool_min=0;").build().close(); + QuestDB.builder().fromConfig( + "ws::addr=127.0.0.1:1;auto_flush=off;auto_flush_bytes=off;sender_pool_min=0;query_pool_min=0;") + .build().close(); } @Test @@ -229,18 +237,24 @@ public void testMalformedIngressConfigRejectedAtBuildWithMinZero() { // Sender -- yet it must still reject a malformed ingress key in the // single cluster config up front. Covers a typed enum (tls_verify), a // registry-STRING value that only the real Sender parse validates - // (auto_flush_rows), and WebSocket build-time checks: auto_flush=off disables - // auto-flush (unsupported on WebSocket), also when auto_flush_bytes is set, and - // sf_durability=flush is not yet supported. + // (auto_flush_rows), and WebSocket build-time checks: a sender with no + // auto-flush trigger is in strict transactional mode, so it cannot combine + // with transaction=off, auto_flush=off cannot combine with a byte trigger, + // and sf_durability=flush is not yet supported. assertBuildRejected( "wss::addr=127.0.0.1:1;tls_verify=strict;sender_pool_min=0;query_pool_min=0;", "tls_verify"); assertBuildRejected( "ws::addr=127.0.0.1:1;auto_flush_rows=abc;sender_pool_min=0;query_pool_min=0;", "auto_flush_rows"); assertBuildRejected( - "ws::addr=127.0.0.1:1;auto_flush=off;sender_pool_min=0;query_pool_min=0;", "auto-flush"); + "ws::addr=127.0.0.1:1;auto_flush=off;transaction=off;sender_pool_min=0;query_pool_min=0;", + "which on WebSocket selects strict transactional mode and cannot be combined with transaction=off"); + assertBuildRejected( + "ws::addr=127.0.0.1:1;auto_flush_rows=off;auto_flush_interval=off;transaction=off;" + + "sender_pool_min=0;query_pool_min=0;", + "which on WebSocket selects strict transactional mode and cannot be combined with transaction=off"); assertBuildRejected( "ws::addr=127.0.0.1:1;auto_flush=off;auto_flush_bytes=65536;sender_pool_min=0;query_pool_min=0;", - "auto-flush"); + "auto flush bytes was already configured [autoFlushBytes=65536]"); assertBuildRejected( "ws::addr=127.0.0.1:1;sf_durability=flush;sender_pool_min=0;query_pool_min=0;", "not yet supported"); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java index cc8c78aa..fdde0e56 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -189,6 +189,32 @@ public void testExtractMaxBatchSizeParsesPositive() throws Exception { Assert.assertEquals(16 * 1024 * 1024, invokeExtractMaxBatchSize(response)); } + @Test + public void testExtractMaxUncommittedRowsAbsentHeaderReturnsZero() throws Exception { + String response = "HTTP/1.1 101 Switching Protocols\r\n" + + "X-QWP-Version: 1\r\n" + + "X-QWP-Max-Batch-Size: 16777216\r\n" + + "\r\n"; + Assert.assertEquals(0, invokeExtractMaxUncommittedRows(response)); + } + + @Test + public void testExtractMaxUncommittedRowsMalformedOrNegativeReturnsZero() throws Exception { + // 0 means "not advertised": a strict transaction then assumes the default. + Assert.assertEquals(0, invokeExtractMaxUncommittedRows("HTTP/1.1 101 Switching Protocols\r\n" + + "X-QWP-Max-Uncommitted-Rows: lots\r\n\r\n")); + Assert.assertEquals(0, invokeExtractMaxUncommittedRows("HTTP/1.1 101 Switching Protocols\r\n" + + "X-QWP-Max-Uncommitted-Rows: -5\r\n\r\n")); + } + + @Test + public void testExtractMaxUncommittedRowsParsesBeyondIntRange() throws Exception { + String response = "HTTP/1.1 101 Switching Protocols\r\n" + + "x-qwp-max-uncommitted-rows: 5000000000 \r\n" + + "\r\n"; + Assert.assertEquals(5_000_000_000L, invokeExtractMaxUncommittedRows(response)); + } + @Test public void testFrameBurstIsParsedInPlace() throws Exception { assertFrameBurstIsParsedInPlace(false); @@ -602,6 +628,12 @@ private static int invokeExtractMaxBatchSize(String response) throws Exception { return (int) m.invoke(null, response); } + private static long invokeExtractMaxUncommittedRows(String response) throws Exception { + Method m = WebSocketClient.class.getDeclaredMethod("extractMaxUncommittedRows", String.class); + m.setAccessible(true); + return (long) m.invoke(null, response); + } + private static void setUpgradedTrue(Object obj) throws Exception { Class clazz = obj.getClass(); while (clazz != null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index fbc833d8..390168fd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -1052,7 +1052,10 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception // Small segment; a heavily padded row's frame cannot fit, so // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no // backpressure timing needed. The server never acks (SilentHandler). - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_segment_bytes=1024;"; + // auto_flush_bytes=off: the oversized row must reach flush() whole, + // as the byte trigger would otherwise publish it from at(). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_segment_bytes=1024;auto_flush_bytes=off;"; String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment Sender sender = Sender.fromConfig(cfg); try { @@ -1187,9 +1190,12 @@ public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { int port = silent.getPort(); silent.start(); Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off: the oversized row must reach flush() whole, + // as the byte trigger would otherwise publish it from at(). String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_segment_bytes=4096" + + ";auto_flush_bytes=off" + ";close_flush_timeout_millis=0;"; Sender s1 = Sender.fromConfig(cfg); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 1d83dff2..f69a5851 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -425,12 +425,28 @@ public void testDefaultIsAsync() { } @Test - public void testDisableAutoFlush_notSupportedForWebSocket() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) + public void testDisableAutoFlush_rejectsPositiveAutoFlushBytes() { + // disableAutoFlush() turns every auto-flush trigger off, so a byte trigger + // contradicts it in either order, as a row or interval trigger does. + assertThrows("cannot set auto flush bytes when auto-flush is already disabled", + () -> Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .disableAutoFlush(), - "not supported for WebSocket"); + .disableAutoFlush() + .autoFlushBytes(65_536)); + assertThrows("auto flush bytes was already configured [autoFlushBytes=65536]", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(65_536) + .disableAutoFlush()); + // 0 turns the byte trigger off, which agrees with it. + Assert.assertNotNull(Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush() + .autoFlushBytes(0)); + Assert.assertNotNull(Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(0) + .disableAutoFlush()); } @Test @@ -790,6 +806,50 @@ public void testNoAddress_fails() { "address not set"); } + @Test + public void testNoAutoFlushTrigger_rejectsTransactionOff() { + // A WebSocket sender with no auto-flush trigger commits only on flush(): it is + // strict unless transactional(true) says otherwise, and transactional(false) + // contradicts it, however the triggers were turned off. + String expected = "auto-flush is off (auto_flush=off, or auto_flush_rows=off and auto_flush_interval=off), " + + "which on WebSocket selects strict transactional mode and cannot be combined with " + + "transaction=off; set transaction=on or turn on an auto-flush trigger"; + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush() + .transactional(false), + expected); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .transactional(false) + .disableAutoFlush(), + expected); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0) + .autoFlushIntervalMillis(Integer.MAX_VALUE) + .transactional(false), + expected); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0) + .autoFlushIntervalMillis(Integer.MAX_VALUE) + .autoFlushBytes(0) + .transactional(false), + expected); + // strictTransactional(false) keeps transactional mode as set: off here. + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush() + .strictTransactional(false), + expected); + } + @Test public void testPortMismatch_fails() { assertThrowsAny( diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java new file mode 100644 index 00000000..95d003c1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -0,0 +1,1704 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.TransactionTooLargeException; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Batches of any size over QWP/WebSocket: the byte trigger that keeps every batch + * within one frame, transactional and strict transactional mode, and rollback. + *

+ * {@link TxnServer} answers frames the way the QuestDB server does: a + * FLAG_DEFER_COMMIT frame gets no ack until a commit frame closes its group, a + * commit frame gets a cumulative OK ack, and the rollback control frame gets a + * cumulative OK ack once the server negotiated rollback. + */ +public class QwpLargeTransactionTest { + private static final int CAP = 32 * 1024; + // The settings a refusal at the client buffer names. + private static final String CLIENT_BUFFER_REMEDY = "Raise sf_max_total_bytes, or set sf_dir to keep the " + + "transaction on disk"; + // What closing a sender or returning a pooled lease logs when it discards an open + // strict transaction of 1000 rows, behind the action that discarded it. + private static final String DISCARD_WARNING = " discarded an uncommitted transaction of 1000 rows: a strict " + + "transactional sender (transaction=strict, or one with no auto-flush trigger) commits only on " + + "flush(). Call flush() before close() to keep the rows, or ignore this if the discard is intentional"; + private static final String NO_TRIGGER_TRANSACTION_OFF_ERROR = "auto-flush is off (auto_flush=off, or " + + "auto_flush_rows=off and auto_flush_interval=off), which on WebSocket selects strict transactional " + + "mode and cannot be combined with transaction=off; set transaction=on or turn on an auto-flush trigger"; + // ~100 encoded bytes per row: 1000 rows (the default row trigger) are ~100 KB, + // three times the 32 KiB cap. + private static final String PAD = TestUtils.repeat("x", 90); + // transaction=on in frames of exactly 280 rows (~28 KB): two fill a 64 KiB segment, + // so the 256 KiB client buffer holds eight frames of a transaction. A blocked + // append would wait 30 s. + private static final String TXN_ON_FRAMES_OF_280 = "transaction=on;auto_flush_rows=280;auto_flush_bytes=off;" + + "auto_flush_interval=off;sf_max_segment_bytes=64k;sf_max_total_bytes=256k;" + + "sf_append_deadline_millis=30000;"; + + @Test + public void testAutoFlushBytesAloneCommitsEachBatch() throws Exception { + // A positive auto_flush_bytes is an auto-flush trigger: with the row and + // interval triggers off, every batch it cuts commits on its own, as before. + // transaction=off may say so explicitly. + assertMemoryLeak(() -> { + String[] configs = { + "auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=8192;", + "auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=8192;transaction=off;" + }; + for (String config : configs) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + config)) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertFalse(config, ws.isTransactional()); + Assert.assertEquals(config, 8_192, ws.getEffectiveAutoFlushBytes()); + // ~20 KB: the byte trigger sends at least two batches, and each + // commits before any flush(). + writeRows(s, "t", 0, 200); + Assert.assertTrue(config, h.awaitFrames(2)); + for (TxnFrame f : h.frames) { + Assert.assertTrue(config + ": " + h.frames, f.isCommit()); + } + } + // close() commits the rest. + Assert.assertEquals(config, h.frames, h.committedFrames()); + } + } + }); + } + + @Test + public void testAutoFlushOffRejectsPositiveAutoFlushBytes() throws Exception { + // auto_flush=off turns every auto-flush trigger off, so a byte trigger + // contradicts it, as a row or interval trigger already does. The ws:: parser + // applies the trigger keys before auto_flush, whatever their order. + // auto_flush_bytes=off is no trigger; see + // testNoAutoFlushTriggerWithAutoFlushBytesOffSendsOneFramePerFlush. + assertMemoryLeak(() -> { + assertConfigRejected("auto_flush=off;auto_flush_bytes=65536;", + "auto flush bytes was already configured [autoFlushBytes=65536]"); + assertConfigRejected("auto_flush_bytes=65536;auto_flush=off;", + "auto flush bytes was already configured [autoFlushBytes=65536]"); + }); + } + + @Test + public void testDefaultSenderSplitsWideRowsIntoFrames() throws Exception { + // Before: no byte trigger by default, so the 1000-row batch outgrew the cap + // and flush threw BatchTooLargeForCapException. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server))) { + writeRows(s, "t", 0, 2_000); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals((long) CAP * 9 / 10, ws.getEffectiveAutoFlushBytes()); + } + assertFramesWithinCap(h, CAP); + Assert.assertTrue("rows must span frames: " + h.frames, h.frames.size() >= 7); + // Not transactional: every frame commits on its own. + Assert.assertEquals(h.frames.size(), h.commitFrames()); + } + }); + } + + @Test + public void testExplicitAutoFlushBytesOffKeepsNoByteTrigger() throws Exception { + // auto_flush_bytes=off still opts out of the byte trigger, as documented. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_bytes=off;")) { + Assert.assertEquals(0, ((QwpWebSocketSender) s).getEffectiveAutoFlushBytes()); + try { + writeRows(s, "t", 0, 1_000); + Assert.fail("a 1000-row batch over the cap must be rejected"); + } catch (BatchTooLargeForCapException expected) { + TestUtils.assertContains(expected.getMessage(), "serverMaxBatchSize=" + CAP); + } + s.reset(); + } + Assert.assertEquals(0, h.frames.size()); + } + }); + } + + @Test + public void testByteTriggerFollowsSegmentSize() throws Exception { + // The server accepts 48 KiB frames, but a 16 KiB segment cannot hold them: + // batches must be sized to the segment, not fail at the append. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 48 * 1024)) { + try (Sender s = Sender.fromConfig(conf(server) + "sf_max_segment_bytes=16k;")) { + writeRows(s, "t", 0, 600); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + int segmentPayload = 16 * 1024 - 24 - 8; + Assert.assertEquals(segmentPayload * 9 / 10, ((QwpWebSocketSender) s).getEffectiveAutoFlushBytes()); + } + assertFramesWithinCap(h, 16 * 1024 - 24 - 8); + Assert.assertTrue(h.frames.size() >= 4); + } + }); + } + + @Test + public void testNewSymbolsCountTowardByteTrigger() throws Exception { + // Symbol columns buffer 4-byte ids; the 300-byte strings ride in the frame's + // dictionary section. Counting them keeps the frame within the cap. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server))) { + String prefix = TestUtils.repeat("s", 290); + for (int i = 0; i < 1_000; i++) { + s.table("t").symbol("sym", prefix + i).longColumn("v", i).atNow(); + } + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, CAP); + Assert.assertTrue(h.frames.size() >= 10); + } + }); + } + + @Test + public void testNoAutoFlushTriggerIsStrictTransactionOfAnySize() throws Exception { + // Every way to turn off all auto-flush triggers builds the same sender: a + // strict transaction of any size, which only flush() commits. + // auto_flush_rows=off with auto_flush_interval=off used to send each flush as + // one batch, which failed past the frame cap, and to commit on close(). + assertMemoryLeak(() -> { + for (NoTriggerSpelling spelling : noTriggerSpellings()) { + TxnServer h = new TxnServer(); + List committed = null; + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = spelling.build(conf(server))) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(spelling.name, ws.isStrictTransactional()); + Assert.assertEquals(spelling.name, CAP * 9 / 10, ws.getEffectiveAutoFlushBytes()); + writeRows(s, "t", 0, 2_000); + // ~200 KB went out as deferred frames before flush(); none commits. + Assert.assertTrue(spelling.name, h.awaitFrames(5)); + Assert.assertEquals(spelling.name, 0, h.commitFrames()); + s.flush(); + Assert.assertTrue(spelling.name, s.drain(5_000)); + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + committed = h.committedFrames(); + Assert.assertEquals(spelling.name, h.frames, committed); + // close() without flush() discards the next transaction, its + // frames already sent included. + writeRows(s, "t", 2_000, 1_000); + Assert.assertTrue(spelling.name, h.awaitFrames(committed.size() + 2)); + } + Assert.assertEquals(spelling.name, committed, h.committedFrames()); + } + } + }); + } + + @Test + public void testNoAutoFlushTriggerRejectsTransactionOff() throws Exception { + // With no auto-flush trigger only flush() commits, and a non-transactional + // flush has to fit one frame, so every spelling refuses transaction=off. + assertMemoryLeak(() -> { + assertConfigRejected("auto_flush=off;transaction=off;", NO_TRIGGER_TRANSACTION_OFF_ERROR); + assertConfigRejected("auto_flush_rows=off;auto_flush_interval=off;transaction=off;", + NO_TRIGGER_TRANSACTION_OFF_ERROR); + assertConfigRejected("auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=off;transaction=off;", + NO_TRIGGER_TRANSACTION_OFF_ERROR); + }); + } + + @Test + public void testNoAutoFlushTriggerWithAutoFlushBytesOffSendsOneFramePerFlush() throws Exception { + // auto_flush_bytes=off opts out of sizing batches to the frame: the + // transaction still commits only on flush(), but the application keeps it + // within one frame per table, and a flush past the cap is refused and keeps + // the rows, as auto_flush_bytes=off documents. + assertMemoryLeak(() -> { + String[] configs = { + "auto_flush=off;auto_flush_bytes=off;", + "auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=off;" + }; + for (String config : configs) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + config)) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(config, ws.isStrictTransactional()); + Assert.assertEquals(config, 0, ws.getEffectiveAutoFlushBytes()); + // ~20 KB fits the 32 KiB cap: one frame, which commits. + writeRows(s, "t", 0, 200); + s.flush(); + Assert.assertTrue(config, s.drain(5_000)); + Assert.assertEquals(config, 1, h.frames.size()); + Assert.assertTrue(config, h.frames.get(0).isCommit()); + // ~100 KB does not. + writeRows(s, "t", 200, 1_000); + try { + s.flush(); + Assert.fail(config + ": a flush past the cap must be refused"); + } catch (BatchTooLargeForCapException e) { + TestUtils.assertContains(e.getMessage(), "serverMaxBatchSize=" + CAP); + } + s.reset(); + } + Assert.assertEquals(config, 1, h.frames.size()); + } + } + }); + } + + @Test + public void testNoAutoFlushTriggerWithTransactionOnCommitsOnClose() throws Exception { + // An explicit transaction=on wins over the strict mode a sender with no + // auto-flush trigger otherwise gets: close() commits, as transaction=on does. + assertMemoryLeak(() -> { + for (NoTriggerSpelling spelling : noTriggerSpellings()) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = spelling.build(conf(server) + "transaction=on;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(spelling.name, ws.isTransactional()); + Assert.assertFalse(spelling.name, ws.isStrictTransactional()); + writeRows(s, "t", 0, 2_000); + Assert.assertTrue(spelling.name, h.awaitFrames(5)); + Assert.assertEquals(spelling.name, 0, h.commitFrames()); + } + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + Assert.assertEquals(spelling.name, h.frames, h.committedFrames()); + } + } + }); + } + + @Test + public void testOneAutoFlushTriggerLeftKeepsEveryBatchCommitting() throws Exception { + // Turning off the row or the interval trigger alone leaves the other one, so + // the sender is not transactional: every batch it sends commits on its own. + assertMemoryLeak(() -> { + String[] configs = {"auto_flush_rows=off;", "auto_flush_interval=off;"}; + for (String config : configs) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + config)) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertFalse(config, ws.isTransactional()); + Assert.assertEquals(config, CAP * 9 / 10, ws.getEffectiveAutoFlushBytes()); + // ~100 KB: the byte trigger sends at least three batches before + // any flush(), and each commits. + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(config, h.awaitFrames(3)); + for (TxnFrame f : h.frames) { + Assert.assertTrue(config + ": " + h.frames, f.isCommit()); + } + } + Assert.assertEquals(config, h.frames, h.committedFrames()); + } + } + }); + } + + @Test + public void testPooledLeaseWithAbortedTransactionRetiresQuietly() throws Exception { + // reset() could not undo the frames already sent, so the transaction is + // aborted and every flush already said so. Returning the lease retires the + // connection, which discards the frames, without warning about them again. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QuestDB db = QuestDB.connect(conf(server) + + "auto_flush_rows=off;auto_flush_interval=off;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + lease.reset(); + assertAborted(lease::flush); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + List committed = h.committedFrames(); + Assert.assertEquals("only the next lease commits", 1, committed.size()); + Assert.assertTrue(committed.get(0).isCommit()); + Assert.assertEquals("the aborted lease's connection was retired", 2, server.upgradeRequestCount()); + } + Assert.assertEquals(Collections.emptyList(), discardWarnings(log)); + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testPooledLeaseWithoutAutoFlushTriggerDiscardsOnReturn() throws Exception { + // Returning a lease of a pool with no auto-flush trigger used to commit its rows. + // The lease now holds a strict transaction, which only flush() commits, so + // returning it discards the rows, and says so once: whether the server rolls + // back the frames already sent, or the pool retires the connection to discard them. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + String[] spellings = {"auto_flush=off;", "auto_flush_rows=off;auto_flush_interval=off;"}; + for (String spelling : spellings) { + for (boolean isRollbackSupported : new boolean[]{false, true}) { + String name = spelling + " rollback=" + isRollbackSupported; + clearLog(log); + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(isRollbackSupported); + try (QuestDB db = QuestDB.connect(conf(server) + spelling + "query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(name, h.awaitFrames(2)); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(name, next.drain(5_000)); + next.close(); + } + List committed = h.committedFrames(); + Assert.assertEquals(name + ": only the next lease commits", 1, committed.size()); + Assert.assertTrue(name, committed.get(0).isCommit()); + Assert.assertEquals(name, isRollbackSupported ? 1 : 0, h.rollbackFrames()); + } + Assert.assertEquals(name, + Collections.singletonList("returning the sender to the pool" + DISCARD_WARNING), + discardWarnings(log)); + } + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testPooledStrictLeaseCloseDuringOutageKeepsTheConnection() throws Exception { + // Returning a strict lease discards its open transaction. During an outage that + // must neither wait for the server nor cost the pool its connection: the next + // borrower's transaction goes out behind the rollback frame once the server is back. + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + TxnServer before = new TxnServer(); + TxnServer after = new TxnServer(); + TestWebSocketServer revived = null; + try (TestWebSocketServer first = start(before, CAP, port, true)) { + try (QuestDB db = QuestDB.connect(outageConf(port) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(before.awaitFrames(2)); + first.close(); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + revived = start(after, CAP, port, true); + Assert.assertTrue(next.drain(10_000)); + next.close(); + } + } finally { + if (revived != null) { + revived.close(); + } + } + assertDiscardedThenCommitted(before, after); + }); + } + + @Test + public void testPooledStrictLeaseCloseRetiresConnectionWithoutServerRollback() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QuestDB db = QuestDB.connect(conf(server) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + // The frames already sent cannot be rolled back: the pool retires + // the connection, which makes the server discard them. + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + Assert.assertEquals(1, h.commitFrames()); + Assert.assertEquals(0, h.rollbackFrames()); + } + }); + } + + @Test + public void testPooledStrictLeaseCloseRollsBackUncommitted() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (QuestDB db = QuestDB.connect(conf(server) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + // Returning a strict lease without flush() discards the transaction, + // without waiting for the server. + lease.close(); + Assert.assertTrue(h.awaitRollbackFrames(1)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isRollback()); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + Assert.assertEquals(1, h.commitFrames()); + Assert.assertEquals(1, h.rollbackFrames()); + assertRolledBackThenCommitted(h); + } + }); + } + + @Test + public void testPooledStrictLeaseCloseWithOnlyPendingRowsKeepsConnection() throws Exception { + // Nothing reached the server yet: returning the lease drops the pending + // rows without a rollback frame and keeps the connection pooled. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QuestDB db = QuestDB.connect(conf(server) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 10); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 5); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + Assert.assertEquals(1, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isCommit()); + Assert.assertEquals(1, server.upgradeRequestCount()); + } + }); + } + + @Test + public void testStrictMultiTableFlushSplitsWithinTheCap() throws Exception { + // Ten tables in one batch carry ten schemas, so the batch the byte trigger + // cuts outgrows the cap and splits per table. Every split frame of the + // open transaction stays deferred. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 4_096)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + for (int i = 0; i < 400; i++) { + s.table("table_with_a_long_name_" + (i % 10)) + .longColumn("first_long_column_name", i) + .longColumn("second_long_column_name", i) + .stringColumn("s", "abc") + .atNow(); + } + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, 4_096); + assertOneTransaction(h); + Assert.assertTrue(h.frames.size() > 10); + } + }); + } + + @Test + public void testResetAbortsStrictTransactionWithoutServerRollback() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.reset(); + assertAborted(s::flush); + assertAborted(() -> s.table("t").longColumn("v", 1).atNow()); + } + Assert.assertEquals("the reset transaction must never commit", 0, h.commitFrames()); + } + }); + } + + @Test + public void testResetDuringOutageDoesNotStallTheSender() throws Exception { + // Store-and-forward keeps every producer call working through an outage. reset() + // used to wait 30 s for the rollback ack and then abort the transaction, so every + // later row and flush threw, even after the server came back. + assertDiscardDuringOutage("transaction=on;", Sender::reset); + } + + @Test + public void testResetKeepsLegacyBehaviourWithoutServerRollback() throws Exception { + // transaction=on against a server without rollback: reset() cannot reach the + // frames already sent, and the next flush() commits them, as before. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.reset(); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // The frames sent before reset() are deferred, and the flush after it + // committed them with an empty commit frame. + assertOneTransaction(h); + } + }); + } + + @Test + public void testResetRollsBackSentFramesWhenServerSupportsIt() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.reset(); + // reset() does not wait for the server; drain() returns once the server + // acked the rollback frame, which follows every frame of the transaction. + Assert.assertTrue(s.drain(5_000)); + int rollbackIndex = h.frames.size() - 1; + Assert.assertTrue(h.frames.get(rollbackIndex).isRollback()); + for (int i = 0; i < rollbackIndex; i++) { + Assert.assertTrue(h.frames.get(i).isDeferred()); + } + Assert.assertEquals(1, h.rolledBackGroups); + // The sender stays usable for the next transaction. + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + } + assertRolledBackThenCommitted(h); + } + }); + } + + @Test + public void testRollbackFrameNeverReachesServerWithoutRollbackSupport() throws Exception { + // The transaction is discarded while the server that confirmed rollback is away, + // and the endpoint comes back as a server without rollback support, as after a + // failover to an older node. That server would read the rollback frame as an + // empty deferred frame and commit the discarded rows with the next transaction. + // The client closes the connection that holds them instead, and sends the next + // transaction on a new one. + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + TxnServer before = new TxnServer(); + TxnServer after = new TxnServer(); + TestWebSocketServer revived = null; + try (TestWebSocketServer first = start(before, CAP, port, true)) { + try (Sender s = Sender.fromConfig(outageConf(port) + "transaction=on;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(before.awaitFrames(2)); + first.close(); + s.rollback(); + writeRows(s, "t", 0, 10); + s.flush(); + revived = start(after, CAP, port, false); + Assert.assertTrue(s.drain(10_000)); + Assert.assertFalse(((QwpWebSocketSender) s).isServerRollbackEnabled()); + } + } finally { + if (revived != null) { + revived.close(); + } + } + Assert.assertEquals(0, before.committedFrames().size()); + Assert.assertEquals("the rollback frame must not reach a server without rollback support: " + + after.frames, 0, after.rollbackFrames()); + List> connections = after.framesByConnection(); + Assert.assertEquals("one connection for the discarded transaction, one for the next: " + + after.frames, 2, connections.size()); + // The reader threads of the two connections may log their first frames in + // either order: tell them apart by what they carry. + List discarded = connections.get(0); + List next = connections.get(1); + if (next.get(next.size() - 1).isDeferred()) { + discarded = connections.get(1); + next = connections.get(0); + } + Assert.assertTrue("the discarded transaction is replayed: " + after.frames, discarded.size() >= 2); + for (TxnFrame f : discarded) { + Assert.assertTrue("frame must be deferred: " + after.frames, f.isDeferred()); + } + Assert.assertTrue(next.get(next.size() - 1).isCommit()); + Assert.assertEquals("only the next transaction commits", next, after.committedFrames()); + }); + } + + @Test + public void testRollbackDuringOutageDoesNotStallTheSender() throws Exception { + assertDiscardDuringOutage("transaction=strict;", Sender::rollback); + } + + @Test + public void testRollbackDiscardsSentFrames() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(ws.isServerRollbackEnabled()); + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + Assert.assertEquals(0, h.commitFrames()); + s.rollback(); + // rollback() does not wait for the server; drain() returns once the + // server acked the rollback frame, which follows every frame of the + // transaction. + Assert.assertTrue(s.drain(5_000)); + int sent = h.frames.size() - 1; + TxnFrame rollback = h.frames.get(sent); + Assert.assertTrue(rollback.isRollback()); + for (int i = 0; i < sent; i++) { + Assert.assertTrue(h.frames.get(i).isDeferred()); + } + Assert.assertEquals(QwpConstants.HEADER_SIZE, rollback.size); + Assert.assertEquals(1, h.rolledBackGroups); + Assert.assertFalse(ws.hasDeferredMessagesForTesting()); + // Nothing pending is left behind: an empty flush sends nothing. + s.flush(); + Assert.assertEquals(sent + 1, h.frames.size()); + // The next transaction starts clean and commits. + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + } + assertRolledBackThenCommitted(h); + } + }); + } + + @Test + public void testRollbackOnNonWebSocketSenderThrows() throws Exception { + assertMemoryLeak(() -> { + try (Sender s = Sender.fromConfig("http::addr=localhost:1;protocol_version=2;")) { + s.rollback(); + Assert.fail("rollback must be WebSocket-only"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), "rollback is only supported for WebSocket transport"); + } + }); + } + + @Test + public void testRollbackUnsupportedAbortsTransaction() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + Assert.assertFalse(((QwpWebSocketSender) s).isServerRollbackEnabled()); + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + assertAborted(s::rollback); + assertAborted(s::flush); + assertAborted(() -> s.table("t").longColumn("v", 1).atNow()); + assertAborted(s::rollback); + } + // close() of an aborted transaction must not commit it either. + Assert.assertEquals(0, h.commitFrames()); + Assert.assertEquals(0, h.rollbackFrames()); + } + }); + } + + @Test + public void testRollbackWithoutSentFramesSendsNothing() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 10); + s.rollback(); + s.flush(); + writeRows(s, "t", 0, 5); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // Only the second transaction reached the server, as one commit frame. + Assert.assertEquals(1, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isCommit()); + } + }); + } + + @Test + public void testServerLimitsReachTheSender() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setAdvertisedMaxUncommittedRows(12_345); + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server))) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals(12_345, ws.getServerMaxUncommittedRows()); + Assert.assertTrue(ws.isServerRollbackEnabled()); + } + Assert.assertEquals("true", server.pollRollbackRequest(5, TimeUnit.SECONDS)); + + // A server that does not confirm rollback, nor advertise its row limit. + server.setAdvertisedMaxUncommittedRows(0); + server.setRollbackSupported(false); + try (Sender s = Sender.fromConfig(conf(server))) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals(0, ws.getServerMaxUncommittedRows()); + Assert.assertFalse(ws.isServerRollbackEnabled()); + } + } + }); + } + + @Test + public void testStrictCloseWithoutFlushDiscardsTransaction() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;close_flush_timeout_millis=1000;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + } + Assert.assertEquals("close() must not commit a strict transaction", 0, h.commitFrames()); + Assert.assertEquals(0, h.rollbackFrames()); + } + }); + } + + @Test + public void testStrictCloseWithoutFlushWarnsOfDiscardedRows() throws Exception { + // A sender with no auto-flush trigger used to commit on close(). Now close() + // discards its open transaction, and says so once, counting the rows already + // sent with those still buffered. A committed transaction leaves nothing to + // discard, and an aborted one already failed every flush. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_rows=off;auto_flush_interval=off;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + } + Assert.assertEquals(Collections.singletonList("close()" + DISCARD_WARNING), discardWarnings(log)); + clearLog(log); + + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_rows=off;auto_flush_interval=off;")) { + writeRows(s, "t", 0, 1_000); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + // Frames went out and the server cannot roll them back. + s.reset(); + assertAborted(s::flush); + } + Assert.assertEquals(Collections.emptyList(), discardWarnings(log)); + Assert.assertEquals(1, h.commitFrames()); + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testStrictCloseWithoutFlushRollsBackWhenServerSupportsIt() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + } + // close() drained until the rollback frame's ack. + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isRollback()); + Assert.assertEquals(0, h.commitFrames()); + Assert.assertEquals(1, h.rolledBackGroups); + } + }); + } + + @Test + public void testStrictRowLimitFromServer() throws Exception { + // The server advertises 100: a strict transaction holds at most 99 rows + // of a table, since the server would commit the 100th on its own. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 4_096)) { + server.setAdvertisedMaxUncommittedRows(100); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + // 4 KiB frames: the 99 rows of "a" go out over several deferred frames. + writeRows(s, "a", 0, 99); + Assert.assertTrue(h.awaitFrames(2)); + try { + writeRows(s, "a", 99, 1); + Assert.fail("the 100th row of a table must be refused"); + } catch (TransactionTooLargeException e) { + Assert.assertEquals(serverRowLimitError("a", 100), e.getMessage()); + } + // Other tables keep their own count. + writeRows(s, "b", 0, 99); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + // A new transaction starts from zero again. + writeRows(s, "a", 0, 99); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + Assert.assertEquals(0, h.rollbackFrames()); + Assert.assertEquals(2, h.commitFrames()); + } + }); + } + + @Test + public void testStrictRowLimitDefaultsWhenNotAdvertised() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals(0, ws.getServerMaxUncommittedRows()); + // Pretend the server default is small to keep the test fast: the + // assumed default applies only while nothing is advertised. + ws.applyServerTransactionLimits(10, false); + writeRows(s, "a", 0, 9); + try { + writeRows(s, "a", 9, 1); + Assert.fail("the 10th row must be refused"); + } catch (TransactionTooLargeException e) { + Assert.assertEquals(serverRowLimitError("a", 10), e.getMessage()); + } + ws.applyServerTransactionLimits(0, false); + // 0 means "not advertised": the server default applies. + writeRows(s, "a", 9, 1); + s.flush(); + } + } + }); + } + + @Test + public void testStrictCommitTooLargeForClientBufferIsNotSent() throws Exception { + // With auto_flush_bytes=off on top of auto_flush=off, the whole transaction + // waits for flush(), which sends one frame per table. Ten tables of ~28 KB need five 64 KiB + // segments, and sf_max_total_bytes holds four: flush() sends nothing. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush=off;auto_flush_bytes=off;" + + "sf_max_segment_bytes=64k;sf_max_total_bytes=256k;")) { + writeTables(s, 0, 10); + long refusedBytes = -1; + try { + s.flush(); + Assert.fail("the client buffer cannot hold the commit"); + } catch (TransactionTooLargeException e) { + refusedBytes = assertClientBufferRefusal(e, 262_144, "so the flush was not sent"); + } + Assert.assertEquals("nothing was sent", 0, h.frames.size()); + s.rollback(); + // The same rows, one table per transaction: the refusal reported + // what these frames add up to. + for (int t = 0; t < 10; t++) { + writeTables(s, t, 1); + s.flush(); + } + Assert.assertTrue(s.drain(5_000)); + Assert.assertEquals(10, h.commitFrames()); + Assert.assertEquals(refusedBytes, frameBytes(h)); + } + } + }); + } + + @Test + public void testStrictDeferredFlushTooLargeForClientBufferIsNotSent() throws Exception { + // setDeferCommit(true) makes flush() send rows without committing them. A + // strict transaction keeps room for its commit next to such a frame. With + // frames as large as a segment that takes four segments, and + // sf_max_total_bytes holds three: flush() sends nothing, and the transaction + // takes no further rows. The commit itself fits. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 64 * 1024)) { + long refusedBytes = -1; + try (QwpWebSocketSender s = (QwpWebSocketSender) Sender.fromConfig(conf(server) + + "transaction=strict;sf_max_segment_bytes=64k;sf_max_total_bytes=192k;")) { + s.setDeferCommit(true); + writeRows(s, "t", 0, 1); + try { + s.flush(); + Assert.fail("the client buffer cannot hold a deferred frame and the commit"); + } catch (TransactionTooLargeException e) { + refusedBytes = assertClientBufferRefusal(e, 196_608, "so the flush was not sent"); + } + Assert.assertEquals("nothing was sent", 0, h.frames.size()); + try { + writeRows(s, "t", 1, 1); + Assert.fail("the transaction must take no further rows"); + } catch (TransactionTooLargeException e) { + Assert.assertEquals(refusedBytes, + assertClientBufferRefusal(e, 196_608, "so the row was not added")); + } + s.setDeferCommit(false); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // The commit carries the row the refused flush held, and only that row. + Assert.assertEquals(1, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isCommit()); + Assert.assertEquals(refusedBytes, h.frames.get(0).size); + } + }); + } + + @Test + public void testStrictTransactionTooLargeForClientBufferFailsFast() throws Exception { + // 64 KiB segments, 256 KiB budget: four segments. A strict transaction keeps + // room for its commit, so once its frames fill the rest it must refuse rows + // instead of blocking for sf_append_deadline_millis. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;" + + "sf_max_segment_bytes=64k;sf_max_total_bytes=256k;sf_append_deadline_millis=30000;")) { + long start = System.nanoTime(); + long refusedBytes = -1; + try { + for (int row = 0; row < 100_000; row++) { + s.table("t").longColumn("v", row).stringColumn("s", PAD).atNow(); + } + Assert.fail("a transaction larger than the client buffer must be refused"); + } catch (TransactionTooLargeException e) { + // As at the server's row limit, the row that hit the limit is not + // in the transaction. + refusedBytes = assertClientBufferRefusal(e, 262_144, "so the row was not added"); + } + Assert.assertTrue("must fail fast, not wait for the append deadline", + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); + // Further rows are refused the same way. + try { + s.table("t").longColumn("v", -1).stringColumn("s", PAD).atNow(); + Assert.fail("a full transaction must refuse rows"); + } catch (TransactionTooLargeException e) { + Assert.assertEquals(refusedBytes, + assertClientBufferRefusal(e, 262_144, "so the row was not added")); + } + // The reserved room lets the transaction commit. + s.flush(); + Assert.assertTrue(s.drain(5_000)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + Assert.assertEquals(1, h.commitFrames()); + // The refusal reported the size of the transaction that committed: + // the rows accepted before it, the last of which filled the buffer. + Assert.assertEquals(refusedBytes, frameBytes(h)); + // And the next transaction starts with an empty buffer. + writeRows(s, "t", 0, 100); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + } + }); + } + + @Test + public void testUnacknowledgedRollbackDoesNotStallTheSender() throws Exception { + // The server never acks the rollback frame on its own. rollback() used to wait + // for that ack and abort the transaction when it did not come. Nothing waits for + // it now: the next transaction goes out behind it, and the commit's cumulative + // ack covers it. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + h.isAckingRollback = false; + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.rollback(); + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertRolledBackThenCommitted(h); + } + }); + } + + @Test + public void testTransactionalCommitsInPartsAtClientBuffer() throws Exception { + // The client holds a transaction's frames until the server commits it. The row + // whose frame found no room used to block for sf_append_deadline_millis and then + // throw, and close() then committed the frames already sent and dropped the rest. + // Now the sender commits the frames already sent and carries on, as the server + // does at qwp.max.uncommitted.rows. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QwpWebSocketSender s = (QwpWebSocketSender) Sender.fromConfig(conf(server) + TXN_ON_FRAMES_OF_280)) { + long start = System.nanoTime(); + // 17 frames of 280 rows, then 240 rows that flush() commits. + writeRows(s, "t", 0, 5_000); + Assert.assertEquals("two parts of eight frames", 4_480, s.getTxnRowsCommittedInPartsForTesting()); + s.flush(); + Assert.assertEquals(0, s.getTxnRowsCommittedInPartsForTesting()); + Assert.assertTrue("must not wait for sf_append_deadline_millis", + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); + Assert.assertTrue(s.drain(5_000)); + } + // The buffer holds eight frames, so the ninth and the seventeenth each + // start a new part, behind a bare commit frame for the frames before them. + Assert.assertEquals("DDDDDDDDcDDDDDDDDcDC", frameKinds(h)); + Assert.assertEquals("every frame commits", h.frames, h.committedFrames()); + // Logged once for the transaction, with what the first commit covered. + Assert.assertEquals(Collections.singletonList(commitInPartsWarning(2_240, frameBytes(h, 0, 8))), + commitInPartsWarnings(log)); + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testTransactionalCommitsInPartsBeforeFlushThatDoesNotFit() throws Exception { + // Eight frames fill the client buffer, and the frame flush() sends with the last + // 279 rows, which commits the transaction, does not fit beside them. flush() used + // to block for sf_append_deadline_millis and throw. Now it commits the eight + // frames first. Each transaction that commits in parts logs once. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + TXN_ON_FRAMES_OF_280)) { + writeRows(s, "t", 0, 2_519); + Assert.assertTrue(h.awaitFrames(8)); + long start = System.nanoTime(); + s.flush(); + Assert.assertTrue("must not wait for sf_append_deadline_millis", + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); + Assert.assertTrue(s.drain(5_000)); + Assert.assertEquals("DDDDDDDDcC", frameKinds(h)); + Assert.assertEquals(Collections.singletonList( + commitInPartsWarning(2_240, frameBytes(h, 0, 8))), commitInPartsWarnings(log)); + + // The next transaction's first frame shares a segment with the + // previous commit frame, so seven frames fill the buffer this time. + writeRows(s, "t", 0, 2_519); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + Assert.assertEquals("DDDDDDDDcC" + "DDDDDDDcDC", frameKinds(h)); + Assert.assertEquals("every frame commits", h.frames, h.committedFrames()); + Assert.assertEquals(Arrays.asList( + commitInPartsWarning(2_240, frameBytes(h, 0, 8)), + commitInPartsWarning(1_960, frameBytes(h, 10, 17)) + ), commitInPartsWarnings(log)); + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testTransactionalDiscardAfterCommitInPartsWarns() throws Exception { + // The frames committed when the transaction outgrew the client buffer cannot be + // rolled back. rollback() and reset() discard the rest of the transaction, and + // say that the part already committed stays. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + String[] actions = {"rollback()", "reset()"}; + for (String action : actions) { + clearLog(log); + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (QwpWebSocketSender s = (QwpWebSocketSender) Sender.fromConfig(conf(server) + TXN_ON_FRAMES_OF_280)) { + // Nine frames, the ninth in a new part, and 100 rows pending. + writeRows(s, "t", 0, 2_620); + Assert.assertEquals(action, 2_240, s.getTxnRowsCommittedInPartsForTesting()); + if ("rollback()".equals(action)) { + s.rollback(); + } else { + s.reset(); + } + Assert.assertEquals(action, 0, s.getTxnRowsCommittedInPartsForTesting()); + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(action, s.drain(5_000)); + } + Assert.assertEquals(action, "DDDDDDDDcDRC", frameKinds(h)); + Assert.assertEquals(action, 1, h.rolledBackGroups); + // The first part and the next transaction commit, the ninth frame does not. + List committed = new ArrayList<>(h.frames.subList(0, 9)); + committed.add(h.frames.get(11)); + Assert.assertEquals(action, committed, h.committedFrames()); + Assert.assertEquals(action, Arrays.asList( + commitInPartsWarning(2_240, frameBytes(h, 0, 8)), + action + " discarded the rows of the open transaction sent since the sender last " + + "committed part of it, but the 2240 rows it committed when the transaction " + + "outgrew the client buffer stay committed" + ), commitInPartsWarnings(log)); + } + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testTransactionalDefaultTriggersSplitWideRows() throws Exception { + // Before: transaction=on with the default triggers threw + // BatchTooLargeForCapException at the 1000th wide row. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + writeRows(s, "t", 0, 2_000); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + } + }); + } + + // Captures what QwpWebSocketSender logs, so a test can assert its warnings. + private static ListAppender captureSenderLog() { + ListAppender appender = new ListAppender<>(); + appender.start(); + senderLogger().addAppender(appender); + return appender; + } + + // The appender appends under its own lock, and I/O threads log into it too. + private static void clearLog(ListAppender log) { + synchronized (log) { + log.list.clear(); + } + } + + // What a transaction=on sender logs the first time a transaction outgrows the + // 256 KiB client buffer and it commits the part already sent. + private static String commitInPartsWarning(long rows, long transactionBytes) { + return "the open transaction outgrew the client buffer, so the sender committed the " + rows + + " rows it had already sent, and the rest of the transaction commits separately [transactionBytes=" + + transactionBytes + ", ringCapacityBytes=262144]. A transaction=on transaction commits in parts when " + + "sf_max_total_bytes cannot hold it: raise sf_max_total_bytes or set sf_dir to keep such transactions " + + "whole, or use transaction=strict to refuse their rows instead. Logged once per transaction"; + } + + // The warnings about a transaction the sender committed in parts at the client + // buffer logged since the last clearLog(), in order. + private static List commitInPartsWarnings(ListAppender log) { + List warnings = new ArrayList<>(); + synchronized (log) { + for (ILoggingEvent e : log.list) { + String message = e.getFormattedMessage(); + if (e.getLevel() == Level.WARN && message.contains("outgrew the client buffer")) { + warnings.add(message); + } + } + } + return warnings; + } + + // The discarded-transaction warnings logged since the last clearLog(). + private static List discardWarnings(ListAppender log) { + List warnings = new ArrayList<>(); + synchronized (log) { + for (ILoggingEvent e : log.list) { + String message = e.getFormattedMessage(); + if (e.getLevel() == Level.WARN && message.contains("discarded an uncommitted transaction")) { + warnings.add(message); + } + } + } + return warnings; + } + + // Every way to configure a WebSocket sender with no auto-flush trigger, applied + // on top of a config string. + private static List noTriggerSpellings() { + List spellings = new ArrayList<>(); + spellings.add(new NoTriggerSpelling("auto_flush=off", c -> Sender.builder(c + "auto_flush=off;"))); + spellings.add(new NoTriggerSpelling("auto_flush_rows=off;auto_flush_interval=off", + c -> Sender.builder(c + "auto_flush_rows=off;auto_flush_interval=off;"))); + spellings.add(new NoTriggerSpelling("auto_flush_interval=off;auto_flush_rows=off", + c -> Sender.builder(c + "auto_flush_interval=off;auto_flush_rows=off;"))); + spellings.add(new NoTriggerSpelling("disableAutoFlush()", + c -> Sender.builder(c).disableAutoFlush())); + spellings.add(new NoTriggerSpelling("autoFlushRows(0).autoFlushIntervalMillis(Integer.MAX_VALUE)", + c -> Sender.builder(c).autoFlushRows(0).autoFlushIntervalMillis(Integer.MAX_VALUE))); + return spellings; + } + + private static void releaseSenderLog(ListAppender log) { + senderLogger().detachAppender(log); + log.stop(); + } + + private static Logger senderLogger() { + return (Logger) LoggerFactory.getLogger(QwpWebSocketSender.class); + } + + private static void assertAborted(ThrowingRunnable r) { + try { + r.run(); + Assert.fail("expected the aborted-transaction error"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), "transaction aborted"); + } + } + + // Asserts the error of a strict transaction the client buffer cannot hold, with + // 64 KiB segments, and returns the transaction size it reports. + private static long assertClientBufferRefusal(TransactionTooLargeException e, long ringCapacityBytes, String outcome) { + String message = e.getMessage(); + String prefix = "transaction too large for the client buffer [transactionBytes="; + Assert.assertTrue(message, message.startsWith(prefix)); + long transactionBytes = Long.parseLong(message.substring(prefix.length(), message.indexOf(','))); + Assert.assertTrue(message, transactionBytes > 0); + Assert.assertEquals(prefix + transactionBytes + ", ringCapacityBytes=" + ringCapacityBytes + + ", segmentBytes=65536]; the client holds a strict transaction until the server commits it, " + + "and sf_max_total_bytes cannot hold this one, " + outcome + ". " + CLIENT_BUFFER_REMEDY, message); + return transactionBytes; + } + + // Build-time validation runs before the sender connects, so no server is needed. + private static void assertConfigRejected(String config, String expectedMessage) { + try { + Sender.fromConfig("ws::addr=localhost:1;" + config).close(); + Assert.fail(config + " must be rejected"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedMessage); + } + } + + private static void assertFramesWithinCap(TxnServer h, int cap) { + for (TxnFrame f : h.frames) { + Assert.assertTrue("frame of " + f.size + " bytes exceeds " + cap, f.size <= cap); + } + } + + private static void assertOneTransaction(TxnServer h) { + Assert.assertTrue(h.frames.size() > 1); + for (int i = 0, n = h.frames.size() - 1; i < n; i++) { + Assert.assertTrue("frame " + i + " must be deferred", h.frames.get(i).isDeferred()); + } + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + } + + // Asserts what the server that came back after an outage received and committed: the + // discarded transaction's frames, replayed and then rolled back, and behind the rollback + // frame the next transaction, which alone commits. + private static void assertDiscardedThenCommitted(TxnServer before, TxnServer after) { + Assert.assertEquals("nothing commits before the outage", 0, before.committedFrames().size()); + assertRolledBackThenCommitted(after); + } + + // Asserts what one connection received: a transaction's frames, the rollback frame + // that discarded them, and behind it the next transaction, which alone commits. + private static void assertRolledBackThenCommitted(TxnServer h) { + Assert.assertEquals("one connection: " + h.frames, 1, h.connectionCount()); + int rollbackAt = -1; + for (int i = 0, n = h.frames.size(); i < n; i++) { + if (h.frames.get(i).isRollback()) { + Assert.assertEquals("one rollback frame: " + h.frames, -1, rollbackAt); + rollbackAt = i; + } + } + Assert.assertTrue("the discarded transaction goes out ahead of the rollback frame: " + h.frames, + rollbackAt >= 2); + for (int i = 0; i < rollbackAt; i++) { + Assert.assertTrue("frame " + i + " must be deferred: " + h.frames, h.frames.get(i).isDeferred()); + } + Assert.assertEquals(1, h.rolledBackGroups); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + Assert.assertEquals("only the next transaction commits", + h.frames.subList(rollbackAt + 1, h.frames.size()), h.committedFrames()); + } + + // The server confirmed rollback, received the first frames of a transaction and went + // away. The transaction is discarded during the outage, the next one is committed, and + // the server comes back on the same endpoint. + private static void assertDiscardDuringOutage(String mode, Consumer discard) throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + TxnServer before = new TxnServer(); + TxnServer after = new TxnServer(); + TestWebSocketServer revived = null; + try (TestWebSocketServer first = start(before, CAP, port, true)) { + try (Sender s = Sender.fromConfig(outageConf(port) + mode)) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(before.awaitFrames(2)); + first.close(); + // Must neither wait for the server nor fail: the server is away. + discard.accept(s); + writeRows(s, "t", 0, 10); + s.flush(); + revived = start(after, CAP, port, true); + Assert.assertTrue(s.drain(10_000)); + } + } finally { + if (revived != null) { + revived.close(); + } + } + assertDiscardedThenCommitted(before, after); + }); + } + + private static String conf(TestWebSocketServer server) { + return "ws::addr=localhost:" + server.getPort() + ";"; + } + + // Reconnects within milliseconds once the server is back. Each test drains before it + // closes, so the short close drain only bounds how long a failing test takes. + private static String outageConf(int port) { + return "ws::addr=localhost:" + port + + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;close_flush_timeout_millis=1000;"; + } + + private static TestWebSocketServer start(TxnServer h, int cap) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(h); + server.setAdvertisedMaxBatchSize(cap); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + // A server on a fixed port, so a test can take it down and bring it back on the same + // endpoint, as one that supports rollback or as one that predates it. + private static TestWebSocketServer start(TxnServer h, int cap, int port, boolean isRollbackSupported) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(h, false, null, port); + server.setAdvertisedMaxBatchSize(cap); + server.setRollbackSupported(isRollbackSupported); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static long frameBytes(TxnServer h) { + return frameBytes(h, 0, h.frames.size()); + } + + // Bytes of the frames [from, to) the server received. + private static long frameBytes(TxnServer h, int from, int to) { + long bytes = 0; + for (int i = from; i < to; i++) { + bytes += h.frames.get(i).size; + } + return bytes; + } + + // One letter per frame the server received: D deferred, c bare commit, C commit + // with rows, R rollback. + private static String frameKinds(TxnServer h) { + StringBuilder kinds = new StringBuilder(); + for (TxnFrame f : h.frames) { + kinds.append(f.isRollback() ? 'R' : f.isDeferred() ? 'D' : f.isBareCommit() ? 'c' : 'C'); + } + return kinds.toString(); + } + + private static String serverRowLimitError(String table, long maxUncommittedRows) { + return "transaction too large for the server [table=" + table + ", rows=" + maxUncommittedRows + + ", maxUncommittedRows=" + maxUncommittedRows + "]; the server commits a table's rows on its " + + "own once qwp.max.uncommitted.rows of them are uncommitted, which would split the transaction, " + + "so the row was not added. Raise qwp.max.uncommitted.rows on the server"; + } + + private static void writeRows(Sender s, String table, int from, int count) { + for (int i = from, n = from + count; i < n; i++) { + s.table(table).longColumn("v", i).stringColumn("s", PAD).atNow(); + } + } + + // ~28 KB in each of count tables, from table t: each fits a 32 KiB frame. + private static void writeTables(Sender s, int first, int count) { + for (int t = first, n = first + count; t < n; t++) { + writeRows(s, "t" + t, 0, 280); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run(); + } + + private static final class NoTriggerSpelling { + final String name; + private final Function builder; + + NoTriggerSpelling(String name, Function builder) { + this.name = name; + this.builder = builder; + } + + Sender build(String config) { + return builder.apply(config).build(); + } + } + + private static final class TxnFrame { + // Index of the connection the frame arrived on, in connection order. + final int connection; + final byte flags; + final int size; + + TxnFrame(byte flags, int size, int connection) { + this.flags = flags; + this.size = size; + this.connection = connection; + } + + // A commit frame with no table: the header and an empty symbol delta. + boolean isBareCommit() { + return isCommit() && size <= QwpConstants.HEADER_SIZE + 6; + } + + boolean isCommit() { + return (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0; + } + + boolean isDeferred() { + return !isRollback() && (flags & QwpConstants.FLAG_DEFER_COMMIT) != 0; + } + + boolean isRollback() { + return size == QwpConstants.HEADER_SIZE && flags == QwpConstants.ROLLBACK_FRAME_FLAGS; + } + + @Override + public String toString() { + return connection + ":" + (isRollback() ? "R" : isDeferred() ? "D" : "C") + size; + } + } + + /** + * Acks frames the way the QuestDB server does, per connection: wire sequences + * restart at 0 on every connection, deferred frames wait for the commit frame + * that closes their group, which acks them cumulatively, and the rollback frame + * discards the group and acks it too. + *

+ * It also models what the server commits: a commit frame commits the group open + * on its connection, while a rollback frame or a closed connection leaves that + * group uncommitted. On a connection that did not confirm rollback the rollback + * frame is what a server without rollback support makes of it, an empty deferred + * frame: the group stays open, and the next commit on the connection commits it. + */ + private static final class TxnServer implements TestWebSocketServer.WebSocketServerHandler { + // Every frame received, over all connections, in arrival order. + final List frames = new CopyOnWriteArrayList<>(); + volatile boolean isAckingRollback = true; + volatile int rolledBackGroups; + // Frames a commit frame committed, in commit order. + private final List committed = new ArrayList<>(); + private final Map connections = new IdentityHashMap<>(); + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + Connection connection = connections.get(client); + if (connection == null) { + connection = new Connection(connections.size()); + connections.put(client, connection); + } + long seq = connection.nextSeq++; + TxnFrame frame = new TxnFrame(data[QwpConstants.HEADER_OFFSET_FLAGS], data.length, connection.index); + frames.add(frame); + notifyAll(); + try { + if (frame.isCommit()) { + committed.addAll(connection.group); + committed.add(frame); + connection.group.clear(); + client.sendBinary(ack(seq)); + } else if (frame.isRollback() && client.isRollbackConfirmed()) { + rolledBackGroups++; + connection.group.clear(); + if (isAckingRollback) { + client.sendBinary(ack(seq)); + } + } else { + connection.group.add(frame); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + synchronized boolean awaitRollbackFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (rollbackFrames() < count) { + long left = deadline - System.nanoTime(); + if (left <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, left); + } + return true; + } + + synchronized List committedFrames() { + return new ArrayList<>(committed); + } + + synchronized int connectionCount() { + return connections.size(); + } + + // The frames of each connection, in arrival order, by connection index. + synchronized List> framesByConnection() { + List> byConnection = new ArrayList<>(); + for (int i = 0, n = connections.size(); i < n; i++) { + byConnection.add(new ArrayList<>()); + } + for (TxnFrame f : frames) { + byConnection.get(f.connection).add(f); + } + return byConnection; + } + + synchronized boolean awaitFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (frames.size() < count) { + long left = deadline - System.nanoTime(); + if (left <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, left); + } + return true; + } + + synchronized int commitFrames() { + int n = 0; + for (TxnFrame f : frames) { + if (f.isCommit()) { + n++; + } + } + return n; + } + + synchronized int rollbackFrames() { + int n = 0; + for (TxnFrame f : frames) { + if (f.isRollback()) { + n++; + } + } + return n; + } + + private static byte[] ack(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static final class Connection { + // Deferred frames the connection holds uncommitted. + final List group = new ArrayList<>(); + final int index; + long nextSeq; + + Connection(int index) { + this.index = index; + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 44ceb6d4..16f8a3bf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -487,8 +487,10 @@ public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off: the batch must reach flush() whole, as the + // byte trigger would otherwise publish it from at(). String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";sf_max_segment_bytes=64;"; + + ";sf_max_segment_bytes=64;auto_flush_bytes=off;"; String pad = TestUtils.repeat("x", 60); Sender sender = Sender.fromConfig(config); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java new file mode 100644 index 00000000..7468fde7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java @@ -0,0 +1,802 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.qwp.client.DurableAckTiers; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import io.questdb.client.test.tools.TestWebSocketClient; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +/** + * The rollback control frame on a connection whose server did not confirm + * {@code X-QWP-Rollback}. The producer publishes the frame without waiting for + * its ack, so a reconnect can reach a server without rollback support, such as + * an older node after a failover, before the frame is acked. That server would + * read it as an empty deferred frame and commit the rolled-back transaction + * with the next commit frame. + *

+ * Contract under test: the I/O loop never sends the rollback frame to such a + * server. Once the server acked every commit-bearing frame sent before it, the + * loop acknowledges the rollback frame itself and recycles the connection, + * which discards the rolled-back frames server-side, and the next connection + * starts past the rollback frame. A server that confirmed rollback receives the + * frame. The same holds for a log recovered after a restart and for the + * background drainer, which replay through the same loop. + *

+ * {@link ServerModel} is the oracle: it commits what the QuestDB server commits, + * and drops the frames a closed connection still had in flight, so a recycle + * before the acks of earlier commits shows as a lost commit. + */ +public class CursorWebSocketSendLoopRollbackTest { + // Random transaction logs per run of the fuzz test; raise it for a local soak. + private static final int FUZZ_ITERATIONS = Integer.getInteger("questdb.fuzz.rollback.iterations", 100); + private static final long SEGMENT_SIZE_BYTES = 64 * 1024; + private static final String TABLE = "t"; + + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testBackgroundDrainerRetiresRollbackOnServerWithoutRollback() throws Exception { + // An orphaned slot whose producer rolled back a transaction and committed the + // next one before it went away, with nothing acked yet. + String slotPath = temporaryFolder.newFolder("slot").getAbsolutePath(); + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slotPath, SEGMENT_SIZE_BYTES)) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + } + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_SIZE_BYTES, + 1L << 20, + () -> newClient(server, clients, false, false), + 5_000L, + 1L, + 5L, + DurableAckTiers.NONE, + 200L); + + // A drainer retries until it drains, so bound the run: a slot that never + // drains fails the test instead of hanging it. + Thread drain = new Thread(drainer::run, "rollback-drainer"); + drain.start(); + drain.join(10_000); + if (drain.isAlive()) { + drainer.requestStop(); + drain.join(); + Assert.fail("the slot did not drain: " + server.received()); + } + + Assert.assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "1:D3", "1:C4"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + }); + } + + @Test + public void testConsecutiveRollbacksRecycleOnceEach() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendRollback(engine); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 5L); + Assert.assertEquals(Arrays.asList("0:D1", "1:D2", "2:D3", "2:C4"), server.received()); + Assert.assertEquals(Arrays.asList("D3", "C4"), server.committed()); + Assert.assertEquals(3, clientCount(clients)); + } + }); + } + + @Test + public void testRandomTransactionsOverServersWithAndWithoutRollback() throws Exception { + // Transactions of zero to three deferred frames, each closed by a commit frame or + // a rollback frame, over connections that reach a server with or without rollback + // support at random. Each rollback frame goes out to a server that confirmed + // rollback, or costs exactly one recycle; only committed transactions commit, + // each once. + long s0 = System.nanoTime(); + long s1 = System.currentTimeMillis(); + Rnd rnd = TestUtils.generateRandom(null, s0, s1); + try { + TestUtils.assertMemoryLeak(() -> { + for (int iteration = 0; iteration < FUZZ_ITERATIONS; iteration++) { + runRandomTransactions(rnd); + } + }); + } catch (Throwable t) { + throw new AssertionError("seeds=" + s0 + "L, " + s1 + "L", t); + } + } + + @Test + public void testRecoveredLogRetiresRollbackOnServerWithoutRollback() throws Exception { + // A restart recovers the log with the rollback frame between two transactions, so + // the orphaned-tail retirement does not apply: the frame is replayed like any other. + String dir = temporaryFolder.newFolder("recovered").getAbsolutePath(); + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(dir, SEGMENT_SIZE_BYTES)) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + } + try (CursorSendEngine engine = new CursorSendEngine(dir, SEGMENT_SIZE_BYTES)) { + Assert.assertTrue(engine.wasRecoveredFromDisk()); + Assert.assertEquals("no orphaned tail", -1L, engine.recoveredOrphanTipFsn()); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 4L); + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "1:D3", "1:C4"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testRollbackAtReplayStartRetiresBeforeSendingAnything() throws Exception { + // The frames ahead of the rollback frame are acked already, as when the server + // committed the transaction's rows itself at its uncommitted-row cap, so the + // rollback frame is the first frame of the connection. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendRollback(engine); + appendDeferred(engine, 2); + appendCommit(engine, 3); + engine.acknowledge(0L); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 3L); + Assert.assertEquals(Arrays.asList("1:D2", "1:C3"), server.received()); + Assert.assertEquals(Arrays.asList("D2", "C3"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testRollbackFrameGoesToServerWithRollback() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendDeferred(engine, 3); + appendRollback(engine); + appendDeferred(engine, 4); + appendCommit(engine, 5); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, true), 5L); + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "0:D3", "0:R", "0:D4", "0:C5"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D4", "C5"), server.committed()); + Assert.assertEquals(1, clientCount(clients)); + // Nothing changes on this path: the whole log goes out in one write. + Assert.assertEquals(1, clients.get(0).flushes); + } + }); + } + + @Test + public void testRollbackFrameNeverReachesServerWithoutRollback() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendDeferred(engine, 3); + appendRollback(engine); + appendDeferred(engine, 4); + appendCommit(engine, 5); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 5L); + // The frames ahead of the rollback frame went out in one batch with it + // held back; closing the connection discarded 2 and 3, and the next + // connection starts past the rollback frame. + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "0:D3", "1:D4", "1:C5"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D4", "C5"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + Assert.assertEquals(1, clients.get(0).flushes); + Assert.assertEquals(1, clients.get(1).flushes); + } + }); + } + + @Test + public void testRollbackSupportIsDecidedPerConnection() throws Exception { + // The first connection reaches a server without rollback support, the second one + // a server with it: the second rollback frame goes out. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendRollback(engine); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + boolean[] isRollbackConfirmed = {false, true}; + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, + () -> newClient(server, clients, isRollbackConfirmed[clientCount(clients)], false), + 1L, 5L); + runToAck(engine, loop, 5L); + Assert.assertEquals(Arrays.asList("0:D1", "1:D2", "1:R", "1:D3", "1:C4"), server.received()); + Assert.assertEquals(Arrays.asList("D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testOnlyTheRollbackFrameIsHeldBack() throws Exception { + // Bare 12-byte headers that are not the rollback frame -- an empty commit, an empty + // deferred frame and a foreign payload -- go out like any frame. Mistaking the + // empty commit for a rollback frame would discard the transaction it commits. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendHeader(engine, QwpConstants.MAGIC_MESSAGE, (byte) 0); + appendHeader(engine, QwpConstants.MAGIC_MESSAGE, QwpConstants.FLAG_DEFER_COMMIT); + appendHeader(engine, 0, QwpConstants.ROLLBACK_FRAME_FLAGS); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 7L); + Assert.assertEquals(Arrays.asList("0:D1", "0:C0", "0:D0", "0:X12", "0:D2", "1:D3", "1:C4"), + server.received()); + Assert.assertEquals(Arrays.asList("D1", "C0", "D0", "X12", "D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testRollbackWaitsForCommitAcksBeforeRecycling() throws Exception { + // The server holds the frames ahead of the rollback frame unprocessed. Recycling + // now would lose commit 1 with the connection: the self-acknowledge of the + // rollback frame covers it, so it would never be replayed. + assertRollbackWaitsForAckOf(false, engine -> appendCommit(engine, 1), "C1"); + } + + @Test + public void testRollbackWaitsForDurableAckBeforeRecycling() throws Exception { + // In durable-ack mode the trim watermark advances only on the durable ack, which + // the server holds back. The OK of commit 1 alone must not let the loop recycle, + // or it would trim commit 1 before the server made it durable. + assertRollbackWaitsForAckOf(true, engine -> appendCommit(engine, 1), "C1"); + } + + @Test + public void testRollbackWaitsForForeignFrameAck() throws Exception { + // A payload that is not a QWP message cannot be told to be deferred, so the loop + // waits for its ack like for a commit's: the safe side. + assertRollbackWaitsForAckOf(false, engine -> appendForeign(engine, 16), "X16"); + } + + @Test + public void testRollbackWaitsForShortFrameAck() throws Exception { + // Same for a payload too short to hold a QWP header. + assertRollbackWaitsForAckOf(false, engine -> appendForeign(engine, 8), "X8"); + } + + private static void appendCommit(CursorSendEngine engine, int id) { + appendFrame(engine, (byte) 0, id); + } + + private static void appendDeferred(CursorSendEngine engine, int id) { + appendFrame(engine, QwpConstants.FLAG_DEFER_COMMIT, id); + } + + // A payload without the QWP magic. + private static void appendForeign(CursorSendEngine engine, int size) { + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, size, (byte) 'x'); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // A QWP header whose flags byte says what the frame is, followed by its id. + private static void appendFrame(CursorSendEngine engine, byte flags, int id) { + int size = QwpConstants.HEADER_SIZE + 4; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, size, (byte) 0); + Unsafe.getUnsafe().putInt(buf, QwpConstants.MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + 4, QwpConstants.VERSION); + Unsafe.getUnsafe().putByte(buf + QwpConstants.HEADER_OFFSET_FLAGS, flags); + Unsafe.getUnsafe().putInt(buf + QwpConstants.HEADER_SIZE, id); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // A bare 12-byte header: no table blocks, no payload. + private static void appendHeader(CursorSendEngine engine, int magic, byte flags) { + int size = QwpConstants.HEADER_SIZE; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, size, (byte) 0); + Unsafe.getUnsafe().putInt(buf, magic); + Unsafe.getUnsafe().putByte(buf + 4, QwpConstants.VERSION); + Unsafe.getUnsafe().putByte(buf + QwpConstants.HEADER_OFFSET_FLAGS, flags); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // The rollback control frame, as QwpWebSocketEncoder.encodeRollbackMessage writes it. + private static void appendRollback(CursorSendEngine engine) { + appendHeader(engine, QwpConstants.MAGIC_MESSAGE, QwpConstants.ROLLBACK_FRAME_FLAGS); + } + + // The frame appendAhead writes is the only commit-bearing frame ahead of the rolled-back + // transaction, and the server holds it unprocessed, or in durable mode holds back its + // durable ack. The loop must not recycle until that ack arrives. + private static void assertRollbackWaitsForAckOf( + boolean isDurable, + Consumer appendAhead, + String ahead + ) throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendAhead.accept(engine); + appendDeferred(engine, 2); + appendDeferred(engine, 3); + appendRollback(engine); + appendDeferred(engine, 4); + appendCommit(engine, 5); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + ModelClient first = newClient(server, clients, false, isDurable); + // Durable mode: the server processes and OKs at once, but holds back the + // durable ack. Otherwise it holds back processing, OKs included. + first.isHoldingAcks = true; + AtomicBoolean isFirstConnect = new AtomicBoolean(true); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, + () -> isFirstConnect.getAndSet(false) ? first : newClient(server, clients, false, isDurable), + 1L, 5L, isDurable ? DurableAckTiers.REPLICATED : DurableAckTiers.NONE, 0L); + try { + loop.start(); + awaitTrue("the frames ahead of the rollback frame go out", () -> first.flushedFrames == 3); + // The loop keeps polling the first connection while it waits for the acks. + int polls = first.polls; + awaitTrue("the loop keeps waiting on its connection", () -> first.polls >= polls + 50); + Assert.assertEquals("no recycle while " + ahead + " is not acked", 1, clientCount(clients)); + Assert.assertEquals(-1L, engine.ackedFsn()); + first.releaseAcks(); + awaitAckedFsn(engine, 5L); + } finally { + loop.close(); + } + Assert.assertEquals(Arrays.asList("0:" + ahead, "0:D2", "0:D3", "1:D4", "1:C5"), server.received()); + Assert.assertEquals(Arrays.asList(ahead, "D4", "C5"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { + awaitTrue("ackedFsn reaches " + target, () -> engine.ackedFsn() >= target); + Assert.assertEquals(target, engine.ackedFsn()); + } + + private static void awaitTrue(String what, BooleanSupplier condition) throws InterruptedException { + long deadline = System.nanoTime() + 10_000_000_000L; + while (!condition.getAsBoolean()) { + if (System.nanoTime() > deadline) { + Assert.fail("timed out: " + what); + } + Thread.sleep(1); + } + } + + private static int clientCount(List clients) { + synchronized (clients) { + return clients.size(); + } + } + + private static ModelClient newClient(ServerModel server, List clients, boolean isRollbackConfirmed, boolean isDurable) { + synchronized (clients) { + ModelClient client = new ModelClient(server, clients.size(), isRollbackConfirmed, isDurable); + clients.add(client); + return client; + } + } + + private static CursorSendEngine newEngine() { + return new CursorSendEngine(null, SEGMENT_SIZE_BYTES); + } + + private static CursorWebSocketSendLoop newLoop(CursorSendEngine engine, ServerModel server, List clients, boolean isRollbackConfirmed) { + return new CursorWebSocketSendLoop( + null, engine, 0L, + () -> newClient(server, clients, isRollbackConfirmed, false), + 1L, 5L); + } + + private static void runRandomTransactions(Rnd rnd) throws Exception { + try (CursorSendEngine engine = newEngine()) { + List expectedCommitted = new ArrayList<>(); + List transaction = new ArrayList<>(); + int rollbacks = 0; + int nextId = 1; + // Appended before the loop starts, the rest while it runs. + List appends = new ArrayList<>(); + for (int t = 0, n = 1 + rnd.nextInt(12); t < n; t++) { + transaction.clear(); + for (int i = 0, k = rnd.nextInt(4); i < k; i++) { + int id = nextId++; + transaction.add("D" + id); + appends.add(() -> appendDeferred(engine, id)); + } + if (rnd.nextInt(5) < 3) { + int id = nextId++; + transaction.add("C" + id); + expectedCommitted.addAll(transaction); + appends.add(() -> appendCommit(engine, id)); + } else { + rollbacks++; + appends.add(() -> appendRollback(engine)); + } + } + int beforeStart = rnd.nextInt(appends.size() + 1); + for (int i = 0; i < beforeStart; i++) { + appends.get(i).run(); + } + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + Rnd ioRnd = new Rnd(rnd.nextLong(), rnd.nextLong()); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, + () -> newClient(server, clients, ioRnd.nextBoolean(), false), + 1L, 5L); + int retired; + try { + loop.start(); + for (int i = beforeStart, n = appends.size(); i < n; i++) { + appends.get(i).run(); + } + awaitAckedFsn(engine, appends.size() - 1L); + // Every rollback frame is received or retired by now. A retired one acks + // itself before its recycle connects, so when it is the last frame the + // recycle may still be under way. + retired = rollbacks - server.rollbacksReceived(); + awaitTrue("one recycle per retired rollback frame", () -> clientCount(clients) == 1 + retired); + } finally { + loop.close(); + } + String shape = "frames=" + appends.size() + ", received=" + server.received(); + Assert.assertEquals(shape, expectedCommitted, server.committed()); + Assert.assertEquals(shape, 0, server.rollbacksToServerWithoutRollback()); + Assert.assertEquals(shape, 1 + retired, clientCount(clients)); + } + } + + private static void runToAck(CursorSendEngine engine, CursorWebSocketSendLoop loop, long lastFsn) throws Exception { + try { + loop.start(); + awaitAckedFsn(engine, lastFsn); + } finally { + loop.close(); + } + } + + /** + * One connection to {@link ServerModel}: frames a batch flush hands over are in + * flight until the I/O loop polls, which is when the server processes them, unless + * it holds its acks. Acks are cumulative, so only the highest pending one goes out. + */ + private static final class ModelClient extends TestWebSocketClient { + private final List batch = new ArrayList<>(); + private final int connection; + private final ArrayDeque inFlight = new ArrayDeque<>(); + private final boolean isDurable; + private final boolean isRollbackConfirmed; + private final ServerModel server; + volatile int flushedFrames; + volatile int flushes; + volatile boolean isHoldingAcks; + volatile int polls; + private long durableReported; + private long nextSeq; + private long pendingOkSeq = -1L; + private long pendingOkSeqTxn; + + ModelClient(ServerModel server, int connection, boolean isRollbackConfirmed, boolean isDurable) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.server = server; + this.connection = connection; + this.isRollbackConfirmed = isRollbackConfirmed; + this.isDurable = isDurable; + } + + @Override + public synchronized void batchAppendBinary(long dataPtr, int length) { + batch.add(Frame.read(dataPtr, length)); + } + + @Override + public synchronized void batchFlush() { + inFlight.addAll(batch); + flushedFrames += batch.size(); + flushes++; + batch.clear(); + } + + @Override + public synchronized void close() { + // Whatever the server has not processed yet never arrives. + inFlight.clear(); + server.close(connection); + super.close(); + } + + @Override + public boolean isServerRollbackEnabled() { + return isRollbackConfirmed; + } + + synchronized void releaseAcks() { + isHoldingAcks = false; + } + + @Override + public void sendPing() { + } + + @Override + public synchronized boolean tryReceiveFrame(WebSocketFrameHandler handler) { + polls++; + // Durable mode: the server processes and OKs at once and holds back the + // durable ack. Otherwise holding back the acks means not processing at all. + if (isDurable || !isHoldingAcks) { + for (Frame frame; (frame = inFlight.pollFirst()) != null; ) { + long seq = nextSeq++; + if (server.process(connection, isRollbackConfirmed, frame)) { + pendingOkSeq = seq; + pendingOkSeqTxn = frame.kind == 'R' ? 0 : server.seqTxn(); + } + } + } + if (pendingOkSeq >= 0) { + sendOk(handler, pendingOkSeq, isDurable && pendingOkSeqTxn > 0 ? pendingOkSeqTxn : 0); + pendingOkSeq = -1L; + return true; + } + if (isDurable && !isHoldingAcks && server.seqTxn() > durableReported) { + durableReported = server.seqTxn(); + sendDurableAck(handler, durableReported); + return true; + } + return false; + } + + // STATUS_DURABLE_ACK: status, then the table entries. + private static void sendDurableAck(WebSocketFrameHandler handler, long seqTxn) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + int size = 1 + 2 + 2 + name.length + 8; + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_DURABLE_ACK); + putTableEntry(ptr + 1, name, seqTxn); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // STATUS_OK: status, cumulative wire sequence, then the table entries -- one for + // a commit in durable mode, so the OK waits for the durable ack covering it. + private static void sendOk(WebSocketFrameHandler handler, long seq, long seqTxn) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + int size = 1 + 8 + 2 + (seqTxn > 0 ? 2 + name.length + 8 : 0); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK); + Unsafe.getUnsafe().putLong(ptr + 1, seq); + if (seqTxn > 0) { + putTableEntry(ptr + 9, name, seqTxn); + } else { + Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); + } + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void putTableEntry(long p, byte[] name, long seqTxn) { + Unsafe.getUnsafe().putShort(p, (short) 1); + Unsafe.getUnsafe().putShort(p + 2, (short) name.length); + for (int i = 0; i < name.length; i++) { + Unsafe.getUnsafe().putByte(p + 4 + i, name[i]); + } + Unsafe.getUnsafe().putLong(p + 4 + name.length, seqTxn); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + + // A frame as the server sees it: D (deferred), C (commit), R (the rollback frame), or + // X, a payload that is not a QWP message, labelled with its size. A frame's id + // follows its header; a bare header has id 0. + private static final class Frame { + final char kind; + final String label; + + Frame(char kind, String label) { + this.kind = kind; + this.label = label; + } + + static Frame read(long ptr, int length) { + if (length < QwpConstants.HEADER_SIZE || Unsafe.getUnsafe().getInt(ptr) != QwpConstants.MAGIC_MESSAGE) { + return new Frame('X', "X" + length); + } + byte flags = Unsafe.getUnsafe().getByte(ptr + QwpConstants.HEADER_OFFSET_FLAGS); + if (length == QwpConstants.HEADER_SIZE && flags == QwpConstants.ROLLBACK_FRAME_FLAGS) { + return new Frame('R', "R"); + } + char kind = (flags & QwpConstants.FLAG_DEFER_COMMIT) != 0 ? 'D' : 'C'; + int id = length > QwpConstants.HEADER_SIZE ? Unsafe.getUnsafe().getInt(ptr + QwpConstants.HEADER_SIZE) : 0; + return new Frame(kind, kind + Integer.toString(id)); + } + + @Override + public String toString() { + return label; + } + } + + /** + * What the QuestDB server makes of the frames, per connection: a deferred frame + * joins the connection's open group; a commit frame commits the group and itself + * and is acked; the rollback frame discards the group and is acked on a connection + * that confirmed rollback. A server without rollback support reads the rollback + * frame as an empty deferred frame, so the group stays open and the next commit on + * the connection commits it. A closed connection leaves its group uncommitted. A + * foreign payload is acked like a commit, so the loop's wait for it ends. + */ + private static final class ServerModel { + private final List committed = new ArrayList<>(); + private final List> groups = new ArrayList<>(); + private final List received = new ArrayList<>(); + private int rollbacksReceived; + private int rollbacksToServerWithoutRollback; + private long seqTxn; + + synchronized void close(int connection) { + group(connection).clear(); + } + + synchronized List committed() { + return new ArrayList<>(committed); + } + + // Whether the server acks the frame. + synchronized boolean process(int connection, boolean isRollbackConfirmed, Frame frame) { + received.add(connection + ":" + frame); + List group = group(connection); + switch (frame.kind) { + case 'C': + case 'X': + committed.addAll(group); + committed.add(frame.label); + group.clear(); + seqTxn++; + return true; + case 'R': + rollbacksReceived++; + if (isRollbackConfirmed) { + group.clear(); + return true; + } + rollbacksToServerWithoutRollback++; + return false; + default: + group.add(frame.label); + return false; + } + } + + synchronized List received() { + return new ArrayList<>(received); + } + + synchronized int rollbacksReceived() { + return rollbacksReceived; + } + + synchronized int rollbacksToServerWithoutRollback() { + return rollbacksToServerWithoutRollback; + } + + synchronized long seqTxn() { + return seqTxn; + } + + private List group(int connection) { + while (groups.size() <= connection) { + groups.add(new ArrayList<>()); + } + return groups.get(connection); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 102922d6..81bf0144 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -146,6 +146,16 @@ public class TestWebSocketServer implements Closeable { // builds. Lets a test force the delta-dictionary catch-up to split across // several frames. Live-updatable via setAdvertisedMaxBatchSize(). private volatile int advertisedMaxBatchSize; + // When > 0, 101 upgrade responses advertise this value as the + // X-QWP-Max-Uncommitted-Rows header. Live-updatable. + private volatile long advertisedMaxUncommittedRows; + // X-QWP-Request-Rollback header value captured from each well-formed upgrade + // request ("" when absent). + private final BlockingQueue capturedRollbackRequests = new LinkedBlockingQueue<>(); + // When true, a 101 upgrade response to a request carrying + // X-QWP-Request-Rollback: true confirms it with X-QWP-Rollback: enabled. + // Live-updatable. + private volatile boolean rollbackSupported; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -349,6 +359,31 @@ public void setAdvertisedMaxBatchSize(int maxBatchSize) { this.advertisedMaxBatchSize = maxBatchSize; } + /** + * Advertises {@code X-QWP-Max-Uncommitted-Rows: } on subsequent + * handshakes (live update). Pass {@code 0} to stop advertising it. + */ + public void setAdvertisedMaxUncommittedRows(long rows) { + this.advertisedMaxUncommittedRows = rows; + } + + /** + * When {@code true}, subsequent handshakes that request rollback + * ({@code X-QWP-Request-Rollback: true}) are confirmed with + * {@code X-QWP-Rollback: enabled} (live update). + */ + public void setRollbackSupported(boolean supported) { + this.rollbackSupported = supported; + } + + /** + * Blocks up to {@code timeout} for the next captured X-QWP-Request-Rollback + * header value ("" when the upgrade request carried none). + */ + public String pollRollbackRequest(long timeout, TimeUnit unit) throws InterruptedException { + return capturedRollbackRequests.poll(timeout, unit); + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -548,6 +583,8 @@ public class ClientHandler implements Closeable { private final Socket socket; private InputStream in; private boolean isClosed; + // True once this connection's 101 response confirmed X-QWP-Rollback: enabled. + private volatile boolean isRollbackConfirmed; private OutputStream out; private Thread readThread; // Request path from the WebSocket upgrade GET line (e.g. /write/v4, @@ -602,6 +639,14 @@ private void drainInput() throws IOException { } } + /** + * Whether this connection's upgrade confirmed {@code X-QWP-Rollback: enabled}, so + * the client may send it the rollback control frame. + */ + public boolean isRollbackConfirmed() { + return isRollbackConfirmed; + } + public synchronized void sendBinary(byte[] data) throws IOException { writeFrame(WebSocketOpcode.BINARY, data, data.length); } @@ -762,6 +807,7 @@ private boolean performHandshake() throws IOException { String key = null; String authorization = ""; String durableAckRequest = ""; + String rollbackRequest = ""; String[] lines = request.toString().split("\r\n"); if (lines.length > 0) { // GET HTTP/1.1 @@ -778,6 +824,8 @@ private boolean performHandshake() throws IOException { authorization = line.substring("authorization:".length()).trim(); } else if (lower.startsWith("x-qwp-request-durable-ack:")) { durableAckRequest = line.substring("x-qwp-request-durable-ack:".length()).trim(); + } else if (lower.startsWith("x-qwp-request-rollback:")) { + rollbackRequest = line.substring("x-qwp-request-rollback:".length()).trim(); } } @@ -787,6 +835,7 @@ private boolean performHandshake() throws IOException { upgradeRequestCount.incrementAndGet(); capturedAuthHeaders.add(authorization); capturedDurableAckRequests.add(durableAckRequest); + capturedRollbackRequests.add(rollbackRequest); // Read-path reject: drop the egress upgrade before the 101 so the // query pool's connect fails fast, while ingest write-path upgrades @@ -866,6 +915,14 @@ private boolean performHandshake() throws IOException { if (maxBatch > 0) { sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n"); } + long maxUncommittedRows = advertisedMaxUncommittedRows; + if (maxUncommittedRows > 0) { + sb.append("X-QWP-Max-Uncommitted-Rows: ").append(maxUncommittedRows).append("\r\n"); + } + if (rollbackSupported && "true".equalsIgnoreCase(rollbackRequest)) { + sb.append("X-QWP-Rollback: enabled\r\n"); + isRollbackConfirmed = true; + } sb.append("\r\n"); out.write(sb.toString().getBytes(StandardCharsets.US_ASCII)); out.flush(); diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index 39695ec2..d9190ba6 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -54,11 +54,12 @@ public void testEveryIngressKeyIsHonored() { assertHonored("auto_flush_interval=250", "auto_flush_interval", 250); // auto_flush=off reaches disableAutoFlush(), which sets the interval to // MAX_VALUE; auto_flush_rows=off leaves the interval unset, so this field - // distinguishes the two. That config is build-rejected on WebSocket; see - // QuestDBBuilderTest.testMalformedIngressConfigRejectedAtBuildWithMinZero. + // distinguishes the two. On WebSocket it selects strict transactional mode + // at build(); see QwpLargeTransactionTest. assertHonored("auto_flush=off", "auto_flush_interval", Integer.MAX_VALUE); assertHonored("max_name_len=99", "max_name_len", 99); - assertHonored("transaction=on", "transaction", true); + assertHonored("transaction=on", "transaction", "on"); + assertHonored("transaction=strict", "transaction", "strict"); assertHonored("request_durable_ack=on", "request_durable_ack", "on"); assertHonored("request_durable_ack=local", "request_durable_ack", "local"); assertHonored("request_durable_ack=replicated", "request_durable_ack", "replicated");