From a0a0d892f0531434eab6690552c51f3f08ec20b4 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 11 Nov 2026 23:24:11 +0000 Subject: [PATCH 1/7] Let QWP senders send batches of any size The WebSocket sender's default byte trigger now sizes every batch to 90% of the frame the connection can carry: the smaller of the server's advertised X-QWP-Max-Batch-Size and the store-and-forward segment payload, or the server's default cap when nothing is advertised. Rows wider than about 2 KiB no longer fail with BatchTooLargeForCapException, in streaming and in transactional mode. auto_flush_bytes=off and a sender with every auto-flush trigger off keep the old behaviour. New symbols count toward the trigger, since their strings ride in the frame's dictionary section. Add strict transactional mode: transaction=strict, or auto_flush=off on WebSocket. Rows go out as deferred frames sized for the connection and only flush() commits, so a transaction can be any size, but it never commits in parts: a row that would take a table to the server's qwp.max.uncommitted.rows (advertised in X-QWP-Max-Uncommitted-Rows, or the 1M default) and a transaction that outgrows sf_max_total_bytes are refused with TransactionTooLargeException before anything reaches the server, leaving the transaction open. close() without flush() discards a strict transaction, and a pooled strict lease discards it on return. Add Sender.rollback() and the QWP rollback control frame. The sender asks for it with X-QWP-Request-Rollback and sends the frame only to a server that confirmed X-QWP-Rollback: enabled; rollback() waits for the ack, so a reconnect can never replay the rolled-back frames. reset() in transactional mode now rolls back frames already sent when the server supports it. Without server support rollback() aborts the transaction, and a strict reset() does too; a non-strict reset() keeps committing those frames on the next flush, as before. --- .../main/java/io/questdb/client/Sender.java | 100 ++- .../cutlass/http/client/WebSocketClient.java | 89 ++ .../qwp/client/QwpWebSocketEncoder.java | 20 + .../qwp/client/QwpWebSocketSender.java | 680 ++++++++++++++-- .../client/TransactionTooLargeException.java | 49 ++ .../client/sf/cursor/CursorSendEngine.java | 11 + .../qwp/client/sf/cursor/SegmentManager.java | 8 + .../cutlass/qwp/protocol/QwpConstants.java | 20 + .../cutlass/qwp/protocol/QwpTableBuffer.java | 18 + .../io/questdb/client/impl/PooledSender.java | 20 +- .../client/test/QuestDBBuilderTest.java | 10 +- .../http/client/WebSocketClientTest.java | 32 + .../qwp/client/DeltaDictRecoveryTest.java | 8 +- .../LineSenderBuilderWebSocketTest.java | 9 +- .../qwp/client/QwpLargeTransactionTest.java | 759 ++++++++++++++++++ .../qwp/client/SelfSufficientFramesTest.java | 4 +- .../qwp/websocket/TestWebSocketServer.java | 46 ++ .../test/impl/WsSenderConfigHonoredTest.java | 7 +- 18 files changed, 1802 insertions(+), 88 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index d16e97804..b3ffb3484 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -780,6 +780,26 @@ default void resetSymbolDictionary() { */ void reset(); + /** + * Discard the open transaction: the rows buffered since the last flush and, in + * WebSocket transactional mode ({@code transaction=on} or {@code strict}), the + * rows of the transaction that auto-flush already sent to the server, which + * the server discards on request. Blocks until the server confirms. + *
+ * Discarding rows already sent needs a server that supports QWP rollback. When + * the server does not, this method throws and aborts the transaction: every + * later flush throws too, and closing the sender discards the rows, because the + * server rolls back uncommitted rows when the connection closes. + *
+ * Only the WebSocket transport supports this method. + * + * @throws LineSenderException if the transport does not support rollback, or the + * server cannot discard rows already sent + */ + default void rollback() { + throw new LineSenderException("rollback is only supported for WebSocket transport"); + } + /** * Add a column with a 16-bit signed integer value. * @@ -1075,7 +1095,10 @@ final class LineSenderBuilder { private static final int DEFAULT_TCP_PORT = 9009; private static final int DEFAULT_UDP_PORT = 9007; private static final int DEFAULT_WEBSOCKET_PORT = 9000; - private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 0; + // Unset auto_flush_bytes on WebSocket sizes the byte trigger from what one + // frame can carry, so no batch outgrows a frame; see + // QwpWebSocketSender.AUTO_FLUSH_BYTES_AUTO. + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = QwpWebSocketSender.AUTO_FLUSH_BYTES_AUTO; private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 1_000; // Cadence at which the SF cursor I/O loop sends a keepalive PING @@ -1234,7 +1257,12 @@ public int getConnectTimeout() { private long reconnectMaxDurationMillis = PARAMETER_NOT_SET_EXPLICITLY; private int durableAckTiers = DurableAckTiers.NONE; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; + private boolean strictTransactional; private boolean transactional; + // True once transaction=... (or transactional()/strictTransactional()) + // set the mode explicitly, so auto_flush=off can refuse to override an + // explicit transaction=off. + private boolean transactionModeSet; private String senderId = DEFAULT_SENDER_ID; // Per-append deadline for SF appendBlocking spin-then-throw. Used to // be a hardcoded 30s constant; expose so tight-SLA users can lower @@ -1868,9 +1896,11 @@ public LineSenderBuilder connectionListenerInboxCapacity(int capacity) { *

* It is essential to ensure the maximum buffer capacity is sufficient to accommodate all locally buffered data. *

- * This method should only be used when communicating via the HTTP protocol. Calling this method is illegal when - * communicating over the TCP or WebSocket protocol. On WebSocket, {@code auto_flush_interval=off} turns off - * only the time-based trigger. + * This method can be used with the HTTP and WebSocket protocols; calling it is illegal when communicating over + * the TCP protocol. On WebSocket it selects strict transactional mode (see + * {@link #strictTransactional(boolean)}): rows still go to the server in frames sized for the connection, so a + * batch can be any size, but only {@code flush()} commits them, and closing without a flush discards them. + * There, {@code auto_flush_interval=off} alone turns off only the time-based trigger. * * @return this instance for method chaining * @see #autoFlushRows(int) @@ -2757,12 +2787,43 @@ private LineSenderBuilder requestDurableAckTiers(int tiers) { return this; } + /** + * Enables strict transactional mode, the builder form of + * {@code transaction=strict}. Like {@link #transactional(boolean)}, auto-flush + * sends rows to the server without committing them and {@code flush()} + * commits, so a transaction can be any size. In addition the transaction + * never commits in parts: a row that would take a table past the server's + * {@code qwp.max.uncommitted.rows}, or a transaction that outgrows + * {@code sf_max_total_bytes}, throws + * {@link io.questdb.client.cutlass.qwp.client.TransactionTooLargeException} + * and leaves the transaction open, and {@code close()} without + * {@code flush()} discards the transaction instead of committing it. + * {@code auto_flush=off} selects the same mode. WebSocket transport only. + * + * @param enabled true to enable strict transactional mode; false leaves + * {@link #transactional(boolean) transactional} mode as set + * @return this instance for method chaining + */ + public LineSenderBuilder strictTransactional(boolean enabled) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("transactional is only supported for WebSocket transport"); + } + this.strictTransactional = enabled; + if (enabled) { + this.transactional = true; + } + this.transactionModeSet = true; + return this; + } + /** * Enables transactional mode. Auto-flush sends data to the server * with {@code FLAG_DEFER_COMMIT}; only an explicit {@code flush()} - * triggers the server-side WAL commit. This allows accumulating - * datasets larger than the server's recv buffer while committing - * atomically per table. + * (or {@code close()}) triggers the server-side WAL commit. This allows + * accumulating datasets larger than the server's recv buffer while + * committing atomically per table. The server commits a table on its own + * once {@code qwp.max.uncommitted.rows} of its rows are uncommitted; use + * {@link #strictTransactional(boolean)} to fail instead. * * @param enabled true to enable transactional mode * @return this instance for method chaining @@ -2772,6 +2833,10 @@ public LineSenderBuilder transactional(boolean enabled) { throw new LineSenderException("transactional is only supported for WebSocket transport"); } this.transactional = enabled; + if (!enabled) { + this.strictTransactional = false; + } + this.transactionModeSet = true; return this; } @@ -3822,6 +3887,11 @@ private QwpWebSocketSender buildWebSocket(HttpTokenProvider tokenProvider) { // dispatcher daemon, drainer pool, microbatch buffers and // WebSocketClient inside the abandoned `connected`. connected.setTransactional(transactional); + // auto_flush=off on WebSocket means "commit only when I flush()": a + // strict transaction of any size. + if (strictTransactional || autoFlushDisabled) { + connected.setStrictTransactional(true); + } try { final String rebuildSfDir = sfDir; final String rebuildSenderId = senderId; @@ -4230,8 +4300,10 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { transactional(true); } else if (Chars.equalsIgnoreCase("off", sink)) { transactional(false); + } else if (Chars.equalsIgnoreCase("strict", sink)) { + strictTransactional(true); } else { - throw new LineSenderException("invalid transaction [value=").put(sink).put(", allowed-values=[on, off]]"); + throw new LineSenderException("invalid transaction [value=").put(sink).put(", allowed-values=[on, off, strict]]"); } } else if (Chars.equals("sf_dir", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { @@ -4682,8 +4754,10 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) transactional(true); } else if (s.equalsIgnoreCase("off")) { transactional(false); + } else if (s.equalsIgnoreCase("strict")) { + strictTransactional(true); } else { - throw new LineSenderException("invalid transaction [value=").put(s).put(", allowed-values=[on, off]]"); + throw new LineSenderException("invalid transaction [value=").put(s).put(", allowed-values=[on, off, strict]]"); } } s = view.getStr("request_durable_ack"); @@ -4821,7 +4895,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("auto_flush_bytes", autoFlushBytes); m.put("auto_flush_interval", autoFlushIntervalMillis); m.put("max_name_len", maxNameLength); - m.put("transaction", transactional); + m.put("transaction", strictTransactional ? "strict" : transactional ? "on" : "off"); m.put("request_durable_ack", DurableAckTiers.configValue(durableAckTiers)); m.put("sender_id", senderId); m.put("sf_dir", sfDir); @@ -5068,8 +5142,10 @@ private void validateParameters() { if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol version is not supported for WebSocket protocol"); } - if (autoFlushDisabled) { - throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol"); + if (autoFlushDisabled && transactionModeSet && !transactional) { + // On WebSocket, auto_flush=off is strict transactional mode. + throw new LineSenderException("auto_flush=off selects strict transactional mode on WebSocket " + + "and cannot be combined with transaction=off"); } if ((sfDurability == SfDurability.FLUSH || sfDurability == SfDurability.APPEND)) { throw new LineSenderException( diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 938d51752..ab479d602 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -84,6 +84,11 @@ public abstract class WebSocketClient implements QuietCloseable { private static final String QWP_CONTENT_ENCODING_HEADER_NAME = "X-QWP-Content-Encoding:"; private static final String QWP_DURABLE_ACK_HEADER_NAME = "X-QWP-Durable-Ack:"; private static final String QWP_MAX_BATCH_SIZE_HEADER_NAME = "X-QWP-Max-Batch-Size:"; + private static final String QWP_MAX_UNCOMMITTED_ROWS_HEADER_NAME = "X-QWP-Max-Uncommitted-Rows:"; + private static final String QWP_ROLLBACK_HEADER_NAME = "X-QWP-Rollback:"; + // Token the server echoes in X-QWP-Rollback when it accepts the rollback + // control frame on this connection. + private static final String QWP_ROLLBACK_TOKEN_ENABLED = "enabled"; private static final String QWP_VERSION_HEADER_NAME = "X-QWP-Version:"; private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { try { @@ -148,6 +153,9 @@ public abstract class WebSocketClient implements QuietCloseable { // Requested durable-ack tier set (DurableAckTiers bitmask), sent as // X-QWP-Request-Durable-Ack: private int qwpDurableAckTiers = DurableAckTiers.NONE; + // When true, the upgrade request carries X-QWP-Request-Rollback: true, asking + // the server to confirm that it accepts the rollback control frame. + private boolean qwpRequestRollback; // Receive buffer (native memory) private long recvBufPtr; private int recvBufSize; @@ -170,6 +178,11 @@ public abstract class WebSocketClient implements QuietCloseable { // not advertise the header (older builds), in which case the sender falls // back to its locally configured budget. private int serverMaxBatchSize; + // Server's per-table cap on rows a deferred-commit group may hold before the + // server commits them on its own (qwp.max.uncommitted.rows), extracted from + // X-QWP-Max-Uncommitted-Rows on the 101 upgrade response. 0 when the server + // did not advertise it (older builds). + private long serverMaxUncommittedRows; // Zstd compression level the server actually applied for this connection, // extracted from the echoed X-QWP-Content-Encoding response header. // 0 means "no zstd" -- either the server picked raw, the header was absent @@ -178,6 +191,11 @@ public abstract class WebSocketClient implements QuietCloseable { // the wire said without re-clamping so a misconfigured server is observable. private int serverNegotiatedZstdLevel; private int serverQwpVersion = 1; + // Set during upgrade response validation when the client asked for rollback + // (qwpRequestRollback) and the server confirmed it with X-QWP-Rollback: + // enabled. A server that does not confirm treats the rollback control frame + // as an empty deferred frame, so the client must never send it there. + private boolean serverRollbackEnabled; private String upgradeRejectRole; // WWW-Authenticate value(s) from the most recent rejected upgrade, joined with ", " when the server sent // several, or null when absent. Lets a 401 carrying "Bearer error=insufficient_scope" be told apart from an @@ -412,6 +430,16 @@ public int getServerMaxBatchSize() { return serverMaxBatchSize; } + /** + * Server's per-table limit on rows a deferred-commit group may hold before + * the server commits them on its own ({@code qwp.max.uncommitted.rows}), taken + * from the {@code X-QWP-Max-Uncommitted-Rows} response header on the 101 + * upgrade. Returns {@code 0} when the server did not advertise the header. + */ + public long getServerMaxUncommittedRows() { + return serverMaxUncommittedRows; + } + /** * Zstd level the server actually applied for this connection, parsed * from the echoed {@code X-QWP-Content-Encoding} response header. @@ -494,6 +522,16 @@ public boolean isServerDurableAckEnabled() { return serverDurableAckEnabled; } + /** + * Returns true when the client asked for rollback support via + * {@link #setQwpRequestRollback} and the 101 upgrade response confirmed it + * with {@code X-QWP-Rollback: enabled}. Meaningful only after + * {@link #upgrade} returns. + */ + public boolean isServerRollbackEnabled() { + return serverRollbackEnabled; + } + /** * Receives and processes WebSocket frames. * @@ -670,6 +708,16 @@ public void setQwpMaxVersion(int maxVersion) { this.qwpMaxVersion = maxVersion; } + /** + * When {@code true}, the upgrade request carries + * {@code X-QWP-Request-Rollback: true}. A server that accepts the rollback + * control frame confirms it with {@code X-QWP-Rollback: enabled}; see + * {@link #isServerRollbackEnabled()}. + */ + public void setQwpRequestRollback(boolean requestRollback) { + this.qwpRequestRollback = requestRollback; + } + /** * Sets the requested durable-ack tier set ({@link DurableAckTiers} * bitmask) for the opt-in X-QWP-Request-Durable-Ack upgrade header. When @@ -795,6 +843,9 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe sendBuffer.putAscii(DurableAckTiers.requestHeaderValue(qwpDurableAckTiers)); sendBuffer.putAscii("\r\n"); } + if (qwpRequestRollback) { + sendBuffer.putAscii("X-QWP-Request-Rollback: true\r\n"); + } if (authorizationHeader != null) { sendBuffer.putAscii("Authorization: "); sendBuffer.putAscii(authorizationHeader); @@ -912,6 +963,22 @@ private static String extractDurableAckEcho(String response) { return null; } + private static String extractHeaderValue(String response, String headerName) { + int headerLen = headerName.length(); + int responseLen = response.length(); + for (int i = 0; i <= responseLen - headerLen; i++) { + if (response.regionMatches(true, i, headerName, 0, headerLen)) { + int valueStart = i + headerLen; + int lineEnd = response.indexOf('\r', valueStart); + if (lineEnd < 0) { + lineEnd = responseLen; + } + return response.substring(valueStart, lineEnd).trim(); + } + } + return null; + } + private static int extractMaxBatchSize(String response) { int headerLen = QWP_MAX_BATCH_SIZE_HEADER_NAME.length(); int responseLen = response.length(); @@ -934,6 +1001,18 @@ private static int extractMaxBatchSize(String response) { return 0; } + private static long extractMaxUncommittedRows(String response) { + String value = extractHeaderValue(response, QWP_MAX_UNCOMMITTED_ROWS_HEADER_NAME); + if (value == null) { + return 0; + } + try { + return Math.max(Long.parseLong(value), 0); + } catch (NumberFormatException e) { + return 0; + } + } + private static int extractQwpVersion(String response) { int headerLen = QWP_VERSION_HEADER_NAME.length(); int responseLen = response.length(); @@ -1541,6 +1620,16 @@ private void validateUpgradeResponse(int headerEnd) { // sender falls back to its locally configured byte budget in that case. serverMaxBatchSize = extractMaxBatchSize(response); + // Extract X-QWP-Max-Uncommitted-Rows (optional). Older servers omit it; + // a strict transaction then assumes the server's default. + serverMaxUncommittedRows = extractMaxUncommittedRows(response); + + // Extract the X-QWP-Rollback confirmation (optional). Only a client that + // asked for rollback may treat it as granted: the server echoes it only + // on request, and an older server never sends it. + serverRollbackEnabled = qwpRequestRollback + && QWP_ROLLBACK_TOKEN_ENABLED.equalsIgnoreCase(extractHeaderValue(response, QWP_ROLLBACK_HEADER_NAME)); + // Extract X-QWP-Content-Encoding (optional). Surfaces what level the // server actually applied -- which may differ from what this client // asked for if the server has qwp.egress.compression.force.level set diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index f38d9e93d..68c338662 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -184,6 +184,26 @@ public int encodeWithDeltaDict( return finishMessage(); } + /** + * Encodes the rollback control frame into the buffer: a bare header with + * {@code ROLLBACK_FRAME_FLAGS}, no table blocks and an empty payload. The + * server matches it on exactly that shape. + * + * @return the frame size in bytes + */ + public int encodeRollbackMessage() { + buffer.reset(); + buffer.putByte((byte) 'Q'); + buffer.putByte((byte) 'W'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '1'); + buffer.putByte(version); + buffer.putByte(ROLLBACK_FRAME_FLAGS); + buffer.putShort((short) 0); + buffer.putInt(0); + return buffer.getPosition(); + } + public int finishMessage() { int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1fc434840..7d94dc702 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -72,6 +72,7 @@ import io.questdb.client.std.NumericException; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.TestOnly; @@ -135,6 +136,15 @@ */ public class QwpWebSocketSender implements Sender { + /** + * {@code autoFlushBytes} value that sizes the byte trigger from the frames the + * connection can carry: 90% of the smaller of the server's advertised + * {@code X-QWP-Max-Batch-Size} and the store-and-forward segment. It is the + * builder default, so a batch never grows past what one frame can hold. A pure + * manual-flush sender (non-transactional, row and interval triggers off) keeps + * no byte trigger, as before. + */ + public static final int AUTO_FLUSH_BYTES_AUTO = -1; public static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; // Soft per-batch byte budget. Trips a flush when raw column-buffer bytes // cross the threshold, well before the server's 16 MB wire cap. Wide-row @@ -176,6 +186,17 @@ public class QwpWebSocketSender implements Sender { public static final int DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS = 100_000; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + // How long rollback() waits for the server to acknowledge the rollback frame. + private static final long DEFAULT_ROLLBACK_ACK_TIMEOUT_MILLIS = 30_000L; + // Frames of at most the frame cap that a strict transaction keeps room for in + // the client buffer, so the commit that ends it can always be published. + private static final int STRICT_COMMIT_RESERVE_FRAMES = 3; + // Frame cap the byte trigger assumes when the server does not advertise + // X-QWP-Max-Batch-Size: the server's default 2 MiB recv buffer minus the + // worst-case WebSocket frame header, the same value such a server would have + // advertised. Sizes only the soft trigger, never the hard per-frame guard, so an + // older server with a larger buffer still accepts every frame it used to. + private static final int FALLBACK_SERVER_MAX_BATCH_SIZE = 2 * 1024 * 1024 - 14; private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final int MAX_TABLE_NAME_LENGTH = 127; // sf-client.md section 4.4 floor: drop-oldest under bursts needs a wide @@ -678,6 +699,50 @@ public class QwpWebSocketSender implements Sender { // refresh this from the cursor I/O thread on a mid-stream reconnect while // sendRow reads it on the producer thread with no synchronization. private volatile int serverMaxBatchSize; + // Byte length of the symbol-dictionary entries the pending batch introduced: + // the rows reference these ids, so the batch's next frame must carry the + // entries too. Counted toward the byte trigger alongside pendingBytes. Reset + // with pendingBytes. + private long pendingSymbolDictBytes; + // Rollback-ack wait bound; see rollbackOpenTransaction. + private long rollbackAckTimeoutMillis = DEFAULT_ROLLBACK_ACK_TIMEOUT_MILLIS; + // Server's per-table cap on rows a deferred-commit group may hold before it + // commits them on its own (X-QWP-Max-Uncommitted-Rows), 0 when not advertised. + // Captured on each FOREGROUND handshake, like serverMaxBatchSize. + private volatile long serverMaxUncommittedRows; + // True when the current foreground connection confirmed X-QWP-Rollback, so the + // rollback control frame may go out. Captured on each FOREGROUND handshake. + private volatile boolean serverRollbackEnabled; + // Strict transactional mode (transaction=strict, or auto_flush=off on this + // transport): implies transactional, never lets an open transaction commit in + // parts, and close() discards instead of committing. See setStrictTransactional. + private boolean strictTransactional; + // Set once the open transaction can no longer be rolled back or committed as + // one: rollback()/reset() could not undo frames already sent because the + // server does not support rollback. Every later flush throws, and close() + // discards; only closing the sender ends this state. + private boolean txnAborted; + // Strict mode only: set when the client buffer cannot take another deferred + // frame of the open transaction and still commit it. Rejects further rows + // until flush() or rollback() ends the transaction. + private boolean txnBufferFull; + // Strict mode only: simulation of how the ring packs the open transaction's + // frames, mirroring MmapSegment's append rule. txnSimSegments counts the + // segments the transaction's frames occupy (0 = no frame yet) and + // txnSimRemaining is the raw space left in the last of them. + private long txnSimRemaining; + private int txnSimSegments; + // Tables with rows in deferred frames of the open transaction, so a commit + // or rollback resets only those tables' counters. Producer thread only. + private final ObjList txnTables = new ObjList<>(); + // Logs the reset()-cannot-roll-back warning once per sender. + private boolean isResetRollbackWarned; + // Strict mode only: payload bytes of the open transaction's frames on the + // ring, for the error message. Reset with the transaction. + private long txnBytes; + // Scratch for ensureStrictTransactionRoom: the frame sizes a flush will + // publish. Producer thread only. + private final IntList strictFrameSizes = new IntList(); private QwpWebSocketSender( List endpoints, @@ -703,11 +768,12 @@ private QwpWebSocketSender( this.closed = false; this.autoFlushRows = autoFlushRows; this.autoFlushBytes = autoFlushBytes; - // Until the handshake completes, honor the configured budget verbatim. - // applyServerBatchSizeLimit() clamps this on connect once the server's - // X-QWP-Max-Batch-Size is known. - this.effectiveAutoFlushBytes = autoFlushBytes; this.autoFlushIntervalNanos = autoFlushIntervalNanos; + // Until the handshake completes, size the byte trigger without a server + // cap. applyServerBatchSizeLimit() re-sizes it on connect once the + // server's X-QWP-Max-Batch-Size is known, and setCursorEngine() once the + // segment size is. + recomputeFrameBudget(); this.globalSymbolDictionary = new GlobalSymbolDictionary(); int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); @@ -1795,29 +1861,51 @@ private void close0(boolean[] restoreInterrupt) { // sender to discard the batch, so honour that: // discard it, remember the error, and let the rest of // close() run. rethrowTerminal below still surfaces it. - try { - flushPendingRows(deferCommit); - } catch (BatchTooLargeForCapException e) { - resetTableBuffersAfterFlush(); - terminalError = captureCloseError(terminalError, e); - } catch (Throwable t) { - // Same reasoning as the pre-flight rejection above, for the - // failures a size check cannot see: sealAndSwapBuffer's - // buffer-recycle timeout and appendBlocking's backpressure - // deadline. Letting those escape to the outer catch skipped - // sendCommitMessage, sealAndSwapBuffer and drainOnClose -- so a - // flush that had already published deferred dictionary chunks - // left their group open forever, and every row an EARLIER - // successful flush published was abandoned unacked. The batch is - // NOT discarded here (unlike the over-cap case, this failure is - // not a verdict on the batch's contents), but the rest of close() - // must still run. rethrowTerminal below surfaces it. - terminalError = captureCloseError(terminalError, t); - } - if (!deferCommit && hasDeferredMessages) { - sendCommitMessage(); + if (strictTransactional || txnAborted) { + // A strict transaction commits only on an explicit flush(), so + // closing without one discards it, and an aborted one must never + // commit. The pending rows never left the client. The frames of + // the open transaction already sent stay uncommitted: a server + // with rollback support discards them on the rollback frame, whose + // ack the drain below waits for; any other server rolls them back + // when the connection closes, and a recovered slot retires them + // as an aborted transaction. + discardPendingRows(); + if (hasDeferredMessages && serverRollbackEnabled && !txnAborted) { + try { + publishRollbackFrame(); + hasDeferredMessages = false; + lastCommitBoundaryFsn = cursorEngine.publishedFsn(); + clearTransactionState(); + } catch (Throwable t) { + terminalError = captureCloseError(terminalError, t); + } + } + } else { + try { + flushPendingRows(deferCommit); + } catch (BatchTooLargeForCapException e) { + resetTableBuffersAfterFlush(); + terminalError = captureCloseError(terminalError, e); + } catch (Throwable t) { + // Same reasoning as the pre-flight rejection above, for the + // failures a size check cannot see: sealAndSwapBuffer's + // buffer-recycle timeout and appendBlocking's backpressure + // deadline. Letting those escape to the outer catch skipped + // sendCommitMessage, sealAndSwapBuffer and drainOnClose -- so a + // flush that had already published deferred dictionary chunks + // left their group open forever, and every row an EARLIER + // successful flush published was abandoned unacked. The batch is + // NOT discarded here (unlike the over-cap case, this failure is + // not a verdict on the batch's contents), but the rest of close() + // must still run. rethrowTerminal below surfaces it. + terminalError = captureCloseError(terminalError, t); + } + if (!deferCommit && hasDeferredMessages) { + sendCommitMessage(); + } } - if (activeBuffer != null && activeBuffer.hasData()) { + if (!strictTransactional && !txnAborted && activeBuffer != null && activeBuffer.hasData()) { sealAndSwapBuffer(); if (!deferCommit) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); @@ -2272,6 +2360,11 @@ public void flush() { public long flushAndGetSequence() { checkNotClosed(); checkRecycleFailure(); + if (txnAborted) { + // Committing now would commit the frames a failed rollback left on + // the server together with whatever came after them. + throw transactionAbortedError(); + } if (cursorEngine != null) { cursorEngine.checkDurability(); } @@ -2618,7 +2711,16 @@ public SenderError getLastTerminalError() { * @return the global symbol ID */ public int getOrAddGlobalSymbol(CharSequence symbol) { + int sizeBefore = globalSymbolDictionary.size(); int globalId = globalSymbolDictionary.getOrAddSymbol(symbol); + if (globalId >= sizeBefore) { + // A new entry: the batch's next frame carries it in the dictionary + // section ([len varint][utf8]), which the column bytes behind + // pendingBytes do not count. Without this a batch of new long symbols + // could pass the byte trigger and still outgrow the frame cap. + int len = NativeBufferWriter.utf8Length(symbol); + pendingSymbolDictBytes += NativeBufferWriter.varintSize(len) + len; + } if (globalId > currentBatchMaxSymbolId) { currentBatchMaxSymbolId = globalId; } @@ -2687,6 +2789,37 @@ public int getServerMaxBatchSize() { return serverMaxBatchSize; } + /** + * The server's per-table {@code qwp.max.uncommitted.rows} from the last + * foreground handshake ({@code X-QWP-Max-Uncommitted-Rows}), or 0 when the + * server did not advertise it. + */ + public long getServerMaxUncommittedRows() { + return serverMaxUncommittedRows; + } + + /** + * True when the server of the last foreground handshake confirmed QWP + * rollback support, so {@link #rollback()} can discard frames already sent. + */ + public boolean isServerRollbackEnabled() { + return serverRollbackEnabled; + } + + /** + * True in strict transactional mode; see {@link #setStrictTransactional(boolean)}. + */ + public boolean isStrictTransactional() { + return strictTransactional; + } + + /** + * True in transactional mode (strict or not); see {@link #setTransactional(boolean)}. + */ + public boolean isTransactional() { + return transactional; + } + /** Resolved value of {@code symbol_dict_reset_max_wait_millis}. */ @TestOnly public long getSymbolDictResetMaxWaitMillis() { @@ -3250,9 +3383,91 @@ public void setCloseStartedHook(Runnable hook) { this.closeStartedHook = hook; } + /** + * Discards the rows buffered since the last flush, like before. In + * transactional mode it also discards the open transaction's frames already + * sent, when the server supports QWP rollback (see {@link #rollback()}). + * Without that support a strict transaction is aborted -- the next flush + * throws and only closing the sender ends it -- while a non-strict one keeps + * the frames already sent, and the next flush commits them, as before. + */ @Override public void reset() { checkNotClosed(); + discardPendingRows(); + if (transactional && hasDeferredMessages && !txnAborted) { + if (serverRollbackEnabled) { + rollbackOpenTransaction(); + } else if (strictTransactional) { + txnAborted = true; + } else if (!isResetRollbackWarned) { + isResetRollbackWarned = true; + LOG.warn("reset() discarded the rows not yet sent, but the server does not support QWP " + + "rollback, so the open transaction's rows already sent stay on the server and the " + + "next flush() commits them. Close the sender to discard them instead"); + } + } + } + + /** + * Discards the open transaction: the rows buffered since the last flush, + * and the transaction's frames already sent, which the server rolls back on + * the rollback control frame. Blocks until the server acknowledges the + * rollback. Without an open transaction this only discards the buffered + * rows. + * + * @throws LineSenderException when frames of the open transaction were already + * sent and the server does not support QWP rollback + * (the transaction is then aborted: the next flush + * throws, and only closing the sender ends it), or + * when the server does not acknowledge the rollback + * in time + */ + @Override + public void rollback() { + checkNotClosed(); + checkRecycleFailure(); + discardPendingRows(); + if (txnAborted) { + throw transactionAbortedError(); + } + if (!hasDeferredMessages) { + clearTransactionState(); + return; + } + if (!serverRollbackEnabled) { + txnAborted = true; + throw transactionAbortedError(); + } + rollbackOpenTransaction(); + } + + /** + * Pool hook for a strict-mode lease being returned: discards what the + * borrower left uncommitted, so the next borrower starts with no open + * transaction. Returns {@code false} when frames already sent could not be + * rolled back -- the server does not support QWP rollback, or the transaction + * was already aborted -- and the pool must retire the connection instead; the + * server rolls those frames back when the connection closes. + */ + public boolean discardTransactionForReuse() { + checkNotClosed(); + discardPendingRows(); + if (txnAborted) { + return false; + } + if (!hasDeferredMessages) { + clearTransactionState(); + return true; + } + if (!serverRollbackEnabled) { + return false; + } + rollbackOpenTransaction(); + return true; + } + + private void discardPendingRows() { // Reset ALL table buffers, not just the current one ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { @@ -3273,6 +3488,7 @@ public void reset() { currentBatchMaxSymbolId = -1; reclaimUnsentSymbolIds(); pendingBytes = 0; + pendingSymbolDictBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; currentTableBuffer = null; @@ -3282,6 +3498,58 @@ public void reset() { cachedTimestampNanosColumn = null; } + /** + * Publishes the rollback control frame behind the open transaction's + * frames: a bare header the server answers by rolling back every row this + * connection holds uncommitted. Sent only to a server that confirmed + * X-QWP-Rollback. + */ + private void publishRollbackFrame() { + int messageSize = encoder.encodeRollbackMessage(); + QwpBufferWriter buffer = encoder.getBuffer(); + ensureActiveBufferReady(); + activeBuffer.ensureCapacity(messageSize); + activeBuffer.write(buffer.getBufferPtr(), messageSize); + activeBuffer.incrementRowCount(); + sealAndSwapBuffer(); + } + + /** + * Rolls back the open transaction on the server and waits for the ack. + *

+ * The wait is what makes the rollback safe across reconnects. Until the + * server acknowledges the rollback frame, a reconnect replays the + * transaction's frames together with it, and a server reached by failover + * that does not support rollback would keep those frames for the next + * commit. Once acknowledged, the frames are trimmed and can never be + * replayed. A rollback that is not acknowledged in time aborts the + * transaction (see {@link #txnAborted}). + */ + private void rollbackOpenTransaction() { + long previousBoundaryFsn = lastCommitBoundaryFsn; + publishRollbackFrame(); + long rollbackFsn = cursorEngine.publishedFsn(); + hasDeferredMessages = false; + // The rollback frame gets an OK ack covering the transaction's frames, so + // close-time drains may wait for it like for a commit. + lastCommitBoundaryFsn = rollbackFsn; + clearTransactionState(); + boolean isAcked = false; + try { + isAcked = awaitAckedFsn(fsnEpochBase + rollbackFsn, rollbackAckTimeoutMillis); + } finally { + if (!isAcked) { + txnAborted = true; + lastCommitBoundaryFsn = previousBoundaryFsn; + } + } + if (!isAcked) { + throw new LineSenderException("rollback not acknowledged by the server [timeoutMillis=") + .put(rollbackAckTimeoutMillis) + .put("]; the transaction is aborted: close the sender to discard it"); + } + } + /** * Advisory request to start a fresh symbol-dictionary epoch. Sets * {@link #manualResetRequested}; if no flush is in flight @@ -3400,6 +3668,8 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { engine.adoptCounters(counters); engine.setSlotLockReleaseListener(this::onSlotLockReleased); } + // The segment size bounds every frame; re-size the byte trigger for it. + recomputeFrameBudget(); } /** @@ -3549,8 +3819,40 @@ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) { this.recycleDeferredCloseMaxWaitMillis = millis; } + @TestOnly + public void setRollbackAckTimeoutMillisForTesting(long millis) { + this.rollbackAckTimeoutMillis = millis; + } + + /** + * Strict transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames + * like {@link #setTransactional(boolean) transactional} mode, only an explicit + * flush commits, and the open transaction never commits in parts. A row that + * would push a table past the server's {@code qwp.max.uncommitted.rows}, or a + * frame the client buffer could not hold alongside the commit, throws + * {@link TransactionTooLargeException} instead. {@link #close()} discards an + * uncommitted transaction rather than committing it. Enabling it enables + * transactional mode; disabling it leaves transactional mode on. + */ + public void setStrictTransactional(boolean strict) { + this.strictTransactional = strict; + if (strict) { + this.transactional = true; + } + recomputeFrameBudget(); + } + + /** + * Transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames and + * only an explicit flush (or close) commits. Disabling it also disables + * {@link #setStrictTransactional(boolean) strict} mode. + */ public void setTransactional(boolean transactional) { this.transactional = transactional; + if (!transactional) { + this.strictTransactional = false; + } + recomputeFrameBudget(); } /** @@ -3886,44 +4188,30 @@ private static List singleEndpoint(String host, int port) { } /** - * Clamps the soft-flush byte budget to fit under the server's advertised - * X-QWP-Max-Batch-Size minus a safety margin for encoding overhead - * (schema, dict deltas, framing). A 0 advertisement means the server did - * not send the header (older build) and the configured budget is kept - * verbatim. Called on every successful connect because a rolling upgrade - * can leave neighbouring endpoints with different caps. + * Records the server's advertised X-QWP-Max-Batch-Size and re-sizes the + * byte trigger from it. A 0 advertisement means the server did not send + * the header (older build). Called on every successful connect because a + * rolling upgrade can leave neighbouring endpoints with different caps. *

- * Always updates {@link #serverMaxBatchSize} so the single-row hard guard - * in {@link #sendRow} fires against the freshly advertised value. The - * byte trigger, however, is only adjusted when the user left it enabled: - * an explicit {@code auto_flush_bytes=off} (autoFlushBytes == 0) is - * preserved even when the server advertises a cap, so applications that - * opted out keep the contract they asked for. + * Always updates {@link #serverMaxBatchSize}, so the single-row guard in + * {@link #sendRow} fires against the freshly advertised value. See + * {@link #recomputeFrameBudget()} for the byte trigger. */ @TestOnly public void applyServerBatchSizeLimit(int advertisedMaxBatchSize) { serverMaxBatchSize = advertisedMaxBatchSize; - if (autoFlushBytes <= 0) { - // User opted out of byte-based auto-flush; respect that even when - // the server advertises a cap. The single-row guard still protects - // against oversize individual rows via serverMaxBatchSize. - effectiveAutoFlushBytes = 0; - return; - } - if (advertisedMaxBatchSize <= 0) { - effectiveAutoFlushBytes = autoFlushBytes; - return; - } - // Cap at 90% of the server's hard limit. Raw column-buffer bytes are - // a conservative proxy for wire size (compression usually shrinks the - // payload), but schema and dict-delta overhead can push the wire size - // above the raw total in pathological cases. - long safeBudget = (long) advertisedMaxBatchSize * 9 / 10; - if (autoFlushBytes < safeBudget) { - effectiveAutoFlushBytes = autoFlushBytes; - } else { - effectiveAutoFlushBytes = (int) safeBudget; - } + recomputeFrameBudget(); + } + + /** + * Records the transaction limits the server advertised on the last + * foreground handshake: its per-table {@code qwp.max.uncommitted.rows} + * ({@code 0} when not advertised) and whether it confirmed rollback support. + */ + @TestOnly + public void applyServerTransactionLimits(long maxUncommittedRows, boolean rollbackEnabled) { + serverMaxUncommittedRows = maxUncommittedRows; + serverRollbackEnabled = rollbackEnabled; } private void atMicros(long timestampMicros) { @@ -4256,6 +4544,9 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpDurableAckTiers(durableAckTiers); + // Ask every server whether it accepts the rollback control frame; + // only one that confirms ever receives it. + newClient.setQwpRequestRollback(true); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); if (cancellation != null) { // Publish the client we are about to block on so a @@ -4460,6 +4751,8 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo // endpoint's cap; an oversize row then escapes the producer-side // guard and trips a wire-level ws-close[1009] downstream. applyServerBatchSizeLimit(newClient.getServerMaxBatchSize()); + applyServerTransactionLimits( + newClient.getServerMaxUncommittedRows(), newClient.isServerRollbackEnabled()); return newClient; } catch (Error e) { closeQuietlyOnError(newClient); @@ -5202,6 +5495,7 @@ private void flushPendingRows(boolean deferCommit) { if (tableCount == 0) { clearPendingTables(); pendingBytes = 0; + pendingSymbolDictBytes = 0; currentTableBufferSnapshotBytes = currentTableBuffer == null ? 0 : currentTableBuffer.getBufferedBytes(); pendingRowCount = 0; @@ -5233,6 +5527,13 @@ private void flushPendingRows(boolean deferCommit) { int combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); int messageSize = encoder.finishMessage(); + // A strict transaction must be able to commit as one: refuse, before any + // frame of this flush reaches the ring, a flush the client buffer could not + // hold together with the commit that has to follow it. + if (strictTransactional && cursorEngine != null) { + ensureStrictTransactionRoom(cap, deltaBaseline, messageSize, deferCommit); + } + // Full-dict over-cap fallback. A full-dictionary frame carries the whole // dictionary from id 0, so its fixed overhead grows with lifetime symbol // cardinality: eventually the section alone fills the cap, and before that @@ -5302,6 +5603,7 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { if (!deferCommit) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + accountTransactionRows(deferCommit); resetTableBuffersAfterFlush(); } catch (Throwable t) { @@ -5531,6 +5833,7 @@ private void flushPendingRowsSplit( // committed the whole group. lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + accountTransactionRows(deferCommit); resetTableBuffersAfterFlush(); } @@ -5554,6 +5857,7 @@ private void resetTableBuffersAfterFlush() { flushTableBuffers.setPos(0); currentBatchMaxSymbolId = -1; pendingBytes = 0; + pendingSymbolDictBytes = 0; currentTableBufferSnapshotBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; @@ -6461,6 +6765,7 @@ private void sendCommitMessage() { sealAndSwapBuffer(); hasDeferredMessages = false; lastCommitBoundaryFsn = cursorEngine.publishedFsn(); + clearTransactionState(); } private SenderErrorHandler userErrorHandler() { @@ -7110,7 +7415,17 @@ private void sealAndSwapBuffer() { // (durable mmap append, returns once published). try { toSend.markSending(); + // Strict mode tracks how the open transaction's frames pack into + // segments; the first frame's placement depends on the active + // segment's room before the append. + boolean isTxnFrame = strictTransactional + && isTransactionFrame(toSend.getBufferPtr(), toSend.getBufferPos()); + long activeRemaining = isTxnFrame && txnSimSegments == 0 + ? cursorEngine.activeSegment().capacityRemaining() : 0; cursorEngine.appendBlocking(toSend.getBufferPtr(), toSend.getBufferPos()); + if (isTxnFrame) { + recordTransactionFrame(toSend.getBufferPos(), activeRemaining); + } toSend.markRecycled(); } catch (Throwable t) { // appendBlocking failed synchronously on the user thread — the @@ -7135,6 +7450,245 @@ private void sealAndSwapBuffer() { } } + /** + * Adds the rows a just-published FLAG_DEFER_COMMIT flush shipped to each + * flushed table's open-transaction count, or forgets the open transaction + * when the flush committed it. + */ + private void accountTransactionRows(boolean deferred) { + if (!deferred) { + clearTransactionState(); + return; + } + for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { + QwpTableBuffer table = flushTableBuffers.getQuick(i); + long sentRows = table.getTxnSentRows(); + if (sentRows == 0) { + txnTables.add(table); + } + table.setTxnSentRows(sentRows + table.getRowCount()); + } + } + + private void checkTransactionAcceptsRow() { + if (txnAborted) { + throw transactionAbortedError(); + } + if (!strictTransactional) { + return; + } + if (txnBufferFull) { + throw new TransactionTooLargeException("transaction too large for the client buffer") + .put(" [transactionBytes=").put(txnBytes) + .put("]; the buffer cannot hold another frame of the open transaction and still " + + "commit it, so the row was not added. Commit the transaction with flush() or " + + "discard it with rollback() before adding rows"); + } + long maxRows = serverMaxUncommittedRows; + if (maxRows <= 0) { + maxRows = QwpConstants.DEFAULT_SERVER_MAX_UNCOMMITTED_ROWS; + } + // The server commits a table on its own once its uncommitted rows reach + // maxRows, so a strict transaction holds at most maxRows - 1 rows per table. + long rows = currentTableBuffer.getTxnSentRows() + currentTableBuffer.getRowCount() + 1; + if (rows >= maxRows) { + throw new TransactionTooLargeException("transaction too large for the server") + .put(" [table=").put(currentTableName) + .put(", rows=").put(rows) + .put(", maxUncommittedRows=").put(maxRows) + .put("]; the server commits a table's rows on its own once qwp.max.uncommitted.rows " + + "of them are uncommitted, which would split the transaction, so the row was " + + "not added. Commit the transaction with flush() or discard it with rollback(), " + + "then send smaller transactions or raise qwp.max.uncommitted.rows on the server"); + } + } + + /** + * Forgets the open transaction: its tables' row counts, the client-buffer + * simulation and the buffer-full latch. Called when a frame commits it or + * the rollback frame discards it; O(tables in the transaction). + */ + private void clearTransactionState() { + for (int i = 0, n = txnTables.size(); i < n; i++) { + txnTables.getQuick(i).setTxnSentRows(0); + } + txnTables.setPos(0); + txnBufferFull = false; + txnBytes = 0; + txnSimSegments = 0; + txnSimRemaining = 0; + } + + /** + * Strict mode: refuses a flush whose frames, together with the open + * transaction's frames already on the ring and (for a deferred flush) room + * for the commit that must follow, would not fit the ring under + * {@code sf_max_total_bytes}. The server acks none of a transaction's frames + * before its commit, so the ring cannot trim them; a transaction that does + * not fit would block its own commit. + *

+ * Mirrors how the ring packs frames: frames never span segments, and a frame + * that does not fit the current segment opens the next one. The flush's + * frame sizes are derived the way flushPendingRows and the split publish + * them; dictionary chunks of the full-dictionary fallback count as full + * frames. Runs before anything of the flush is published, so a refusal + * leaves the batch pending and the ring untouched. + */ + private void ensureStrictTransactionRoom(int cap, int deltaBaseline, int messageSize, boolean deferCommit) { + CursorSendEngine engine = cursorEngine; + long segmentPayload = segmentPayloadCapacity(); + strictFrameSizes.clear(); + if (cap <= 0 || messageSize <= cap) { + strictFrameSizes.add(messageSize); + } else { + int baseline = deltaBaseline; + if (!deltaDictEnabled && currentBatchMaxSymbolId > deltaBaseline + && !splitFramesFit(cap, deltaBaseline) + && splitFramesFit(cap, currentBatchMaxSymbolId)) { + // The full-dictionary fallback ships the section as chunks of at + // most cap bytes ahead of the table frames. + long dictBytes = encoder.getSplitMessageSize(0, deltaBaseline, currentBatchMaxSymbolId) + - QwpConstants.HEADER_SIZE; + long chunkPayload = Math.max(1, cap - QwpConstants.HEADER_SIZE - 16); + for (long chunks = dictBytes / chunkPayload + 1; chunks > 0; chunks--) { + strictFrameSizes.add(cap); + } + baseline = currentBatchMaxSymbolId; + } + for (int i = 0, n = splitFrameBodyBytes.size(); i < n; i++) { + strictFrameSizes.add(encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(i), baseline, currentBatchMaxSymbolId)); + if (deltaDictEnabled && currentBatchMaxSymbolId > baseline) { + baseline = currentBatchMaxSymbolId; + } + } + } + int segments = txnSimSegments; + long remaining = segments == 0 ? engine.activeSegment().capacityRemaining() : txnSimRemaining; + long bytes = txnBytes; + for (int i = 0, n = strictFrameSizes.size(); i < n; i++) { + long size = strictFrameSizes.getQuick(i); + bytes += size; + if (segments == 0 || size > remaining) { + if (segments > 0 || size > remaining) { + remaining = segmentPayload; + } + segments++; + } + remaining = Math.max(0, remaining - size - MmapSegment.FRAME_HEADER_SIZE); + } + if (deferCommit) { + // Keep room for the commit: the rows pending when this transaction + // ends are at most a frame's worth plus one row, which a split + // ships as a few frames of at most cap bytes. + long reserve = cap > 0 ? Math.min(cap, segmentPayload) : segmentPayload; + for (int i = 0; i < STRICT_COMMIT_RESERVE_FRAMES; i++) { + if (reserve > remaining) { + remaining = segmentPayload; + segments++; + } + remaining = Math.max(0, remaining - reserve - MmapSegment.FRAME_HEADER_SIZE); + } + } + long ringCapacity = engine.ringCapacityBytes(); + long maxSegments = Math.max(2, ringCapacity / engine.segmentSizeBytes()); + if (segments > maxSegments) { + if (deferCommit) { + txnBufferFull = true; + } + throw new TransactionTooLargeException("transaction too large for the client buffer") + .put(" [transactionBytes=").put(bytes) + .put(", ringCapacityBytes=").put(ringCapacity) + .put(", segmentBytes=").put(engine.segmentSizeBytes()) + .put("]; the client holds an uncommitted transaction until the server commits it, so " + + "the transaction must fit sf_max_total_bytes. This flush was not sent: commit " + + "the transaction with flush() or discard it with rollback(), then send smaller " + + "transactions, raise sf_max_total_bytes, or set sf_dir to keep it on disk"); + } + } + + /** + * True for a QWP frame that joins the open transaction: FLAG_DEFER_COMMIT + * set, and not the rollback control frame (which also carries the flag). + */ + private static boolean isTransactionFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DEFER_COMMIT) != 0 && (flags & QwpConstants.FLAG_ROLLBACK) == 0; + } + + /** + * Re-sizes the byte trigger {@link #effectiveAutoFlushBytes} from the frame + * size the connection can carry: the smaller of the advertised + * X-QWP-Max-Batch-Size and the payload a store-and-forward segment can hold. + *

+ */ + private void recomputeFrameBudget() { + long segmentPayload = segmentPayloadCapacity(); + int advertised = serverMaxBatchSize; + long frameSize = Math.min(advertised > 0 ? advertised : FALLBACK_SERVER_MAX_BATCH_SIZE, segmentPayload); + long budget = frameSize * 9 / 10; + if (autoFlushBytes > 0) { + long limit = advertised > 0 ? budget + : (segmentPayload == Long.MAX_VALUE ? Long.MAX_VALUE : segmentPayload * 9 / 10); + effectiveAutoFlushBytes = (int) Math.min(autoFlushBytes, limit); + } else if (autoFlushBytes == AUTO_FLUSH_BYTES_AUTO + && (transactional || autoFlushRows > 0 || autoFlushIntervalNanos > 0)) { + effectiveAutoFlushBytes = (int) budget; + } else { + effectiveAutoFlushBytes = 0; + } + } + + /** + * Strict mode: advances the client-buffer simulation by one open-transaction + * frame that just reached the ring. {@code activeRemaining} is the active + * segment's room before the append, needed for the transaction's first frame. + */ + private void recordTransactionFrame(int payloadLen, long activeRemaining) { + txnBytes += payloadLen; + if (txnSimSegments == 0) { + txnSimRemaining = payloadLen > activeRemaining ? segmentPayloadCapacity() : activeRemaining; + txnSimSegments = 1; + } else if (payloadLen > txnSimRemaining) { + txnSimRemaining = segmentPayloadCapacity(); + txnSimSegments++; + } + txnSimRemaining = Math.max(0, txnSimRemaining - payloadLen - MmapSegment.FRAME_HEADER_SIZE); + } + + /** + * Largest frame payload a store-and-forward segment can hold, or + * {@link Long#MAX_VALUE} before an engine is attached. + */ + private long segmentPayloadCapacity() { + CursorSendEngine engine = cursorEngine; + return engine == null + ? Long.MAX_VALUE + : engine.segmentSizeBytes() - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE; + } + + private LineSenderException transactionAbortedError() { + return new LineSenderException("transaction aborted: rollback could not undo the frames of the " + + "open transaction already sent, because the server does not support QWP rollback. Close " + + "the sender to discard them; the server rolls them back when the connection closes"); + } + /** * Accumulates the current row. * Rows buffer until flush (explicit or auto-flush). @@ -7142,6 +7696,12 @@ private void sealAndSwapBuffer() { private void sendRow() { checkRecycleFailure(); ensureConnected(); + if (transactional) { + // Before nextRow(): a refusal leaves the row uncommitted, so the + // at()/atNow() error path discards it and the transaction keeps the + // rows accepted before it. + checkTransactionAcceptsRow(); + } // Hard guard: a single row whose bytes exceed the server's wire cap // would flush as an oversize WS frame the server closes with @@ -7193,7 +7753,7 @@ private boolean shouldAutoFlush() { if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { return true; } - if (effectiveAutoFlushBytes > 0 && getPendingBytes() >= effectiveAutoFlushBytes) { + if (effectiveAutoFlushBytes > 0 && pendingBytes + pendingSymbolDictBytes >= effectiveAutoFlushBytes) { return true; } // pendingRowCount == 1: sendRow() just timestamped this very row, so its diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java new file mode 100644 index 000000000..4374b7337 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java @@ -0,0 +1,49 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** + * A strict transaction ({@code transaction=strict} or {@code auto_flush=off} on the + * WebSocket transport) grew past a limit that would stop it from committing as one + * transaction: the server's per-table {@code qwp.max.uncommitted.rows}, past which + * the server commits the rows on its own, or the client's {@code sf_max_total_bytes}, + * which must hold the whole uncommitted transaction until the server commits it. + *

+ * Thrown before anything that would cross the limit reaches the server, and the + * transaction stays open: the caller can still commit the rows accepted so far + * with {@code flush()}, or discard them with {@code rollback()}. A row rejected at + * the per-table limit is not part of the transaction. When the client buffer is + * the limit, the row being added when the buffer filled up stays in the + * transaction, and every further row is rejected until {@code flush()} or + * {@code rollback()} ends the transaction. + */ +public class TransactionTooLargeException extends LineSenderException { + + public TransactionTooLargeException(CharSequence message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index ba46365af..16d9296de 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -1900,6 +1900,17 @@ public int sealedSegmentsSnapshot(MmapSegment[] target) { return ring.snapshotSealedSegments(target); } + /** + * Bytes of segments the ring may hold under {@code sf_max_total_bytes}: the + * budget's capacity minus the side-file bytes it also charges (disk mode). + * In a sender pool the budget is shared, so this bounds the rings of all + * pooled senders together, not this ring alone. + */ + public long ringCapacityBytes() { + SegmentBudget budget = manager.getBudget(); + return budget.getCapacityBytes() - budget.getSideFileBytes(); + } + /** * Configured per-segment size in bytes. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 7d3f5f56d..ba4000168 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -472,6 +472,14 @@ public synchronized boolean isWorkerReaped() { return workerThread == null; } + /** + * The {@code sf_max_total_bytes} budget this manager charges its rings' + * segments to; shared with other managers in a sender pool. + */ + public SegmentBudget getBudget() { + return budget; + } + /** * Stop tracking {@code ring}. Idempotent; safe to call from any thread. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 0fa7a6455..ba1ee0e5f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -36,6 +36,13 @@ public final class QwpConstants { * Client identifier sent in the X-QWP-Client-Id upgrade header. */ public static final String CLIENT_ID = "java/1.0.2"; + /** + * Default of the server's {@code qwp.max.uncommitted.rows}: the per-table + * count of rows a deferred-commit group may hold before the server commits + * them on its own. A strict transaction assumes this value when the server + * does not advertise {@code X-QWP-Max-Uncommitted-Rows}. + */ + public static final long DEFAULT_SERVER_MAX_UNCOMMITTED_ROWS = 1_000_000L; /** * Flag bit: defer WAL commit. The server appends rows to WAL writers * but skips the commit until a subsequent message without this flag. @@ -50,6 +57,12 @@ public final class QwpConstants { * Flag bit: Gorilla timestamp encoding enabled. */ public static final byte FLAG_GORILLA = 0x04; + /** + * Flag bit: roll back the connection's open deferred-commit group. Only ever + * set on the rollback control frame (see {@link #ROLLBACK_FRAME_FLAGS}), which + * the client sends only after the server confirmed {@code X-QWP-Rollback}. + */ + public static final byte FLAG_ROLLBACK = 0x20; /** * Flag bit: payload region after the prelude is zstd-compressed. Set only * when the handshake negotiated zstd compression. Mirror of the server-side @@ -108,6 +121,13 @@ public final class QwpConstants { * constant; used by the decoder to reject malformed wire bytes. */ public static final int MAX_TABLE_NAME_LENGTH = 127; + /** + * Flags byte of the rollback control frame: a bare 12-byte header with no + * table blocks and no payload. It carries {@link #FLAG_DEFER_COMMIT} too, so a + * server that does not know {@link #FLAG_ROLLBACK} treats it as an empty + * deferred frame instead of a commit. + */ + public static final byte ROLLBACK_FRAME_FLAGS = FLAG_ROLLBACK | FLAG_DEFER_COMMIT; /** * Status byte on a {@code QUERY_ERROR} frame: the query was cancelled, * either by a client {@code CANCEL} frame or by explicit server-side diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 02f95b914..7e7ce99b6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -78,6 +78,9 @@ public class QwpTableBuffer implements QuietCloseable { // buffer is listed once. Producer thread only. private boolean listedPending; private int rowCount; + // Rows of this table the sender already shipped in FLAG_DEFER_COMMIT frames + // of the open transaction. Producer thread only. + private long txnSentRows; public QwpTableBuffer(String tableName) { this(tableName, null); @@ -282,6 +285,14 @@ public String getTableName() { return tableName; } + /** + * Rows of this table already shipped in FLAG_DEFER_COMMIT frames of the + * sender's open transaction; 0 outside a transaction. + */ + public long getTxnSentRows() { + return txnSentRows; + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -426,6 +437,13 @@ public void setListedPending(boolean listedPending) { this.listedPending = listedPending; } + /** + * Sets the count reported by {@link #getTxnSentRows()}. + */ + public void setTxnSentRows(long txnSentRows) { + this.txnSentRows = txnSentRows; + } + private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { if (column.type != type) { throw new LineSenderException( diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index a9ef418af..1c49b3099 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -28,6 +28,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -167,8 +168,18 @@ public void close() { // above. A stale duplicate throws here, and the finally routes it // to discardBroken, which drops it on the same stale-generation // check under the pool lock. - slot.live(generation).flush(); - flushed = true; + Sender live = slot.live(generation); + if (live instanceof QwpWebSocketSender && ((QwpWebSocketSender) live).isStrictTransactional()) { + // A strict transaction commits only on an explicit flush(), so + // returning the lease discards whatever the borrower left open. + // The next borrower must not inherit it, so a transaction the + // server cannot roll back retires the connection instead (the + // server rolls it back when the connection closes). + flushed = ((QwpWebSocketSender) live).discardTransactionForReuse(); + } else { + live.flush(); + flushed = true; + } } finally { if (flushed) { slot.pool().giveBack(this); @@ -341,6 +352,11 @@ public void reset() { slot.live(generation).reset(); } + @Override + public void rollback() { + slot.live(generation).rollback(); + } + @Override public void resetSymbolDictionary() { slot.live(generation).resetSymbolDictionary(); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index e304df951..4dc06fcd6 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -229,18 +229,16 @@ public void testMalformedIngressConfigRejectedAtBuildWithMinZero() { // Sender -- yet it must still reject a malformed ingress key in the // single cluster config up front. Covers a typed enum (tls_verify), a // registry-STRING value that only the real Sender parse validates - // (auto_flush_rows), and WebSocket build-time checks: auto_flush=off disables - // auto-flush (unsupported on WebSocket), also when auto_flush_bytes is set, and + // (auto_flush_rows), and WebSocket build-time checks: auto_flush=off selects + // strict transactional mode, so it cannot combine with transaction=off, and // sf_durability=flush is not yet supported. assertBuildRejected( "wss::addr=127.0.0.1:1;tls_verify=strict;sender_pool_min=0;query_pool_min=0;", "tls_verify"); assertBuildRejected( "ws::addr=127.0.0.1:1;auto_flush_rows=abc;sender_pool_min=0;query_pool_min=0;", "auto_flush_rows"); assertBuildRejected( - "ws::addr=127.0.0.1:1;auto_flush=off;sender_pool_min=0;query_pool_min=0;", "auto-flush"); - assertBuildRejected( - "ws::addr=127.0.0.1:1;auto_flush=off;auto_flush_bytes=65536;sender_pool_min=0;query_pool_min=0;", - "auto-flush"); + "ws::addr=127.0.0.1:1;auto_flush=off;transaction=off;sender_pool_min=0;query_pool_min=0;", + "auto_flush=off selects strict transactional mode"); assertBuildRejected( "ws::addr=127.0.0.1:1;sf_durability=flush;sender_pool_min=0;query_pool_min=0;", "not yet supported"); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java index cc8c78aa4..fdde0e563 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -189,6 +189,32 @@ public void testExtractMaxBatchSizeParsesPositive() throws Exception { Assert.assertEquals(16 * 1024 * 1024, invokeExtractMaxBatchSize(response)); } + @Test + public void testExtractMaxUncommittedRowsAbsentHeaderReturnsZero() throws Exception { + String response = "HTTP/1.1 101 Switching Protocols\r\n" + + "X-QWP-Version: 1\r\n" + + "X-QWP-Max-Batch-Size: 16777216\r\n" + + "\r\n"; + Assert.assertEquals(0, invokeExtractMaxUncommittedRows(response)); + } + + @Test + public void testExtractMaxUncommittedRowsMalformedOrNegativeReturnsZero() throws Exception { + // 0 means "not advertised": a strict transaction then assumes the default. + Assert.assertEquals(0, invokeExtractMaxUncommittedRows("HTTP/1.1 101 Switching Protocols\r\n" + + "X-QWP-Max-Uncommitted-Rows: lots\r\n\r\n")); + Assert.assertEquals(0, invokeExtractMaxUncommittedRows("HTTP/1.1 101 Switching Protocols\r\n" + + "X-QWP-Max-Uncommitted-Rows: -5\r\n\r\n")); + } + + @Test + public void testExtractMaxUncommittedRowsParsesBeyondIntRange() throws Exception { + String response = "HTTP/1.1 101 Switching Protocols\r\n" + + "x-qwp-max-uncommitted-rows: 5000000000 \r\n" + + "\r\n"; + Assert.assertEquals(5_000_000_000L, invokeExtractMaxUncommittedRows(response)); + } + @Test public void testFrameBurstIsParsedInPlace() throws Exception { assertFrameBurstIsParsedInPlace(false); @@ -602,6 +628,12 @@ private static int invokeExtractMaxBatchSize(String response) throws Exception { return (int) m.invoke(null, response); } + private static long invokeExtractMaxUncommittedRows(String response) throws Exception { + Method m = WebSocketClient.class.getDeclaredMethod("extractMaxUncommittedRows", String.class); + m.setAccessible(true); + return (long) m.invoke(null, response); + } + private static void setUpgradedTrue(Object obj) throws Exception { Class clazz = obj.getClass(); while (clazz != null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index fbc833d86..390168fda 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -1052,7 +1052,10 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception // Small segment; a heavily padded row's frame cannot fit, so // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no // backpressure timing needed. The server never acks (SilentHandler). - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_segment_bytes=1024;"; + // auto_flush_bytes=off: the oversized row must reach flush() whole, + // as the byte trigger would otherwise publish it from at(). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_segment_bytes=1024;auto_flush_bytes=off;"; String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment Sender sender = Sender.fromConfig(cfg); try { @@ -1187,9 +1190,12 @@ public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { int port = silent.getPort(); silent.start(); Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off: the oversized row must reach flush() whole, + // as the byte trigger would otherwise publish it from at(). String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_segment_bytes=4096" + + ";auto_flush_bytes=off" + ";close_flush_timeout_millis=0;"; Sender s1 = Sender.fromConfig(cfg); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 1d83dff2a..0fd081389 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -425,12 +425,15 @@ public void testDefaultIsAsync() { } @Test - public void testDisableAutoFlush_notSupportedForWebSocket() { + public void testDisableAutoFlush_rejectsTransactionOffForWebSocket() { + // On WebSocket, disabling auto-flush selects strict transactional mode, + // which an explicit transactional(false) contradicts. assertThrowsAny( Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) - .disableAutoFlush(), - "not supported for WebSocket"); + .disableAutoFlush() + .transactional(false), + "cannot be combined with transaction=off"); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java new file mode 100644 index 000000000..4b96c8196 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -0,0 +1,759 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.TransactionTooLargeException; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Batches of any size over QWP/WebSocket: the byte trigger that keeps every batch + * within one frame, transactional and strict transactional mode, and rollback. + *

+ * {@link TxnServer} answers frames the way the QuestDB server does: a + * FLAG_DEFER_COMMIT frame gets no ack until a commit frame closes its group, a + * commit frame gets a cumulative OK ack, and the rollback control frame gets a + * cumulative OK ack once the server negotiated rollback. + */ +public class QwpLargeTransactionTest { + private static final int CAP = 32 * 1024; + // ~100 encoded bytes per row: 1000 rows (the default row trigger) are ~100 KB, + // three times the 32 KiB cap. + private static final String PAD = TestUtils.repeat("x", 90); + + @Test + public void testAutoFlushOffIsStrictTransactionOfAnySize() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush=off;")) { + Assert.assertTrue(((QwpWebSocketSender) s).isStrictTransactional()); + writeRows(s, "t", 0, 2_000); + // ~200 KB went out as deferred frames before flush(); none commits. + Assert.assertTrue(h.awaitFrames(5)); + Assert.assertEquals(0, h.commitFrames()); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + } + }); + } + + @Test + public void testAutoFlushOffRejectsTransactionOff() throws Exception { + assertMemoryLeak(() -> { + try { + Sender.fromConfig("ws::addr=localhost:1;auto_flush=off;transaction=off;").close(); + Assert.fail("auto_flush=off must not combine with transaction=off"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), + "auto_flush=off selects strict transactional mode on WebSocket"); + } + }); + } + + @Test + public void testDefaultSenderSplitsWideRowsIntoFrames() throws Exception { + // Before: no byte trigger by default, so the 1000-row batch outgrew the cap + // and flush threw BatchTooLargeForCapException. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server))) { + writeRows(s, "t", 0, 2_000); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals((long) CAP * 9 / 10, ws.getEffectiveAutoFlushBytes()); + } + assertFramesWithinCap(h, CAP); + Assert.assertTrue("rows must span frames: " + h.frames, h.frames.size() >= 7); + // Not transactional: every frame commits on its own. + Assert.assertEquals(h.frames.size(), h.commitFrames()); + } + }); + } + + @Test + public void testExplicitAutoFlushBytesOffKeepsNoByteTrigger() throws Exception { + // auto_flush_bytes=off still opts out of the byte trigger, as documented. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_bytes=off;")) { + Assert.assertEquals(0, ((QwpWebSocketSender) s).getEffectiveAutoFlushBytes()); + try { + writeRows(s, "t", 0, 1_000); + Assert.fail("a 1000-row batch over the cap must be rejected"); + } catch (BatchTooLargeForCapException expected) { + TestUtils.assertContains(expected.getMessage(), "serverMaxBatchSize=" + CAP); + } + s.reset(); + } + Assert.assertEquals(0, h.frames.size()); + } + }); + } + + @Test + public void testByteTriggerFollowsSegmentSize() throws Exception { + // The server accepts 48 KiB frames, but a 16 KiB segment cannot hold them: + // batches must be sized to the segment, not fail at the append. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 48 * 1024)) { + try (Sender s = Sender.fromConfig(conf(server) + "sf_max_segment_bytes=16k;")) { + writeRows(s, "t", 0, 600); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + int segmentPayload = 16 * 1024 - 24 - 8; + Assert.assertEquals(segmentPayload * 9 / 10, ((QwpWebSocketSender) s).getEffectiveAutoFlushBytes()); + } + assertFramesWithinCap(h, 16 * 1024 - 24 - 8); + Assert.assertTrue(h.frames.size() >= 4); + } + }); + } + + @Test + public void testManualFlushKeepsOneCommitPerFlush() throws Exception { + // A non-transactional sender with every auto-flush trigger off keeps its + // flush as one commit: no byte trigger splits the batch early. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_rows=off;auto_flush_interval=off;")) { + Assert.assertEquals(0, ((QwpWebSocketSender) s).getEffectiveAutoFlushBytes()); + writeRows(s, "a", 0, 250); + writeRows(s, "b", 0, 250); + Assert.assertEquals(0, h.frames.size()); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // The 50 KB batch splits per table at flush: one deferred frame, one commit. + Assert.assertEquals(2, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isDeferred()); + Assert.assertTrue(h.frames.get(1).isCommit()); + } + }); + } + + @Test + public void testNewSymbolsCountTowardByteTrigger() throws Exception { + // Symbol columns buffer 4-byte ids; the 300-byte strings ride in the frame's + // dictionary section. Counting them keeps the frame within the cap. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server))) { + String prefix = TestUtils.repeat("s", 290); + for (int i = 0; i < 1_000; i++) { + s.table("t").symbol("sym", prefix + i).longColumn("v", i).atNow(); + } + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, CAP); + Assert.assertTrue(h.frames.size() >= 10); + } + }); + } + + @Test + public void testPooledStrictLeaseCloseRetiresConnectionWithoutServerRollback() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QuestDB db = QuestDB.connect(conf(server) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + // The frames already sent cannot be rolled back: the pool retires + // the connection, which makes the server discard them. + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + Assert.assertEquals(1, h.commitFrames()); + Assert.assertEquals(0, h.rollbackFrames()); + } + }); + } + + @Test + public void testPooledStrictLeaseCloseRollsBackUncommitted() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (QuestDB db = QuestDB.connect(conf(server) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + // Returning a strict lease without flush() discards the transaction. + lease.close(); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isRollback()); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + Assert.assertEquals(1, h.commitFrames()); + Assert.assertEquals(1, h.rollbackFrames()); + } + }); + } + + @Test + public void testResetAbortsStrictTransactionWithoutServerRollback() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.reset(); + assertAborted(s::flush); + assertAborted(() -> s.table("t").longColumn("v", 1).atNow()); + } + Assert.assertEquals("the reset transaction must never commit", 0, h.commitFrames()); + } + }); + } + + @Test + public void testResetKeepsLegacyBehaviourWithoutServerRollback() throws Exception { + // transaction=on against a server without rollback: reset() cannot reach the + // frames already sent, and the next flush() commits them, as before. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.reset(); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // The frames sent before reset() are deferred, and the flush after it + // committed them with an empty commit frame. + assertOneTransaction(h); + } + }); + } + + @Test + public void testResetRollsBackSentFramesWhenServerSupportsIt() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.reset(); + // reset() returned after the server acked the rollback frame, which + // follows every frame of the transaction. + int rollbackIndex = h.frames.size() - 1; + Assert.assertTrue(h.frames.get(rollbackIndex).isRollback()); + for (int i = 0; i < rollbackIndex; i++) { + Assert.assertTrue(h.frames.get(i).isDeferred()); + } + Assert.assertEquals(1, h.rolledBackGroups); + // The sender stays usable for the next transaction. + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + } + } + }); + } + + @Test + public void testRollbackAckTimeoutAbortsTransaction() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + h.isAckingRollback = false; + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + ((QwpWebSocketSender) s).setRollbackAckTimeoutMillisForTesting(300); + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + try { + s.rollback(); + Assert.fail("an unacknowledged rollback must fail"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), "rollback not acknowledged by the server"); + } + assertAborted(s::flush); + } + Assert.assertEquals(0, h.commitFrames()); + } + }); + } + + @Test + public void testRollbackDiscardsSentFrames() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(ws.isServerRollbackEnabled()); + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + Assert.assertEquals(0, h.commitFrames()); + s.rollback(); + // rollback() returned after the ack, so the frame is on the server, + // behind every frame of the transaction. + int sent = h.frames.size() - 1; + TxnFrame rollback = h.frames.get(sent); + Assert.assertTrue(rollback.isRollback()); + for (int i = 0; i < sent; i++) { + Assert.assertTrue(h.frames.get(i).isDeferred()); + } + Assert.assertEquals(QwpConstants.HEADER_SIZE, rollback.size); + Assert.assertEquals(1, h.rolledBackGroups); + Assert.assertFalse(ws.hasDeferredMessagesForTesting()); + // Nothing pending is left behind: an empty flush sends nothing. + s.flush(); + Assert.assertEquals(sent + 1, h.frames.size()); + // The next transaction starts clean and commits. + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + } + } + }); + } + + @Test + public void testRollbackOnNonWebSocketSenderThrows() throws Exception { + assertMemoryLeak(() -> { + try (Sender s = Sender.fromConfig("http::addr=localhost:1;protocol_version=2;")) { + s.rollback(); + Assert.fail("rollback must be WebSocket-only"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), "rollback is only supported for WebSocket transport"); + } + }); + } + + @Test + public void testRollbackUnsupportedAbortsTransaction() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + Assert.assertFalse(((QwpWebSocketSender) s).isServerRollbackEnabled()); + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + assertAborted(s::rollback); + assertAborted(s::flush); + assertAborted(() -> s.table("t").longColumn("v", 1).atNow()); + assertAborted(s::rollback); + } + // close() of an aborted transaction must not commit it either. + Assert.assertEquals(0, h.commitFrames()); + Assert.assertEquals(0, h.rollbackFrames()); + } + }); + } + + @Test + public void testRollbackWithoutSentFramesSendsNothing() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 10); + s.rollback(); + s.flush(); + writeRows(s, "t", 0, 5); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // Only the second transaction reached the server, as one commit frame. + Assert.assertEquals(1, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isCommit()); + } + }); + } + + @Test + public void testServerLimitsReachTheSender() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setAdvertisedMaxUncommittedRows(12_345); + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server))) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals(12_345, ws.getServerMaxUncommittedRows()); + Assert.assertTrue(ws.isServerRollbackEnabled()); + } + Assert.assertEquals("true", server.pollRollbackRequest(5, TimeUnit.SECONDS)); + + // A server that does not confirm rollback, nor advertise its row limit. + server.setAdvertisedMaxUncommittedRows(0); + server.setRollbackSupported(false); + try (Sender s = Sender.fromConfig(conf(server))) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals(0, ws.getServerMaxUncommittedRows()); + Assert.assertFalse(ws.isServerRollbackEnabled()); + } + } + }); + } + + @Test + public void testStrictCloseWithoutFlushDiscardsTransaction() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;close_flush_timeout_millis=1000;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + } + Assert.assertEquals("close() must not commit a strict transaction", 0, h.commitFrames()); + Assert.assertEquals(0, h.rollbackFrames()); + } + }); + } + + @Test + public void testStrictCloseWithoutFlushRollsBackWhenServerSupportsIt() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + } + // close() drained until the rollback frame's ack. + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isRollback()); + Assert.assertEquals(0, h.commitFrames()); + Assert.assertEquals(1, h.rolledBackGroups); + } + }); + } + + @Test + public void testStrictRowLimitFromServer() throws Exception { + // The server advertises 100: a strict transaction holds at most 99 rows + // of a table, since the server would commit the 100th on its own. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 4_096)) { + server.setAdvertisedMaxUncommittedRows(100); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + // 4 KiB frames: the 99 rows of "a" go out over several deferred frames. + writeRows(s, "a", 0, 99); + Assert.assertTrue(h.awaitFrames(2)); + try { + writeRows(s, "a", 99, 1); + Assert.fail("the 100th row of a table must be refused"); + } catch (TransactionTooLargeException e) { + TestUtils.assertContains(e.getMessage(), + "transaction too large for the server [table=a, rows=100, maxUncommittedRows=100]"); + } + // Other tables keep their own count. + writeRows(s, "b", 0, 99); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + // A new transaction starts from zero again. + writeRows(s, "a", 0, 99); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + Assert.assertEquals(0, h.rollbackFrames()); + Assert.assertEquals(2, h.commitFrames()); + } + }); + } + + @Test + public void testStrictRowLimitDefaultsWhenNotAdvertised() throws Exception { + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertEquals(0, ws.getServerMaxUncommittedRows()); + // Pretend the server default is small to keep the test fast: the + // assumed default applies only while nothing is advertised. + ws.applyServerTransactionLimits(10, false); + writeRows(s, "a", 0, 9); + try { + writeRows(s, "a", 9, 1); + Assert.fail("the 10th row must be refused"); + } catch (TransactionTooLargeException e) { + TestUtils.assertContains(e.getMessage(), "maxUncommittedRows=10"); + } + ws.applyServerTransactionLimits(0, false); + // 0 means "not advertised": the server default applies. + writeRows(s, "a", 9, 1); + s.flush(); + } + } + }); + } + + @Test + public void testStrictTransactionTooLargeForClientBufferFailsFast() throws Exception { + // 64 KiB segments, 256 KiB budget: four segments. A strict transaction keeps + // room for its commit, so it must refuse the flush that would take that room + // instead of blocking for sf_append_deadline_millis. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;" + + "sf_max_segment_bytes=64k;sf_max_total_bytes=256k;sf_append_deadline_millis=30000;")) { + long start = System.nanoTime(); + int row = 0; + try { + for (; row < 100_000; row++) { + s.table("t").longColumn("v", row).stringColumn("s", PAD).atNow(); + } + Assert.fail("a transaction larger than the client buffer must be refused"); + } catch (TransactionTooLargeException e) { + TestUtils.assertContains(e.getMessage(), "transaction too large for the client buffer"); + } + Assert.assertTrue("must fail fast, not wait for the append deadline", + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); + // The row being added stays in the transaction; further rows are refused. + try { + s.table("t").longColumn("v", -1).stringColumn("s", PAD).atNow(); + Assert.fail("a full transaction must refuse rows"); + } catch (TransactionTooLargeException e) { + TestUtils.assertContains(e.getMessage(), "the row was not added"); + } + // The reserved room lets the transaction commit. + s.flush(); + Assert.assertTrue(s.drain(5_000)); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + Assert.assertEquals(1, h.commitFrames()); + // And the next transaction starts with an empty buffer. + writeRows(s, "t", 0, 100); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + } + }); + } + + @Test + public void testTransactionalDefaultTriggersSplitWideRows() throws Exception { + // Before: transaction=on with the default triggers threw + // BatchTooLargeForCapException at the 1000th wide row. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=on;")) { + writeRows(s, "t", 0, 2_000); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + } + }); + } + + private static void assertAborted(ThrowingRunnable r) { + try { + r.run(); + Assert.fail("expected the aborted-transaction error"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), "transaction aborted"); + } + } + + private static void assertFramesWithinCap(TxnServer h, int cap) { + for (TxnFrame f : h.frames) { + Assert.assertTrue("frame of " + f.size + " bytes exceeds " + cap, f.size <= cap); + } + } + + private static void assertOneTransaction(TxnServer h) { + Assert.assertTrue(h.frames.size() > 1); + for (int i = 0, n = h.frames.size() - 1; i < n; i++) { + Assert.assertTrue("frame " + i + " must be deferred", h.frames.get(i).isDeferred()); + } + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); + } + + private static String conf(TestWebSocketServer server) { + return "ws::addr=localhost:" + server.getPort() + ";"; + } + + private static TestWebSocketServer start(TxnServer h, int cap) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(h); + server.setAdvertisedMaxBatchSize(cap); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static void writeRows(Sender s, String table, int from, int count) { + for (int i = from, n = from + count; i < n; i++) { + s.table(table).longColumn("v", i).stringColumn("s", PAD).atNow(); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run(); + } + + private static final class TxnFrame { + final byte flags; + final int size; + + TxnFrame(byte flags, int size) { + this.flags = flags; + this.size = size; + } + + boolean isCommit() { + return (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0; + } + + boolean isDeferred() { + return !isRollback() && (flags & QwpConstants.FLAG_DEFER_COMMIT) != 0; + } + + boolean isRollback() { + return size == QwpConstants.HEADER_SIZE && flags == QwpConstants.ROLLBACK_FRAME_FLAGS; + } + + @Override + public String toString() { + return (isRollback() ? "R" : isDeferred() ? "D" : "C") + size; + } + } + + /** + * Acks frames the way the QuestDB server does for one connection: deferred + * frames wait for the commit frame that closes their group, which acks them + * cumulatively; the rollback frame discards the group and acks it too. + */ + private static final class TxnServer implements TestWebSocketServer.WebSocketServerHandler { + final List frames = new CopyOnWriteArrayList<>(); + volatile boolean isAckingRollback = true; + volatile int rolledBackGroups; + private long nextSeq; + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long seq = nextSeq++; + TxnFrame frame = new TxnFrame(data[QwpConstants.HEADER_OFFSET_FLAGS], data.length); + frames.add(frame); + notifyAll(); + try { + if (frame.isRollback()) { + rolledBackGroups++; + if (isAckingRollback) { + client.sendBinary(ack(seq)); + } + } else if (frame.isCommit()) { + client.sendBinary(ack(seq)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + synchronized boolean awaitFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (frames.size() < count) { + long left = deadline - System.nanoTime(); + if (left <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, left); + } + return true; + } + + synchronized int commitFrames() { + int n = 0; + for (TxnFrame f : frames) { + if (f.isCommit()) { + n++; + } + } + return n; + } + + synchronized int rollbackFrames() { + int n = 0; + for (TxnFrame f : frames) { + if (f.isRollback()) { + n++; + } + } + return n; + } + + private static byte[] ack(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 44ceb6d42..16f8a3bff 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -487,8 +487,10 @@ public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off: the batch must reach flush() whole, as the + // byte trigger would otherwise publish it from at(). String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";sf_max_segment_bytes=64;"; + + ";sf_max_segment_bytes=64;auto_flush_bytes=off;"; String pad = TestUtils.repeat("x", 60); Sender sender = Sender.fromConfig(config); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 102922d6b..ee6b1f4a7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -146,6 +146,16 @@ public class TestWebSocketServer implements Closeable { // builds. Lets a test force the delta-dictionary catch-up to split across // several frames. Live-updatable via setAdvertisedMaxBatchSize(). private volatile int advertisedMaxBatchSize; + // When > 0, 101 upgrade responses advertise this value as the + // X-QWP-Max-Uncommitted-Rows header. Live-updatable. + private volatile long advertisedMaxUncommittedRows; + // X-QWP-Request-Rollback header value captured from each well-formed upgrade + // request ("" when absent). + private final BlockingQueue capturedRollbackRequests = new LinkedBlockingQueue<>(); + // When true, a 101 upgrade response to a request carrying + // X-QWP-Request-Rollback: true confirms it with X-QWP-Rollback: enabled. + // Live-updatable. + private volatile boolean rollbackSupported; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -349,6 +359,31 @@ public void setAdvertisedMaxBatchSize(int maxBatchSize) { this.advertisedMaxBatchSize = maxBatchSize; } + /** + * Advertises {@code X-QWP-Max-Uncommitted-Rows: } on subsequent + * handshakes (live update). Pass {@code 0} to stop advertising it. + */ + public void setAdvertisedMaxUncommittedRows(long rows) { + this.advertisedMaxUncommittedRows = rows; + } + + /** + * When {@code true}, subsequent handshakes that request rollback + * ({@code X-QWP-Request-Rollback: true}) are confirmed with + * {@code X-QWP-Rollback: enabled} (live update). + */ + public void setRollbackSupported(boolean supported) { + this.rollbackSupported = supported; + } + + /** + * Blocks up to {@code timeout} for the next captured X-QWP-Request-Rollback + * header value ("" when the upgrade request carried none). + */ + public String pollRollbackRequest(long timeout, TimeUnit unit) throws InterruptedException { + return capturedRollbackRequests.poll(timeout, unit); + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -762,6 +797,7 @@ private boolean performHandshake() throws IOException { String key = null; String authorization = ""; String durableAckRequest = ""; + String rollbackRequest = ""; String[] lines = request.toString().split("\r\n"); if (lines.length > 0) { // GET HTTP/1.1 @@ -778,6 +814,8 @@ private boolean performHandshake() throws IOException { authorization = line.substring("authorization:".length()).trim(); } else if (lower.startsWith("x-qwp-request-durable-ack:")) { durableAckRequest = line.substring("x-qwp-request-durable-ack:".length()).trim(); + } else if (lower.startsWith("x-qwp-request-rollback:")) { + rollbackRequest = line.substring("x-qwp-request-rollback:".length()).trim(); } } @@ -787,6 +825,7 @@ private boolean performHandshake() throws IOException { upgradeRequestCount.incrementAndGet(); capturedAuthHeaders.add(authorization); capturedDurableAckRequests.add(durableAckRequest); + capturedRollbackRequests.add(rollbackRequest); // Read-path reject: drop the egress upgrade before the 101 so the // query pool's connect fails fast, while ingest write-path upgrades @@ -866,6 +905,13 @@ private boolean performHandshake() throws IOException { if (maxBatch > 0) { sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n"); } + long maxUncommittedRows = advertisedMaxUncommittedRows; + if (maxUncommittedRows > 0) { + sb.append("X-QWP-Max-Uncommitted-Rows: ").append(maxUncommittedRows).append("\r\n"); + } + if (rollbackSupported && "true".equalsIgnoreCase(rollbackRequest)) { + sb.append("X-QWP-Rollback: enabled\r\n"); + } sb.append("\r\n"); out.write(sb.toString().getBytes(StandardCharsets.US_ASCII)); out.flush(); diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index 39695ec22..d9190ba6b 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -54,11 +54,12 @@ public void testEveryIngressKeyIsHonored() { assertHonored("auto_flush_interval=250", "auto_flush_interval", 250); // auto_flush=off reaches disableAutoFlush(), which sets the interval to // MAX_VALUE; auto_flush_rows=off leaves the interval unset, so this field - // distinguishes the two. That config is build-rejected on WebSocket; see - // QuestDBBuilderTest.testMalformedIngressConfigRejectedAtBuildWithMinZero. + // distinguishes the two. On WebSocket it selects strict transactional mode + // at build(); see QwpLargeTransactionTest. assertHonored("auto_flush=off", "auto_flush_interval", Integer.MAX_VALUE); assertHonored("max_name_len=99", "max_name_len", 99); - assertHonored("transaction=on", "transaction", true); + assertHonored("transaction=on", "transaction", "on"); + assertHonored("transaction=strict", "transaction", "strict"); assertHonored("request_durable_ack=on", "request_durable_ack", "on"); assertHonored("request_durable_ack=local", "request_durable_ack", "local"); assertHonored("request_durable_ack=replicated", "request_durable_ack", "replicated"); From d5028e534556d261e34df49277f55ac65e54cbdc Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 11 Nov 2026 23:45:23 +0000 Subject: [PATCH 2/7] Test strict split frames and pooled lease reuse Cover a strict transaction whose batch splits per table within the cap, and a strict pooled lease returned with only unsent rows, which keeps its connection and sends no rollback frame. --- .../qwp/client/QwpLargeTransactionTest.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java index 4b96c8196..191ab3912 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -249,6 +249,56 @@ public void testPooledStrictLeaseCloseRollsBackUncommitted() throws Exception { }); } + @Test + public void testPooledStrictLeaseCloseWithOnlyPendingRowsKeepsConnection() throws Exception { + // Nothing reached the server yet: returning the lease drops the pending + // rows without a rollback frame and keeps the connection pooled. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QuestDB db = QuestDB.connect(conf(server) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 10); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 5); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + Assert.assertEquals(1, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isCommit()); + Assert.assertEquals(1, server.upgradeRequestCount()); + } + }); + } + + @Test + public void testStrictMultiTableFlushSplitsWithinTheCap() throws Exception { + // Ten tables in one batch carry ten schemas, so the batch the byte trigger + // cuts outgrows the cap and splits per table. Every split frame of the + // open transaction stays deferred. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 4_096)) { + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + for (int i = 0; i < 400; i++) { + s.table("table_with_a_long_name_" + (i % 10)) + .longColumn("first_long_column_name", i) + .longColumn("second_long_column_name", i) + .stringColumn("s", "abc") + .atNow(); + } + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertFramesWithinCap(h, 4_096); + assertOneTransaction(h); + Assert.assertTrue(h.frames.size() > 10); + } + }); + } + @Test public void testResetAbortsStrictTransactionWithoutServerRollback() throws Exception { assertMemoryLeak(() -> { From ce20002c0966eb7660a95d34718b1ed418acde33 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Thu, 12 Nov 2026 11:59:35 +0000 Subject: [PATCH 3/7] Test discarding a transaction during an outage reset(), rollback() and returning a strict pooled lease discard the open transaction by publishing the rollback frame. During a server outage they wait up to 30 s for its ack, then throw and leave the sender aborted: every later row, flush, drain and rollback throws, even after the server comes back and acks the replayed rollback frame. Store-and-forward promises that an outage never fails a producer call. The new tests take the server down after a transaction's first frames went out, discard the transaction, commit the next one and bring the server back on the same endpoint. They fail until discarding stops waiting for the server. TxnServer now models the server per connection: wire sequences restart on every connection, and it records which frames a commit frame commits, so a test can assert that discarded rows never commit. A connection that did not confirm rollback reads the rollback frame as an empty deferred frame, as a server without rollback support does. TestWebSocketServer records which connections confirmed rollback. --- .../qwp/client/QwpLargeTransactionTest.java | 185 ++++++++++++++++-- .../qwp/websocket/TestWebSocketServer.java | 11 ++ 2 files changed, 185 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java index 191ab3912..14692d7e8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -39,9 +39,13 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -200,6 +204,39 @@ public void testNewSymbolsCountTowardByteTrigger() throws Exception { }); } + @Test + public void testPooledStrictLeaseCloseDuringOutageKeepsTheConnection() throws Exception { + // Returning a strict lease discards its open transaction. During an outage that + // must neither wait for the server nor cost the pool its connection: the next + // borrower's transaction goes out behind the rollback frame once the server is back. + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + TxnServer before = new TxnServer(); + TxnServer after = new TxnServer(); + TestWebSocketServer revived = null; + try (TestWebSocketServer first = start(before, CAP, port, true)) { + try (QuestDB db = QuestDB.connect(outageConf(port) + "transaction=strict;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(before.awaitFrames(2)); + first.close(); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + revived = start(after, CAP, port, true); + Assert.assertTrue(next.drain(10_000)); + next.close(); + } + } finally { + if (revived != null) { + revived.close(); + } + } + assertDiscardedThenCommitted(before, after); + }); + } + @Test public void testPooledStrictLeaseCloseRetiresConnectionWithoutServerRollback() throws Exception { assertMemoryLeak(() -> { @@ -316,6 +353,14 @@ public void testResetAbortsStrictTransactionWithoutServerRollback() throws Excep }); } + @Test + public void testResetDuringOutageDoesNotStallTheSender() throws Exception { + // Store-and-forward keeps every producer call working through an outage. reset() + // used to wait 30 s for the rollback ack and then abort the transaction, so every + // later row and flush threw, even after the server came back. + assertDiscardDuringOutage("transaction=on;", Sender::reset); + } + @Test public void testResetKeepsLegacyBehaviourWithoutServerRollback() throws Exception { // transaction=on against a server without rollback: reset() cannot reach the @@ -389,6 +434,11 @@ public void testRollbackAckTimeoutAbortsTransaction() throws Exception { }); } + @Test + public void testRollbackDuringOutageDoesNotStallTheSender() throws Exception { + assertDiscardDuringOutage("transaction=strict;", Sender::rollback); + } + @Test public void testRollbackDiscardsSentFrames() throws Exception { assertMemoryLeak(() -> { @@ -684,10 +734,69 @@ private static void assertOneTransaction(TxnServer h) { Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); } + // Asserts what the server that came back after an outage received and committed: the + // discarded transaction's frames, replayed and then rolled back, and behind the rollback + // frame the next transaction, which alone commits. + private static void assertDiscardedThenCommitted(TxnServer before, TxnServer after) { + Assert.assertEquals("nothing commits before the outage", 0, before.committedFrames().size()); + Assert.assertEquals("one connection after the outage: " + after.frames, 1, after.connectionCount()); + int rollbackAt = -1; + for (int i = 0, n = after.frames.size(); i < n; i++) { + if (after.frames.get(i).isRollback()) { + Assert.assertEquals("one rollback frame: " + after.frames, -1, rollbackAt); + rollbackAt = i; + } + } + Assert.assertTrue("the discarded transaction is replayed ahead of the rollback frame: " + after.frames, + rollbackAt >= 2); + for (int i = 0; i < rollbackAt; i++) { + Assert.assertTrue("frame " + i + " must be deferred: " + after.frames, after.frames.get(i).isDeferred()); + } + Assert.assertEquals(1, after.rolledBackGroups); + Assert.assertTrue(after.frames.get(after.frames.size() - 1).isCommit()); + Assert.assertEquals("only the next transaction commits", + after.frames.subList(rollbackAt + 1, after.frames.size()), after.committedFrames()); + } + + // The server confirmed rollback, received the first frames of a transaction and went + // away. The transaction is discarded during the outage, the next one is committed, and + // the server comes back on the same endpoint. + private static void assertDiscardDuringOutage(String mode, Consumer discard) throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + TxnServer before = new TxnServer(); + TxnServer after = new TxnServer(); + TestWebSocketServer revived = null; + try (TestWebSocketServer first = start(before, CAP, port, true)) { + try (Sender s = Sender.fromConfig(outageConf(port) + mode)) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(before.awaitFrames(2)); + first.close(); + // Must neither wait for the server nor fail: the server is away. + discard.accept(s); + writeRows(s, "t", 0, 10); + s.flush(); + revived = start(after, CAP, port, true); + Assert.assertTrue(s.drain(10_000)); + } + } finally { + if (revived != null) { + revived.close(); + } + } + assertDiscardedThenCommitted(before, after); + }); + } + private static String conf(TestWebSocketServer server) { return "ws::addr=localhost:" + server.getPort() + ";"; } + // Reconnects within milliseconds once the server is back. + private static String outageConf(int port) { + return "ws::addr=localhost:" + port + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;"; + } + private static TestWebSocketServer start(TxnServer h, int cap) throws Exception { TestWebSocketServer server = new TestWebSocketServer(h); server.setAdvertisedMaxBatchSize(cap); @@ -696,6 +805,17 @@ private static TestWebSocketServer start(TxnServer h, int cap) throws Exception return server; } + // A server on a fixed port, so a test can take it down and bring it back on the same + // endpoint, as one that supports rollback or as one that predates it. + private static TestWebSocketServer start(TxnServer h, int cap, int port, boolean isRollbackSupported) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(h, false, null, port); + server.setAdvertisedMaxBatchSize(cap); + server.setRollbackSupported(isRollbackSupported); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + private static void writeRows(Sender s, String table, int from, int count) { for (int i = from, n = from + count; i < n; i++) { s.table(table).longColumn("v", i).stringColumn("s", PAD).atNow(); @@ -708,12 +828,15 @@ private interface ThrowingRunnable { } private static final class TxnFrame { + // Index of the connection the frame arrived on, in connection order. + final int connection; final byte flags; final int size; - TxnFrame(byte flags, int size) { + TxnFrame(byte flags, int size, int connection) { this.flags = flags; this.size = size; + this.connection = connection; } boolean isCommit() { @@ -730,41 +853,70 @@ boolean isRollback() { @Override public String toString() { - return (isRollback() ? "R" : isDeferred() ? "D" : "C") + size; + return connection + ":" + (isRollback() ? "R" : isDeferred() ? "D" : "C") + size; } } /** - * Acks frames the way the QuestDB server does for one connection: deferred - * frames wait for the commit frame that closes their group, which acks them - * cumulatively; the rollback frame discards the group and acks it too. + * Acks frames the way the QuestDB server does, per connection: wire sequences + * restart at 0 on every connection, deferred frames wait for the commit frame + * that closes their group, which acks them cumulatively, and the rollback frame + * discards the group and acks it too. + *

+ * It also models what the server commits: a commit frame commits the group open + * on its connection, while a rollback frame or a closed connection leaves that + * group uncommitted. On a connection that did not confirm rollback the rollback + * frame is what a server without rollback support makes of it, an empty deferred + * frame: the group stays open, and the next commit on the connection commits it. */ private static final class TxnServer implements TestWebSocketServer.WebSocketServerHandler { + // Every frame received, over all connections, in arrival order. final List frames = new CopyOnWriteArrayList<>(); volatile boolean isAckingRollback = true; volatile int rolledBackGroups; - private long nextSeq; + // Frames a commit frame committed, in commit order. + private final List committed = new ArrayList<>(); + private final Map connections = new IdentityHashMap<>(); @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - long seq = nextSeq++; - TxnFrame frame = new TxnFrame(data[QwpConstants.HEADER_OFFSET_FLAGS], data.length); + Connection connection = connections.get(client); + if (connection == null) { + connection = new Connection(connections.size()); + connections.put(client, connection); + } + long seq = connection.nextSeq++; + TxnFrame frame = new TxnFrame(data[QwpConstants.HEADER_OFFSET_FLAGS], data.length, connection.index); frames.add(frame); notifyAll(); try { - if (frame.isRollback()) { + if (frame.isCommit()) { + committed.addAll(connection.group); + committed.add(frame); + connection.group.clear(); + client.sendBinary(ack(seq)); + } else if (frame.isRollback() && client.isRollbackConfirmed()) { rolledBackGroups++; + connection.group.clear(); if (isAckingRollback) { client.sendBinary(ack(seq)); } - } else if (frame.isCommit()) { - client.sendBinary(ack(seq)); + } else { + connection.group.add(frame); } } catch (IOException e) { throw new RuntimeException(e); } } + synchronized List committedFrames() { + return new ArrayList<>(committed); + } + + synchronized int connectionCount() { + return connections.size(); + } + synchronized boolean awaitFrames(int count) throws InterruptedException { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (frames.size() < count) { @@ -805,5 +957,16 @@ private static byte[] ack(long seq) { bb.putShort((short) 0); return buf; } + + private static final class Connection { + // Deferred frames the connection holds uncommitted. + final List group = new ArrayList<>(); + final int index; + long nextSeq; + + Connection(int index) { + this.index = index; + } + } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index ee6b1f4a7..81bf01442 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -583,6 +583,8 @@ public class ClientHandler implements Closeable { private final Socket socket; private InputStream in; private boolean isClosed; + // True once this connection's 101 response confirmed X-QWP-Rollback: enabled. + private volatile boolean isRollbackConfirmed; private OutputStream out; private Thread readThread; // Request path from the WebSocket upgrade GET line (e.g. /write/v4, @@ -637,6 +639,14 @@ private void drainInput() throws IOException { } } + /** + * Whether this connection's upgrade confirmed {@code X-QWP-Rollback: enabled}, so + * the client may send it the rollback control frame. + */ + public boolean isRollbackConfirmed() { + return isRollbackConfirmed; + } + public synchronized void sendBinary(byte[] data) throws IOException { writeFrame(WebSocketOpcode.BINARY, data, data.length); } @@ -911,6 +921,7 @@ private boolean performHandshake() throws IOException { } if (rollbackSupported && "true".equalsIgnoreCase(rollbackRequest)) { sb.append("X-QWP-Rollback: enabled\r\n"); + isRollbackConfirmed = true; } sb.append("\r\n"); out.write(sb.toString().getBytes(StandardCharsets.US_ASCII)); From fa1c788a1d76dae58411de8668a8aee39eb1a430 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Thu, 12 Nov 2026 13:33:15 +0000 Subject: [PATCH 4/7] Discard transactions without waiting on the server reset(), rollback() and returning a strict pooled lease waited up to 30 s for the rollback frame's ack, then latched the transaction as aborted: every later row, flush, drain and rollback threw, even after the server came back and acked the replayed rollback frame, and the error blamed missing server rollback support. A transient outage broke the producer for good, which store-and-forward must never do. The wait stood in for a guarantee the producer cannot see: a reconnect that reaches a server without rollback support, such as an older node after a failover, must not replay the rolled-back frames there, as that server reads the rollback frame as an empty deferred frame and commits the frames with the next commit. The I/O loop knows which server each connection reached, so the guarantee moves there. The store-and-forward log already orders the rollback frame ahead of every later frame on every connection. On a connection whose server did not confirm X-QWP-Rollback the loop never sends a rollback frame: once the server acked every commit-bearing frame sent before it, the loop acknowledges the frame itself and recycles the connection. Closing the connection discards the rolled-back frames server-side, as every server does with what a closed connection leaves uncommitted, and the next connection starts past the frame, the way the orphaned-tail retirement already works. Replays after a restart and background drains run through the same loop, so they get the same guard. rollbackOpenTransaction() now only publishes the frame; the timeout, its test setter and the abort on timeout are gone. Sender.rollback() no longer promises to block, and Sender.reset() documents the transactional discard. On a connection that confirmed rollback, trySendBatch adds one cached boolean check per frame and no extra writes. CursorWebSocketSendLoopRollbackTest drives the loop against a model of the server's commits that drops what a closed connection had in flight: the rollback frame never reaches a server without rollback support, the loop waits for commit acks (durable ones in durable-ack mode) before recycling, only the exact rollback frame is held back, consecutive rollbacks, a recovered log, the background drainer, a per-connection capability, and a fuzz test over random transactions and mixed servers (questdb.fuzz.rollback.iterations). A sender-level test fails over to a server without rollback support. testRollbackAckTimeoutAbortsTransaction asserted the abort this commit removes; testUnacknowledgedRollbackDoesNotStallTheSender replaces it. Tests that read the server's frames right after reset(), rollback() or a lease close now wait with drain() or for the rollback frame, since those calls no longer block, and assert what the server committed. --- .../main/java/io/questdb/client/Sender.java | 15 +- .../qwp/client/QwpWebSocketSender.java | 77 +- .../sf/cursor/CursorWebSocketSendLoop.java | 86 ++ .../qwp/client/QwpLargeTransactionTest.java | 157 +++- .../CursorWebSocketSendLoopRollbackTest.java | 802 ++++++++++++++++++ 5 files changed, 1057 insertions(+), 80 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index b3ffb3484..bb5709e69 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -773,8 +773,11 @@ default void resetSymbolDictionary() { * data format) and want to continue sending new data without retrying the problematic data. * After calling this method, you can start building a new row by calling {@link #table(CharSequence)}. *
- * Note: This method is only available for HTTP transport. TCP transport doesn't support - * this operation. + * In WebSocket transactional mode ({@code transaction=on} or {@code strict}) it also + * discards the rows of the open transaction that auto-flush already sent, as + * {@link #rollback()} does, when the server supports QWP rollback. + *
+ * Note: the ILP TCP and UDP transports don't support this operation. * * @see #flush() */ @@ -784,7 +787,13 @@ default void resetSymbolDictionary() { * Discard the open transaction: the rows buffered since the last flush and, in * WebSocket transactional mode ({@code transaction=on} or {@code strict}), the * rows of the transaction that auto-flush already sent to the server, which - * the server discards on request. Blocks until the server confirms. + * the server discards on request. + *
+ * Like {@link #flush()}, this returns without waiting for the server, so a + * server outage makes it fail only when the store-and-forward buffer is full, + * as it would a flush. The request goes out ahead of anything sent later, so + * later rows never commit the discarded ones, even across reconnects. Call + * {@link #drain(long)} to wait until the server confirms. *
* Discarding rows already sent needs a server that supports QWP rollback. When * the server does not, this method throws and aborts the transaction: every diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 7d94dc702..0e0b28c02 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -186,8 +186,6 @@ public class QwpWebSocketSender implements Sender { public static final int DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS = 100_000; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB - // How long rollback() waits for the server to acknowledge the rollback frame. - private static final long DEFAULT_ROLLBACK_ACK_TIMEOUT_MILLIS = 30_000L; // Frames of at most the frame cap that a strict transaction keeps room for in // the client buffer, so the commit that ends it can always be published. private static final int STRICT_COMMIT_RESERVE_FRAMES = 3; @@ -704,8 +702,6 @@ public class QwpWebSocketSender implements Sender { // entries too. Counted toward the byte trigger alongside pendingBytes. Reset // with pendingBytes. private long pendingSymbolDictBytes; - // Rollback-ack wait bound; see rollbackOpenTransaction. - private long rollbackAckTimeoutMillis = DEFAULT_ROLLBACK_ACK_TIMEOUT_MILLIS; // Server's per-table cap on rows a deferred-commit group may hold before it // commits them on its own (X-QWP-Max-Uncommitted-Rows), 0 when not advertised. // Captured on each FOREGROUND handshake, like serverMaxBatchSize. @@ -3386,10 +3382,11 @@ public void setCloseStartedHook(Runnable hook) { /** * Discards the rows buffered since the last flush, like before. In * transactional mode it also discards the open transaction's frames already - * sent, when the server supports QWP rollback (see {@link #rollback()}). - * Without that support a strict transaction is aborted -- the next flush - * throws and only closing the sender ends it -- while a non-strict one keeps - * the frames already sent, and the next flush commits them, as before. + * sent, when the server supports QWP rollback (see {@link #rollback()}), and + * like {@link #rollback()} it does not wait for the server. Without that + * support a strict transaction is aborted -- the next flush throws and only + * closing the sender ends it -- while a non-strict one keeps the frames + * already sent, and the next flush commits them, as before. */ @Override public void reset() { @@ -3412,16 +3409,21 @@ public void reset() { /** * Discards the open transaction: the rows buffered since the last flush, * and the transaction's frames already sent, which the server rolls back on - * the rollback control frame. Blocks until the server acknowledges the - * rollback. Without an open transaction this only discards the buffered - * rows. + * the rollback control frame. Without an open transaction this only discards + * the buffered rows. + *

+ * Like {@link #flush()}, it hands the rollback frame to store-and-forward and + * returns without waiting for the server, so a server outage makes it fail + * only when the store-and-forward buffer is full, as it would a flush. The + * frame goes out behind the transaction's frames and ahead of anything sent + * later, on every connection and after a restart, so nothing sent later can + * commit the discarded rows. Call {@link #drain(long)} to wait for the server + * to confirm. * * @throws LineSenderException when frames of the open transaction were already * sent and the server does not support QWP rollback * (the transaction is then aborted: the next flush - * throws, and only closing the sender ends it), or - * when the server does not acknowledge the rollback - * in time + * throws, and only closing the sender ends it) */ @Override public void rollback() { @@ -3515,39 +3517,29 @@ private void publishRollbackFrame() { } /** - * Rolls back the open transaction on the server and waits for the ack. + * Rolls back the open transaction: publishes the rollback frame behind its + * frames and ends it, without waiting for the server. *

- * The wait is what makes the rollback safe across reconnects. Until the - * server acknowledges the rollback frame, a reconnect replays the - * transaction's frames together with it, and a server reached by failover - * that does not support rollback would keep those frames for the next - * commit. Once acknowledged, the frames are trimmed and can never be - * replayed. A rollback that is not acknowledged in time aborts the - * transaction (see {@link #txnAborted}). + * No wait is needed to keep the rolled-back frames from ever committing. The + * store-and-forward log orders the rollback frame ahead of every frame + * published later, and every connection replays the log in that order, so a + * server that receives a later commit has rolled the transaction back first. + * Only a server that does not support rollback, reached by a reconnect + * before the frame was acked, could commit them: it would read the frame as + * an empty deferred frame. The I/O loop never sends it there; it discards + * the transaction by closing that connection instead (see + * {@code CursorWebSocketSendLoop.retireRollbackFrame}). The same holds for + * frames replayed after a restart or by a background drainer, which use the + * same loop. */ private void rollbackOpenTransaction() { - long previousBoundaryFsn = lastCommitBoundaryFsn; publishRollbackFrame(); - long rollbackFsn = cursorEngine.publishedFsn(); hasDeferredMessages = false; - // The rollback frame gets an OK ack covering the transaction's frames, so + // The rollback frame gets an OK ack covering the transaction's frames, or the + // I/O loop acknowledges it when it discards the transaction itself, so // close-time drains may wait for it like for a commit. - lastCommitBoundaryFsn = rollbackFsn; + lastCommitBoundaryFsn = cursorEngine.publishedFsn(); clearTransactionState(); - boolean isAcked = false; - try { - isAcked = awaitAckedFsn(fsnEpochBase + rollbackFsn, rollbackAckTimeoutMillis); - } finally { - if (!isAcked) { - txnAborted = true; - lastCommitBoundaryFsn = previousBoundaryFsn; - } - } - if (!isAcked) { - throw new LineSenderException("rollback not acknowledged by the server [timeoutMillis=") - .put(rollbackAckTimeoutMillis) - .put("]; the transaction is aborted: close the sender to discard it"); - } } /** @@ -3819,11 +3811,6 @@ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) { this.recycleDeferredCloseMaxWaitMillis = millis; } - @TestOnly - public void setRollbackAckTimeoutMillisForTesting(long millis) { - this.rollbackAckTimeoutMillis = millis; - } - /** * Strict transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames * like {@link #setTransactional(boolean) transactional} mode, only an explicit diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 8a76edcc3..e45e53c8a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -591,6 +591,16 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // and by the I/O thread afterwards -- never concurrently. private long orphanSkipStartFsn = -1L; private long orphanSkipTipFsn = -1L; + // Whether the current connection's server confirmed X-QWP-Rollback, so a rollback + // control frame may go out on it. setWireBaselineWithCatchUp sets it for every + // connection from the client it runs on; false until then, the safe side (see + // retireRollbackFrame). Written by start() on the user thread before the I/O thread + // exists, and by the I/O thread afterwards -- never concurrently. + private boolean isRollbackConfirmed; + // FSN of the last commit-bearing frame sent on the current connection, -1 when none. + // Tracked only while !isRollbackConfirmed, for retireRollbackFrame. Reset with + // isRollbackConfirmed; same threading. + private long lastSentCommitBearingFsn = -1L; // Poison-frame detector state (I/O thread only). poisonFsn is the FSN of the // frame implicated by the most recent server-active rejection: the NACK-named // frame, or the OK-level head-of-line frame (highestOkFsn+1) for a @@ -3198,6 +3208,10 @@ private void setWireBaselineWithCatchUp(long replayStart) { // handleServerRejection can tell "only the catch-up went out" from "the // head data frame went out". dataFrameSentThisConnection = false; + // The server this connection reached decides whether a rollback frame may go + // out on it; until a client exists no frame goes out at all. + isRollbackConfirmed = client != null && client.isServerRollbackEnabled(); + lastSentCommitBearingFsn = -1L; // Do not gate solely on isDeltaDictEnabled(): a recovered delta slot can // lose access to its side-file while its on-disk frames still start above // zero. hasReplayDictionaryDependency preserves that distinction while @@ -3230,6 +3244,26 @@ private int frameDeltaStart(long payloadAddr, int payloadLen) { return encoded < 0L ? -1 : (int) (encoded >>> 3); } + // True for a QWP frame carrying FLAG_DEFER_COMMIT. A short or foreign payload counts + // as commit-bearing, so retireRollbackFrame waits for its ack: the safe side, and the + // rule RecoveredFrameAnalysis applies to the orphaned deferred tail. + private static boolean isDeferredFrame(long payloadAddr, int payloadLen) { + return payloadLen >= QwpConstants.HEADER_SIZE + && Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE + && (Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) + & QwpConstants.FLAG_DEFER_COMMIT) != 0; + } + + // True only for the rollback control frame (QwpWebSocketEncoder.encodeRollbackMessage): + // a bare QWP header whose flags byte equals ROLLBACK_FRAME_FLAGS, compared whole as the + // server matches it. + private static boolean isRollbackFrame(long payloadAddr, int payloadLen) { + return payloadLen == QwpConstants.HEADER_SIZE + && Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE + && Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) + == QwpConstants.ROLLBACK_FRAME_FLAGS; + } + // True only for a well-formed QWP frame this encoder produced that carries a // delta symbol-dict section. The magic check keeps the dict logic from // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose @@ -4001,6 +4035,16 @@ private boolean trySendBatch() { break; // fixed byte cap: the next call carries on from this frame } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + if (!isRollbackConfirmed && isRollbackFrame(frameAddr, payloadLen)) { + if (batchFrames > 0) { + break; + } + // This server cannot roll back, so the frame never goes out on this + // connection: the loop waits for acks, or it discarded the rolled-back + // transaction and recycled. + retireRollbackFrame(); + return false; + } // Torn-dictionary guard. sentDictCount is this loop's model of how many ids the // CURRENT server has been told about. A frame whose delta starts ABOVE that // coverage references ids the server was never given, and the server now rejects @@ -4063,6 +4107,9 @@ private boolean trySendBatch() { sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; nextWireSeq++; + if (!isRollbackConfirmed && !isDeferredFrame(frameAddr, payloadLen)) { + lastSentCommitBearingFsn = fsnSent; + } batchFrames++; batchBytes += wireLen; if (replayTargetFsn >= 0) { @@ -4114,6 +4161,45 @@ void positionCursorForStart() { positionCursorAt(replayStart); } + /** + * Retires the rollback control frame at the send cursor of a connection whose + * server did not confirm X-QWP-Rollback. The producer sends that frame only after + * a server confirmed rollback, so this connection is a reconnect that reached a + * server without rollback support, such as an older node after a failover, with + * the frame not yet acked. That server would read the frame as an empty deferred + * frame and commit the rolled-back transaction, whose frames it holds uncommitted + * on this connection, with the next commit frame. So the frame never goes out: + * closing the connection discards those frames instead, as every server does with + * the rows a closed connection leaves uncommitted. + *

+ * The close waits until the server acks every commit-bearing frame sent on this + * connection before the rollback frame: the self-acknowledge below covers those + * frames too, so a commit the server had not yet processed would be lost with the + * connection. Until then this returns without doing anything, and tryReceiveAcks + * runs every loop iteration. Once they are acked, the frames this connection holds + * unacked are exactly the rolled-back transaction's deferred frames. The cumulative + * self-acknowledge of the rollback frame retires them, and the recycle discards + * them server-side and re-anchors the next connection past the rollback frame, the + * way {@link #tryRetireOrphanTail} handles an orphaned deferred tail. + */ + private void retireRollbackFrame() { + long rollbackFsn = fsnAtZero + nextWireSeq; + long ackedFsn = engine.ackedFsn(); + if (ackedFsn < lastSentCommitBearingFsn) { + return; + } + LOG.info("server does not support QWP rollback: discarding the rolled-back transaction " + + "[fsn {}..{}] by closing the connection that holds it uncommitted", + externalFsnBase + ackedFsn + 1L, externalFsnBase + rollbackFsn); + boolean isAdvanced = engine.acknowledge(rollbackFsn); + // Every ack is clamped to the frames this loop sent, and the cursor stands on + // the rollback frame, so the self-acknowledge always advances the watermark. + assert isAdvanced : "rollback frame already acknowledged [fsn=" + rollbackFsn + ']'; + dispatchProgress(rollbackFsn); + fail(new LineSenderException( + "recycling connection to discard a rolled-back transaction the server cannot roll back")); + } + /** * Retires the recovered orphaned deferred tail once retirable. *

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java index 14692d7e8..1bdfa9718 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -271,8 +271,10 @@ public void testPooledStrictLeaseCloseRollsBackUncommitted() throws Exception { Sender lease = db.borrowSender(); writeRows(lease, "t", 0, 1_000); Assert.assertTrue(h.awaitFrames(2)); - // Returning a strict lease without flush() discards the transaction. + // Returning a strict lease without flush() discards the transaction, + // without waiting for the server. lease.close(); + Assert.assertTrue(h.awaitRollbackFrames(1)); Assert.assertTrue(h.frames.get(h.frames.size() - 1).isRollback()); Sender next = db.borrowSender(); writeRows(next, "t", 0, 10); @@ -282,6 +284,7 @@ public void testPooledStrictLeaseCloseRollsBackUncommitted() throws Exception { } Assert.assertEquals(1, h.commitFrames()); Assert.assertEquals(1, h.rollbackFrames()); + assertRolledBackThenCommitted(h); } }); } @@ -392,8 +395,9 @@ public void testResetRollsBackSentFramesWhenServerSupportsIt() throws Exception writeRows(s, "t", 0, 1_000); Assert.assertTrue(h.awaitFrames(2)); s.reset(); - // reset() returned after the server acked the rollback frame, which - // follows every frame of the transaction. + // reset() does not wait for the server; drain() returns once the server + // acked the rollback frame, which follows every frame of the transaction. + Assert.assertTrue(s.drain(5_000)); int rollbackIndex = h.frames.size() - 1; Assert.assertTrue(h.frames.get(rollbackIndex).isRollback()); for (int i = 0; i < rollbackIndex; i++) { @@ -406,31 +410,61 @@ public void testResetRollsBackSentFramesWhenServerSupportsIt() throws Exception Assert.assertTrue(s.drain(5_000)); Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); } + assertRolledBackThenCommitted(h); } }); } @Test - public void testRollbackAckTimeoutAbortsTransaction() throws Exception { + public void testRollbackFrameNeverReachesServerWithoutRollbackSupport() throws Exception { + // The transaction is discarded while the server that confirmed rollback is away, + // and the endpoint comes back as a server without rollback support, as after a + // failover to an older node. That server would read the rollback frame as an + // empty deferred frame and commit the discarded rows with the next transaction. + // The client closes the connection that holds them instead, and sends the next + // transaction on a new one. assertMemoryLeak(() -> { - TxnServer h = new TxnServer(); - h.isAckingRollback = false; - try (TestWebSocketServer server = start(h, CAP)) { - server.setRollbackSupported(true); - try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { - ((QwpWebSocketSender) s).setRollbackAckTimeoutMillisForTesting(300); + int port = TestPorts.findUnusedPort(); + TxnServer before = new TxnServer(); + TxnServer after = new TxnServer(); + TestWebSocketServer revived = null; + try (TestWebSocketServer first = start(before, CAP, port, true)) { + try (Sender s = Sender.fromConfig(outageConf(port) + "transaction=on;")) { writeRows(s, "t", 0, 1_000); - Assert.assertTrue(h.awaitFrames(2)); - try { - s.rollback(); - Assert.fail("an unacknowledged rollback must fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "rollback not acknowledged by the server"); - } - assertAborted(s::flush); + Assert.assertTrue(before.awaitFrames(2)); + first.close(); + s.rollback(); + writeRows(s, "t", 0, 10); + s.flush(); + revived = start(after, CAP, port, false); + Assert.assertTrue(s.drain(10_000)); + Assert.assertFalse(((QwpWebSocketSender) s).isServerRollbackEnabled()); } - Assert.assertEquals(0, h.commitFrames()); + } finally { + if (revived != null) { + revived.close(); + } + } + Assert.assertEquals(0, before.committedFrames().size()); + Assert.assertEquals("the rollback frame must not reach a server without rollback support: " + + after.frames, 0, after.rollbackFrames()); + List> connections = after.framesByConnection(); + Assert.assertEquals("one connection for the discarded transaction, one for the next: " + + after.frames, 2, connections.size()); + // The reader threads of the two connections may log their first frames in + // either order: tell them apart by what they carry. + List discarded = connections.get(0); + List next = connections.get(1); + if (next.get(next.size() - 1).isDeferred()) { + discarded = connections.get(1); + next = connections.get(0); + } + Assert.assertTrue("the discarded transaction is replayed: " + after.frames, discarded.size() >= 2); + for (TxnFrame f : discarded) { + Assert.assertTrue("frame must be deferred: " + after.frames, f.isDeferred()); } + Assert.assertTrue(next.get(next.size() - 1).isCommit()); + Assert.assertEquals("only the next transaction commits", next, after.committedFrames()); }); } @@ -452,8 +486,10 @@ public void testRollbackDiscardsSentFrames() throws Exception { Assert.assertTrue(h.awaitFrames(2)); Assert.assertEquals(0, h.commitFrames()); s.rollback(); - // rollback() returned after the ack, so the frame is on the server, - // behind every frame of the transaction. + // rollback() does not wait for the server; drain() returns once the + // server acked the rollback frame, which follows every frame of the + // transaction. + Assert.assertTrue(s.drain(5_000)); int sent = h.frames.size() - 1; TxnFrame rollback = h.frames.get(sent); Assert.assertTrue(rollback.isRollback()); @@ -472,6 +508,7 @@ public void testRollbackDiscardsSentFrames() throws Exception { Assert.assertTrue(s.drain(5_000)); Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); } + assertRolledBackThenCommitted(h); } }); } @@ -693,6 +730,30 @@ public void testStrictTransactionTooLargeForClientBufferFailsFast() throws Excep }); } + @Test + public void testUnacknowledgedRollbackDoesNotStallTheSender() throws Exception { + // The server never acks the rollback frame on its own. rollback() used to wait + // for that ack and abort the transaction when it did not come. Nothing waits for + // it now: the next transaction goes out behind it, and the commit's cumulative + // ack covers it. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + h.isAckingRollback = false; + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + s.rollback(); + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + assertRolledBackThenCommitted(h); + } + }); + } + @Test public void testTransactionalDefaultTriggersSplitWideRows() throws Exception { // Before: transaction=on with the default triggers threw @@ -739,23 +800,29 @@ private static void assertOneTransaction(TxnServer h) { // frame the next transaction, which alone commits. private static void assertDiscardedThenCommitted(TxnServer before, TxnServer after) { Assert.assertEquals("nothing commits before the outage", 0, before.committedFrames().size()); - Assert.assertEquals("one connection after the outage: " + after.frames, 1, after.connectionCount()); + assertRolledBackThenCommitted(after); + } + + // Asserts what one connection received: a transaction's frames, the rollback frame + // that discarded them, and behind it the next transaction, which alone commits. + private static void assertRolledBackThenCommitted(TxnServer h) { + Assert.assertEquals("one connection: " + h.frames, 1, h.connectionCount()); int rollbackAt = -1; - for (int i = 0, n = after.frames.size(); i < n; i++) { - if (after.frames.get(i).isRollback()) { - Assert.assertEquals("one rollback frame: " + after.frames, -1, rollbackAt); + for (int i = 0, n = h.frames.size(); i < n; i++) { + if (h.frames.get(i).isRollback()) { + Assert.assertEquals("one rollback frame: " + h.frames, -1, rollbackAt); rollbackAt = i; } } - Assert.assertTrue("the discarded transaction is replayed ahead of the rollback frame: " + after.frames, + Assert.assertTrue("the discarded transaction goes out ahead of the rollback frame: " + h.frames, rollbackAt >= 2); for (int i = 0; i < rollbackAt; i++) { - Assert.assertTrue("frame " + i + " must be deferred: " + after.frames, after.frames.get(i).isDeferred()); + Assert.assertTrue("frame " + i + " must be deferred: " + h.frames, h.frames.get(i).isDeferred()); } - Assert.assertEquals(1, after.rolledBackGroups); - Assert.assertTrue(after.frames.get(after.frames.size() - 1).isCommit()); + Assert.assertEquals(1, h.rolledBackGroups); + Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); Assert.assertEquals("only the next transaction commits", - after.frames.subList(rollbackAt + 1, after.frames.size()), after.committedFrames()); + h.frames.subList(rollbackAt + 1, h.frames.size()), h.committedFrames()); } // The server confirmed rollback, received the first frames of a transaction and went @@ -792,9 +859,11 @@ private static String conf(TestWebSocketServer server) { return "ws::addr=localhost:" + server.getPort() + ";"; } - // Reconnects within milliseconds once the server is back. + // Reconnects within milliseconds once the server is back. Each test drains before it + // closes, so the short close drain only bounds how long a failing test takes. private static String outageConf(int port) { - return "ws::addr=localhost:" + port + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;"; + return "ws::addr=localhost:" + port + + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;close_flush_timeout_millis=1000;"; } private static TestWebSocketServer start(TxnServer h, int cap) throws Exception { @@ -909,6 +978,18 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } } + synchronized boolean awaitRollbackFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (rollbackFrames() < count) { + long left = deadline - System.nanoTime(); + if (left <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, left); + } + return true; + } + synchronized List committedFrames() { return new ArrayList<>(committed); } @@ -917,6 +998,18 @@ synchronized int connectionCount() { return connections.size(); } + // The frames of each connection, in arrival order, by connection index. + synchronized List> framesByConnection() { + List> byConnection = new ArrayList<>(); + for (int i = 0, n = connections.size(); i < n; i++) { + byConnection.add(new ArrayList<>()); + } + for (TxnFrame f : frames) { + byConnection.get(f.connection).add(f); + } + return byConnection; + } + synchronized boolean awaitFrames(int count) throws InterruptedException { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (frames.size() < count) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java new file mode 100644 index 000000000..7468fde72 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopRollbackTest.java @@ -0,0 +1,802 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.qwp.client.DurableAckTiers; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import io.questdb.client.test.tools.TestWebSocketClient; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +/** + * The rollback control frame on a connection whose server did not confirm + * {@code X-QWP-Rollback}. The producer publishes the frame without waiting for + * its ack, so a reconnect can reach a server without rollback support, such as + * an older node after a failover, before the frame is acked. That server would + * read it as an empty deferred frame and commit the rolled-back transaction + * with the next commit frame. + *

+ * Contract under test: the I/O loop never sends the rollback frame to such a + * server. Once the server acked every commit-bearing frame sent before it, the + * loop acknowledges the rollback frame itself and recycles the connection, + * which discards the rolled-back frames server-side, and the next connection + * starts past the rollback frame. A server that confirmed rollback receives the + * frame. The same holds for a log recovered after a restart and for the + * background drainer, which replay through the same loop. + *

+ * {@link ServerModel} is the oracle: it commits what the QuestDB server commits, + * and drops the frames a closed connection still had in flight, so a recycle + * before the acks of earlier commits shows as a lost commit. + */ +public class CursorWebSocketSendLoopRollbackTest { + // Random transaction logs per run of the fuzz test; raise it for a local soak. + private static final int FUZZ_ITERATIONS = Integer.getInteger("questdb.fuzz.rollback.iterations", 100); + private static final long SEGMENT_SIZE_BYTES = 64 * 1024; + private static final String TABLE = "t"; + + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testBackgroundDrainerRetiresRollbackOnServerWithoutRollback() throws Exception { + // An orphaned slot whose producer rolled back a transaction and committed the + // next one before it went away, with nothing acked yet. + String slotPath = temporaryFolder.newFolder("slot").getAbsolutePath(); + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slotPath, SEGMENT_SIZE_BYTES)) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + } + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_SIZE_BYTES, + 1L << 20, + () -> newClient(server, clients, false, false), + 5_000L, + 1L, + 5L, + DurableAckTiers.NONE, + 200L); + + // A drainer retries until it drains, so bound the run: a slot that never + // drains fails the test instead of hanging it. + Thread drain = new Thread(drainer::run, "rollback-drainer"); + drain.start(); + drain.join(10_000); + if (drain.isAlive()) { + drainer.requestStop(); + drain.join(); + Assert.fail("the slot did not drain: " + server.received()); + } + + Assert.assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "1:D3", "1:C4"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + }); + } + + @Test + public void testConsecutiveRollbacksRecycleOnceEach() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendRollback(engine); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 5L); + Assert.assertEquals(Arrays.asList("0:D1", "1:D2", "2:D3", "2:C4"), server.received()); + Assert.assertEquals(Arrays.asList("D3", "C4"), server.committed()); + Assert.assertEquals(3, clientCount(clients)); + } + }); + } + + @Test + public void testRandomTransactionsOverServersWithAndWithoutRollback() throws Exception { + // Transactions of zero to three deferred frames, each closed by a commit frame or + // a rollback frame, over connections that reach a server with or without rollback + // support at random. Each rollback frame goes out to a server that confirmed + // rollback, or costs exactly one recycle; only committed transactions commit, + // each once. + long s0 = System.nanoTime(); + long s1 = System.currentTimeMillis(); + Rnd rnd = TestUtils.generateRandom(null, s0, s1); + try { + TestUtils.assertMemoryLeak(() -> { + for (int iteration = 0; iteration < FUZZ_ITERATIONS; iteration++) { + runRandomTransactions(rnd); + } + }); + } catch (Throwable t) { + throw new AssertionError("seeds=" + s0 + "L, " + s1 + "L", t); + } + } + + @Test + public void testRecoveredLogRetiresRollbackOnServerWithoutRollback() throws Exception { + // A restart recovers the log with the rollback frame between two transactions, so + // the orphaned-tail retirement does not apply: the frame is replayed like any other. + String dir = temporaryFolder.newFolder("recovered").getAbsolutePath(); + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(dir, SEGMENT_SIZE_BYTES)) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + } + try (CursorSendEngine engine = new CursorSendEngine(dir, SEGMENT_SIZE_BYTES)) { + Assert.assertTrue(engine.wasRecoveredFromDisk()); + Assert.assertEquals("no orphaned tail", -1L, engine.recoveredOrphanTipFsn()); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 4L); + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "1:D3", "1:C4"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testRollbackAtReplayStartRetiresBeforeSendingAnything() throws Exception { + // The frames ahead of the rollback frame are acked already, as when the server + // committed the transaction's rows itself at its uncommitted-row cap, so the + // rollback frame is the first frame of the connection. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendRollback(engine); + appendDeferred(engine, 2); + appendCommit(engine, 3); + engine.acknowledge(0L); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 3L); + Assert.assertEquals(Arrays.asList("1:D2", "1:C3"), server.received()); + Assert.assertEquals(Arrays.asList("D2", "C3"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testRollbackFrameGoesToServerWithRollback() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendDeferred(engine, 3); + appendRollback(engine); + appendDeferred(engine, 4); + appendCommit(engine, 5); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, true), 5L); + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "0:D3", "0:R", "0:D4", "0:C5"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D4", "C5"), server.committed()); + Assert.assertEquals(1, clientCount(clients)); + // Nothing changes on this path: the whole log goes out in one write. + Assert.assertEquals(1, clients.get(0).flushes); + } + }); + } + + @Test + public void testRollbackFrameNeverReachesServerWithoutRollback() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendCommit(engine, 1); + appendDeferred(engine, 2); + appendDeferred(engine, 3); + appendRollback(engine); + appendDeferred(engine, 4); + appendCommit(engine, 5); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 5L); + // The frames ahead of the rollback frame went out in one batch with it + // held back; closing the connection discarded 2 and 3, and the next + // connection starts past the rollback frame. + Assert.assertEquals(Arrays.asList("0:C1", "0:D2", "0:D3", "1:D4", "1:C5"), server.received()); + Assert.assertEquals(Arrays.asList("C1", "D4", "C5"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + Assert.assertEquals(1, clients.get(0).flushes); + Assert.assertEquals(1, clients.get(1).flushes); + } + }); + } + + @Test + public void testRollbackSupportIsDecidedPerConnection() throws Exception { + // The first connection reaches a server without rollback support, the second one + // a server with it: the second rollback frame goes out. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendRollback(engine); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + boolean[] isRollbackConfirmed = {false, true}; + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, + () -> newClient(server, clients, isRollbackConfirmed[clientCount(clients)], false), + 1L, 5L); + runToAck(engine, loop, 5L); + Assert.assertEquals(Arrays.asList("0:D1", "1:D2", "1:R", "1:D3", "1:C4"), server.received()); + Assert.assertEquals(Arrays.asList("D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testOnlyTheRollbackFrameIsHeldBack() throws Exception { + // Bare 12-byte headers that are not the rollback frame -- an empty commit, an empty + // deferred frame and a foreign payload -- go out like any frame. Mistaking the + // empty commit for a rollback frame would discard the transaction it commits. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeferred(engine, 1); + appendHeader(engine, QwpConstants.MAGIC_MESSAGE, (byte) 0); + appendHeader(engine, QwpConstants.MAGIC_MESSAGE, QwpConstants.FLAG_DEFER_COMMIT); + appendHeader(engine, 0, QwpConstants.ROLLBACK_FRAME_FLAGS); + appendDeferred(engine, 2); + appendRollback(engine); + appendDeferred(engine, 3); + appendCommit(engine, 4); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + runToAck(engine, newLoop(engine, server, clients, false), 7L); + Assert.assertEquals(Arrays.asList("0:D1", "0:C0", "0:D0", "0:X12", "0:D2", "1:D3", "1:C4"), + server.received()); + Assert.assertEquals(Arrays.asList("D1", "C0", "D0", "X12", "D3", "C4"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + @Test + public void testRollbackWaitsForCommitAcksBeforeRecycling() throws Exception { + // The server holds the frames ahead of the rollback frame unprocessed. Recycling + // now would lose commit 1 with the connection: the self-acknowledge of the + // rollback frame covers it, so it would never be replayed. + assertRollbackWaitsForAckOf(false, engine -> appendCommit(engine, 1), "C1"); + } + + @Test + public void testRollbackWaitsForDurableAckBeforeRecycling() throws Exception { + // In durable-ack mode the trim watermark advances only on the durable ack, which + // the server holds back. The OK of commit 1 alone must not let the loop recycle, + // or it would trim commit 1 before the server made it durable. + assertRollbackWaitsForAckOf(true, engine -> appendCommit(engine, 1), "C1"); + } + + @Test + public void testRollbackWaitsForForeignFrameAck() throws Exception { + // A payload that is not a QWP message cannot be told to be deferred, so the loop + // waits for its ack like for a commit's: the safe side. + assertRollbackWaitsForAckOf(false, engine -> appendForeign(engine, 16), "X16"); + } + + @Test + public void testRollbackWaitsForShortFrameAck() throws Exception { + // Same for a payload too short to hold a QWP header. + assertRollbackWaitsForAckOf(false, engine -> appendForeign(engine, 8), "X8"); + } + + private static void appendCommit(CursorSendEngine engine, int id) { + appendFrame(engine, (byte) 0, id); + } + + private static void appendDeferred(CursorSendEngine engine, int id) { + appendFrame(engine, QwpConstants.FLAG_DEFER_COMMIT, id); + } + + // A payload without the QWP magic. + private static void appendForeign(CursorSendEngine engine, int size) { + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, size, (byte) 'x'); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // A QWP header whose flags byte says what the frame is, followed by its id. + private static void appendFrame(CursorSendEngine engine, byte flags, int id) { + int size = QwpConstants.HEADER_SIZE + 4; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, size, (byte) 0); + Unsafe.getUnsafe().putInt(buf, QwpConstants.MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + 4, QwpConstants.VERSION); + Unsafe.getUnsafe().putByte(buf + QwpConstants.HEADER_OFFSET_FLAGS, flags); + Unsafe.getUnsafe().putInt(buf + QwpConstants.HEADER_SIZE, id); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // A bare 12-byte header: no table blocks, no payload. + private static void appendHeader(CursorSendEngine engine, int magic, byte flags) { + int size = QwpConstants.HEADER_SIZE; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, size, (byte) 0); + Unsafe.getUnsafe().putInt(buf, magic); + Unsafe.getUnsafe().putByte(buf + 4, QwpConstants.VERSION); + Unsafe.getUnsafe().putByte(buf + QwpConstants.HEADER_OFFSET_FLAGS, flags); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // The rollback control frame, as QwpWebSocketEncoder.encodeRollbackMessage writes it. + private static void appendRollback(CursorSendEngine engine) { + appendHeader(engine, QwpConstants.MAGIC_MESSAGE, QwpConstants.ROLLBACK_FRAME_FLAGS); + } + + // The frame appendAhead writes is the only commit-bearing frame ahead of the rolled-back + // transaction, and the server holds it unprocessed, or in durable mode holds back its + // durable ack. The loop must not recycle until that ack arrives. + private static void assertRollbackWaitsForAckOf( + boolean isDurable, + Consumer appendAhead, + String ahead + ) throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendAhead.accept(engine); + appendDeferred(engine, 2); + appendDeferred(engine, 3); + appendRollback(engine); + appendDeferred(engine, 4); + appendCommit(engine, 5); + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + ModelClient first = newClient(server, clients, false, isDurable); + // Durable mode: the server processes and OKs at once, but holds back the + // durable ack. Otherwise it holds back processing, OKs included. + first.isHoldingAcks = true; + AtomicBoolean isFirstConnect = new AtomicBoolean(true); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, + () -> isFirstConnect.getAndSet(false) ? first : newClient(server, clients, false, isDurable), + 1L, 5L, isDurable ? DurableAckTiers.REPLICATED : DurableAckTiers.NONE, 0L); + try { + loop.start(); + awaitTrue("the frames ahead of the rollback frame go out", () -> first.flushedFrames == 3); + // The loop keeps polling the first connection while it waits for the acks. + int polls = first.polls; + awaitTrue("the loop keeps waiting on its connection", () -> first.polls >= polls + 50); + Assert.assertEquals("no recycle while " + ahead + " is not acked", 1, clientCount(clients)); + Assert.assertEquals(-1L, engine.ackedFsn()); + first.releaseAcks(); + awaitAckedFsn(engine, 5L); + } finally { + loop.close(); + } + Assert.assertEquals(Arrays.asList("0:" + ahead, "0:D2", "0:D3", "1:D4", "1:C5"), server.received()); + Assert.assertEquals(Arrays.asList(ahead, "D4", "C5"), server.committed()); + Assert.assertEquals(2, clientCount(clients)); + } + }); + } + + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { + awaitTrue("ackedFsn reaches " + target, () -> engine.ackedFsn() >= target); + Assert.assertEquals(target, engine.ackedFsn()); + } + + private static void awaitTrue(String what, BooleanSupplier condition) throws InterruptedException { + long deadline = System.nanoTime() + 10_000_000_000L; + while (!condition.getAsBoolean()) { + if (System.nanoTime() > deadline) { + Assert.fail("timed out: " + what); + } + Thread.sleep(1); + } + } + + private static int clientCount(List clients) { + synchronized (clients) { + return clients.size(); + } + } + + private static ModelClient newClient(ServerModel server, List clients, boolean isRollbackConfirmed, boolean isDurable) { + synchronized (clients) { + ModelClient client = new ModelClient(server, clients.size(), isRollbackConfirmed, isDurable); + clients.add(client); + return client; + } + } + + private static CursorSendEngine newEngine() { + return new CursorSendEngine(null, SEGMENT_SIZE_BYTES); + } + + private static CursorWebSocketSendLoop newLoop(CursorSendEngine engine, ServerModel server, List clients, boolean isRollbackConfirmed) { + return new CursorWebSocketSendLoop( + null, engine, 0L, + () -> newClient(server, clients, isRollbackConfirmed, false), + 1L, 5L); + } + + private static void runRandomTransactions(Rnd rnd) throws Exception { + try (CursorSendEngine engine = newEngine()) { + List expectedCommitted = new ArrayList<>(); + List transaction = new ArrayList<>(); + int rollbacks = 0; + int nextId = 1; + // Appended before the loop starts, the rest while it runs. + List appends = new ArrayList<>(); + for (int t = 0, n = 1 + rnd.nextInt(12); t < n; t++) { + transaction.clear(); + for (int i = 0, k = rnd.nextInt(4); i < k; i++) { + int id = nextId++; + transaction.add("D" + id); + appends.add(() -> appendDeferred(engine, id)); + } + if (rnd.nextInt(5) < 3) { + int id = nextId++; + transaction.add("C" + id); + expectedCommitted.addAll(transaction); + appends.add(() -> appendCommit(engine, id)); + } else { + rollbacks++; + appends.add(() -> appendRollback(engine)); + } + } + int beforeStart = rnd.nextInt(appends.size() + 1); + for (int i = 0; i < beforeStart; i++) { + appends.get(i).run(); + } + ServerModel server = new ServerModel(); + List clients = new ArrayList<>(); + Rnd ioRnd = new Rnd(rnd.nextLong(), rnd.nextLong()); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, + () -> newClient(server, clients, ioRnd.nextBoolean(), false), + 1L, 5L); + int retired; + try { + loop.start(); + for (int i = beforeStart, n = appends.size(); i < n; i++) { + appends.get(i).run(); + } + awaitAckedFsn(engine, appends.size() - 1L); + // Every rollback frame is received or retired by now. A retired one acks + // itself before its recycle connects, so when it is the last frame the + // recycle may still be under way. + retired = rollbacks - server.rollbacksReceived(); + awaitTrue("one recycle per retired rollback frame", () -> clientCount(clients) == 1 + retired); + } finally { + loop.close(); + } + String shape = "frames=" + appends.size() + ", received=" + server.received(); + Assert.assertEquals(shape, expectedCommitted, server.committed()); + Assert.assertEquals(shape, 0, server.rollbacksToServerWithoutRollback()); + Assert.assertEquals(shape, 1 + retired, clientCount(clients)); + } + } + + private static void runToAck(CursorSendEngine engine, CursorWebSocketSendLoop loop, long lastFsn) throws Exception { + try { + loop.start(); + awaitAckedFsn(engine, lastFsn); + } finally { + loop.close(); + } + } + + /** + * One connection to {@link ServerModel}: frames a batch flush hands over are in + * flight until the I/O loop polls, which is when the server processes them, unless + * it holds its acks. Acks are cumulative, so only the highest pending one goes out. + */ + private static final class ModelClient extends TestWebSocketClient { + private final List batch = new ArrayList<>(); + private final int connection; + private final ArrayDeque inFlight = new ArrayDeque<>(); + private final boolean isDurable; + private final boolean isRollbackConfirmed; + private final ServerModel server; + volatile int flushedFrames; + volatile int flushes; + volatile boolean isHoldingAcks; + volatile int polls; + private long durableReported; + private long nextSeq; + private long pendingOkSeq = -1L; + private long pendingOkSeqTxn; + + ModelClient(ServerModel server, int connection, boolean isRollbackConfirmed, boolean isDurable) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.server = server; + this.connection = connection; + this.isRollbackConfirmed = isRollbackConfirmed; + this.isDurable = isDurable; + } + + @Override + public synchronized void batchAppendBinary(long dataPtr, int length) { + batch.add(Frame.read(dataPtr, length)); + } + + @Override + public synchronized void batchFlush() { + inFlight.addAll(batch); + flushedFrames += batch.size(); + flushes++; + batch.clear(); + } + + @Override + public synchronized void close() { + // Whatever the server has not processed yet never arrives. + inFlight.clear(); + server.close(connection); + super.close(); + } + + @Override + public boolean isServerRollbackEnabled() { + return isRollbackConfirmed; + } + + synchronized void releaseAcks() { + isHoldingAcks = false; + } + + @Override + public void sendPing() { + } + + @Override + public synchronized boolean tryReceiveFrame(WebSocketFrameHandler handler) { + polls++; + // Durable mode: the server processes and OKs at once and holds back the + // durable ack. Otherwise holding back the acks means not processing at all. + if (isDurable || !isHoldingAcks) { + for (Frame frame; (frame = inFlight.pollFirst()) != null; ) { + long seq = nextSeq++; + if (server.process(connection, isRollbackConfirmed, frame)) { + pendingOkSeq = seq; + pendingOkSeqTxn = frame.kind == 'R' ? 0 : server.seqTxn(); + } + } + } + if (pendingOkSeq >= 0) { + sendOk(handler, pendingOkSeq, isDurable && pendingOkSeqTxn > 0 ? pendingOkSeqTxn : 0); + pendingOkSeq = -1L; + return true; + } + if (isDurable && !isHoldingAcks && server.seqTxn() > durableReported) { + durableReported = server.seqTxn(); + sendDurableAck(handler, durableReported); + return true; + } + return false; + } + + // STATUS_DURABLE_ACK: status, then the table entries. + private static void sendDurableAck(WebSocketFrameHandler handler, long seqTxn) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + int size = 1 + 2 + 2 + name.length + 8; + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_DURABLE_ACK); + putTableEntry(ptr + 1, name, seqTxn); + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + // STATUS_OK: status, cumulative wire sequence, then the table entries -- one for + // a commit in durable mode, so the OK waits for the durable ack covering it. + private static void sendOk(WebSocketFrameHandler handler, long seq, long seqTxn) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + int size = 1 + 8 + 2 + (seqTxn > 0 ? 2 + name.length + 8 : 0); + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK); + Unsafe.getUnsafe().putLong(ptr + 1, seq); + if (seqTxn > 0) { + putTableEntry(ptr + 9, name, seqTxn); + } else { + Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); + } + handler.onBinaryMessage(ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void putTableEntry(long p, byte[] name, long seqTxn) { + Unsafe.getUnsafe().putShort(p, (short) 1); + Unsafe.getUnsafe().putShort(p + 2, (short) name.length); + for (int i = 0; i < name.length; i++) { + Unsafe.getUnsafe().putByte(p + 4 + i, name[i]); + } + Unsafe.getUnsafe().putLong(p + 4 + name.length, seqTxn); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + + // A frame as the server sees it: D (deferred), C (commit), R (the rollback frame), or + // X, a payload that is not a QWP message, labelled with its size. A frame's id + // follows its header; a bare header has id 0. + private static final class Frame { + final char kind; + final String label; + + Frame(char kind, String label) { + this.kind = kind; + this.label = label; + } + + static Frame read(long ptr, int length) { + if (length < QwpConstants.HEADER_SIZE || Unsafe.getUnsafe().getInt(ptr) != QwpConstants.MAGIC_MESSAGE) { + return new Frame('X', "X" + length); + } + byte flags = Unsafe.getUnsafe().getByte(ptr + QwpConstants.HEADER_OFFSET_FLAGS); + if (length == QwpConstants.HEADER_SIZE && flags == QwpConstants.ROLLBACK_FRAME_FLAGS) { + return new Frame('R', "R"); + } + char kind = (flags & QwpConstants.FLAG_DEFER_COMMIT) != 0 ? 'D' : 'C'; + int id = length > QwpConstants.HEADER_SIZE ? Unsafe.getUnsafe().getInt(ptr + QwpConstants.HEADER_SIZE) : 0; + return new Frame(kind, kind + Integer.toString(id)); + } + + @Override + public String toString() { + return label; + } + } + + /** + * What the QuestDB server makes of the frames, per connection: a deferred frame + * joins the connection's open group; a commit frame commits the group and itself + * and is acked; the rollback frame discards the group and is acked on a connection + * that confirmed rollback. A server without rollback support reads the rollback + * frame as an empty deferred frame, so the group stays open and the next commit on + * the connection commits it. A closed connection leaves its group uncommitted. A + * foreign payload is acked like a commit, so the loop's wait for it ends. + */ + private static final class ServerModel { + private final List committed = new ArrayList<>(); + private final List> groups = new ArrayList<>(); + private final List received = new ArrayList<>(); + private int rollbacksReceived; + private int rollbacksToServerWithoutRollback; + private long seqTxn; + + synchronized void close(int connection) { + group(connection).clear(); + } + + synchronized List committed() { + return new ArrayList<>(committed); + } + + // Whether the server acks the frame. + synchronized boolean process(int connection, boolean isRollbackConfirmed, Frame frame) { + received.add(connection + ":" + frame); + List group = group(connection); + switch (frame.kind) { + case 'C': + case 'X': + committed.addAll(group); + committed.add(frame.label); + group.clear(); + seqTxn++; + return true; + case 'R': + rollbacksReceived++; + if (isRollbackConfirmed) { + group.clear(); + return true; + } + rollbacksToServerWithoutRollback++; + return false; + default: + group.add(frame.label); + return false; + } + } + + synchronized List received() { + return new ArrayList<>(received); + } + + synchronized int rollbacksReceived() { + return rollbacksReceived; + } + + synchronized int rollbacksToServerWithoutRollback() { + return rollbacksToServerWithoutRollback; + } + + synchronized long seqTxn() { + return seqTxn; + } + + private List group(int connection) { + while (groups.size() <= connection) { + groups.add(new ArrayList<>()); + } + return groups.get(connection); + } + } +} From 4749d5d5b7067521bb92660a85e4e72628564e09 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Thu, 12 Nov 2026 16:20:01 +0000 Subject: [PATCH 5/7] Decide strict mode from the auto-flush triggers auto_flush=off made a WebSocket sender a strict transaction of any size, while auto_flush_rows=off with auto_flush_interval=off, the same trigger state, built a non-transactional sender with no byte trigger: each flush went out as one batch, which failed past about 2 MiB per table, and close() committed. The builder keyed the mode on the auto_flush=off spelling, and recomputeFrameBudget() turned the byte trigger off for exactly that non-transactional sender. HTTP already treats both spellings alike. The builder now reads the trigger state. A sender with no auto-flush trigger (no row or interval trigger, no positive auto_flush_bytes) is in strict transactional mode unless transaction= picks the mode. transaction=on keeps it non-strict, so close() commits; auto_flush=off used to override it to strict. transaction=off is rejected for every spelling, as nothing but flush() could commit and a non-transactional flush must fit one frame. auto_flush=off with a positive auto_flush_bytes is rejected, as auto_flush=off with auto_flush_rows=N already was. auto_flush_bytes=off keeps its contract: no frame sizing, so each batch must fit one frame per table. recomputeFrameBudget() no longer special-cases the manual-flush sender, which the builder can no longer build. Behaviour change: a sender, or a pool, configured with auto_flush_rows=off and auto_flush_interval=off now discards the rows it did not flush when it closes or returns a lease, like HTTP and auto_flush=off, where it used to commit them. close() and lease return now log a WARN with the row count when they discard an uncommitted strict transaction, so a caller that relied on close() to commit sees it. A lease returned without server rollback support now aborts its transaction before the pool retires the connection, so the retiring close() does not report the same rows again. QwpLargeTransactionTest runs every spelling of "no trigger" (both config strings and both builder forms) through the same strict transaction of any size and the close() discard, and covers transaction=on committing on close, the rejections, auto_flush_bytes as the only trigger, auto_flush_bytes=off, one trigger left, the WARN on close and on lease return with and without server rollback, and an aborted lease retiring quietly. It replaces testManualFlushKeepsOneCommitPerFlush, which pinned the old split. LineSenderBuilderWebSocketTest and QuestDBBuilderTest cover the builder and the pool config paths. --- .../main/java/io/questdb/client/QuestDB.java | 6 +- .../main/java/io/questdb/client/Sender.java | 80 +++- .../qwp/client/QwpWebSocketSender.java | 50 +- .../client/TransactionTooLargeException.java | 11 +- .../io/questdb/client/impl/PooledSender.java | 4 +- .../client/test/QuestDBBuilderTest.java | 26 +- .../LineSenderBuilderWebSocketTest.java | 71 ++- .../qwp/client/QwpLargeTransactionTest.java | 429 ++++++++++++++++-- 8 files changed, 583 insertions(+), 94 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 8cfc437b7..02a0cb6de 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -139,7 +139,11 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token * {@link Sender#close()} on the returned instance to release it back to * the pool. {@code close()} on a pooled Sender flushes pending rows * before returning to the pool; a real disconnect only happens at - * {@link #close()} on this {@code QuestDB} handle. + * {@link #close()} on this {@code QuestDB} handle. A Sender in strict + * transactional mode ({@code transaction=strict}, or a WebSocket config + * with no auto-flush trigger, such as {@code auto_flush=off}) commits only + * on {@link Sender#flush()}, so its {@code close()} discards the rows it + * did not flush instead. *

* Allocation: zero at steady state -- the returned instance is a * pre-allocated decorator backed by a pre-allocated underlying Sender. diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index bb5709e69..c65f15412 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1150,10 +1150,12 @@ final class LineSenderBuilder { private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY; - // Set by disableAutoFlush(). auto_flush_interval=off also stores the interval as - // Integer.MAX_VALUE, so the WebSocket validation keys on this flag to reject only the - // full disable and accept the interval-only opt-out. - private boolean autoFlushDisabled; + // disableAutoFlush() (auto_flush=off) sets it. That call turns off every + // auto-flush trigger, so a positive auto_flush_bytes contradicts it, and only + // that conflict check reads this flag. auto_flush_rows=off with + // auto_flush_interval=off reaches the same trigger state, which is what + // hasAutoFlushTrigger() reads to pick the sender's mode. + private boolean isAutoFlushDisabled; private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; private int autoFlushRows = PARAMETER_NOT_SET_EXPLICITLY; private int bufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; @@ -1422,7 +1424,13 @@ public LineSenderBuilder authTimeoutMillis(long millis) { *
* This is only used when communicating over WebSocket transport. *
- * Default value is 0, which disables byte-based auto-flush. + * By default the byte trigger fires at 90% of the largest frame the connection can carry: the server's + * advertised maximum batch size, or the store-and-forward segment when that is smaller. A positive value sets + * the trigger, capped at that size, and counts as an auto-flush trigger: unless the sender is transactional, + * every batch it sends commits. 0 turns the byte trigger off, and then every batch must fit one frame per + * table. + *
+ * You cannot set a positive value when auto-flush is disabled. See {@link #disableAutoFlush()}. * * @param bytes maximum bytes per batch * @return this instance for method chaining @@ -1434,6 +1442,8 @@ public LineSenderBuilder autoFlushBytes(int bytes) { if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("auto flush bytes was already configured") .put("[bytes=").put(this.autoFlushBytes).put("]"); + } else if (isAutoFlushDisabled && bytes > 0) { + throw new LineSenderException("cannot set auto flush bytes when auto-flush is already disabled"); } if (bytes < 0) { throw new LineSenderException("auto flush bytes cannot be negative") @@ -1492,7 +1502,9 @@ public LineSenderBuilder autoFlushIntervalMillis(int autoFlushIntervalMillis) { * Setting this to 1 means that the Sender will send each row to a server immediately after it is added. This * effectively disables batching and may lead to a significant performance degradation. *
- * Setting this to 0 disables row-based auto-flush. Interval-based auto-flush remains enabled. + * Setting this to 0 disables row-based auto-flush. Interval-based auto-flush remains enabled. On WebSocket, a + * sender whose interval trigger is off too, and that has no positive {@link #autoFlushBytes(int)}, has no + * auto-flush trigger left and works like one built with {@link #disableAutoFlush()}. *

* You cannot set this value when auto-flush is disabled. See {@link #disableAutoFlush()}. * @@ -1906,10 +1918,16 @@ public LineSenderBuilder connectionListenerInboxCapacity(int capacity) { * It is essential to ensure the maximum buffer capacity is sufficient to accommodate all locally buffered data. *

* This method can be used with the HTTP and WebSocket protocols; calling it is illegal when communicating over - * the TCP protocol. On WebSocket it selects strict transactional mode (see - * {@link #strictTransactional(boolean)}): rows still go to the server in frames sized for the connection, so a - * batch can be any size, but only {@code flush()} commits them, and closing without a flush discards them. - * There, {@code auto_flush_interval=off} alone turns off only the time-based trigger. + * the TCP protocol. Turning off both the row and the interval trigger ({@code auto_flush_rows=off} and + * {@code auto_flush_interval=off}) has the same effect. + *

+ * On WebSocket, a sender with no auto-flush trigger is in strict transactional mode (see + * {@link #strictTransactional(boolean)}) unless {@link #transactional(boolean)} selects the mode: rows still go + * to the server in frames sized for the connection, so a batch can be any size, but only {@code flush()} + * commits them, and closing the sender or returning a pooled sender without a flush discards them. + * {@code transactional(true)} keeps such a sender non-strict, so closing it commits, and + * {@code transactional(false)} cannot be combined with it. Nor can a positive {@link #autoFlushBytes(int)}, + * which is an auto-flush trigger. * * @return this instance for method chaining * @see #autoFlushRows(int) @@ -1924,10 +1942,14 @@ public LineSenderBuilder disableAutoFlush() { throw new LineSenderException("auto flush interval was already configured ") .put("[autoFlushIntervalMillis=").put(this.autoFlushIntervalMillis).put("]"); } + if (this.autoFlushBytes > 0) { + throw new LineSenderException("auto flush bytes was already configured ") + .put("[autoFlushBytes=").put(this.autoFlushBytes).put("]"); + } this.autoFlushRows = AUTO_FLUSH_DISABLED; this.autoFlushIntervalMillis = Integer.MAX_VALUE; - this.autoFlushDisabled = true; + this.isAutoFlushDisabled = true; return this; } @@ -2807,7 +2829,9 @@ private LineSenderBuilder requestDurableAckTiers(int tiers) { * {@link io.questdb.client.cutlass.qwp.client.TransactionTooLargeException} * and leaves the transaction open, and {@code close()} without * {@code flush()} discards the transaction instead of committing it. - * {@code auto_flush=off} selects the same mode. WebSocket transport only. + * A sender with no auto-flush trigger (see {@link #disableAutoFlush()}) is + * in this mode unless {@link #transactional(boolean)} selects the mode. + * WebSocket transport only. * * @param enabled true to enable strict transactional mode; false leaves * {@link #transactional(boolean) transactional} mode as set @@ -2833,6 +2857,11 @@ public LineSenderBuilder strictTransactional(boolean enabled) { * committing atomically per table. The server commits a table on its own * once {@code qwp.max.uncommitted.rows} of its rows are uncommitted; use * {@link #strictTransactional(boolean)} to fail instead. + *

+ * A sender with no auto-flush trigger (see {@link #disableAutoFlush()}) is + * in strict transactional mode unless this method selects the mode: + * {@code true} keeps it in this non-strict mode, and {@code false} cannot + * be combined with it. * * @param enabled true to enable transactional mode * @return this instance for method chaining @@ -3896,9 +3925,9 @@ private QwpWebSocketSender buildWebSocket(HttpTokenProvider tokenProvider) { // dispatcher daemon, drainer pool, microbatch buffers and // WebSocketClient inside the abandoned `connected`. connected.setTransactional(transactional); - // auto_flush=off on WebSocket means "commit only when I flush()": a - // strict transaction of any size. - if (strictTransactional || autoFlushDisabled) { + // With no auto-flush trigger only flush() commits: a strict transaction of + // any size, unless transaction= chose the mode. + if (strictTransactional || (!transactionModeSet && !hasAutoFlushTrigger())) { connected.setStrictTransactional(true); } try { @@ -4948,6 +4977,16 @@ public java.util.Map wsConfigSnapshotForTest() { return m; } + // True when a configured trigger flushes rows on its own: the row or interval + // trigger, or a positive auto_flush_bytes. auto_flush=off, and + // auto_flush_rows=off with auto_flush_interval=off, leave none. The WebSocket + // default byte trigger does not count: it only sizes frames. + private boolean hasAutoFlushTrigger() { + return autoFlushRows != AUTO_FLUSH_DISABLED + || autoFlushIntervalMillis != Integer.MAX_VALUE + || autoFlushBytes > 0; + } + /** * Use HTTP protocol as transport. *
@@ -5151,10 +5190,13 @@ private void validateParameters() { if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("protocol version is not supported for WebSocket protocol"); } - if (autoFlushDisabled && transactionModeSet && !transactional) { - // On WebSocket, auto_flush=off is strict transactional mode. - throw new LineSenderException("auto_flush=off selects strict transactional mode on WebSocket " - + "and cannot be combined with transaction=off"); + if (transactionModeSet && !transactional && !hasAutoFlushTrigger()) { + // Only flush() can commit this sender's rows, and a non-transactional + // flush has to fit one frame per table. + throw new LineSenderException("auto-flush is off (auto_flush=off, or auto_flush_rows=off and " + + "auto_flush_interval=off), which on WebSocket selects strict transactional mode and " + + "cannot be combined with transaction=off; set transaction=on or turn on an auto-flush " + + "trigger"); } if ((sfDurability == SfDurability.FLUSH || sfDurability == SfDurability.APPEND)) { throw new LineSenderException( diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 0e0b28c02..624d2bc80 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -140,9 +140,7 @@ public class QwpWebSocketSender implements Sender { * {@code autoFlushBytes} value that sizes the byte trigger from the frames the * connection can carry: 90% of the smaller of the server's advertised * {@code X-QWP-Max-Batch-Size} and the store-and-forward segment. It is the - * builder default, so a batch never grows past what one frame can hold. A pure - * manual-flush sender (non-transactional, row and interval triggers off) keeps - * no byte trigger, as before. + * builder default, so a batch never grows past what one frame can hold. */ public static final int AUTO_FLUSH_BYTES_AUTO = -1; public static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; @@ -709,8 +707,8 @@ public class QwpWebSocketSender implements Sender { // True when the current foreground connection confirmed X-QWP-Rollback, so the // rollback control frame may go out. Captured on each FOREGROUND handshake. private volatile boolean serverRollbackEnabled; - // Strict transactional mode (transaction=strict, or auto_flush=off on this - // transport): implies transactional, never lets an open transaction commit in + // Strict transactional mode (transaction=strict, or a sender with no auto-flush + // trigger): implies transactional, never lets an open transaction commit in // parts, and close() discards instead of committing. See setStrictTransactional. private boolean strictTransactional; // Set once the open transaction can no longer be rolled back or committed as @@ -1866,6 +1864,9 @@ private void close0(boolean[] restoreInterrupt) { // ack the drain below waits for; any other server rolls them back // when the connection closes, and a recovered slot retires them // as an aborted transaction. + if (!txnAborted) { + warnDiscardedTransaction("close()"); + } discardPendingRows(); if (hasDeferredMessages && serverRollbackEnabled && !txnAborted) { try { @@ -3454,6 +3455,9 @@ public void rollback() { */ public boolean discardTransactionForReuse() { checkNotClosed(); + if (!txnAborted) { + warnDiscardedTransaction("returning the sender to the pool"); + } discardPendingRows(); if (txnAborted) { return false; @@ -3463,6 +3467,10 @@ public boolean discardTransactionForReuse() { return true; } if (!serverRollbackEnabled) { + // Only closing the connection discards the frames already sent, so the + // transaction is over. Aborting it also keeps the close() that retires + // the connection from warning about the same rows again. + txnAborted = true; return false; } rollbackOpenTransaction(); @@ -3826,7 +3834,6 @@ public void setStrictTransactional(boolean strict) { if (strict) { this.transactional = true; } - recomputeFrameBudget(); } /** @@ -3839,7 +3846,6 @@ public void setTransactional(boolean transactional) { if (!transactional) { this.strictTransactional = false; } - recomputeFrameBudget(); } /** @@ -7620,9 +7626,9 @@ private static boolean isTransactionFrame(long payloadAddr, int payloadLen) { * trigger off. *

  • {@link #AUTO_FLUSH_BYTES_AUTO} sets the trigger to 90% of the frame * size, assuming the server's default cap when nothing is advertised, so - * that a batch never grows past what one frame can carry. A pure - * manual-flush sender (not transactional, row and interval triggers off) - * keeps no byte trigger, so its flush stays one commit as before.
    • + * that a batch never grows past what one frame can carry. The builder makes + * a sender with no other auto-flush trigger transactional, so this trigger + * never splits such a sender's flush into several commits. * */ private void recomputeFrameBudget() { @@ -7634,8 +7640,7 @@ private void recomputeFrameBudget() { long limit = advertised > 0 ? budget : (segmentPayload == Long.MAX_VALUE ? Long.MAX_VALUE : segmentPayload * 9 / 10); effectiveAutoFlushBytes = (int) Math.min(autoFlushBytes, limit); - } else if (autoFlushBytes == AUTO_FLUSH_BYTES_AUTO - && (transactional || autoFlushRows > 0 || autoFlushIntervalNanos > 0)) { + } else if (autoFlushBytes == AUTO_FLUSH_BYTES_AUTO) { effectiveAutoFlushBytes = (int) budget; } else { effectiveAutoFlushBytes = 0; @@ -7676,6 +7681,27 @@ private LineSenderException transactionAbortedError() { + "the sender to discard them; the server rolls them back when the connection closes"); } + /** + * Strict mode commits only on flush(), so closing the sender or returning a + * pooled lease discards the open transaction. Logs it when the transaction holds + * rows: a caller that relied on close() to commit, as a sender with auto-flush + * triggers does, would otherwise lose them without a trace. Counts the rows + * already sent in deferred frames and those still buffered; call it before + * discardPendingRows() clears the latter. + */ + private void warnDiscardedTransaction(String action) { + long rows = pendingRowCount; + for (int i = 0, n = txnTables.size(); i < n; i++) { + rows += txnTables.getQuick(i).getTxnSentRows(); + } + if (rows > 0) { + LOG.warn("{} discarded an uncommitted transaction of {} rows: a strict transactional sender " + + "(transaction=strict, or one with no auto-flush trigger) commits only on flush(). Call " + + "flush() before close() to keep the rows, or ignore this if the discard is intentional", + action, rows); + } + } + /** * Accumulates the current row. * Rows buffer until flush (explicit or auto-flush). diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java index 4374b7337..2c2b0fc8f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java @@ -27,11 +27,12 @@ import io.questdb.client.cutlass.line.LineSenderException; /** - * A strict transaction ({@code transaction=strict} or {@code auto_flush=off} on the - * WebSocket transport) grew past a limit that would stop it from committing as one - * transaction: the server's per-table {@code qwp.max.uncommitted.rows}, past which - * the server commits the rows on its own, or the client's {@code sf_max_total_bytes}, - * which must hold the whole uncommitted transaction until the server commits it. + * A strict transaction ({@code transaction=strict}, or a sender with no auto-flush + * trigger such as {@code auto_flush=off}, on the WebSocket transport) grew past a + * limit that would stop it from committing as one transaction: the server's + * per-table {@code qwp.max.uncommitted.rows}, past which the server commits the + * rows on its own, or the client's {@code sf_max_total_bytes}, which must hold the + * whole uncommitted transaction until the server commits it. *

    * Thrown before anything that would cross the limit reaches the server, and the * transaction stays open: the caller can still commit the rows accepted so far diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 1c49b3099..e25bad14f 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -48,7 +48,9 @@ *

    * Behaviour difference from a raw Sender: {@link #close()} flushes the buffer * and returns the slot to the pool. The underlying Sender is only truly closed - * when {@link io.questdb.client.QuestDB#close()} shuts the pool down. + * when {@link io.questdb.client.QuestDB#close()} shuts the pool down. A + * WebSocket Sender in strict transactional mode commits only on flush(), so + * there close() discards the open transaction instead of flushing it. *

    * Because the slot is reused across borrows, this wrapper -- not the slot -- * carries the lease identity. A stale handle (held after {@link #close()}, with diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index 4dc06fcd6..d15713900 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -215,12 +215,20 @@ public void testMalformedEgressConfigRejectedAtBuildWithMinZero() { @Test public void testAutoFlushIntervalOffAcceptedAtBuildWithMinZero() { // auto_flush_interval=off turns off only the time-based trigger and is legal on - // WebSocket, alone or with the row trigger off too (only auto_flush=off is rejected). + // WebSocket: alone, with the row trigger off too (a sender with no auto-flush + // trigger, in strict transactional mode), and with a byte trigger left as the + // only one. QuestDB.builder().fromConfig( "ws::addr=127.0.0.1:1;auto_flush_interval=off;sender_pool_min=0;query_pool_min=0;").build().close(); + QuestDB.builder().fromConfig( + "ws::addr=127.0.0.1:1;auto_flush_rows=off;auto_flush_interval=off;" + + "sender_pool_min=0;query_pool_min=0;").build().close(); QuestDB.builder().fromConfig( "ws::addr=127.0.0.1:1;auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=65536;" + "sender_pool_min=0;query_pool_min=0;").build().close(); + QuestDB.builder().fromConfig( + "ws::addr=127.0.0.1:1;auto_flush=off;auto_flush_bytes=off;sender_pool_min=0;query_pool_min=0;") + .build().close(); } @Test @@ -229,16 +237,24 @@ public void testMalformedIngressConfigRejectedAtBuildWithMinZero() { // Sender -- yet it must still reject a malformed ingress key in the // single cluster config up front. Covers a typed enum (tls_verify), a // registry-STRING value that only the real Sender parse validates - // (auto_flush_rows), and WebSocket build-time checks: auto_flush=off selects - // strict transactional mode, so it cannot combine with transaction=off, and - // sf_durability=flush is not yet supported. + // (auto_flush_rows), and WebSocket build-time checks: a sender with no + // auto-flush trigger is in strict transactional mode, so it cannot combine + // with transaction=off, auto_flush=off cannot combine with a byte trigger, + // and sf_durability=flush is not yet supported. assertBuildRejected( "wss::addr=127.0.0.1:1;tls_verify=strict;sender_pool_min=0;query_pool_min=0;", "tls_verify"); assertBuildRejected( "ws::addr=127.0.0.1:1;auto_flush_rows=abc;sender_pool_min=0;query_pool_min=0;", "auto_flush_rows"); assertBuildRejected( "ws::addr=127.0.0.1:1;auto_flush=off;transaction=off;sender_pool_min=0;query_pool_min=0;", - "auto_flush=off selects strict transactional mode"); + "which on WebSocket selects strict transactional mode and cannot be combined with transaction=off"); + assertBuildRejected( + "ws::addr=127.0.0.1:1;auto_flush_rows=off;auto_flush_interval=off;transaction=off;" + + "sender_pool_min=0;query_pool_min=0;", + "which on WebSocket selects strict transactional mode and cannot be combined with transaction=off"); + assertBuildRejected( + "ws::addr=127.0.0.1:1;auto_flush=off;auto_flush_bytes=65536;sender_pool_min=0;query_pool_min=0;", + "auto flush bytes was already configured [autoFlushBytes=65536]"); assertBuildRejected( "ws::addr=127.0.0.1:1;sf_durability=flush;sender_pool_min=0;query_pool_min=0;", "not yet supported"); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 0fd081389..f69a58516 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -425,15 +425,28 @@ public void testDefaultIsAsync() { } @Test - public void testDisableAutoFlush_rejectsTransactionOffForWebSocket() { - // On WebSocket, disabling auto-flush selects strict transactional mode, - // which an explicit transactional(false) contradicts. - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) + public void testDisableAutoFlush_rejectsPositiveAutoFlushBytes() { + // disableAutoFlush() turns every auto-flush trigger off, so a byte trigger + // contradicts it in either order, as a row or interval trigger does. + assertThrows("cannot set auto flush bytes when auto-flush is already disabled", + () -> Sender.builder(Sender.Transport.WEBSOCKET) .address(LOCALHOST) .disableAutoFlush() - .transactional(false), - "cannot be combined with transaction=off"); + .autoFlushBytes(65_536)); + assertThrows("auto flush bytes was already configured [autoFlushBytes=65536]", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(65_536) + .disableAutoFlush()); + // 0 turns the byte trigger off, which agrees with it. + Assert.assertNotNull(Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush() + .autoFlushBytes(0)); + Assert.assertNotNull(Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(0) + .disableAutoFlush()); } @Test @@ -793,6 +806,50 @@ public void testNoAddress_fails() { "address not set"); } + @Test + public void testNoAutoFlushTrigger_rejectsTransactionOff() { + // A WebSocket sender with no auto-flush trigger commits only on flush(): it is + // strict unless transactional(true) says otherwise, and transactional(false) + // contradicts it, however the triggers were turned off. + String expected = "auto-flush is off (auto_flush=off, or auto_flush_rows=off and auto_flush_interval=off), " + + "which on WebSocket selects strict transactional mode and cannot be combined with " + + "transaction=off; set transaction=on or turn on an auto-flush trigger"; + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush() + .transactional(false), + expected); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .transactional(false) + .disableAutoFlush(), + expected); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0) + .autoFlushIntervalMillis(Integer.MAX_VALUE) + .transactional(false), + expected); + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0) + .autoFlushIntervalMillis(Integer.MAX_VALUE) + .autoFlushBytes(0) + .transactional(false), + expected); + // strictTransactional(false) keeps transactional mode as set: off here. + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush() + .strictTransactional(false), + expected); + } + @Test public void testPortMismatch_fails() { assertThrowsAny( diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java index 1bdfa9718..8393c2f6a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -24,6 +24,10 @@ package io.questdb.client.test.cutlass.qwp.client; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import io.questdb.client.QuestDB; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; @@ -35,17 +39,20 @@ import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Function; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -60,40 +67,62 @@ */ public class QwpLargeTransactionTest { private static final int CAP = 32 * 1024; + // What closing a sender or returning a pooled lease logs when it discards an open + // strict transaction of 1000 rows, behind the action that discarded it. + private static final String DISCARD_WARNING = " discarded an uncommitted transaction of 1000 rows: a strict " + + "transactional sender (transaction=strict, or one with no auto-flush trigger) commits only on " + + "flush(). Call flush() before close() to keep the rows, or ignore this if the discard is intentional"; + private static final String NO_TRIGGER_TRANSACTION_OFF_ERROR = "auto-flush is off (auto_flush=off, or " + + "auto_flush_rows=off and auto_flush_interval=off), which on WebSocket selects strict transactional " + + "mode and cannot be combined with transaction=off; set transaction=on or turn on an auto-flush trigger"; // ~100 encoded bytes per row: 1000 rows (the default row trigger) are ~100 KB, // three times the 32 KiB cap. private static final String PAD = TestUtils.repeat("x", 90); @Test - public void testAutoFlushOffIsStrictTransactionOfAnySize() throws Exception { + public void testAutoFlushBytesAloneCommitsEachBatch() throws Exception { + // A positive auto_flush_bytes is an auto-flush trigger: with the row and + // interval triggers off, every batch it cuts commits on its own, as before. + // transaction=off may say so explicitly. assertMemoryLeak(() -> { - TxnServer h = new TxnServer(); - try (TestWebSocketServer server = start(h, CAP)) { - try (Sender s = Sender.fromConfig(conf(server) + "auto_flush=off;")) { - Assert.assertTrue(((QwpWebSocketSender) s).isStrictTransactional()); - writeRows(s, "t", 0, 2_000); - // ~200 KB went out as deferred frames before flush(); none commits. - Assert.assertTrue(h.awaitFrames(5)); - Assert.assertEquals(0, h.commitFrames()); - s.flush(); - Assert.assertTrue(s.drain(5_000)); + String[] configs = { + "auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=8192;", + "auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=8192;transaction=off;" + }; + for (String config : configs) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + config)) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertFalse(config, ws.isTransactional()); + Assert.assertEquals(config, 8_192, ws.getEffectiveAutoFlushBytes()); + // ~20 KB: the byte trigger sends at least two batches, and each + // commits before any flush(). + writeRows(s, "t", 0, 200); + Assert.assertTrue(config, h.awaitFrames(2)); + for (TxnFrame f : h.frames) { + Assert.assertTrue(config + ": " + h.frames, f.isCommit()); + } + } + // close() commits the rest. + Assert.assertEquals(config, h.frames, h.committedFrames()); } - assertFramesWithinCap(h, CAP); - assertOneTransaction(h); } }); } @Test - public void testAutoFlushOffRejectsTransactionOff() throws Exception { + public void testAutoFlushOffRejectsPositiveAutoFlushBytes() throws Exception { + // auto_flush=off turns every auto-flush trigger off, so a byte trigger + // contradicts it, as a row or interval trigger already does. The ws:: parser + // applies the trigger keys before auto_flush, whatever their order. + // auto_flush_bytes=off is no trigger; see + // testNoAutoFlushTriggerWithAutoFlushBytesOffSendsOneFramePerFlush. assertMemoryLeak(() -> { - try { - Sender.fromConfig("ws::addr=localhost:1;auto_flush=off;transaction=off;").close(); - Assert.fail("auto_flush=off must not combine with transaction=off"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), - "auto_flush=off selects strict transactional mode on WebSocket"); - } + assertConfigRejected("auto_flush=off;auto_flush_bytes=65536;", + "auto flush bytes was already configured [autoFlushBytes=65536]"); + assertConfigRejected("auto_flush_bytes=65536;auto_flush=off;", + "auto flush bytes was already configured [autoFlushBytes=65536]"); }); } @@ -160,29 +189,6 @@ public void testByteTriggerFollowsSegmentSize() throws Exception { }); } - @Test - public void testManualFlushKeepsOneCommitPerFlush() throws Exception { - // A non-transactional sender with every auto-flush trigger off keeps its - // flush as one commit: no byte trigger splits the batch early. - assertMemoryLeak(() -> { - TxnServer h = new TxnServer(); - try (TestWebSocketServer server = start(h, CAP)) { - try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_rows=off;auto_flush_interval=off;")) { - Assert.assertEquals(0, ((QwpWebSocketSender) s).getEffectiveAutoFlushBytes()); - writeRows(s, "a", 0, 250); - writeRows(s, "b", 0, 250); - Assert.assertEquals(0, h.frames.size()); - s.flush(); - Assert.assertTrue(s.drain(5_000)); - } - // The 50 KB batch splits per table at flush: one deferred frame, one commit. - Assert.assertEquals(2, h.frames.size()); - Assert.assertTrue(h.frames.get(0).isDeferred()); - Assert.assertTrue(h.frames.get(1).isCommit()); - } - }); - } - @Test public void testNewSymbolsCountTowardByteTrigger() throws Exception { // Symbol columns buffer 4-byte ids; the 300-byte strings ride in the frame's @@ -204,6 +210,225 @@ public void testNewSymbolsCountTowardByteTrigger() throws Exception { }); } + @Test + public void testNoAutoFlushTriggerIsStrictTransactionOfAnySize() throws Exception { + // Every way to turn off all auto-flush triggers builds the same sender: a + // strict transaction of any size, which only flush() commits. + // auto_flush_rows=off with auto_flush_interval=off used to send each flush as + // one batch, which failed past the frame cap, and to commit on close(). + assertMemoryLeak(() -> { + for (NoTriggerSpelling spelling : noTriggerSpellings()) { + TxnServer h = new TxnServer(); + List committed = null; + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = spelling.build(conf(server))) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(spelling.name, ws.isStrictTransactional()); + Assert.assertEquals(spelling.name, CAP * 9 / 10, ws.getEffectiveAutoFlushBytes()); + writeRows(s, "t", 0, 2_000); + // ~200 KB went out as deferred frames before flush(); none commits. + Assert.assertTrue(spelling.name, h.awaitFrames(5)); + Assert.assertEquals(spelling.name, 0, h.commitFrames()); + s.flush(); + Assert.assertTrue(spelling.name, s.drain(5_000)); + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + committed = h.committedFrames(); + Assert.assertEquals(spelling.name, h.frames, committed); + // close() without flush() discards the next transaction, its + // frames already sent included. + writeRows(s, "t", 2_000, 1_000); + Assert.assertTrue(spelling.name, h.awaitFrames(committed.size() + 2)); + } + Assert.assertEquals(spelling.name, committed, h.committedFrames()); + } + } + }); + } + + @Test + public void testNoAutoFlushTriggerRejectsTransactionOff() throws Exception { + // With no auto-flush trigger only flush() commits, and a non-transactional + // flush has to fit one frame, so every spelling refuses transaction=off. + assertMemoryLeak(() -> { + assertConfigRejected("auto_flush=off;transaction=off;", NO_TRIGGER_TRANSACTION_OFF_ERROR); + assertConfigRejected("auto_flush_rows=off;auto_flush_interval=off;transaction=off;", + NO_TRIGGER_TRANSACTION_OFF_ERROR); + assertConfigRejected("auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=off;transaction=off;", + NO_TRIGGER_TRANSACTION_OFF_ERROR); + }); + } + + @Test + public void testNoAutoFlushTriggerWithAutoFlushBytesOffSendsOneFramePerFlush() throws Exception { + // auto_flush_bytes=off opts out of sizing batches to the frame: the + // transaction still commits only on flush(), but the application keeps it + // within one frame per table, and a flush past the cap is refused and keeps + // the rows, as auto_flush_bytes=off documents. + assertMemoryLeak(() -> { + String[] configs = { + "auto_flush=off;auto_flush_bytes=off;", + "auto_flush_rows=off;auto_flush_interval=off;auto_flush_bytes=off;" + }; + for (String config : configs) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + config)) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(config, ws.isStrictTransactional()); + Assert.assertEquals(config, 0, ws.getEffectiveAutoFlushBytes()); + // ~20 KB fits the 32 KiB cap: one frame, which commits. + writeRows(s, "t", 0, 200); + s.flush(); + Assert.assertTrue(config, s.drain(5_000)); + Assert.assertEquals(config, 1, h.frames.size()); + Assert.assertTrue(config, h.frames.get(0).isCommit()); + // ~100 KB does not. + writeRows(s, "t", 200, 1_000); + try { + s.flush(); + Assert.fail(config + ": a flush past the cap must be refused"); + } catch (BatchTooLargeForCapException e) { + TestUtils.assertContains(e.getMessage(), "serverMaxBatchSize=" + CAP); + } + s.reset(); + } + Assert.assertEquals(config, 1, h.frames.size()); + } + } + }); + } + + @Test + public void testNoAutoFlushTriggerWithTransactionOnCommitsOnClose() throws Exception { + // An explicit transaction=on wins over the strict mode a sender with no + // auto-flush trigger otherwise gets: close() commits, as transaction=on does. + assertMemoryLeak(() -> { + for (NoTriggerSpelling spelling : noTriggerSpellings()) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = spelling.build(conf(server) + "transaction=on;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertTrue(spelling.name, ws.isTransactional()); + Assert.assertFalse(spelling.name, ws.isStrictTransactional()); + writeRows(s, "t", 0, 2_000); + Assert.assertTrue(spelling.name, h.awaitFrames(5)); + Assert.assertEquals(spelling.name, 0, h.commitFrames()); + } + assertFramesWithinCap(h, CAP); + assertOneTransaction(h); + Assert.assertEquals(spelling.name, h.frames, h.committedFrames()); + } + } + }); + } + + @Test + public void testOneAutoFlushTriggerLeftKeepsEveryBatchCommitting() throws Exception { + // Turning off the row or the interval trigger alone leaves the other one, so + // the sender is not transactional: every batch it sends commits on its own. + assertMemoryLeak(() -> { + String[] configs = {"auto_flush_rows=off;", "auto_flush_interval=off;"}; + for (String config : configs) { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + config)) { + QwpWebSocketSender ws = (QwpWebSocketSender) s; + Assert.assertFalse(config, ws.isTransactional()); + Assert.assertEquals(config, CAP * 9 / 10, ws.getEffectiveAutoFlushBytes()); + // ~100 KB: the byte trigger sends at least three batches before + // any flush(), and each commits. + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(config, h.awaitFrames(3)); + for (TxnFrame f : h.frames) { + Assert.assertTrue(config + ": " + h.frames, f.isCommit()); + } + } + Assert.assertEquals(config, h.frames, h.committedFrames()); + } + } + }); + } + + @Test + public void testPooledLeaseWithAbortedTransactionRetiresQuietly() throws Exception { + // reset() could not undo the frames already sent, so the transaction is + // aborted and every flush already said so. Returning the lease retires the + // connection, which discards the frames, without warning about them again. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QuestDB db = QuestDB.connect(conf(server) + + "auto_flush_rows=off;auto_flush_interval=off;query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + lease.reset(); + assertAborted(lease::flush); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(next.drain(5_000)); + next.close(); + } + List committed = h.committedFrames(); + Assert.assertEquals("only the next lease commits", 1, committed.size()); + Assert.assertTrue(committed.get(0).isCommit()); + Assert.assertEquals("the aborted lease's connection was retired", 2, server.upgradeRequestCount()); + } + Assert.assertEquals(Collections.emptyList(), discardWarnings(log)); + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testPooledLeaseWithoutAutoFlushTriggerDiscardsOnReturn() throws Exception { + // Returning a lease of a pool with no auto-flush trigger used to commit its rows. + // The lease now holds a strict transaction, which only flush() commits, so + // returning it discards the rows, and says so once: whether the server rolls + // back the frames already sent, or the pool retires the connection to discard them. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + String[] spellings = {"auto_flush=off;", "auto_flush_rows=off;auto_flush_interval=off;"}; + for (String spelling : spellings) { + for (boolean isRollbackSupported : new boolean[]{false, true}) { + String name = spelling + " rollback=" + isRollbackSupported; + clearLog(log); + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(isRollbackSupported); + try (QuestDB db = QuestDB.connect(conf(server) + spelling + "query_pool_min=0;")) { + Sender lease = db.borrowSender(); + writeRows(lease, "t", 0, 1_000); + Assert.assertTrue(name, h.awaitFrames(2)); + lease.close(); + Sender next = db.borrowSender(); + writeRows(next, "t", 0, 10); + next.flush(); + Assert.assertTrue(name, next.drain(5_000)); + next.close(); + } + List committed = h.committedFrames(); + Assert.assertEquals(name + ": only the next lease commits", 1, committed.size()); + Assert.assertTrue(name, committed.get(0).isCommit()); + Assert.assertEquals(name, isRollbackSupported ? 1 : 0, h.rollbackFrames()); + } + Assert.assertEquals(name, + Collections.singletonList("returning the sender to the pool" + DISCARD_WARNING), + discardWarnings(log)); + } + } + } finally { + releaseSenderLog(log); + } + }); + } + @Test public void testPooledStrictLeaseCloseDuringOutageKeepsTheConnection() throws Exception { // Returning a strict lease discards its open transaction. During an outage that @@ -608,6 +833,44 @@ public void testStrictCloseWithoutFlushDiscardsTransaction() throws Exception { }); } + @Test + public void testStrictCloseWithoutFlushWarnsOfDiscardedRows() throws Exception { + // A sender with no auto-flush trigger used to commit on close(). Now close() + // discards its open transaction, and says so once, counting the rows already + // sent with those still buffered. A committed transaction leaves nothing to + // discard, and an aborted one already failed every flush. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_rows=off;auto_flush_interval=off;")) { + writeRows(s, "t", 0, 1_000); + Assert.assertTrue(h.awaitFrames(2)); + } + Assert.assertEquals(Collections.singletonList("close()" + DISCARD_WARNING), discardWarnings(log)); + clearLog(log); + + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush_rows=off;auto_flush_interval=off;")) { + writeRows(s, "t", 0, 1_000); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;")) { + writeRows(s, "t", 0, 1_000); + // Frames went out and the server cannot roll them back. + s.reset(); + assertAborted(s::flush); + } + Assert.assertEquals(Collections.emptyList(), discardWarnings(log)); + Assert.assertEquals(1, h.commitFrames()); + } + } finally { + releaseSenderLog(log); + } + }); + } + @Test public void testStrictCloseWithoutFlushRollsBackWhenServerSupportsIt() throws Exception { assertMemoryLeak(() -> { @@ -772,6 +1035,60 @@ public void testTransactionalDefaultTriggersSplitWideRows() throws Exception { }); } + // Captures what QwpWebSocketSender logs, so a test can assert its warnings. + private static ListAppender captureSenderLog() { + ListAppender appender = new ListAppender<>(); + appender.start(); + senderLogger().addAppender(appender); + return appender; + } + + // The appender appends under its own lock, and I/O threads log into it too. + private static void clearLog(ListAppender log) { + synchronized (log) { + log.list.clear(); + } + } + + // The discarded-transaction warnings logged since the last clearLog(). + private static List discardWarnings(ListAppender log) { + List warnings = new ArrayList<>(); + synchronized (log) { + for (ILoggingEvent e : log.list) { + String message = e.getFormattedMessage(); + if (e.getLevel() == Level.WARN && message.contains("discarded an uncommitted transaction")) { + warnings.add(message); + } + } + } + return warnings; + } + + // Every way to configure a WebSocket sender with no auto-flush trigger, applied + // on top of a config string. + private static List noTriggerSpellings() { + List spellings = new ArrayList<>(); + spellings.add(new NoTriggerSpelling("auto_flush=off", c -> Sender.builder(c + "auto_flush=off;"))); + spellings.add(new NoTriggerSpelling("auto_flush_rows=off;auto_flush_interval=off", + c -> Sender.builder(c + "auto_flush_rows=off;auto_flush_interval=off;"))); + spellings.add(new NoTriggerSpelling("auto_flush_interval=off;auto_flush_rows=off", + c -> Sender.builder(c + "auto_flush_interval=off;auto_flush_rows=off;"))); + spellings.add(new NoTriggerSpelling("disableAutoFlush()", + c -> Sender.builder(c).disableAutoFlush())); + spellings.add(new NoTriggerSpelling("autoFlushRows(0).autoFlushIntervalMillis(Integer.MAX_VALUE)", + c -> Sender.builder(c).autoFlushRows(0).autoFlushIntervalMillis(Integer.MAX_VALUE))); + return spellings; + } + + private static void releaseSenderLog(ListAppender log) { + senderLogger().detachAppender(log); + log.stop(); + } + + private static Logger senderLogger() { + return (Logger) LoggerFactory.getLogger(QwpWebSocketSender.class); + } + private static void assertAborted(ThrowingRunnable r) { try { r.run(); @@ -781,6 +1098,16 @@ private static void assertAborted(ThrowingRunnable r) { } } + // Build-time validation runs before the sender connects, so no server is needed. + private static void assertConfigRejected(String config, String expectedMessage) { + try { + Sender.fromConfig("ws::addr=localhost:1;" + config).close(); + Assert.fail(config + " must be rejected"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedMessage); + } + } + private static void assertFramesWithinCap(TxnServer h, int cap) { for (TxnFrame f : h.frames) { Assert.assertTrue("frame of " + f.size + " bytes exceeds " + cap, f.size <= cap); @@ -896,6 +1223,20 @@ private interface ThrowingRunnable { void run(); } + private static final class NoTriggerSpelling { + final String name; + private final Function builder; + + NoTriggerSpelling(String name, Function builder) { + this.name = name; + this.builder = builder; + } + + Sender build(String config) { + return builder.apply(config).build(); + } + } + private static final class TxnFrame { // Index of the connection the frame arrived on, in connection order. final int connection; From a6ed99894fc967e868a1793407c4578449fb82f1 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Thu, 12 Nov 2026 17:57:59 +0000 Subject: [PATCH 6/7] Keep refused rows out of strict transactions A strict transaction refused a row inconsistently. At the server's qwp.max.uncommitted.rows the row that hit the limit stayed out of the transaction, but at the client's sf_max_total_bytes the row whose auto-flush found no room stayed in it, although at() threw all the same. A caller could not tell whether flush() would commit that row. The auto-flush that finds no room now keeps its rows pending and returns, since the room kept for the commit holds them, and the next row throws. A row whose at() throws TransactionTooLargeException is never part of the transaction, at either limit. flush() still throws when the client buffer cannot hold it, and sends nothing. The errors and the exception's javadoc name the setting to raise instead of suggesting code that commits part of the transaction. A row refused at the client buffer reports the size the transaction would have reached, as a refused flush does, rather than the bytes already sent, which read 0 when nothing had been sent yet. --- .../main/java/io/questdb/client/Sender.java | 6 +- .../qwp/client/QwpWebSocketSender.java | 109 ++++++++------ .../client/TransactionTooLargeException.java | 19 +-- .../qwp/client/QwpLargeTransactionTest.java | 137 ++++++++++++++++-- 4 files changed, 208 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c65f15412..b7c511f91 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -2824,11 +2824,11 @@ private LineSenderBuilder requestDurableAckTiers(int tiers) { * sends rows to the server without committing them and {@code flush()} * commits, so a transaction can be any size. In addition the transaction * never commits in parts: a row that would take a table past the server's - * {@code qwp.max.uncommitted.rows}, or a transaction that outgrows + * {@code qwp.max.uncommitted.rows}, or the transaction past * {@code sf_max_total_bytes}, throws * {@link io.questdb.client.cutlass.qwp.client.TransactionTooLargeException} - * and leaves the transaction open, and {@code close()} without - * {@code flush()} discards the transaction instead of committing it. + * and is not added, and {@code close()} without {@code flush()} discards the + * transaction instead of committing it. * A sender with no auto-flush trigger (see {@link #disableAutoFlush()}) is * in this mode unless {@link #transactional(boolean)} selects the mode. * WebSocket transport only. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 624d2bc80..a61661f4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -718,7 +718,9 @@ public class QwpWebSocketSender implements Sender { private boolean txnAborted; // Strict mode only: set when the client buffer cannot take another deferred // frame of the open transaction and still commit it. Rejects further rows - // until flush() or rollback() ends the transaction. + // until flush() or rollback() ends the transaction. The rows accepted before + // it stay in the transaction, including the one whose auto-flush found no + // room: the room kept for the commit holds them. private boolean txnBufferFull; // Strict mode only: simulation of how the ring packs the open transaction's // frames, mirroring MmapSegment's append rule. txnSimSegments counts the @@ -732,9 +734,13 @@ public class QwpWebSocketSender implements Sender { // Logs the reset()-cannot-roll-back warning once per sender. private boolean isResetRollbackWarned; // Strict mode only: payload bytes of the open transaction's frames on the - // ring, for the error message. Reset with the transaction. + // ring. Reset with the transaction. private long txnBytes; - // Scratch for ensureStrictTransactionRoom: the frame sizes a flush will + // Strict mode only: the size the open transaction would have reached with the + // flush the client buffer last refused, for the error messages. Reset with the + // transaction. + private long txnRefusedBytes; + // Scratch for hasStrictTransactionRoom: the frame sizes a flush will // publish. Producer thread only. private final IntList strictFrameSizes = new IntList(); @@ -1880,7 +1886,7 @@ private void close0(boolean[] restoreInterrupt) { } } else { try { - flushPendingRows(deferCommit); + flushPendingRows(deferCommit, false); } catch (BatchTooLargeForCapException e) { resetTableBuffersAfterFlush(); terminalError = captureCloseError(terminalError, e); @@ -2375,7 +2381,7 @@ public long flushAndGetSequence() { // is published in its mmap'd segment. PERIODIC stable-storage barriers // run independently in the manager. No processingCount to drain and no // awaitPendingAcks here; just surface any I/O thread error. - flushPendingRows(deferCommit); + flushPendingRows(deferCommit, false); if (!deferCommit && hasDeferredMessages) { sendCommitMessage(); } @@ -3823,11 +3829,12 @@ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) { * Strict transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames * like {@link #setTransactional(boolean) transactional} mode, only an explicit * flush commits, and the open transaction never commits in parts. A row that - * would push a table past the server's {@code qwp.max.uncommitted.rows}, or a - * frame the client buffer could not hold alongside the commit, throws - * {@link TransactionTooLargeException} instead. {@link #close()} discards an - * uncommitted transaction rather than committing it. Enabling it enables - * transactional mode; disabling it leaves transactional mode on. + * would push a table past the server's {@code qwp.max.uncommitted.rows}, or the + * open transaction past what the client buffer can hold alongside the commit, + * throws {@link TransactionTooLargeException} instead and is not added. + * {@link #close()} discards an uncommitted transaction rather than committing + * it. Enabling it enables transactional mode; disabling it leaves transactional + * mode on. */ public void setStrictTransactional(boolean strict) { this.strictTransactional = strict; @@ -5475,8 +5482,12 @@ private void ensureNoInProgressRow() { * @param deferCommit when true, the message carries FLAG_DEFER_COMMIT * so the server appends rows but does not commit. * Used by auto-flush in transactional mode. + * @param isAutoFlush true when a row triggered the flush: when the client + * buffer cannot take it, a strict transaction keeps the + * rows pending instead of throwing, and refuses the next + * row. */ - private void flushPendingRows(boolean deferCommit) { + private void flushPendingRows(boolean deferCommit, boolean isAutoFlush) { if (pendingRowCount <= 0) { return; } @@ -5523,8 +5534,19 @@ private void flushPendingRows(boolean deferCommit) { // A strict transaction must be able to commit as one: refuse, before any // frame of this flush reaches the ring, a flush the client buffer could not // hold together with the commit that has to follow it. - if (strictTransactional && cursorEngine != null) { - ensureStrictTransactionRoom(cap, deltaBaseline, messageSize, deferCommit); + if (strictTransactional && cursorEngine != null + && !hasStrictTransactionRoom(cap, deltaBaseline, messageSize, deferCommit)) { + if (deferCommit) { + txnBufferFull = true; + if (isAutoFlush) { + // The rows stay pending, the one that triggered this flush + // included, and the room kept for the commit holds them. The + // next row is refused, so a refused row is never part of the + // transaction, as at the server's row limit. + return; + } + } + throw transactionTooLargeForClientBuffer("so the flush was not sent"); } // Full-dict over-cap fallback. A full-dictionary frame carries the whole @@ -7471,11 +7493,7 @@ private void checkTransactionAcceptsRow() { return; } if (txnBufferFull) { - throw new TransactionTooLargeException("transaction too large for the client buffer") - .put(" [transactionBytes=").put(txnBytes) - .put("]; the buffer cannot hold another frame of the open transaction and still " - + "commit it, so the row was not added. Commit the transaction with flush() or " - + "discard it with rollback() before adding rows"); + throw transactionTooLargeForClientBuffer("so the row was not added"); } long maxRows = serverMaxUncommittedRows; if (maxRows <= 0) { @@ -7491,8 +7509,7 @@ private void checkTransactionAcceptsRow() { .put(", maxUncommittedRows=").put(maxRows) .put("]; the server commits a table's rows on its own once qwp.max.uncommitted.rows " + "of them are uncommitted, which would split the transaction, so the row was " - + "not added. Commit the transaction with flush() or discard it with rollback(), " - + "then send smaller transactions or raise qwp.max.uncommitted.rows on the server"); + + "not added. Raise qwp.max.uncommitted.rows on the server"); } } @@ -7508,26 +7525,27 @@ private void clearTransactionState() { txnTables.setPos(0); txnBufferFull = false; txnBytes = 0; + txnRefusedBytes = 0; txnSimSegments = 0; txnSimRemaining = 0; } /** - * Strict mode: refuses a flush whose frames, together with the open - * transaction's frames already on the ring and (for a deferred flush) room - * for the commit that must follow, would not fit the ring under - * {@code sf_max_total_bytes}. The server acks none of a transaction's frames - * before its commit, so the ring cannot trim them; a transaction that does - * not fit would block its own commit. + * Strict mode: whether the ring under {@code sf_max_total_bytes} can hold a + * flush's frames together with the open transaction's frames already on it + * and, for a deferred flush, room for the commit that must follow. The server + * acks none of a transaction's frames before its commit, so the ring cannot + * trim them; a transaction that does not fit would block its own commit. *

    * Mirrors how the ring packs frames: frames never span segments, and a frame * that does not fit the current segment opens the next one. The flush's * frame sizes are derived the way flushPendingRows and the split publish * them; dictionary chunks of the full-dictionary fallback count as full * frames. Runs before anything of the flush is published, so a refusal - * leaves the batch pending and the ring untouched. + * leaves the batch pending and the ring untouched. On a refusal it records + * the size the transaction would have reached in {@link #txnRefusedBytes}. */ - private void ensureStrictTransactionRoom(int cap, int deltaBaseline, int messageSize, boolean deferCommit) { + private boolean hasStrictTransactionRoom(int cap, int deltaBaseline, int messageSize, boolean deferCommit) { CursorSendEngine engine = cursorEngine; long segmentPayload = segmentPayloadCapacity(); strictFrameSizes.clear(); @@ -7583,21 +7601,12 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { remaining = Math.max(0, remaining - reserve - MmapSegment.FRAME_HEADER_SIZE); } } - long ringCapacity = engine.ringCapacityBytes(); - long maxSegments = Math.max(2, ringCapacity / engine.segmentSizeBytes()); + long maxSegments = Math.max(2, engine.ringCapacityBytes() / engine.segmentSizeBytes()); if (segments > maxSegments) { - if (deferCommit) { - txnBufferFull = true; - } - throw new TransactionTooLargeException("transaction too large for the client buffer") - .put(" [transactionBytes=").put(bytes) - .put(", ringCapacityBytes=").put(ringCapacity) - .put(", segmentBytes=").put(engine.segmentSizeBytes()) - .put("]; the client holds an uncommitted transaction until the server commits it, so " - + "the transaction must fit sf_max_total_bytes. This flush was not sent: commit " - + "the transaction with flush() or discard it with rollback(), then send smaller " - + "transactions, raise sf_max_total_bytes, or set sf_dir to keep it on disk"); + txnRefusedBytes = bytes; + return false; } + return true; } /** @@ -7681,6 +7690,22 @@ private LineSenderException transactionAbortedError() { + "the sender to discard them; the server rolls them back when the connection closes"); } + /** + * The error for a strict transaction the client buffer cannot hold: the size + * the transaction would have reached, the buffer, what was refused, and the + * settings that make room. + */ + private LineSenderException transactionTooLargeForClientBuffer(String outcome) { + CursorSendEngine engine = cursorEngine; + return new TransactionTooLargeException("transaction too large for the client buffer") + .put(" [transactionBytes=").put(txnRefusedBytes) + .put(", ringCapacityBytes=").put(engine.ringCapacityBytes()) + .put(", segmentBytes=").put(engine.segmentSizeBytes()) + .put("]; the client holds a strict transaction until the server commits it, and " + + "sf_max_total_bytes cannot hold this one, ").put(outcome) + .put(". Raise sf_max_total_bytes, or set sf_dir to keep the transaction on disk"); + } + /** * Strict mode commits only on flush(), so closing the sender or returning a * pooled lease discards the open transaction. Logs it when the transaction holds @@ -7752,7 +7777,7 @@ private void sendRow() { currentTableBufferSnapshotBytes = bufferedNow; if (shouldAutoFlush()) { - flushPendingRows(transactional); + flushPendingRows(transactional, true); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java index 2c2b0fc8f..c7cc37f81 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/TransactionTooLargeException.java @@ -28,19 +28,20 @@ /** * A strict transaction ({@code transaction=strict}, or a sender with no auto-flush - * trigger such as {@code auto_flush=off}, on the WebSocket transport) grew past a + * trigger such as {@code auto_flush=off}, on the WebSocket transport) outgrew a * limit that would stop it from committing as one transaction: the server's * per-table {@code qwp.max.uncommitted.rows}, past which the server commits the * rows on its own, or the client's {@code sf_max_total_bytes}, which must hold the - * whole uncommitted transaction until the server commits it. + * whole uncommitted transaction until the server commits it. To send transactions + * of that size, raise the limit the message names: {@code qwp.max.uncommitted.rows} + * on the server, or {@code sf_max_total_bytes} on the client, whose default is + * larger with {@code sf_dir} set. *

    - * Thrown before anything that would cross the limit reaches the server, and the - * transaction stays open: the caller can still commit the rows accepted so far - * with {@code flush()}, or discard them with {@code rollback()}. A row rejected at - * the per-table limit is not part of the transaction. When the client buffer is - * the limit, the row being added when the buffer filled up stays in the - * transaction, and every further row is rejected until {@code flush()} or - * {@code rollback()} ends the transaction. + * {@code at()} and {@code atNow()} throw it for the row that would take the + * transaction past either limit, and that row is not part of the transaction. + * {@code flush()} throws it when the client buffer cannot hold the flush, and + * sends nothing. Either way nothing past the limit reaches the server, and the + * transaction keeps the rows accepted before the error. */ public class TransactionTooLargeException extends LineSenderException { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java index 8393c2f6a..d42307354 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -67,6 +67,9 @@ */ public class QwpLargeTransactionTest { private static final int CAP = 32 * 1024; + // The settings a refusal at the client buffer names. + private static final String CLIENT_BUFFER_REMEDY = "Raise sf_max_total_bytes, or set sf_dir to keep the " + + "transaction on disk"; // What closing a sender or returning a pooled lease logs when it discards an open // strict transaction of 1000 rows, behind the action that discarded it. private static final String DISCARD_WARNING = " discarded an uncommitted transaction of 1000 rows: a strict " @@ -905,8 +908,7 @@ public void testStrictRowLimitFromServer() throws Exception { writeRows(s, "a", 99, 1); Assert.fail("the 100th row of a table must be refused"); } catch (TransactionTooLargeException e) { - TestUtils.assertContains(e.getMessage(), - "transaction too large for the server [table=a, rows=100, maxUncommittedRows=100]"); + Assert.assertEquals(serverRowLimitError("a", 100), e.getMessage()); } // Other tables keep their own count. writeRows(s, "b", 0, 99); @@ -939,7 +941,7 @@ public void testStrictRowLimitDefaultsWhenNotAdvertised() throws Exception { writeRows(s, "a", 9, 1); Assert.fail("the 10th row must be refused"); } catch (TransactionTooLargeException e) { - TestUtils.assertContains(e.getMessage(), "maxUncommittedRows=10"); + Assert.assertEquals(serverRowLimitError("a", 10), e.getMessage()); } ws.applyServerTransactionLimits(0, false); // 0 means "not advertised": the server default applies. @@ -950,10 +952,85 @@ public void testStrictRowLimitDefaultsWhenNotAdvertised() throws Exception { }); } + @Test + public void testStrictCommitTooLargeForClientBufferIsNotSent() throws Exception { + // With auto_flush_bytes=off on top of auto_flush=off, the whole transaction + // waits for flush(), which sends one frame per table. Ten tables of ~28 KB need five 64 KiB + // segments, and sf_max_total_bytes holds four: flush() sends nothing. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + "auto_flush=off;auto_flush_bytes=off;" + + "sf_max_segment_bytes=64k;sf_max_total_bytes=256k;")) { + writeTables(s, 0, 10); + long refusedBytes = -1; + try { + s.flush(); + Assert.fail("the client buffer cannot hold the commit"); + } catch (TransactionTooLargeException e) { + refusedBytes = assertClientBufferRefusal(e, 262_144, "so the flush was not sent"); + } + Assert.assertEquals("nothing was sent", 0, h.frames.size()); + s.rollback(); + // The same rows, one table per transaction: the refusal reported + // what these frames add up to. + for (int t = 0; t < 10; t++) { + writeTables(s, t, 1); + s.flush(); + } + Assert.assertTrue(s.drain(5_000)); + Assert.assertEquals(10, h.commitFrames()); + Assert.assertEquals(refusedBytes, frameBytes(h)); + } + } + }); + } + + @Test + public void testStrictDeferredFlushTooLargeForClientBufferIsNotSent() throws Exception { + // setDeferCommit(true) makes flush() send rows without committing them. A + // strict transaction keeps room for its commit next to such a frame. With + // frames as large as a segment that takes four segments, and + // sf_max_total_bytes holds three: flush() sends nothing, and the transaction + // takes no further rows. The commit itself fits. + assertMemoryLeak(() -> { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, 64 * 1024)) { + long refusedBytes = -1; + try (QwpWebSocketSender s = (QwpWebSocketSender) Sender.fromConfig(conf(server) + + "transaction=strict;sf_max_segment_bytes=64k;sf_max_total_bytes=192k;")) { + s.setDeferCommit(true); + writeRows(s, "t", 0, 1); + try { + s.flush(); + Assert.fail("the client buffer cannot hold a deferred frame and the commit"); + } catch (TransactionTooLargeException e) { + refusedBytes = assertClientBufferRefusal(e, 196_608, "so the flush was not sent"); + } + Assert.assertEquals("nothing was sent", 0, h.frames.size()); + try { + writeRows(s, "t", 1, 1); + Assert.fail("the transaction must take no further rows"); + } catch (TransactionTooLargeException e) { + Assert.assertEquals(refusedBytes, + assertClientBufferRefusal(e, 196_608, "so the row was not added")); + } + s.setDeferCommit(false); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + // The commit carries the row the refused flush held, and only that row. + Assert.assertEquals(1, h.frames.size()); + Assert.assertTrue(h.frames.get(0).isCommit()); + Assert.assertEquals(refusedBytes, h.frames.get(0).size); + } + }); + } + @Test public void testStrictTransactionTooLargeForClientBufferFailsFast() throws Exception { // 64 KiB segments, 256 KiB budget: four segments. A strict transaction keeps - // room for its commit, so it must refuse the flush that would take that room + // room for its commit, so once its frames fill the rest it must refuse rows // instead of blocking for sf_append_deadline_millis. assertMemoryLeak(() -> { TxnServer h = new TxnServer(); @@ -961,29 +1038,35 @@ public void testStrictTransactionTooLargeForClientBufferFailsFast() throws Excep try (Sender s = Sender.fromConfig(conf(server) + "transaction=strict;" + "sf_max_segment_bytes=64k;sf_max_total_bytes=256k;sf_append_deadline_millis=30000;")) { long start = System.nanoTime(); - int row = 0; + long refusedBytes = -1; try { - for (; row < 100_000; row++) { + for (int row = 0; row < 100_000; row++) { s.table("t").longColumn("v", row).stringColumn("s", PAD).atNow(); } Assert.fail("a transaction larger than the client buffer must be refused"); } catch (TransactionTooLargeException e) { - TestUtils.assertContains(e.getMessage(), "transaction too large for the client buffer"); + // As at the server's row limit, the row that hit the limit is not + // in the transaction. + refusedBytes = assertClientBufferRefusal(e, 262_144, "so the row was not added"); } Assert.assertTrue("must fail fast, not wait for the append deadline", System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); - // The row being added stays in the transaction; further rows are refused. + // Further rows are refused the same way. try { s.table("t").longColumn("v", -1).stringColumn("s", PAD).atNow(); Assert.fail("a full transaction must refuse rows"); } catch (TransactionTooLargeException e) { - TestUtils.assertContains(e.getMessage(), "the row was not added"); + Assert.assertEquals(refusedBytes, + assertClientBufferRefusal(e, 262_144, "so the row was not added")); } // The reserved room lets the transaction commit. s.flush(); Assert.assertTrue(s.drain(5_000)); Assert.assertTrue(h.frames.get(h.frames.size() - 1).isCommit()); Assert.assertEquals(1, h.commitFrames()); + // The refusal reported the size of the transaction that committed: + // the rows accepted before it, the last of which filled the buffer. + Assert.assertEquals(refusedBytes, frameBytes(h)); // And the next transaction starts with an empty buffer. writeRows(s, "t", 0, 100); s.flush(); @@ -1098,6 +1181,20 @@ private static void assertAborted(ThrowingRunnable r) { } } + // Asserts the error of a strict transaction the client buffer cannot hold, with + // 64 KiB segments, and returns the transaction size it reports. + private static long assertClientBufferRefusal(TransactionTooLargeException e, long ringCapacityBytes, String outcome) { + String message = e.getMessage(); + String prefix = "transaction too large for the client buffer [transactionBytes="; + Assert.assertTrue(message, message.startsWith(prefix)); + long transactionBytes = Long.parseLong(message.substring(prefix.length(), message.indexOf(','))); + Assert.assertTrue(message, transactionBytes > 0); + Assert.assertEquals(prefix + transactionBytes + ", ringCapacityBytes=" + ringCapacityBytes + + ", segmentBytes=65536]; the client holds a strict transaction until the server commits it, " + + "and sf_max_total_bytes cannot hold this one, " + outcome + ". " + CLIENT_BUFFER_REMEDY, message); + return transactionBytes; + } + // Build-time validation runs before the sender connects, so no server is needed. private static void assertConfigRejected(String config, String expectedMessage) { try { @@ -1212,12 +1309,34 @@ private static TestWebSocketServer start(TxnServer h, int cap, int port, boolean return server; } + private static long frameBytes(TxnServer h) { + long bytes = 0; + for (TxnFrame f : h.frames) { + bytes += f.size; + } + return bytes; + } + + private static String serverRowLimitError(String table, long maxUncommittedRows) { + return "transaction too large for the server [table=" + table + ", rows=" + maxUncommittedRows + + ", maxUncommittedRows=" + maxUncommittedRows + "]; the server commits a table's rows on its " + + "own once qwp.max.uncommitted.rows of them are uncommitted, which would split the transaction, " + + "so the row was not added. Raise qwp.max.uncommitted.rows on the server"; + } + private static void writeRows(Sender s, String table, int from, int count) { for (int i = from, n = from + count; i < n; i++) { s.table(table).longColumn("v", i).stringColumn("s", PAD).atNow(); } } + // ~28 KB in each of count tables, from table t: each fits a 32 KiB frame. + private static void writeTables(Sender s, int first, int count) { + for (int t = first, n = first + count; t < n; t++) { + writeRows(s, "t" + t, 0, 280); + } + } + @FunctionalInterface private interface ThrowingRunnable { void run(); From 83eec1e81f66d06b7dac1dba061f6370a86f7d75 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Thu, 12 Nov 2026 19:13:59 +0000 Subject: [PATCH 7/7] Commit transaction=on in parts at client buffer A transaction=on transaction holds its frames in the store-and-forward buffer until the server commits it, and the server acks none of them before that, so the buffer cannot trim them. The row whose frame no longer fit blocked for sf_append_deadline_millis (30 s by default) and then threw, and close() then committed the frames already sent and dropped the rest, under an error that did not say so. Before a flush that does not fit beside the open transaction's frames, the sender now commits those frames with a bare commit frame and goes on, as the server already does at qwp.max.uncommitted.rows. Every deferred flush leaves room for that commit frame, so it never waits for the buffer to drain. The sender logs a warning on the first part of each transaction it commits this way, and rollback() and reset() warn that the parts already committed stay. Strict mode still refuses such transactions. The client-buffer simulation strict mode uses now tracks transaction=on transactions too. A single flush larger than the whole buffer still blocks, as before; only auto_flush_bytes=off builds one. --- .../main/java/io/questdb/client/Sender.java | 12 +- .../qwp/client/QwpWebSocketSender.java | 200 +++++++++++++----- .../qwp/client/QwpLargeTransactionTest.java | 183 +++++++++++++++- 3 files changed, 340 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index b7c511f91..fb21442f6 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -795,6 +795,10 @@ default void resetSymbolDictionary() { * later rows never commit the discarded ones, even across reconnects. Call * {@link #drain(long)} to wait until the server confirms. *
    + * A {@code transaction=on} transaction that already committed in parts (see + * {@link LineSenderBuilder#transactional(boolean)}) keeps the parts it committed; + * this discards only the rest. + *
    * Discarding rows already sent needs a server that supports QWP rollback. When * the server does not, this method throws and aborts the transaction: every * later flush throws too, and closing the sender discards the rows, because the @@ -2854,8 +2858,12 @@ public LineSenderBuilder strictTransactional(boolean enabled) { * with {@code FLAG_DEFER_COMMIT}; only an explicit {@code flush()} * (or {@code close()}) triggers the server-side WAL commit. This allows * accumulating datasets larger than the server's recv buffer while - * committing atomically per table. The server commits a table on its own - * once {@code qwp.max.uncommitted.rows} of its rows are uncommitted; use + * committing atomically per table. A transaction too large for either side + * commits in parts: the server commits a table on its own once + * {@code qwp.max.uncommitted.rows} of its rows are uncommitted, and the + * sender, which holds the transaction in {@code sf_max_total_bytes} until + * the server commits it, commits the rows it already sent once the + * transaction outgrows that buffer, and logs a warning. Use * {@link #strictTransactional(boolean)} to fail instead. *

    * A sender with no auto-flush trigger (see {@link #disableAutoFlush()}) is diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index a61661f4c..7e6f01a6f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -182,6 +182,12 @@ public class QwpWebSocketSender implements Sender { // Default for symbol_dict_reset_threshold: distinct-symbol count that // triggers a recycle once symbol_dict_reset is on. public static final int DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS = 100_000; + // Largest bare commit frame: the header and an empty symbol-dictionary delta, + // whose start id takes at most five varint bytes and whose count one. The + // rollback control frame is the header alone. A transaction=on transaction keeps + // room for one such frame in the client buffer, so the frame that ends it never + // waits for the buffer to drain. + private static final int COMMIT_FRAME_MAX_BYTES = QwpConstants.HEADER_SIZE + 6; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB // Frames of at most the frame cap that a strict transaction keeps room for in @@ -722,27 +728,31 @@ public class QwpWebSocketSender implements Sender { // it stay in the transaction, including the one whose auto-flush found no // room: the room kept for the commit holds them. private boolean txnBufferFull; - // Strict mode only: simulation of how the ring packs the open transaction's + // Transactional mode: simulation of how the ring packs the open transaction's // frames, mirroring MmapSegment's append rule. txnSimSegments counts the // segments the transaction's frames occupy (0 = no frame yet) and // txnSimRemaining is the raw space left in the last of them. private long txnSimRemaining; private int txnSimSegments; + // transaction=on: rows of the open transaction the sender already committed + // because the transaction outgrew the client buffer, 0 when it did not. Reset + // with the transaction; committing a part keeps it. + private long txnRowsCommittedInParts; // Tables with rows in deferred frames of the open transaction, so a commit // or rollback resets only those tables' counters. Producer thread only. private final ObjList txnTables = new ObjList<>(); // Logs the reset()-cannot-roll-back warning once per sender. private boolean isResetRollbackWarned; - // Strict mode only: payload bytes of the open transaction's frames on the + // Transactional mode: payload bytes of the open transaction's frames on the // ring. Reset with the transaction. private long txnBytes; // Strict mode only: the size the open transaction would have reached with the // flush the client buffer last refused, for the error messages. Reset with the // transaction. private long txnRefusedBytes; - // Scratch for hasStrictTransactionRoom: the frame sizes a flush will - // publish. Producer thread only. - private final IntList strictFrameSizes = new IntList(); + // Scratch for hasTransactionRoom: the frame sizes a flush will publish. + // Producer thread only. + private final IntList transactionFrameSizes = new IntList(); private QwpWebSocketSender( List endpoints, @@ -3112,6 +3122,16 @@ public long getTotalServerErrors() { return counters.serverErrors.get(); } + /** + * Rows of the open transaction the sender already committed because the + * transaction outgrew the client buffer (see {@link #setTransactional}); 0 + * when it did not. + */ + @TestOnly + public long getTxnRowsCommittedInPartsForTesting() { + return txnRowsCommittedInParts; + } + /** * Adds an INT column value to the current row. * @@ -3401,7 +3421,7 @@ public void reset() { discardPendingRows(); if (transactional && hasDeferredMessages && !txnAborted) { if (serverRollbackEnabled) { - rollbackOpenTransaction(); + rollbackOpenTransaction("reset()"); } else if (strictTransactional) { txnAborted = true; } else if (!isResetRollbackWarned) { @@ -3426,6 +3446,10 @@ public void reset() { * later, on every connection and after a restart, so nothing sent later can * commit the discarded rows. Call {@link #drain(long)} to wait for the server * to confirm. + *

    + * A transaction=on transaction that already committed in parts, at the + * client buffer or the server's {@code qwp.max.uncommitted.rows}, keeps the + * parts it committed; this discards only the rest. * * @throws LineSenderException when frames of the open transaction were already * sent and the server does not support QWP rollback @@ -3448,7 +3472,7 @@ public void rollback() { txnAborted = true; throw transactionAbortedError(); } - rollbackOpenTransaction(); + rollbackOpenTransaction("rollback()"); } /** @@ -3479,7 +3503,7 @@ public boolean discardTransactionForReuse() { txnAborted = true; return false; } - rollbackOpenTransaction(); + rollbackOpenTransaction("returning the sender to the pool"); return true; } @@ -3545,8 +3569,12 @@ private void publishRollbackFrame() { * {@code CursorWebSocketSendLoop.retireRollbackFrame}). The same holds for * frames replayed after a restart or by a background drainer, which use the * same loop. + *

    + * The rows a transaction=on transaction committed in parts stay committed; + * {@code action} names the call in the warning that says so. */ - private void rollbackOpenTransaction() { + private void rollbackOpenTransaction(String action) { + long committedRows = txnRowsCommittedInParts; publishRollbackFrame(); hasDeferredMessages = false; // The rollback frame gets an OK ack covering the transaction's frames, or the @@ -3554,6 +3582,11 @@ private void rollbackOpenTransaction() { // close-time drains may wait for it like for a commit. lastCommitBoundaryFsn = cursorEngine.publishedFsn(); clearTransactionState(); + if (committedRows > 0) { + LOG.warn("{} discarded the rows of the open transaction sent since the sender last committed part of " + + "it, but the {} rows it committed when the transaction outgrew the client buffer stay " + + "committed", action, committedRows); + } } /** @@ -3845,8 +3878,11 @@ public void setStrictTransactional(boolean strict) { /** * Transactional mode: auto-flush ships rows as FLAG_DEFER_COMMIT frames and - * only an explicit flush (or close) commits. Disabling it also disables - * {@link #setStrictTransactional(boolean) strict} mode. + * only an explicit flush (or close) commits. A transaction that outgrows the + * client buffer commits in parts: before a flush that does not fit beside the + * frames already sent, the sender commits those frames and logs a warning. + * Disabling it also disables {@link #setStrictTransactional(boolean) strict} + * mode, which refuses rows instead. */ public void setTransactional(boolean transactional) { this.transactional = transactional; @@ -5531,22 +5567,37 @@ private void flushPendingRows(boolean deferCommit, boolean isAutoFlush) { int combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); int messageSize = encoder.finishMessage(); - // A strict transaction must be able to commit as one: refuse, before any - // frame of this flush reaches the ring, a flush the client buffer could not - // hold together with the commit that has to follow it. - if (strictTransactional && cursorEngine != null - && !hasStrictTransactionRoom(cap, deltaBaseline, messageSize, deferCommit)) { - if (deferCommit) { - txnBufferFull = true; - if (isAutoFlush) { - // The rows stay pending, the one that triggered this flush - // included, and the room kept for the commit holds them. The - // next row is refused, so a refused row is never part of the - // transaction, as at the server's row limit. - return; + // The client buffer holds a transaction's frames until the server commits it, + // so it cannot trim them to make room. Before any frame of this flush reaches + // the ring, check that the buffer holds the flush beside them. + if (transactional && cursorEngine != null) { + if (strictTransactional) { + // A strict transaction must be able to commit as one: refuse a flush + // the client buffer could not hold together with the commit that has + // to follow it. + if (!hasTransactionRoom(cap, deltaBaseline, messageSize, deferCommit)) { + if (deferCommit) { + txnBufferFull = true; + if (isAutoFlush) { + // The rows stay pending, the one that triggered this flush + // included, and the room kept for the commit holds them. The + // next row is refused, so a refused row is never part of the + // transaction, as at the server's row limit. + return; + } + } + throw transactionTooLargeForClientBuffer("so the flush was not sent"); } + } else if (txnSimSegments > 0 && !hasTransactionRoom(cap, deltaBaseline, messageSize, deferCommit)) { + // transaction=on commits in parts instead: commit the frames already + // sent, which lets the buffer trim them, and send this flush as the + // start of the next part. A flush that does not fit even alone goes + // out as it is. + commitTransactionPart(); + // The commit frame went through the encoder: encode the batch again. + combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); + messageSize = encoder.finishMessage(); } - throw transactionTooLargeForClientBuffer("so the flush was not sent"); } // Full-dict over-cap fallback. A full-dictionary frame carries the whole @@ -7430,10 +7481,10 @@ private void sealAndSwapBuffer() { // (durable mmap append, returns once published). try { toSend.markSending(); - // Strict mode tracks how the open transaction's frames pack into - // segments; the first frame's placement depends on the active + // Transactional mode tracks how the open transaction's frames pack + // into segments; the first frame's placement depends on the active // segment's room before the append. - boolean isTxnFrame = strictTransactional + boolean isTxnFrame = transactional && isTransactionFrame(toSend.getBufferPtr(), toSend.getBufferPos()); long activeRemaining = isTxnFrame && txnSimSegments == 0 ? cursorEngine.activeSegment().capacityRemaining() : 0; @@ -7526,31 +7577,62 @@ private void clearTransactionState() { txnBufferFull = false; txnBytes = 0; txnRefusedBytes = 0; + txnRowsCommittedInParts = 0; txnSimSegments = 0; txnSimRemaining = 0; } /** - * Strict mode: whether the ring under {@code sf_max_total_bytes} can hold a - * flush's frames together with the open transaction's frames already on it - * and, for a deferred flush, room for the commit that must follow. The server - * acks none of a transaction's frames before its commit, so the ring cannot - * trim them; a transaction that does not fit would block its own commit. + * transaction=on: commits the frames of the open transaction already sent, + * because the next flush does not fit beside them in the client buffer, which + * cannot trim them before the server commits them. The transaction goes on + * with the rows sent from here on, so it commits in parts, as it does when the + * server commits a table at {@code qwp.max.uncommitted.rows}. Logs the first + * part each transaction commits this way. + */ + private void commitTransactionPart() { + long rows = sentTransactionRows(); + long bytes = txnBytes; + long committedRows = txnRowsCommittedInParts; + sendCommitMessage(); + // sendCommitMessage() forgot the transaction, but it goes on. + txnRowsCommittedInParts = committedRows + rows; + if (committedRows == 0) { + LOG.warn("the open transaction outgrew the client buffer, so the sender committed the {} rows it had " + + "already sent, and the rest of the transaction commits separately [transactionBytes={}, " + + "ringCapacityBytes={}]. A transaction=on transaction commits in parts when " + + "sf_max_total_bytes cannot hold it: raise sf_max_total_bytes or set sf_dir to keep such " + + "transactions whole, or use transaction=strict to refuse their rows instead. Logged once " + + "per transaction", + rows, bytes, cursorEngine.ringCapacityBytes()); + } + } + + /** + * Transactional mode: whether the ring under {@code sf_max_total_bytes} can hold + * a flush's frames together with the open transaction's frames already on it + * and, for a deferred flush, room for the frame that ends the transaction. The + * server acks none of a transaction's frames before its commit, so the ring + * cannot trim them; a transaction that does not fit would block its own commit. + * A strict transaction keeps room for a commit that carries the rows still + * pending when it ends, a transaction=on transaction for a bare commit frame: + * it commits its frames with one when the next flush does not fit. *

    * Mirrors how the ring packs frames: frames never span segments, and a frame * that does not fit the current segment opens the next one. The flush's * frame sizes are derived the way flushPendingRows and the split publish * them; dictionary chunks of the full-dictionary fallback count as full * frames. Runs before anything of the flush is published, so a refusal - * leaves the batch pending and the ring untouched. On a refusal it records - * the size the transaction would have reached in {@link #txnRefusedBytes}. + * leaves the batch pending and the ring untouched. On a strict refusal it + * records the size the transaction would have reached in + * {@link #txnRefusedBytes}. */ - private boolean hasStrictTransactionRoom(int cap, int deltaBaseline, int messageSize, boolean deferCommit) { + private boolean hasTransactionRoom(int cap, int deltaBaseline, int messageSize, boolean deferCommit) { CursorSendEngine engine = cursorEngine; long segmentPayload = segmentPayloadCapacity(); - strictFrameSizes.clear(); + transactionFrameSizes.clear(); if (cap <= 0 || messageSize <= cap) { - strictFrameSizes.add(messageSize); + transactionFrameSizes.add(messageSize); } else { int baseline = deltaBaseline; if (!deltaDictEnabled && currentBatchMaxSymbolId > deltaBaseline @@ -7562,23 +7644,28 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { - QwpConstants.HEADER_SIZE; long chunkPayload = Math.max(1, cap - QwpConstants.HEADER_SIZE - 16); for (long chunks = dictBytes / chunkPayload + 1; chunks > 0; chunks--) { - strictFrameSizes.add(cap); + transactionFrameSizes.add(cap); } baseline = currentBatchMaxSymbolId; } for (int i = 0, n = splitFrameBodyBytes.size(); i < n; i++) { - strictFrameSizes.add(encoder.getSplitMessageSize( + transactionFrameSizes.add(encoder.getSplitMessageSize( splitFrameBodyBytes.getQuick(i), baseline, currentBatchMaxSymbolId)); if (deltaDictEnabled && currentBatchMaxSymbolId > baseline) { baseline = currentBatchMaxSymbolId; } } } + if (deferCommit && !strictTransactional) { + // Keep room for the bare commit or rollback frame that ends the + // transaction: the next flush that does not fit commits it with one. + transactionFrameSizes.add(COMMIT_FRAME_MAX_BYTES); + } int segments = txnSimSegments; long remaining = segments == 0 ? engine.activeSegment().capacityRemaining() : txnSimRemaining; long bytes = txnBytes; - for (int i = 0, n = strictFrameSizes.size(); i < n; i++) { - long size = strictFrameSizes.getQuick(i); + for (int i = 0, n = transactionFrameSizes.size(); i < n; i++) { + long size = transactionFrameSizes.getQuick(i); bytes += size; if (segments == 0 || size > remaining) { if (segments > 0 || size > remaining) { @@ -7588,7 +7675,7 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { } remaining = Math.max(0, remaining - size - MmapSegment.FRAME_HEADER_SIZE); } - if (deferCommit) { + if (deferCommit && strictTransactional) { // Keep room for the commit: the rows pending when this transaction // ends are at most a frame's worth plus one row, which a split // ships as a few frames of at most cap bytes. @@ -7603,7 +7690,9 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { } long maxSegments = Math.max(2, engine.ringCapacityBytes() / engine.segmentSizeBytes()); if (segments > maxSegments) { - txnRefusedBytes = bytes; + if (strictTransactional) { + txnRefusedBytes = bytes; + } return false; } return true; @@ -7657,9 +7746,10 @@ private void recomputeFrameBudget() { } /** - * Strict mode: advances the client-buffer simulation by one open-transaction - * frame that just reached the ring. {@code activeRemaining} is the active - * segment's room before the append, needed for the transaction's first frame. + * Transactional mode: advances the client-buffer simulation by one + * open-transaction frame that just reached the ring. {@code activeRemaining} + * is the active segment's room before the append, needed for the + * transaction's first frame. */ private void recordTransactionFrame(int payloadLen, long activeRemaining) { txnBytes += payloadLen; @@ -7684,6 +7774,17 @@ private long segmentPayloadCapacity() { : engine.segmentSizeBytes() - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE; } + /** + * Rows the open transaction's deferred frames already sent, over all its tables. + */ + private long sentTransactionRows() { + long rows = 0; + for (int i = 0, n = txnTables.size(); i < n; i++) { + rows += txnTables.getQuick(i).getTxnSentRows(); + } + return rows; + } + private LineSenderException transactionAbortedError() { return new LineSenderException("transaction aborted: rollback could not undo the frames of the " + "open transaction already sent, because the server does not support QWP rollback. Close " @@ -7715,10 +7816,7 @@ private LineSenderException transactionTooLargeForClientBuffer(String outcome) { * discardPendingRows() clears the latter. */ private void warnDiscardedTransaction(String action) { - long rows = pendingRowCount; - for (int i = 0, n = txnTables.size(); i < n; i++) { - rows += txnTables.getQuick(i).getTxnSentRows(); - } + long rows = pendingRowCount + sentTransactionRows(); if (rows > 0) { LOG.warn("{} discarded an uncommitted transaction of {} rows: a strict transactional sender " + "(transaction=strict, or one with no auto-flush trigger) commits only on flush(). Call " diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java index d42307354..95d003c11 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpLargeTransactionTest.java @@ -45,6 +45,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; @@ -81,6 +82,12 @@ public class QwpLargeTransactionTest { // ~100 encoded bytes per row: 1000 rows (the default row trigger) are ~100 KB, // three times the 32 KiB cap. private static final String PAD = TestUtils.repeat("x", 90); + // transaction=on in frames of exactly 280 rows (~28 KB): two fill a 64 KiB segment, + // so the 256 KiB client buffer holds eight frames of a transaction. A blocked + // append would wait 30 s. + private static final String TXN_ON_FRAMES_OF_280 = "transaction=on;auto_flush_rows=280;auto_flush_bytes=off;" + + "auto_flush_interval=off;sf_max_segment_bytes=64k;sf_max_total_bytes=256k;" + + "sf_append_deadline_millis=30000;"; @Test public void testAutoFlushBytesAloneCommitsEachBatch() throws Exception { @@ -1100,6 +1107,133 @@ public void testUnacknowledgedRollbackDoesNotStallTheSender() throws Exception { }); } + @Test + public void testTransactionalCommitsInPartsAtClientBuffer() throws Exception { + // The client holds a transaction's frames until the server commits it. The row + // whose frame found no room used to block for sf_append_deadline_millis and then + // throw, and close() then committed the frames already sent and dropped the rest. + // Now the sender commits the frames already sent and carries on, as the server + // does at qwp.max.uncommitted.rows. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (QwpWebSocketSender s = (QwpWebSocketSender) Sender.fromConfig(conf(server) + TXN_ON_FRAMES_OF_280)) { + long start = System.nanoTime(); + // 17 frames of 280 rows, then 240 rows that flush() commits. + writeRows(s, "t", 0, 5_000); + Assert.assertEquals("two parts of eight frames", 4_480, s.getTxnRowsCommittedInPartsForTesting()); + s.flush(); + Assert.assertEquals(0, s.getTxnRowsCommittedInPartsForTesting()); + Assert.assertTrue("must not wait for sf_append_deadline_millis", + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); + Assert.assertTrue(s.drain(5_000)); + } + // The buffer holds eight frames, so the ninth and the seventeenth each + // start a new part, behind a bare commit frame for the frames before them. + Assert.assertEquals("DDDDDDDDcDDDDDDDDcDC", frameKinds(h)); + Assert.assertEquals("every frame commits", h.frames, h.committedFrames()); + // Logged once for the transaction, with what the first commit covered. + Assert.assertEquals(Collections.singletonList(commitInPartsWarning(2_240, frameBytes(h, 0, 8))), + commitInPartsWarnings(log)); + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testTransactionalCommitsInPartsBeforeFlushThatDoesNotFit() throws Exception { + // Eight frames fill the client buffer, and the frame flush() sends with the last + // 279 rows, which commits the transaction, does not fit beside them. flush() used + // to block for sf_append_deadline_millis and throw. Now it commits the eight + // frames first. Each transaction that commits in parts logs once. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + try (Sender s = Sender.fromConfig(conf(server) + TXN_ON_FRAMES_OF_280)) { + writeRows(s, "t", 0, 2_519); + Assert.assertTrue(h.awaitFrames(8)); + long start = System.nanoTime(); + s.flush(); + Assert.assertTrue("must not wait for sf_append_deadline_millis", + System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)); + Assert.assertTrue(s.drain(5_000)); + Assert.assertEquals("DDDDDDDDcC", frameKinds(h)); + Assert.assertEquals(Collections.singletonList( + commitInPartsWarning(2_240, frameBytes(h, 0, 8))), commitInPartsWarnings(log)); + + // The next transaction's first frame shares a segment with the + // previous commit frame, so seven frames fill the buffer this time. + writeRows(s, "t", 0, 2_519); + s.flush(); + Assert.assertTrue(s.drain(5_000)); + } + Assert.assertEquals("DDDDDDDDcC" + "DDDDDDDcDC", frameKinds(h)); + Assert.assertEquals("every frame commits", h.frames, h.committedFrames()); + Assert.assertEquals(Arrays.asList( + commitInPartsWarning(2_240, frameBytes(h, 0, 8)), + commitInPartsWarning(1_960, frameBytes(h, 10, 17)) + ), commitInPartsWarnings(log)); + } + } finally { + releaseSenderLog(log); + } + }); + } + + @Test + public void testTransactionalDiscardAfterCommitInPartsWarns() throws Exception { + // The frames committed when the transaction outgrew the client buffer cannot be + // rolled back. rollback() and reset() discard the rest of the transaction, and + // say that the part already committed stays. + assertMemoryLeak(() -> { + ListAppender log = captureSenderLog(); + try { + String[] actions = {"rollback()", "reset()"}; + for (String action : actions) { + clearLog(log); + TxnServer h = new TxnServer(); + try (TestWebSocketServer server = start(h, CAP)) { + server.setRollbackSupported(true); + try (QwpWebSocketSender s = (QwpWebSocketSender) Sender.fromConfig(conf(server) + TXN_ON_FRAMES_OF_280)) { + // Nine frames, the ninth in a new part, and 100 rows pending. + writeRows(s, "t", 0, 2_620); + Assert.assertEquals(action, 2_240, s.getTxnRowsCommittedInPartsForTesting()); + if ("rollback()".equals(action)) { + s.rollback(); + } else { + s.reset(); + } + Assert.assertEquals(action, 0, s.getTxnRowsCommittedInPartsForTesting()); + writeRows(s, "t", 0, 10); + s.flush(); + Assert.assertTrue(action, s.drain(5_000)); + } + Assert.assertEquals(action, "DDDDDDDDcDRC", frameKinds(h)); + Assert.assertEquals(action, 1, h.rolledBackGroups); + // The first part and the next transaction commit, the ninth frame does not. + List committed = new ArrayList<>(h.frames.subList(0, 9)); + committed.add(h.frames.get(11)); + Assert.assertEquals(action, committed, h.committedFrames()); + Assert.assertEquals(action, Arrays.asList( + commitInPartsWarning(2_240, frameBytes(h, 0, 8)), + action + " discarded the rows of the open transaction sent since the sender last " + + "committed part of it, but the 2240 rows it committed when the transaction " + + "outgrew the client buffer stay committed" + ), commitInPartsWarnings(log)); + } + } + } finally { + releaseSenderLog(log); + } + }); + } + @Test public void testTransactionalDefaultTriggersSplitWideRows() throws Exception { // Before: transaction=on with the default triggers threw @@ -1133,6 +1267,31 @@ private static void clearLog(ListAppender log) { } } + // What a transaction=on sender logs the first time a transaction outgrows the + // 256 KiB client buffer and it commits the part already sent. + private static String commitInPartsWarning(long rows, long transactionBytes) { + return "the open transaction outgrew the client buffer, so the sender committed the " + rows + + " rows it had already sent, and the rest of the transaction commits separately [transactionBytes=" + + transactionBytes + ", ringCapacityBytes=262144]. A transaction=on transaction commits in parts when " + + "sf_max_total_bytes cannot hold it: raise sf_max_total_bytes or set sf_dir to keep such transactions " + + "whole, or use transaction=strict to refuse their rows instead. Logged once per transaction"; + } + + // The warnings about a transaction the sender committed in parts at the client + // buffer logged since the last clearLog(), in order. + private static List commitInPartsWarnings(ListAppender log) { + List warnings = new ArrayList<>(); + synchronized (log) { + for (ILoggingEvent e : log.list) { + String message = e.getFormattedMessage(); + if (e.getLevel() == Level.WARN && message.contains("outgrew the client buffer")) { + warnings.add(message); + } + } + } + return warnings; + } + // The discarded-transaction warnings logged since the last clearLog(). private static List discardWarnings(ListAppender log) { List warnings = new ArrayList<>(); @@ -1310,13 +1469,28 @@ private static TestWebSocketServer start(TxnServer h, int cap, int port, boolean } private static long frameBytes(TxnServer h) { + return frameBytes(h, 0, h.frames.size()); + } + + // Bytes of the frames [from, to) the server received. + private static long frameBytes(TxnServer h, int from, int to) { long bytes = 0; - for (TxnFrame f : h.frames) { - bytes += f.size; + for (int i = from; i < to; i++) { + bytes += h.frames.get(i).size; } return bytes; } + // One letter per frame the server received: D deferred, c bare commit, C commit + // with rows, R rollback. + private static String frameKinds(TxnServer h) { + StringBuilder kinds = new StringBuilder(); + for (TxnFrame f : h.frames) { + kinds.append(f.isRollback() ? 'R' : f.isDeferred() ? 'D' : f.isBareCommit() ? 'c' : 'C'); + } + return kinds.toString(); + } + private static String serverRowLimitError(String table, long maxUncommittedRows) { return "transaction too large for the server [table=" + table + ", rows=" + maxUncommittedRows + ", maxUncommittedRows=" + maxUncommittedRows + "]; the server commits a table's rows on its " @@ -1368,6 +1542,11 @@ private static final class TxnFrame { this.connection = connection; } + // A commit frame with no table: the header and an empty symbol delta. + boolean isBareCommit() { + return isCommit() && size <= QwpConstants.HEADER_SIZE + 6; + } + boolean isCommit() { return (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0; }