From 10be4166eeb739adac1b2a5fbc3f3f1b61a6e90a Mon Sep 17 00:00:00 2001 From: Matthias Kurz Date: Sun, 8 Nov 2026 22:16:33 +0100 Subject: [PATCH] Upgrade zstd-jni to 1.5.7-26 zstd-jni before 1.5.7-26 took the end of the input for a clean end at a frame boundary when the stream was cut inside a frame that started in the same read as the end of the previous frame (luben/zstd-jni#435). AHC hands zstd-jni one chunk per read, so a zstd body of several frames that was cut inside a frame starting in the same chunk as the end of the previous one completed without an error. 1.5.7-26 fails such a body. Add a test for that cut, with the end of the first frame and the start of the second in one chunk, and a test that two complete frames in one chunk decode in full. The first fails with 1.5.7-25. zstd-jni is an optional dependency, so the application decides which version is used. Limit the note in the README and the Javadoc of Http1ContentDecompressor to versions before 1.5.7-26, recommend 1.5.7-26 or later, and show that version in the README. Claude Code on behalf of Matthias Kurz Co-Authored-By: Claude Opus 5.5 --- README.md | 11 +++-- .../handler/Http1ContentDecompressor.java | 6 +-- .../handler/Http1ContentDecompressorTest.java | 46 +++++++++++++++++++ pom.xml | 2 +- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c3eba5854..29a5184ea 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ AsyncHttpClient client = asyncHttpClient(config().setUseNativeTransport(true)); com.github.luben zstd-jni - 1.5.7-7 + 1.5.7-26 ``` @@ -118,10 +118,11 @@ deflate are not affected. Over HTTP/1.1, a compressed response body that ends in the middle of its compressed stream, for example because a body delimited by the connection was cut off, fails the request with a `DecompressionException`; a body without any bytes completes. Not every such end is noticed: Snappy has no end marker, so a body cut between two of its chunks looks -complete, and a Zstd body of several frames completes when it is cut inside a frame that started in the same chunk as -the end of the previous one. HTTP/2 response bodies are not checked, and neither are transfer codings such as -`Transfer-Encoding: gzip`, nor Brotli, Zstd and Snappy bodies decompressed as soon as they are read, as described -above. +complete. With zstd-jni older than 1.5.7-26, a Zstd body of several frames also completes when it is cut inside a frame +that started in the same chunk as the end of the previous one. zstd-jni is an optional dependency, so the version your +application declares is the one used: use 1.5.7-26 or later. HTTP/2 response bodies are not checked, and neither are +transfer codings such as `Transfer-Encoding: gzip`, nor Brotli, Zstd and Snappy bodies decompressed as soon as they are +read, as described above. diff --git a/client/src/main/java/org/asynchttpclient/netty/handler/Http1ContentDecompressor.java b/client/src/main/java/org/asynchttpclient/netty/handler/Http1ContentDecompressor.java index a0efe5760..968d658b8 100644 --- a/client/src/main/java/org/asynchttpclient/netty/handler/Http1ContentDecompressor.java +++ b/client/src/main/java/org/asynchttpclient/netty/handler/Http1ContentDecompressor.java @@ -97,9 +97,9 @@ * {@code zstd} also goes there while {@code io.netty.compression.defaultMaxForwardBytes} is not positive. * {@code gzip} and {@code deflate} never use the API. A body that ends inside its compressed stream fails on the * pull path too, and still ends quietly on the paths through {@link HttpContentDecompressor}. Not every such end is - * noticed: {@code snappy} has no end marker, so a body cut between two of its chunks looks complete, and a - * {@code zstd} body of several frames completes when it is cut inside a frame that started in the same chunk as the - * end of the previous one. + * noticed: {@code snappy} has no end marker, so a body cut between two of its chunks looks complete, and with + * zstd-jni before 1.5.7-26, a {@code zstd} body of several frames completes when it is cut inside a frame that started + * in the same chunk as the end of the previous one. *

* Every other content coding, and a transfer coding such as {@code Transfer-Encoding: gzip}, still goes through * {@link HttpContentDecompressor}, which decompresses it as soon as it is read. There the counting is done diff --git a/client/src/test/java/org/asynchttpclient/netty/handler/Http1ContentDecompressorTest.java b/client/src/test/java/org/asynchttpclient/netty/handler/Http1ContentDecompressorTest.java index 1bf16494d..745560c05 100644 --- a/client/src/test/java/org/asynchttpclient/netty/handler/Http1ContentDecompressorTest.java +++ b/client/src/test/java/org/asynchttpclient/netty/handler/Http1ContentDecompressorTest.java @@ -1279,6 +1279,52 @@ void truncatedBrZstdAndSnappyBodiesFailAndBodiesWithoutBytesEnd() { assertTrue(checked > 30, "only " + checked + " scenarios ran"); } + private static final byte[] FIRST_ZSTD_FRAME = Zstd.compress(Arrays.copyOf(RANDOM, 20_000)); + private static final byte[] SECOND_ZSTD_FRAME = Zstd.compress(Arrays.copyOfRange(RANDOM, 20_000, 40_000)); + + // Each ends the first frame and starts the second in the same chunk. + private static List splitsAcrossTheFirstZstdFrameEnd(int length) { + int half = FIRST_ZSTD_FRAME.length / 2; + return List.of(new int[]{length}, new int[]{length, 0}, new int[]{half, length - half}); + } + + @Test + void zstdBodyCutInAFrameThatStartedInTheChunkThatEndedThePreviousOneFails() { + byte[] cut = concat(FIRST_ZSTD_FRAME, Arrays.copyOf(SECOND_ZSTD_FRAME, SECOND_ZSTD_FRAME.length / 2)); + for (int[] sizes : splitsAcrossTheFirstZstdFrameEnd(cut.length)) { + for (Kind kind : List.of(Kind.HEAP, Kind.DIRECT)) { + String scenario = Arrays.toString(sizes) + ", " + kind; + UnpooledByteBufAllocator allocator = allocator(); + EmbeddedChannel channel = channel(new Http1ContentDecompressor(false, 0), allocator); + Outcome outcome = exchange(channel, "zstd", cut, sizes, kind); + assertInstanceOf(DecompressionException.class, outcome.failure, scenario); + assertFalse(outcome.ended, scenario + ": a truncated body must not end normally"); + channel.finishAndReleaseAll(); + assertEquals(0, allocator.metric().usedHeapMemory(), scenario + ": heap left allocated"); + assertInputsReleased(outcome, scenario); + } + } + } + + @Test + void zstdFramesThatShareAChunkDecodeInFull() { + byte[] body = concat(FIRST_ZSTD_FRAME, SECOND_ZSTD_FRAME); + for (int[] sizes : splitsAcrossTheFirstZstdFrameEnd(body.length)) { + for (Kind kind : List.of(Kind.HEAP, Kind.DIRECT)) { + String scenario = Arrays.toString(sizes) + ", " + kind; + UnpooledByteBufAllocator allocator = allocator(); + EmbeddedChannel channel = channel(new Http1ContentDecompressor(false, 0), allocator); + Outcome outcome = exchange(channel, "zstd", body, sizes, kind); + assertNull(outcome.failure, scenario); + assertTrue(outcome.ended, scenario); + assertArrayEquals(Arrays.copyOf(RANDOM, 40_000), outcome.body.toByteArray(), scenario); + assertFalse(channel.finishAndReleaseAll(), scenario); + assertEquals(0, allocator.metric().usedHeapMemory(), scenario + ": heap left allocated"); + assertInputsReleased(outcome, scenario); + } + } + } + @Test void outputThatOnlyFollowsTheEndOfInputIsHandedOn() { UnpooledByteBufAllocator allocator = allocator(); diff --git a/pom.xml b/pom.xml index d478f6e4d..43608e774 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 4.2.20.Final 1.24.0 2.0.20 - 1.5.7-25 + 1.5.7-26 1.6.6 26.1.0 2.0.5