diff --git a/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java b/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java index f2b8df4ad..800e8544d 100755 --- a/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java +++ b/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java @@ -235,9 +235,17 @@ private void mergeCookies(List cookies, HttpHeaders headers) { private final AsyncHttpClientConfig config; private final ClientCookieEncoder cookieEncoder; private final @Nullable AsciiString userAgent; + private final boolean brotliAvailable; + private final boolean zstdAvailable; NettyRequestFactory(AsyncHttpClientConfig config) { + this(config, Brotli.isAvailable(), Zstd.isAvailable()); + } + + NettyRequestFactory(AsyncHttpClientConfig config, boolean brotliAvailable, boolean zstdAvailable) { this.config = config; + this.brotliAvailable = brotliAvailable; + this.zstdAvailable = zstdAvailable; cookieEncoder = config.isUseLaxCookieEncoder() ? ClientCookieEncoder.LAX : ClientCookieEncoder.STRICT; // An AsciiString is bulk-copied by the encoder; a String is encoded char by char. Same bytes either // way, and the validating request headers still reject an illegal character on every request. @@ -359,27 +367,18 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque // Unconditional: it also folds several caller-set Cookie fields into one. mergeCookies(request.getCookies(), headers); - String userDefinedAcceptEncoding = headers.get(ACCEPT_ENCODING); - if (userDefinedAcceptEncoding != null) { + if (headers.contains(ACCEPT_ENCODING)) { + // For manual decompression by the user any encoding may suit, so leave it untouched if (config.isEnableAutomaticDecompression()) { - if (!Brotli.isAvailable()) { - // Brotli is not available. - // For manual decompression by user, any encoding may suite, so leave untouched - headers.set(ACCEPT_ENCODING, filterOutBrotliFromAcceptEncoding(userDefinedAcceptEncoding)); - } - if (!Zstd.isAvailable()) { - // zstd is not available. - // For manual decompression by user, any encoding may suit, so leave untouched - headers.set(ACCEPT_ENCODING, filterOutZstdFromAcceptEncoding(userDefinedAcceptEncoding)); - } + filterOutUndecodableAcceptEncodings(headers, brotliAvailable, zstdAvailable); } } else if (config.isCompressionEnforced()) { // Add Accept Encoding header if compression is enforced headers.set(ACCEPT_ENCODING, GZIP_DEFLATE); - if (Brotli.isAvailable()) { + if (brotliAvailable) { headers.add(ACCEPT_ENCODING, HttpHeaderValues.BR); } - if (Zstd.isAvailable()) { + if (zstdAvailable) { headers.add(ACCEPT_ENCODING, HttpHeaderValues.ZSTD); } } @@ -495,4 +494,26 @@ private static CharSequence connectionHeader(boolean keepAlive, HttpVersion http return keepAlive ? HttpHeaderValues.KEEP_ALIVE : null; } } + + /** + * Without its library a coding would reach the user still compressed, so it is not offered. Several fields are + * filtered as their comma-joined list, which means the same (RFC 9110 section 5.3). + */ + static void filterOutUndecodableAcceptEncodings(HttpHeaders headers, boolean brotliAvailable, boolean zstdAvailable) { + if (brotliAvailable && zstdAvailable) { + return; + } + List values = headers.getAll(ACCEPT_ENCODING); + String acceptEncoding = values.size() == 1 ? values.get(0) : String.join(", ", values); + String filtered = acceptEncoding; + if (!brotliAvailable) { + filtered = filterOutBrotliFromAcceptEncoding(filtered).toString(); + } + if (!zstdAvailable) { + filtered = filterOutZstdFromAcceptEncoding(filtered).toString(); + } + if (!filtered.equals(acceptEncoding)) { + headers.set(ACCEPT_ENCODING, filtered); + } + } } diff --git a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java index 308e73677..fd7e76db5 100644 --- a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java +++ b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java @@ -41,8 +41,6 @@ public final class HttpUtils { public static final AsciiString GZIP_DEFLATE_HPACK = new AsciiString(HttpHeaderValues.GZIP + ", " + HttpHeaderValues.DEFLATE); private static final String CONTENT_TYPE_CHARSET_ATTRIBUTE = "charset="; private static final String CONTENT_TYPE_BOUNDARY_ATTRIBUTE = "boundary="; - private static final String BROTLY_ACCEPT_ENCODING_SUFFIX = ", br"; - private static final String ZSTD_ACCEPT_ENCODING_SUFFIX = ", zstd"; private HttpUtils() { // Prevent outside initialization @@ -196,18 +194,75 @@ public static String cookiePairName(String pair) { } public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncoding) { - // we don't support Brotly ATM - if (acceptEncoding.endsWith(BROTLY_ACCEPT_ENCODING_SUFFIX)) { - return acceptEncoding.subSequence(0, acceptEncoding.length() - BROTLY_ACCEPT_ENCODING_SUFFIX.length()); - } - return acceptEncoding; + return filterOutAcceptEncoding(acceptEncoding, HttpHeaderValues.BR); } public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding) { - // we don't support zstd ATM - if (acceptEncoding.endsWith(ZSTD_ACCEPT_ENCODING_SUFFIX)) { - return acceptEncoding.subSequence(0, acceptEncoding.length() - ZSTD_ACCEPT_ENCODING_SUFFIX.length()); + return filterOutAcceptEncoding(acceptEncoding, HttpHeaderValues.ZSTD); + } + + /** + * Drops every element offering {@code coding}, which may appear anywhere in the list, in any case and with a weight + * (RFC 9110 section 12.5.3). Dropping the last element leaves an empty value, which accepts no content coding. + * An element with weight 0 refuses the coding and is kept. Next to a "*" element, which would still offer the coding + * at its own weight, the dropped elements are replaced by one such refusal. + */ + private static String filterOutAcceptEncoding(String acceptEncoding, AsciiString coding) { + if (!AsciiString.containsIgnoreCase(acceptEncoding, coding)) { + return acceptEncoding; + } + String[] elements = acceptEncoding.split(","); + boolean wildcard = false; + for (String element : elements) { + wildcard |= offers(element.trim(), "*"); + } + StringBuilder kept = new StringBuilder(acceptEncoding.length()); + boolean dropped = false; + for (String element : elements) { + String trimmed = element.trim(); + if (offers(trimmed, coding)) { + if (wildcard && !dropped) { + appendElement(kept, coding + ";q=0"); + } + dropped = true; + } else if (!trimmed.isEmpty()) { + appendElement(kept, trimmed); + } + } + return dropped ? kept.toString() : acceptEncoding; + } + + private static boolean offers(String element, CharSequence coding) { + int semicolon = element.indexOf(';'); + String token = semicolon < 0 ? element : element.substring(0, semicolon).trim(); + return AsciiString.contentEqualsIgnoreCase(token, coding) + && (semicolon < 0 || !hasZeroWeight(element.substring(semicolon + 1))); + } + + private static void appendElement(StringBuilder list, String element) { + if (list.length() > 0) { + list.append(", "); + } + list.append(element); + } + + private static boolean hasZeroWeight(String parameters) { + for (String parameter : parameters.split(";")) { + int eq = parameter.indexOf('='); + if (eq >= 0 && "q".equalsIgnoreCase(parameter.substring(0, eq).trim())) { + String weight = parameter.substring(eq + 1).trim(); + if (weight.isEmpty() || weight.charAt(0) != '0') { + return false; + } + for (int i = 1; i < weight.length(); i++) { + char c = weight.charAt(i); + if (c != '0' && (c != '.' || i != 1)) { + return false; + } + } + return true; + } } - return acceptEncoding; + return false; } } diff --git a/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java b/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java new file mode 100644 index 000000000..9ba0e5576 --- /dev/null +++ b/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2026 AsyncHttpClient Project. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.asynchttpclient.netty.request; + +import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.handler.codec.http.HttpHeaders; +import org.asynchttpclient.AsyncHttpClientConfig; +import org.asynchttpclient.DefaultAsyncHttpClientConfig; +import org.asynchttpclient.RequestBuilder; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT_ENCODING; +import static org.asynchttpclient.netty.request.NettyRequestFactory.filterOutUndecodableAcceptEncodings; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Brotli and zstd are both on the test classpath, so the availability flags are passed in, to the helper or to the + * factory, rather than taken from Netty's {@code Brotli.isAvailable()} and {@code Zstd.isAvailable()}. + */ +public class NettyRequestFactoryAcceptEncodingTest { + + @Test + public void dropsBothWhenNeitherIsAvailable() { + assertEquals(List.of("gzip, deflate"), filter(false, false, "gzip, deflate, br, zstd")); + assertEquals(List.of("gzip, deflate"), filter(false, false, "gzip, deflate, zstd, br")); + } + + @Test + public void dropsOnlyTheUnavailableOne() { + assertEquals(List.of("gzip, deflate, zstd"), filter(false, true, "gzip, deflate, br, zstd")); + assertEquals(List.of("gzip, deflate, br"), filter(true, false, "gzip, deflate, br, zstd")); + } + + @Test + public void leavesHeaderUntouchedWhenBothAreAvailable() { + assertEquals(List.of("gzip", "br, zstd"), filter(true, true, "gzip", "br, zstd")); + } + + @Test + public void leavesHeaderUntouchedWhenNothingIsDropped() { + assertEquals(List.of("gzip", "deflate"), filter(false, false, "gzip", "deflate")); + } + + @Test + public void filtersEveryHeaderFieldAndKeepsTheOthers() { + assertEquals(List.of("gzip, deflate"), filter(false, false, "gzip", "br", "deflate, zstd")); + } + + @Test + public void newRequestDropsTheUnavailableCodingsWithAutomaticDecompression() { + assertEquals(List.of("gzip, deflate"), newRequest(true, false, false, "gzip, deflate, br, zstd")); + assertEquals(List.of("gzip, deflate, zstd"), newRequest(true, false, true, "gzip, deflate, br, zstd")); + assertEquals(List.of("gzip, deflate, br"), newRequest(true, true, false, "gzip, deflate, br, zstd")); + assertEquals(List.of("gzip, deflate, br, zstd"), newRequest(true, true, true, "gzip, deflate, br, zstd")); + } + + @Test + public void newRequestLeavesTheHeaderToTheUserWithoutAutomaticDecompression() { + assertEquals(List.of("gzip, deflate, br, zstd"), newRequest(false, false, false, "gzip, deflate, br, zstd")); + } + + @Test + public void newRequestOffersOnlyAvailableCodingsWhenCompressionIsEnforced() { + assertEquals(List.of("gzip,deflate"), newRequest(true, false, false)); + assertEquals(List.of("gzip,deflate", "br"), newRequest(true, true, false)); + assertEquals(List.of("gzip,deflate", "zstd"), newRequest(true, false, true)); + } + + private static List newRequest(boolean automaticDecompression, boolean brotliAvailable, boolean zstdAvailable, + String... acceptEncodings) { + AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder() + .setEnableAutomaticDecompression(automaticDecompression) + .setCompressionEnforced(true) + .build(); + RequestBuilder request = new RequestBuilder("GET").setUrl("http://example.com/"); + for (String acceptEncoding : acceptEncodings) { + request.addHeader(ACCEPT_ENCODING, acceptEncoding); + } + return new NettyRequestFactory(config, brotliAvailable, zstdAvailable) + .newNettyRequest(request.build(), false, null, null, null) + .getHttpRequest() + .headers() + .getAll(ACCEPT_ENCODING); + } + + private static List filter(boolean brotliAvailable, boolean zstdAvailable, String... acceptEncodings) { + HttpHeaders headers = new DefaultHttpHeaders(); + headers.add(ACCEPT_ENCODING, Arrays.asList(acceptEncodings)); + filterOutUndecodableAcceptEncodings(headers, brotliAvailable, zstdAvailable); + return headers.getAll(ACCEPT_ENCODING); + } +} diff --git a/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java b/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java index 7ed607dde..e2fc630fa 100644 --- a/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java +++ b/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java @@ -194,4 +194,57 @@ public void cookiePairsEndAtEverySemicolonAndKeepTheirText() { assertEquals("flag", HttpUtils.cookiePairName(" flag")); assertEquals("", HttpUtils.cookiePairName("")); } + + @Test + public void filterOutBrotliDropsTheCodingWherever() { + assertEquals("gzip, deflate", filterOutBrotli("gzip, deflate, br")); + assertEquals("gzip, deflate, zstd", filterOutBrotli("gzip, deflate, br, zstd")); + assertEquals("gzip, deflate", filterOutBrotli("br, gzip, deflate")); + assertEquals("gzip, deflate", filterOutBrotli("gzip,br,deflate")); + assertEquals("", filterOutBrotli("br")); + } + + @Test + public void filterOutBrotliIgnoresCaseWhitespaceAndWeight() { + assertEquals("gzip;q=0.8, deflate", filterOutBrotli("gzip;q=0.8, BR ; q=1.0 ,\tdeflate")); + assertEquals("gzip", filterOutBrotli("gzip, Br;q=0.5")); + } + + @Test + public void filterOutBrotliKeepsOtherCodings() { + assertEquals("gzip,deflate", filterOutBrotli("gzip,deflate")); + assertEquals("gzip, brotli, x-br", filterOutBrotli("gzip, brotli, x-br")); + } + + @Test + public void filterOutZstdDropsTheCodingWherever() { + assertEquals("gzip, deflate", filterOutZstd("gzip, deflate, zstd")); + assertEquals("gzip, deflate, br", filterOutZstd("gzip, deflate, zstd, br")); + assertEquals("gzip;q=0.8, br", filterOutZstd("ZSTD;q=1, gzip;q=0.8, br")); + assertEquals("gzip, zstd-x", filterOutZstd("gzip, zstd-x")); + assertEquals("", filterOutZstd(" zstd ")); + } + + @Test + public void filterOutKeepsARefusalOfTheCoding() { + assertEquals("*, br;q=0", filterOutBrotli("*, br;q=0")); + assertEquals("*, BR ; Q = 0.000", filterOutBrotli("*, BR ; Q = 0.000")); + assertEquals("gzip, zstd;q=0., *", filterOutZstd("gzip, zstd;q=0., *")); + assertEquals("gzip, br;q=0", filterOutBrotli("gzip, br;q=0.001, br;q=0")); + } + + @Test + public void filterOutRefusesTheCodingNextToAWildcard() { + assertEquals("*, zstd;q=0", filterOutZstd("*, zstd;q=0.5")); + assertEquals("gzip, br;q=0, *", filterOutBrotli("gzip, BR, br;q=0.2, *")); + assertEquals("*;q=0", filterOutBrotli("*;q=0, br")); + } + + private static String filterOutBrotli(String acceptEncoding) { + return HttpUtils.filterOutBrotliFromAcceptEncoding(acceptEncoding).toString(); + } + + private static String filterOutZstd(String acceptEncoding) { + return HttpUtils.filterOutZstdFromAcceptEncoding(acceptEncoding).toString(); + } }