diff --git a/client/src/jmh/java/org/asynchttpclient/bench/FormDigestEncodeBenchmark.java b/client/src/jmh/java/org/asynchttpclient/bench/FormDigestEncodeBenchmark.java new file mode 100644 index 000000000..62185c17d --- /dev/null +++ b/client/src/jmh/java/org/asynchttpclient/bench/FormDigestEncodeBenchmark.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 AsyncHttpClient Project. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.asynchttpclient.bench; + +import org.asynchttpclient.Param; +import org.asynchttpclient.Realm; +import org.asynchttpclient.uri.Uri; +import org.asynchttpclient.util.HttpUtils; +import org.asynchttpclient.util.StringUtils; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Production paths that encode a pooled StringBuilder to bytes through + * {@code StringUtils.charSequence2ByteBuffer}: an url-encoded form body, and the three + * hashes of a Digest response, end to end and the encoding step alone. Run against two builds to + * compare them. + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class FormDigestEncodeBenchmark { + + private static final String UTF8_USER = "\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c"; + private static final String UTF8_PASSWORD = "\u043f\u0430\u0440\u043e\u043b\u044c"; + + private List smallForm; + private List largeForm; + private Realm.Builder asciiDigest; + private Realm.Builder utf8Digest; + private StringBuilder formBuilder; + private StringBuilder digestBuilder; + private StringBuilder utf8DigestBuilder; + + @Setup(Level.Trial) + public void setup() { + smallForm = form(4); + largeForm = form(256); + ByteBuffer formBody = HttpUtils.urlEncodeFormParams(largeForm, StandardCharsets.UTF_8); + formBuilder = new StringBuilder(StandardCharsets.US_ASCII.decode(formBody)); + // HA1 ":" nonce ":" nc ":" cnonce ":" qop ":" HA2 + digestBuilder = new StringBuilder("939e7578ed9e3c518a452acee763bce9:dcd98b7102dd2f0e8b11d0f600bfb0c093:00000001:" + + "0a4f113b5c8e1d2a:auth:39aff3a2bab6126f332b942af96d3366"); + // username ":" realm ":" password + utf8DigestBuilder = new StringBuilder(UTF8_USER + ":protected:" + UTF8_PASSWORD); + Uri uri = Uri.create("http://localhost:8080/protected/resource?id=42"); + String challenge = "Digest realm=\"protected\", nonce=\"dcd98b7102dd2f0e8b11d0f600bfb0c093\", qop=\"auth\""; + asciiDigest = new Realm.Builder("user", "secret-password") + .setUri(uri) + .setMethodName("GET") + .parseWWWAuthenticateHeader(challenge); + utf8Digest = new Realm.Builder(UTF8_USER, UTF8_PASSWORD) + .setUri(uri) + .setMethodName("GET") + .parseWWWAuthenticateHeader(challenge + ", charset=UTF-8"); + } + + private static List form(int params) { + List list = new ArrayList<>(params); + for (int i = 0; i < params; i++) { + list.add(new Param("field" + i, "value " + i + "&x=y")); + } + return list; + } + + /** 4 params, a 91 byte body. */ + @Benchmark + public ByteBuffer formSmall() { + return HttpUtils.urlEncodeFormParams(smallForm, StandardCharsets.UTF_8); + } + + /** 256 params, a 6,691 byte body. */ + @Benchmark + public ByteBuffer formLarge() { + return HttpUtils.urlEncodeFormParams(largeForm, StandardCharsets.UTF_8); + } + + /** ISO-8859-1 digest of ASCII credentials. */ + @Benchmark + public Realm digestAscii() { + return asciiDigest.build(); + } + + /** UTF-8 digest of Cyrillic credentials: HA1 is not ASCII. */ + @Benchmark + public Realm digestUtf8() { + return utf8Digest.build(); + } + + /** The encoding step of {@link #formLarge} alone. */ + @Benchmark + public ByteBuffer encodeFormBuilder() { + return StringUtils.charSequence2ByteBuffer(formBuilder, StandardCharsets.US_ASCII); + } + + /** The encoding step of the last {@link #digestAscii} hash alone. */ + @Benchmark + public ByteBuffer encodeDigestBuilder() { + return StringUtils.charSequence2ByteBuffer(digestBuilder, StandardCharsets.ISO_8859_1); + } + + /** The encoding step of the first {@link #digestUtf8} hash alone: non-ASCII input. */ + @Benchmark + public ByteBuffer encodeUtf8DigestBuilder() { + return StringUtils.charSequence2ByteBuffer(utf8DigestBuilder, StandardCharsets.UTF_8); + } +} diff --git a/client/src/main/java/org/asynchttpclient/util/StringUtils.java b/client/src/main/java/org/asynchttpclient/util/StringUtils.java index 0abf0b686..35e202d60 100644 --- a/client/src/main/java/org/asynchttpclient/util/StringUtils.java +++ b/client/src/main/java/org/asynchttpclient/util/StringUtils.java @@ -16,9 +16,12 @@ package org.asynchttpclient.util; import java.nio.ByteBuffer; -import java.nio.CharBuffer; import java.nio.charset.Charset; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; + public final class StringUtils { private StringUtils() { @@ -26,7 +29,7 @@ private StringUtils() { } public static ByteBuffer charSequence2ByteBuffer(CharSequence cs, Charset charset) { - return charset.encode(CharBuffer.wrap(cs)); + return ByteBuffer.wrap(charSequence2Bytes(cs, charset)); } public static byte[] byteBuffer2ByteArray(ByteBuffer bb) { @@ -36,8 +39,23 @@ public static byte[] byteBuffer2ByteArray(ByteBuffer bb) { } public static byte[] charSequence2Bytes(CharSequence sb, Charset charset) { - ByteBuffer bb = charSequence2ByteBuffer(sb, charset); - return byteBuffer2ByteArray(bb); + if (sb instanceof String) { + return ((String) sb).getBytes(charset); + } + // Each of these charsets maps a char below U+0080 to that byte, so ASCII input needs no String copy. + if (charset.equals(UTF_8) || charset.equals(ISO_8859_1) || charset.equals(US_ASCII)) { + int length = sb.length(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + char c = sb.charAt(i); + if (c >= 0x80) { + return sb.toString().getBytes(charset); + } + bytes[i] = (byte) c; + } + return bytes; + } + return sb.toString().getBytes(charset); } public static String toHexString(byte[] data) { diff --git a/client/src/test/java/org/asynchttpclient/RealmTest.java b/client/src/test/java/org/asynchttpclient/RealmTest.java index 590def0a8..93ba12243 100644 --- a/client/src/test/java/org/asynchttpclient/RealmTest.java +++ b/client/src/test/java/org/asynchttpclient/RealmTest.java @@ -16,6 +16,8 @@ import org.asynchttpclient.util.StringUtils; import org.junit.jupiter.api.Test; +import java.nio.CharBuffer; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -131,6 +133,36 @@ public void testAuthIntDigestKeepsMethodAndUriInA2() throws Exception { assertEquals(expectedResponse, orig.getResponse()); } + @Test + public void digestHashesNonAsciiInputAsCharsetEncodeDoes() throws Exception { + String user = "\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c"; + String pass = "p\u00e4ss\ud800w\u20ac\ud83d\ude00"; + String realmName = "r\u00e9alm"; + String nonce = "nonce"; + String method = "GET"; + Uri uri = Uri.create("http://ahc.io/foo"); + for (String charsetParam : new String[]{"", ", charset=UTF-8"}) { + Realm realm = digestAuthRealm(user, pass) + .setUri(uri) + .setMethodName(method) + .parseWWWAuthenticateHeader("Digest realm=\"" + realmName + "\", nonce=\"" + nonce + "\", qop=\"auth\"" + charsetParam) + .build(); + + Charset charset = realm.getCharset(); + assertEquals(charsetParam.isEmpty() ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8, charset); + String ha1 = charsetEncodeMd5(user + ':' + realmName + ':' + pass, charset); + String ha2 = charsetEncodeMd5(method + ':' + uri.getPath(), charset); + String expectedResponse = charsetEncodeMd5(ha1 + ':' + nonce + ':' + realm.getNc() + ':' + realm.getCnonce() + ":auth:" + ha2, charset); + assertEquals(expectedResponse, realm.getResponse(), charset.name()); + } + } + + private static String charsetEncodeMd5(String what, Charset charset) throws Exception { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(charset.encode(CharBuffer.wrap(what))); + return StringUtils.toHexString(md.digest()); + } + // Phase 1: matchParam tests @Test public void testMatchParamUnquotedAlgorithm() { diff --git a/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java b/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java index 7ed607dde..8bdaf44e9 100644 --- a/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java +++ b/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java @@ -26,6 +26,7 @@ import java.net.URLEncoder; import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashSet; @@ -157,6 +158,39 @@ public void formUrlEncodingShouldSupportNonUtf8Charset() throws Exception { formUrlEncoding(Charset.forName("GBK")); } + @Test + public void formParamBytesMatchCharsetEncodeOfTheEncodedForm() { + List params = new ArrayList<>(); + params.add(new Param("a b", "caf\u00e9")); + params.add(new Param("\u4e2d\u6587", null)); + params.add(new Param("x\ud83d\ude00y", "a\ud800b")); + params.add(new Param("\udc00", "")); + for (Charset charset : new Charset[]{UTF_8, ISO_8859_1, Charset.forName("GBK")}) { + StringBuilder form = new StringBuilder(); + for (Param param : params) { + appendFormField(form, param.getName(), charset); + if (param.getValue() != null) { + appendFormField(form.append('='), param.getValue(), charset); + } + form.append('&'); + } + form.setLength(form.length() - 1); + ByteBuffer expected = US_ASCII.encode(CharBuffer.wrap(form)); + + // the pooled builder keeps a UTF-16 coder once it has held a char above U+00FF + StringBuilderPool.DEFAULT.stringBuilder().append('\u0100'); + assertEquals(expected, HttpUtils.urlEncodeFormParams(params, charset), charset.name()); + } + } + + private static void appendFormField(StringBuilder sb, String field, Charset charset) { + if (charset.equals(UTF_8)) { + Utf8UrlEncoder.encodeAndAppendFormElement(sb, field); + } else { + sb.append(URLEncoder.encode(field, charset)); + } + } + @Test public void computeOriginForPlainUriWithImplicitPort() { assertEquals("http://foo.com", HttpUtils.originHeader(Uri.create("ws://foo.com/bar"))); diff --git a/client/src/test/java/org/asynchttpclient/util/StringUtilsTest.java b/client/src/test/java/org/asynchttpclient/util/StringUtilsTest.java new file mode 100644 index 000000000..e2cf987c5 --- /dev/null +++ b/client/src/test/java/org/asynchttpclient/util/StringUtilsTest.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2026 AsyncHttpClient Project. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.asynchttpclient.util; + +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * {@link StringUtils#charSequence2Bytes} and {@link StringUtils#charSequence2ByteBuffer} must return + * exactly the bytes {@code charset.encode(CharBuffer.wrap(cs))} produces, including the replacement + * of malformed and unmappable input and the byte order mark of charsets that write one. + */ +public class StringUtilsTest { + + private static final String[] INPUTS = { + "", + "hello", + "caf\u00e9", + "\u20ac 100", + "\u4e2d\u6587", + "\u0000\u007f\u0080\u00ff\u0100", + // supplementary character (surrogate pair) + "x\ud83d\ude00y", + // lone high surrogate at the end, at the start and before another high surrogate + "a\ud800", + "\ud800a", + "\ud800\ud800\udc00", + // lone low surrogate, and a low surrogate followed by a high one + "\udc00a", + "a\udc00\ud800b", + "\ud83d\ude00\ud800", + // byte order mark and replacement character as content + "\ufeffbom", + "\ufffd", + }; + + @Test + public void bytesMatchCharsetEncodeForEveryEncodingCharset() { + List mismatches = new ArrayList<>(); + for (Charset charset : Charset.availableCharsets().values()) { + if (!charset.canEncode()) { + continue; + } + for (String input : INPUTS) { + for (CharSequence cs : sequences(input)) { + byte[] expected = charsetEncode(cs, charset); + if (!Arrays.equals(expected, StringUtils.charSequence2Bytes(cs, charset)) + || !Arrays.equals(expected, remaining(StringUtils.charSequence2ByteBuffer(cs, charset)))) { + mismatches.add(charset.name() + " / " + cs.getClass().getSimpleName() + " / " + escape(input)); + } + } + } + } + assertTrue(mismatches.isEmpty(), "bytes differ from Charset.encode for " + mismatches); + } + + @Test + public void unpairedSurrogatesBecomeQuestionMarks() { + assertArrayEquals(new byte[]{'a', '?', 'b'}, StringUtils.charSequence2Bytes(new StringBuilder("a\ud800b"), StandardCharsets.UTF_8)); + assertArrayEquals(new byte[]{'?', 'a'}, StringUtils.charSequence2Bytes(new StringBuilder("\udc00a"), StandardCharsets.US_ASCII)); + // a valid pair is one unmappable character in a single byte charset + assertArrayEquals(new byte[]{'x', '?', 'y'}, + remaining(StringUtils.charSequence2ByteBuffer(new StringBuilder("x\ud83d\ude00y"), StandardCharsets.ISO_8859_1))); + } + + private static List sequences(String input) { + // a recycled StringBuilder keeps a UTF-16 coder after it once held a char above U+00FF + StringBuilder inflated = new StringBuilder("\u0100"); + inflated.setLength(0); + inflated.append(input); + return Arrays.asList(input, new StringBuilder(input), inflated, CharBuffer.wrap(input)); + } + + private static byte[] charsetEncode(CharSequence cs, Charset charset) { + return remaining(charset.encode(CharBuffer.wrap(cs))); + } + + private static byte[] remaining(ByteBuffer bb) { + byte[] bytes = new byte[bb.remaining()]; + bb.duplicate().get(bytes); + return bytes; + } + + private static String escape(String s) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + sb.append(String.format("U+%04X ", (int) s.charAt(i))); + } + return sb.toString().trim(); + } +}