Encode form and digest input without CharBuffer

client/src/jmh/java/org/asynchttpclient/bench/FormDigestEncodeBenchmark.java

@@ -0,0 +1,132 @@
+/*
+ *    Copyright (c) 2026 AsyncHttpClient Project. All rights reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+package org.asynchttpclient.bench;
+
+import org.asynchttpclient.Param;
+import org.asynchttpclient.Realm;
+import org.asynchttpclient.uri.Uri;
+import org.asynchttpclient.util.HttpUtils;
+import org.asynchttpclient.util.StringUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Production paths that encode a pooled StringBuilder to bytes through
+ * {@code StringUtils.charSequence2ByteBuffer}: an url-encoded form body, and the three
+ * hashes of a Digest response, end to end and the encoding step alone. Run against two builds to
+ * compare them.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+public class FormDigestEncodeBenchmark {
+
+    private static final String UTF8_USER = "\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c";
+    private static final String UTF8_PASSWORD = "\u043f\u0430\u0440\u043e\u043b\u044c";
+
+    private List<Param> smallForm;
+    private List<Param> largeForm;
+    private Realm.Builder asciiDigest;
+    private Realm.Builder utf8Digest;
+    private StringBuilder formBuilder;
+    private StringBuilder digestBuilder;
+    private StringBuilder utf8DigestBuilder;
+
+    @Setup(Level.Trial)
+    public void setup() {
+        smallForm = form(4);
+        largeForm = form(256);
+        ByteBuffer formBody = HttpUtils.urlEncodeFormParams(largeForm, StandardCharsets.UTF_8);
+        formBuilder = new StringBuilder(StandardCharsets.US_ASCII.decode(formBody));
+        // HA1 ":" nonce ":" nc ":" cnonce ":" qop ":" HA2
+        digestBuilder = new StringBuilder("939e7578ed9e3c518a452acee763bce9:dcd98b7102dd2f0e8b11d0f600bfb0c093:00000001:"
+                + "0a4f113b5c8e1d2a:auth:39aff3a2bab6126f332b942af96d3366");
+        // username ":" realm ":" password
+        utf8DigestBuilder = new StringBuilder(UTF8_USER + ":protected:" + UTF8_PASSWORD);
+        Uri uri = Uri.create("http://localhost:8080/protected/resource?id=42");
+        String challenge = "Digest realm=\"protected\", nonce=\"dcd98b7102dd2f0e8b11d0f600bfb0c093\", qop=\"auth\"";
+        asciiDigest = new Realm.Builder("user", "secret-password")
+                .setUri(uri)
+                .setMethodName("GET")
+                .parseWWWAuthenticateHeader(challenge);
+        utf8Digest = new Realm.Builder(UTF8_USER, UTF8_PASSWORD)
+                .setUri(uri)
+                .setMethodName("GET")
+                .parseWWWAuthenticateHeader(challenge + ", charset=UTF-8");
+    }
+
+    private static List<Param> form(int params) {
+        List<Param> list = new ArrayList<>(params);
+        for (int i = 0; i < params; i++) {
+            list.add(new Param("field" + i, "value " + i + "&x=y"));
+        }
+        return list;
+    }
+
+    /** 4 params, a 91 byte body. */
+    @Benchmark
+    public ByteBuffer formSmall() {
+        return HttpUtils.urlEncodeFormParams(smallForm, StandardCharsets.UTF_8);
+    }
+
+    /** 256 params, a 6,691 byte body. */
+    @Benchmark
+    public ByteBuffer formLarge() {
+        return HttpUtils.urlEncodeFormParams(largeForm, StandardCharsets.UTF_8);
+    }
+
+    /** ISO-8859-1 digest of ASCII credentials. */
+    @Benchmark
+    public Realm digestAscii() {
+        return asciiDigest.build();
+    }
+
+    /** UTF-8 digest of Cyrillic credentials: HA1 is not ASCII. */
+    @Benchmark
+    public Realm digestUtf8() {
+        return utf8Digest.build();
+    }
+
+    /** The encoding step of {@link #formLarge} alone. */
+    @Benchmark
+    public ByteBuffer encodeFormBuilder() {
+        return StringUtils.charSequence2ByteBuffer(formBuilder, StandardCharsets.US_ASCII);
+    }
+
+    /** The encoding step of the last {@link #digestAscii} hash alone. */
+    @Benchmark
+    public ByteBuffer encodeDigestBuilder() {
+        return StringUtils.charSequence2ByteBuffer(digestBuilder, StandardCharsets.ISO_8859_1);
+    }
+
+    /** The encoding step of the first {@link #digestUtf8} hash alone: non-ASCII input. */
+    @Benchmark
+    public ByteBuffer encodeUtf8DigestBuilder() {
+        return StringUtils.charSequence2ByteBuffer(utf8DigestBuilder, StandardCharsets.UTF_8);
+    }
+}

client/src/main/java/org/asynchttpclient/util/StringUtils.java

@@ -16,17 +16,20 @@
 package org.asynchttpclient.util;
 
 import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.US_ASCII;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public final class StringUtils {
 
     private StringUtils() {
         // Prevent outside initialization
     }
 
     public static ByteBuffer charSequence2ByteBuffer(CharSequence cs, Charset charset) {
-        return charset.encode(CharBuffer.wrap(cs));
+        return ByteBuffer.wrap(charSequence2Bytes(cs, charset));
     }
 
     public static byte[] byteBuffer2ByteArray(ByteBuffer bb) {
@@ -36,8 +39,23 @@ public static byte[] byteBuffer2ByteArray(ByteBuffer bb) {
     }
 
     public static byte[] charSequence2Bytes(CharSequence sb, Charset charset) {
-        ByteBuffer bb = charSequence2ByteBuffer(sb, charset);
-        return byteBuffer2ByteArray(bb);
+        if (sb instanceof String) {
+            return ((String) sb).getBytes(charset);
+        }
+        // Each of these charsets maps a char below U+0080 to that byte, so ASCII input needs no String copy.
+        if (charset.equals(UTF_8) || charset.equals(ISO_8859_1) || charset.equals(US_ASCII)) {
+            int length = sb.length();
+            byte[] bytes = new byte[length];
+            for (int i = 0; i < length; i++) {
+                char c = sb.charAt(i);
+                if (c >= 0x80) {
+                    return sb.toString().getBytes(charset);
+                }
+                bytes[i] = (byte) c;
+            }
+            return bytes;
+        }
+        return sb.toString().getBytes(charset);
     }
 
     public static String toHexString(byte[] data) {

client/src/test/java/org/asynchttpclient/RealmTest.java

@@ -16,6 +16,8 @@
 import org.asynchttpclient.util.StringUtils;
 import org.junit.jupiter.api.Test;
 
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 
@@ -131,6 +133,36 @@ public void testAuthIntDigestKeepsMethodAndUriInA2() throws Exception {
         assertEquals(expectedResponse, orig.getResponse());
     }
 
+    @Test
+    public void digestHashesNonAsciiInputAsCharsetEncodeDoes() throws Exception {
+        String user = "\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c";
+        String pass = "p\u00e4ss\ud800w\u20ac\ud83d\ude00";
+        String realmName = "r\u00e9alm";
+        String nonce = "nonce";
+        String method = "GET";
+        Uri uri = Uri.create("http://ahc.io/foo");
+        for (String charsetParam : new String[]{"", ", charset=UTF-8"}) {
+            Realm realm = digestAuthRealm(user, pass)
+                    .setUri(uri)
+                    .setMethodName(method)
+                    .parseWWWAuthenticateHeader("Digest realm=\"" + realmName + "\", nonce=\"" + nonce + "\", qop=\"auth\"" + charsetParam)
+                    .build();
+
+            Charset charset = realm.getCharset();
+            assertEquals(charsetParam.isEmpty() ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8, charset);
+            String ha1 = charsetEncodeMd5(user + ':' + realmName + ':' + pass, charset);
+            String ha2 = charsetEncodeMd5(method + ':' + uri.getPath(), charset);
+            String expectedResponse = charsetEncodeMd5(ha1 + ':' + nonce + ':' + realm.getNc() + ':' + realm.getCnonce() + ":auth:" + ha2, charset);
+            assertEquals(expectedResponse, realm.getResponse(), charset.name());
+        }
+    }
+
+    private static String charsetEncodeMd5(String what, Charset charset) throws Exception {
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        md.update(charset.encode(CharBuffer.wrap(what)));
+        return StringUtils.toHexString(md.digest());
+    }
+
     // Phase 1: matchParam tests
     @Test
     public void testMatchParamUnquotedAlgorithm() {

client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java

@@ -26,6 +26,7 @@
 
 import java.net.URLEncoder;
 import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -157,6 +158,39 @@ public void formUrlEncodingShouldSupportNonUtf8Charset() throws Exception {
         formUrlEncoding(Charset.forName("GBK"));
     }
 
+    @Test
+    public void formParamBytesMatchCharsetEncodeOfTheEncodedForm() {
+        List<Param> params = new ArrayList<>();
+        params.add(new Param("a b", "caf\u00e9"));
+        params.add(new Param("\u4e2d\u6587", null));
+        params.add(new Param("x\ud83d\ude00y", "a\ud800b"));
+        params.add(new Param("\udc00", ""));
+        for (Charset charset : new Charset[]{UTF_8, ISO_8859_1, Charset.forName("GBK")}) {
+            StringBuilder form = new StringBuilder();
+            for (Param param : params) {
+                appendFormField(form, param.getName(), charset);
+                if (param.getValue() != null) {
+                    appendFormField(form.append('='), param.getValue(), charset);
+                }
+                form.append('&');
+            }
+            form.setLength(form.length() - 1);
+            ByteBuffer expected = US_ASCII.encode(CharBuffer.wrap(form));
+
+            // the pooled builder keeps a UTF-16 coder once it has held a char above U+00FF
+            StringBuilderPool.DEFAULT.stringBuilder().append('\u0100');
+            assertEquals(expected, HttpUtils.urlEncodeFormParams(params, charset), charset.name());
+        }
+    }
+
+    private static void appendFormField(StringBuilder sb, String field, Charset charset) {
+        if (charset.equals(UTF_8)) {
+            Utf8UrlEncoder.encodeAndAppendFormElement(sb, field);
+        } else {
+            sb.append(URLEncoder.encode(field, charset));
+        }
+    }
+
     @Test
     public void computeOriginForPlainUriWithImplicitPort() {
         assertEquals("http://foo.com", HttpUtils.originHeader(Uri.create("ws://foo.com/bar")));

client/src/test/java/org/asynchttpclient/util/StringUtilsTest.java

@@ -0,0 +1,114 @@
+/*
+ *    Copyright (c) 2026 AsyncHttpClient Project. All rights reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+package org.asynchttpclient.util;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * {@link StringUtils#charSequence2Bytes} and {@link StringUtils#charSequence2ByteBuffer} must return
+ * exactly the bytes {@code charset.encode(CharBuffer.wrap(cs))} produces, including the replacement
+ * of malformed and unmappable input and the byte order mark of charsets that write one.
+ */
+public class StringUtilsTest {
+
+    private static final String[] INPUTS = {
+            "",
+            "hello",
+            "caf\u00e9",
+            "\u20ac 100",
+            "\u4e2d\u6587",
+            "\u0000\u007f\u0080\u00ff\u0100",
+            // supplementary character (surrogate pair)
+            "x\ud83d\ude00y",
+            // lone high surrogate at the end, at the start and before another high surrogate
+            "a\ud800",
+            "\ud800a",
+            "\ud800\ud800\udc00",
+            // lone low surrogate, and a low surrogate followed by a high one
+            "\udc00a",
+            "a\udc00\ud800b",
+            "\ud83d\ude00\ud800",
+            // byte order mark and replacement character as content
+            "\ufeffbom",
+            "\ufffd",
+    };
+
+    @Test
+    public void bytesMatchCharsetEncodeForEveryEncodingCharset() {
+        List<String> mismatches = new ArrayList<>();
+        for (Charset charset : Charset.availableCharsets().values()) {
+            if (!charset.canEncode()) {
+                continue;
+            }
+            for (String input : INPUTS) {
+                for (CharSequence cs : sequences(input)) {
+                    byte[] expected = charsetEncode(cs, charset);
+                    if (!Arrays.equals(expected, StringUtils.charSequence2Bytes(cs, charset))
+                            || !Arrays.equals(expected, remaining(StringUtils.charSequence2ByteBuffer(cs, charset)))) {
+                        mismatches.add(charset.name() + " / " + cs.getClass().getSimpleName() + " / " + escape(input));
+                    }
+                }
+            }
+        }
+        assertTrue(mismatches.isEmpty(), "bytes differ from Charset.encode for " + mismatches);
+    }
+
+    @Test
+    public void unpairedSurrogatesBecomeQuestionMarks() {
+        assertArrayEquals(new byte[]{'a', '?', 'b'}, StringUtils.charSequence2Bytes(new StringBuilder("a\ud800b"), StandardCharsets.UTF_8));
+        assertArrayEquals(new byte[]{'?', 'a'}, StringUtils.charSequence2Bytes(new StringBuilder("\udc00a"), StandardCharsets.US_ASCII));
+        // a valid pair is one unmappable character in a single byte charset
+        assertArrayEquals(new byte[]{'x', '?', 'y'},
+                remaining(StringUtils.charSequence2ByteBuffer(new StringBuilder("x\ud83d\ude00y"), StandardCharsets.ISO_8859_1)));
+    }
+
+    private static List<CharSequence> sequences(String input) {
+        // a recycled StringBuilder keeps a UTF-16 coder after it once held a char above U+00FF
+        StringBuilder inflated = new StringBuilder("\u0100");
+        inflated.setLength(0);
+        inflated.append(input);
+        return Arrays.asList(input, new StringBuilder(input), inflated, CharBuffer.wrap(input));
+    }
+
+    private static byte[] charsetEncode(CharSequence cs, Charset charset) {
+        return remaining(charset.encode(CharBuffer.wrap(cs)));
+    }
+
+    private static byte[] remaining(ByteBuffer bb) {
+        byte[] bytes = new byte[bb.remaining()];
+        bb.duplicate().get(bytes);
+        return bytes;
+    }
+
+    private static String escape(String s) {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < s.length(); i++) {
+            sb.append(String.format("U+%04X ", (int) s.charAt(i)));
+        }
+        return sb.toString().trim();
+    }
+}