Add XXHash64, XXHash3, and XXHash128 to smithy-checksums

packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-4efedfc0ab3648858d67c42495fb580b.json

@@ -0,0 +1,4 @@
+{
+  "type": "feature",
+  "description": "Added XXHash64, XXHash3, and XXHash128 checksums."
+}

packages/smithy-checksums/Cargo.toml

@@ -14,6 +14,7 @@ path = "rust/lib.rs"
 [dependencies]
 crc-fast = { version = "1.10", default-features = false, features = ["std"] }
 pyo3 = { version = "0.30", features = ["abi3-py312", "abi3t-py315"] }
+twox-hash = { version = "2.1", default-features = false, features = ["std", "xxhash64", "xxhash3_64", "xxhash3_128"] }
 
 [profile.release]
 codegen-units = 1

packages/smithy-checksums/LICENSE-THIRD-PARTY

@@ -1,7 +1,9 @@
-Third-party software compiled into the smithy-checksums native extension.
+Third-party software in smithy-checksums: the crates compiled into the native
+extension, and the reference code the pure-Python XXHash implementation is ported
+from.
 
-This file is generated from Cargo.lock by `make third-party-licenses`. Don't edit
-it by hand.
+This file is generated from Cargo.lock and about.hbs by `make third-party-licenses`.
+Don't edit it by hand.
 
 ================================================================================
 Apache License 2.0 (Apache-2.0)
@@ -958,6 +960,35 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+================================================================================
+MIT License (MIT)
+
+Used by:
+  - twox-hash 2.1.5 (https://github.com/shepmaster/twox-hash)
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Jake Goulding
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
 ================================================================================
 MIT License (MIT)
 
@@ -1032,3 +1063,38 @@ not be used in advertising or otherwise to promote the sale, use or other
 dealings in these Data Files or Software without prior written
 authorization of the copyright holder.
 
+
+================================================================================
+BSD 2-Clause "Simplified" License (BSD-2-Clause)
+
+Used by:
+  - The pure-Python XXH64, XXH3, and XXH128 in src/smithy_checksums/_private/xxhash.py,
+    ported from xxHash (https://github.com/Cyan4973/xxHash)
+
+xxHash - Extremely Fast Hash algorithm
+Copyright (C) 2012-2023 Yann Collet
+
+BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+     copyright notice, this list of conditions and the following disclaimer
+     in the documentation and/or other materials provided with the
+     distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/smithy-checksums/README.md

@@ -1,7 +1,8 @@
 # smithy-checksums
 
 This package provides the checksum algorithms used by Smithy clients: CRC32,
-CRC32C, CRC64NVME, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5.
+CRC32C, CRC64NVME, XXHash64, XXHash3, XXHash128, and, through `new()`, SHA-1,
+SHA-256, SHA-512, and MD5.
 
 ```python
 import smithy_checksums
@@ -13,7 +14,7 @@ checksum.digest()
 
 ## Native acceleration
 
-The CRC algorithms are implemented in Rust and published as wheels for
+The CRC and XXHash algorithms are implemented in Rust and published as wheels for
 common platforms. Where no wheel is available, the package installs as pure Python
 and produces identical results, much more slowly. `HAS_NATIVE_EXTENSION` reports
 which implementation is in use. CRC32 always uses `zlib` in pure Python, and the

packages/smithy-checksums/about.hbs

@@ -1,7 +1,9 @@
-Third-party software compiled into the smithy-checksums native extension.
+Third-party software in smithy-checksums: the crates compiled into the native
+extension, and the reference code the pure-Python XXHash implementation is ported
+from.
 
-This file is generated from Cargo.lock by `make third-party-licenses`. Don't edit
-it by hand.
+This file is generated from Cargo.lock and about.hbs by `make third-party-licenses`.
+Don't edit it by hand.
 {{#each licenses}}
 
 ================================================================================
@@ -14,3 +16,38 @@ Used by:
 
 {{{text}}}
 {{/each}}
+
+================================================================================
+BSD 2-Clause "Simplified" License (BSD-2-Clause)
+
+Used by:
+  - The pure-Python XXH64, XXH3, and XXH128 in src/smithy_checksums/_private/xxhash.py,
+    ported from xxHash (https://github.com/Cyan4973/xxHash)
+
+xxHash - Extremely Fast Hash algorithm
+Copyright (C) 2012-2023 Yann Collet
+
+BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+     copyright notice, this list of conditions and the following disclaimer
+     in the documentation and/or other materials provided with the
+     distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/smithy-checksums/build_backend/smithy_checksums_build.py

@@ -36,7 +36,8 @@
 _CARGO_TOML = Path(__file__).parent.parent / "Cargo.toml"
 _PURE_PYTHON_HINT = (
     f"To install the pure-Python implementation instead, set {_ENV_VAR}=never. It "
-    "produces identical checksums, but CRC32C and CRC64NVME are much slower."
+    "produces identical checksums, but CRC32C, CRC64NVME, and the XXHash algorithms "
+    "are much slower."
 )
 
 

packages/smithy-checksums/pyproject.toml

@@ -9,7 +9,7 @@ authors = [
 description = "Checksum algorithms for Smithy tooling, with native acceleration."
 readme = "README.md"
 license = {text = "Apache License 2.0"}
-keywords = ["smithy", "sdk", "checksum", "crc32c", "crc64nvme"]
+keywords = ["smithy", "sdk", "checksum", "crc32c", "crc64nvme", "xxhash"]
 classifiers = [
   "Development Status :: 2 - Pre-Alpha",
   "Intended Audience :: Developers",

packages/smithy-checksums/rust/lib.rs

@@ -4,8 +4,10 @@
 //! Native checksum implementations for `smithy_checksums`.
 //!
 //! Each class mirrors the pure-Python implementation of the same name in
-//! `smithy_checksums._private.crc`, so the two are interchangeable.
+//! `smithy_checksums._private.crc` or `smithy_checksums._private.xxhash`, so
+//! the two are interchangeable.
 
+use std::hash::Hasher as _;
 use std::sync::{Mutex, MutexGuard};
 
 use pyo3::buffer::PyBuffer;
@@ -56,6 +58,63 @@ crc_state!(Crc32State, Crc32IsoHdlc, u32);
 crc_state!(Crc32cState, Crc32Iscsi, u32);
 crc_state!(Crc64NvmeState, Crc64Nvme, u64);
 
+#[derive(Clone)]
+struct XxHash64State(twox_hash::XxHash64);
+
+impl State for XxHash64State {
+    type Digest = [u8; 8];
+
+    fn new() -> Self {
+        Self(twox_hash::XxHash64::with_seed(0))
+    }
+
+    fn update(&mut self, data: &[u8]) {
+        self.0.write(data);
+    }
+
+    fn digest(&self) -> Self::Digest {
+        self.0.finish().to_be_bytes()
+    }
+}
+
+#[derive(Clone)]
+struct XxHash3State(twox_hash::XxHash3_64);
+
+impl State for XxHash3State {
+    type Digest = [u8; 8];
+
+    fn new() -> Self {
+        Self(twox_hash::XxHash3_64::new())
+    }
+
+    fn update(&mut self, data: &[u8]) {
+        self.0.write(data);
+    }
+
+    fn digest(&self) -> Self::Digest {
+        self.0.finish().to_be_bytes()
+    }
+}
+
+#[derive(Clone)]
+struct XxHash128State(twox_hash::XxHash3_128);
+
+impl State for XxHash128State {
+    type Digest = [u8; 16];
+
+    fn new() -> Self {
+        Self(twox_hash::XxHash3_128::new())
+    }
+
+    fn update(&mut self, data: &[u8]) {
+        self.0.write(data);
+    }
+
+    fn digest(&self) -> Self::Digest {
+        self.0.finish_128().to_be_bytes()
+    }
+}
+
 fn lock<S>(state: &Mutex<S>) -> MutexGuard<'_, S> {
     // A panic while holding the lock can't leave a checksum half-updated in a
     // way that matters more than the panic itself, so ignore poisoning.
@@ -124,9 +183,12 @@ macro_rules! checksum_class {
 checksum_class!(Crc32, Crc32State);
 checksum_class!(Crc32C, Crc32cState);
 checksum_class!(Crc64Nvme, Crc64NvmeState);
+checksum_class!(XxHash64, XxHash64State);
+checksum_class!(XxHash3, XxHash3State);
+checksum_class!(XxHash128, XxHash128State);
 
 #[pymodule]
 mod _native {
     #[pymodule_export]
-    use super::{Crc32, Crc32C, Crc64Nvme};
+    use super::{Crc32, Crc32C, Crc64Nvme, XxHash3, XxHash64, XxHash128};
 }

packages/smithy-checksums/src/smithy_checksums/__init__.py

@@ -2,11 +2,11 @@
 #  SPDX-License-Identifier: Apache-2.0
 """Checksum algorithms for Smithy clients.
 
-CRC32, CRC32C, and CRC64NVME use a native extension when one is available for the
-platform and fall back to pure Python otherwise. The fallback produces identical
-results but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`.
-SHA-1, SHA-256, SHA-512, and MD5 are available through :func:`new` and always use
-:mod:`hashlib`.
+CRC32, CRC32C, CRC64NVME, XXHash64, XXHash3, and XXHash128 use a native extension when
+one is available for the platform and fall back to pure Python otherwise. The
+fallback produces identical results but is orders of magnitude slower, except for
+CRC32, which uses :mod:`zlib`. SHA-1, SHA-256, SHA-512, and MD5 are available
+through :func:`new` and always use :mod:`hashlib`.
 """
 
 import hashlib
@@ -26,13 +26,17 @@
     "CRC64NVME",
     "HAS_NATIVE_EXTENSION",
     "Checksum",
+    "XXHash3",
+    "XXHash64",
+    "XXHash128",
     "new",
 )
 
 HAS_NATIVE_EXTENSION: Final[bool] = _NATIVE
 """Whether the native extension is in use.
 
-When it isn't, CRC32C and CRC64NVME are computed in pure Python.
+When it isn't, CRC32C, CRC64NVME, XXHash64, XXHash3, and XXHash128 are computed in
+pure Python.
 """
 
 
@@ -132,7 +136,36 @@ class CRC64NVME(_BaseChecksum):
     digest_size = 8
 
 
-_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C, CRC64NVME)}
+class XXHash64(_BaseChecksum):
+    """XXH64 with a seed of zero."""
+
+    __slots__ = ()
+
+    name = "xxhash64"
+    digest_size = 8
+
+
+class XXHash3(_BaseChecksum):
+    """XXH3 (64-bit) with a seed of zero and the default secret."""
+
+    __slots__ = ()
+
+    name = "xxhash3"
+    digest_size = 8
+
+
+class XXHash128(_BaseChecksum):
+    """XXH3 (128-bit) with a seed of zero and the default secret."""
+
+    __slots__ = ()
+
+    name = "xxhash128"
+    digest_size = 16
+
+
+_CHECKSUMS: Final = {
+    cls.name: cls for cls in (CRC32, CRC32C, CRC64NVME, XXHash64, XXHash3, XXHash128)
+}
 _HASHLIB_ALGORITHMS: Final = frozenset(("sha1", "sha256", "sha512", "md5"))
 
 
@@ -142,8 +175,8 @@ def new(name: str) -> Checksum:
     Names are case-insensitive, so the values of a checksum algorithm enum such as
     ``CRC64NVME`` or ``SHA256`` can be passed directly.
 
-    :param name: One of ``crc32``, ``crc32c``, ``crc64nvme``, ``sha1``, ``sha256``,
-        ``sha512``, or ``md5``.
+    :param name: One of ``crc32``, ``crc32c``, ``crc64nvme``, ``xxhash64``,
+        ``xxhash3``, ``xxhash128``, ``sha1``, ``sha256``, ``sha512``, or ``md5``.
     :raises ValueError: If the algorithm isn't supported, or isn't available in this
         Python build.
     """