diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index cf465e5fdc..b116fe85de 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -33,6 +33,7 @@ set(ICEBERG_SOURCES deletes/position_delete_index.cc deletes/position_delete_range_consumer.cc deletes/roaring_position_bitmap.cc + encryption/encryption_manager.cc encryption/key_management_client.cc encryption/standard_key_metadata.cc expression/aggregate.cc @@ -321,6 +322,7 @@ if(ICEBERG_BUILD_BUNDLE) avro/avro_register.cc avro/avro_schema_util.cc avro/avro_stream_internal.cc + encryption/encryption_register.cc parquet/parquet_data_util.cc parquet/parquet_metrics.cc parquet/parquet_metrics_row_group_filter.cc @@ -335,7 +337,8 @@ if(ICEBERG_BUILD_BUNDLE) ICEBERG_BUNDLE_SOURCES encryption/aes_gcm.cc encryption/aes_gcm_stream.cc - encryption/in_memory_kms.cc) + encryption/in_memory_kms.cc + encryption/standard_encryption_manager.cc) endif() # Libraries to link with exported libiceberg_bundle.{so,a}. @@ -410,6 +413,11 @@ if(ICEBERG_BUILD_BUNDLE) foreach(target iceberg_bundle_static iceberg_bundle_shared) if(TARGET ${target}) + if(ICEBERG_ENCRYPTION) + target_compile_definitions(${target} PRIVATE ICEBERG_ENCRYPTION_ENABLED=1) + else() + target_compile_definitions(${target} PRIVATE ICEBERG_ENCRYPTION_ENABLED=0) + endif() if(ICEBERG_S3) target_compile_definitions(${target} PUBLIC "$") diff --git a/src/iceberg/cpp20_compatibility_internal.h b/src/iceberg/cpp20_compatibility_internal.h index d186e5bb0d..ad9edfb1b1 100644 --- a/src/iceberg/cpp20_compatibility_internal.h +++ b/src/iceberg/cpp20_compatibility_internal.h @@ -49,6 +49,7 @@ #include "iceberg/deletes/position_delete_range_consumer.h" #include "iceberg/deletes/roaring_position_bitmap.h" #include "iceberg/encryption/encrypted_key.h" +#include "iceberg/encryption/encryption_manager.h" #include "iceberg/encryption/key_management_client.h" #include "iceberg/encryption/standard_key_metadata.h" #include "iceberg/exception.h" @@ -213,6 +214,7 @@ # include "iceberg/avro/avro_reader.h" # include "iceberg/avro/avro_register.h" # include "iceberg/avro/avro_writer.h" +# include "iceberg/encryption/encryption_register.h" # include "iceberg/parquet/parquet_reader.h" # include "iceberg/parquet/parquet_register.h" # include "iceberg/parquet/parquet_writer.h" @@ -221,6 +223,7 @@ #ifdef ICEBERG_PUBLIC_HEADERS_ENCRYPTION # include "iceberg/encryption/aes_gcm_stream.h" # include "iceberg/encryption/in_memory_kms.h" +# include "iceberg/encryption/standard_encryption_manager.h" #endif // ICEBERG_PUBLIC_HEADERS_ENCRYPTION #ifdef ICEBERG_PUBLIC_HEADERS_HIVE diff --git a/src/iceberg/encryption/CMakeLists.txt b/src/iceberg/encryption/CMakeLists.txt index c490aa79ca..932b1deb75 100644 --- a/src/iceberg/encryption/CMakeLists.txt +++ b/src/iceberg/encryption/CMakeLists.txt @@ -18,13 +18,22 @@ iceberg_install_all_headers(iceberg/encryption PATTERN encrypted_key.h + encryption_manager.h key_management_client.h standard_key_metadata.h) -if(ICEBERG_BUILD_BUNDLE AND ICEBERG_ENCRYPTION) +if(ICEBERG_BUILD_BUNDLE) iceberg_install_all_headers(iceberg/encryption PATTERN - aes_gcm_stream.h - in_memory_kms.h + encryption_register.h HEADER_GROUP - ICEBERG_PUBLIC_HEADERS_ENCRYPTION) + ICEBERG_PUBLIC_HEADERS_BUNDLE) + if(ICEBERG_ENCRYPTION) + iceberg_install_all_headers(iceberg/encryption + PATTERN + aes_gcm_stream.h + in_memory_kms.h + standard_encryption_manager.h + HEADER_GROUP + ICEBERG_PUBLIC_HEADERS_ENCRYPTION) + endif() endif() diff --git a/src/iceberg/encryption/encryption_manager.cc b/src/iceberg/encryption/encryption_manager.cc new file mode 100644 index 0000000000..a49d86d1fc --- /dev/null +++ b/src/iceberg/encryption/encryption_manager.cc @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/encryption_manager.h" + +#include + +#include "iceberg/util/macros.h" + +namespace iceberg { + +namespace { + +struct RegistryState { + std::mutex mutex; + EncryptionManagerRegistry::Factory factory; +}; + +RegistryState& State() { + static RegistryState state; + return state; +} + +} // namespace + +void EncryptionManagerRegistry::Register(Factory factory) { + auto& state = State(); + std::lock_guard lock(state.mutex); + state.factory = std::move(factory); +} + +Result> EncryptionManagerRegistry::Make( + Options options) { + ICEBERG_PRECHECK(!options.table_key_id.empty(), "Invalid encryption key ID: empty"); + ICEBERG_PRECHECK(options.data_key_length == 16 || options.data_key_length == 24 || + options.data_key_length == 32, + "Invalid data key length: {} (must be 16, 24, or 32)", + options.data_key_length); + ICEBERG_PRECHECK(options.kms != nullptr, + "Cannot create an encryption manager without a KMS client; set the " + "encryption.kms-type or encryption.kms-impl catalog property"); + Factory factory; + { + auto& state = State(); + std::lock_guard lock(state.mutex); + factory = state.factory; + } + if (!factory) { + return NotSupported( + "Table encryption is not available: link iceberg_bundle built with " + "ICEBERG_ENCRYPTION=ON and call iceberg::encryption::RegisterAll()"); + } + return factory(std::move(options)); +} + +} // namespace iceberg diff --git a/src/iceberg/encryption/encryption_manager.h b/src/iceberg/encryption/encryption_manager.h new file mode 100644 index 0000000000..83ccaa99d3 --- /dev/null +++ b/src/iceberg/encryption/encryption_manager.h @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/encryption_manager.h +/// Table encryption manager interface and the registry that creates one. + +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/encryption/encrypted_key.h" +#include "iceberg/encryption/key_management_client.h" +#include "iceberg/encryption/standard_key_metadata.h" +#include "iceberg/file_io.h" +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" + +namespace iceberg { + +/// \brief Keys created when the key metadata of a new manifest list is registered. +/// +/// Both must be added to the table metadata `encryption-keys` in the same commit as +/// the snapshot whose `key-id` is `file_key.key_id`. +struct ICEBERG_EXPORT ManifestListEncryptionKeys { + /// The key encryption key, wrapped by the table master key in the KMS. + EncryptedKey key_encryption_key; + /// The manifest list key metadata, encrypted by the key encryption key. + EncryptedKey file_key; +}; + +/// \brief Encrypts and decrypts the files of one table. +/// +/// The standard implementation (StandardEncryptionManager) needs a crypto provider +/// and lives in the bundle library; the core library reaches it through +/// EncryptionManagerRegistry. +class ICEBERG_EXPORT EncryptionManager { + public: + virtual ~EncryptionManager() = default; + + /// \brief Generate key metadata (random data key and AAD prefix) for a new file. + virtual Result NewKeyMetadata() = 0; + + /// \brief Wrap an AES GCM Stream (AGS1) file in a decrypting InputFile. + /// + /// \param encrypted the encrypted file + /// \param key_metadata serialized key metadata, including the file length + virtual Result> NewDecryptingInputFile( + std::unique_ptr encrypted, std::span key_metadata) = 0; + + /// \brief Wrap a file in an AES GCM Stream (AGS1) encrypting OutputFile. + virtual Result> NewEncryptingOutputFile( + std::unique_ptr plain, const StandardKeyMetadata& key_metadata) = 0; + + /// \brief Decrypt the key metadata of a manifest list. + /// + /// \param manifest_list_key_id the snapshot `key-id` + /// \return the serialized StandardKeyMetadata of the manifest list file + virtual Result> DecryptManifestListKeyMetadata( + std::string_view manifest_list_key_id) = 0; + + /// \brief Encrypt the key metadata of a new manifest list with a key encryption key. + /// + /// Creates a key encryption key (wrapped by the table master key) if there is no + /// unexpired one. + virtual Result RegisterManifestListKeyMetadata( + const StandardKeyMetadata& key_metadata) = 0; +}; + +/// \brief Creates the encryption manager of encrypted tables. +/// +/// The bundle library registers the standard implementation (see +/// iceberg::encryption::RegisterAll()). Without it, encrypted tables cannot be read or +/// written. +class ICEBERG_EXPORT EncryptionManagerRegistry { + public: + struct Options { + /// The table master key ID in the KMS (table property `encryption.key-id`), taken + /// from a trusted source. + std::string table_key_id; + /// Length of generated data keys: 16, 24 or 32 bytes. + int32_t data_key_length = 16; + /// The table metadata `encryption-keys`. + std::vector encryption_keys; + /// The catalog's KMS client. + std::shared_ptr kms; + }; + + using Factory = + std::function>(Options options)>; + + /// \brief Register the encryption manager implementation. + static void Register(Factory factory); + + /// \brief Create an encryption manager. + static Result> Make(Options options); +}; + +} // namespace iceberg diff --git a/src/iceberg/encryption/encryption_register.cc b/src/iceberg/encryption/encryption_register.cc new file mode 100644 index 0000000000..3df7383c2b --- /dev/null +++ b/src/iceberg/encryption/encryption_register.cc @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/encryption_register.h" + +#if ICEBERG_ENCRYPTION_ENABLED +# include "iceberg/encryption/standard_encryption_manager.h" +#endif + +namespace iceberg::encryption { + +void RegisterAll() { +#if ICEBERG_ENCRYPTION_ENABLED + RegisterStandardEncryptionManager(); +#endif +} + +bool IsEncryptionAvailable() { return ICEBERG_ENCRYPTION_ENABLED; } + +} // namespace iceberg::encryption diff --git a/src/iceberg/encryption/encryption_register.h b/src/iceberg/encryption/encryption_register.h new file mode 100644 index 0000000000..beacad013e --- /dev/null +++ b/src/iceberg/encryption/encryption_register.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/encryption_register.h +/// Registration of the table encryption implementation of the bundle library. + +#include "iceberg/iceberg_bundle_export.h" + +namespace iceberg::encryption { + +/// \brief Register the standard encryption manager, if the bundle was built with +/// ICEBERG_ENCRYPTION=ON. Otherwise a no-op, and encrypted tables are not supported. +ICEBERG_BUNDLE_EXPORT void RegisterAll(); + +/// \brief Whether the bundle was built with table encryption support. +ICEBERG_BUNDLE_EXPORT bool IsEncryptionAvailable(); + +} // namespace iceberg::encryption diff --git a/src/iceberg/encryption/standard_encryption_manager.cc b/src/iceberg/encryption/standard_encryption_manager.cc new file mode 100644 index 0000000000..fc4356758f --- /dev/null +++ b/src/iceberg/encryption/standard_encryption_manager.cc @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/standard_encryption_manager.h" + +#include +#include +#include + +#include "iceberg/encryption/aes_gcm_internal.h" +#include "iceberg/encryption/aes_gcm_stream.h" +#include "iceberg/util/base64.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +namespace { + +using encryption::AesGcmCipher; +using encryption::SecureRandomBytes; + +std::span AsBytes(std::string_view s) { + return {reinterpret_cast(s.data()), s.size()}; +} + +std::string ToString(std::span bytes) { + return {reinterpret_cast(bytes.data()), bytes.size()}; +} + +Result GenerateKeyId() { + std::array id{}; + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(id)); + return Base64::Encode(ToString(id)); +} + +std::optional ParseTimestamp(const EncryptedKey& key) { + auto it = key.properties.find(std::string(StandardEncryptionManager::kKeyTimestamp)); + if (it == key.properties.end()) { + return std::nullopt; + } + int64_t value = 0; + const auto& str = it->second; + auto [ptr, ec] = std::from_chars(str.data(), str.data() + str.size(), value); + if (ec != std::errc{} || ptr != str.data() + str.size() || value < 0) { + return std::nullopt; + } + return value; +} + +} // namespace + +Result> StandardEncryptionManager::Make( + EncryptionManagerRegistry::Options options) { + ICEBERG_PRECHECK(!options.table_key_id.empty(), "Invalid encryption key ID: empty"); + ICEBERG_PRECHECK(options.data_key_length == 16 || options.data_key_length == 24 || + options.data_key_length == 32, + "Invalid data key length: {} (must be 16, 24, or 32)", + options.data_key_length); + ICEBERG_PRECHECK(options.kms != nullptr, "Invalid KMS client: null"); + return std::shared_ptr( + new StandardEncryptionManager(std::move(options))); +} + +StandardEncryptionManager::StandardEncryptionManager( + EncryptionManagerRegistry::Options options) + : table_key_id_(std::move(options.table_key_id)), + data_key_length_(options.data_key_length), + kms_(std::move(options.kms)) { + for (auto& key : options.encryption_keys) { + AddKey(std::move(key)); + } +} + +void StandardEncryptionManager::AddKey(EncryptedKey key) { + auto key_id = key.key_id; + if (keys_.insert_or_assign(key_id, std::move(key)).second) { + key_order_.push_back(std::move(key_id)); + } +} + +void StandardEncryptionManager::SetTestTimeShift(std::chrono::milliseconds shift) { + std::lock_guard lock(mutex_); + time_shift_ = shift; +} + +int64_t StandardEncryptionManager::NowMs() const { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + time_shift_) + .count(); +} + +Result StandardEncryptionManager::NewKeyMetadata() { + StandardKeyMetadata metadata; + metadata.encryption_key.resize(data_key_length_); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(metadata.encryption_key)); + metadata.aad_prefix.emplace(kAadPrefixLength); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(*metadata.aad_prefix)); + return metadata; +} + +Result> StandardEncryptionManager::NewDecryptingInputFile( + std::unique_ptr encrypted, std::span key_metadata) { + ICEBERG_PRECHECK(encrypted != nullptr, "Encrypted input file cannot be null"); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, StandardKeyMetadata::Parse(key_metadata)); + if (!metadata.file_length.has_value()) { + // The length must come from trusted key metadata, not from the file system, so that + // truncation is detected (format/gcm-stream-spec.md, "File length"). + return Invalid("Cannot read AES GCM stream {}: key metadata has no file length", + encrypted->location()); + } + return std::make_unique( + std::move(encrypted), std::move(metadata.encryption_key), + metadata.aad_prefix.value_or(std::vector{}), *metadata.file_length); +} + +Result> StandardEncryptionManager::NewEncryptingOutputFile( + std::unique_ptr plain, const StandardKeyMetadata& key_metadata) { + ICEBERG_PRECHECK(plain != nullptr, "Output file cannot be null"); + return std::make_unique( + std::move(plain), key_metadata.encryption_key, + key_metadata.aad_prefix.value_or(std::vector{})); +} + +Result> StandardEncryptionManager::UnwrapKek( + const std::string& kek_id) { + int64_t now = NowMs(); + if (auto it = unwrapped_keks_.find(kek_id); it != unwrapped_keks_.end()) { + if (now - it->second.unwrapped_at_ms < kUnwrappedKekCacheExpiry.count()) { + return it->second.key; + } + unwrapped_keks_.erase(it); + } + auto it = keys_.find(kek_id); + if (it == keys_.end()) { + return Invalid("Cannot find key encryption key with id {}", kek_id); + } + ICEBERG_ASSIGN_OR_RAISE( + auto kek, + kms_->UnwrapKey(AsBytes(it->second.encrypted_key_metadata), table_key_id_)); + unwrapped_keks_.insert_or_assign(kek_id, CachedKek{.key = kek, .unwrapped_at_ms = now}); + return kek; +} + +Result> StandardEncryptionManager::DecryptManifestListKeyMetadata( + std::string_view manifest_list_key_id) { + std::lock_guard lock(mutex_); + auto it = keys_.find(std::string(manifest_list_key_id)); + if (it == keys_.end()) { + return Invalid("Cannot find manifest list key metadata with id {}", + manifest_list_key_id); + } + const EncryptedKey& manifest_list_key = it->second; + if (!manifest_list_key.encrypted_by_id.has_value() || + *manifest_list_key.encrypted_by_id == table_key_id_) { + return Invalid("{} is not manifest list key metadata", manifest_list_key_id); + } + + const std::string& kek_id = *manifest_list_key.encrypted_by_id; + ICEBERG_ASSIGN_OR_RAISE(auto kek, UnwrapKek(kek_id)); + auto timestamp = keys_.at(kek_id).properties.find(std::string(kKeyTimestamp)); + if (timestamp == keys_.at(kek_id).properties.end()) { + return Invalid("Key encryption key {} must be timestamped", kek_id); + } + + // The KEK timestamp is the AAD, to prevent timestamp tampering attacks. + ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(kek)); + return cipher->Decrypt(AsBytes(manifest_list_key.encrypted_key_metadata), + AsBytes(timestamp->second)); +} + +Result StandardEncryptionManager::CurrentKekId() { + int64_t now = NowMs(); + for (const auto& key_id : key_order_) { + const EncryptedKey& key = keys_.at(key_id); + if (key.encrypted_by_id != table_key_id_) { + continue; + } + auto timestamp = ParseTimestamp(key); + if (timestamp.has_value() && now - *timestamp < kKekLifespan.count()) { + return key_id; + } + } + + // No unexpired key encryption key: create one, wrapped by the table master key. + std::vector kek(data_key_length_); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(kek)); + ICEBERG_ASSIGN_OR_RAISE(auto wrapped, kms_->WrapKey(kek, table_key_id_)); + ICEBERG_ASSIGN_OR_RAISE(auto kek_id, GenerateKeyId()); + AddKey(EncryptedKey{.key_id = kek_id, + .encrypted_key_metadata = ToString(wrapped), + .encrypted_by_id = table_key_id_, + .properties = {{std::string(kKeyTimestamp), std::to_string(now)}}}); + unwrapped_keks_.insert_or_assign( + kek_id, CachedKek{.key = std::move(kek), .unwrapped_at_ms = now}); + return kek_id; +} + +Result +StandardEncryptionManager::RegisterManifestListKeyMetadata( + const StandardKeyMetadata& key_metadata) { + ICEBERG_PRECHECK(key_metadata.file_length.has_value(), + "Manifest list key metadata must include the file length"); + std::lock_guard lock(mutex_); + ICEBERG_ASSIGN_OR_RAISE(auto kek_id, CurrentKekId()); + ICEBERG_ASSIGN_OR_RAISE(auto kek, UnwrapKek(kek_id)); + const EncryptedKey& kek_entry = keys_.at(kek_id); + const std::string& timestamp = kek_entry.properties.at(std::string(kKeyTimestamp)); + + ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(kek)); + ICEBERG_ASSIGN_OR_RAISE(auto encrypted, + cipher->Encrypt(key_metadata.Serialize(), AsBytes(timestamp))); + ICEBERG_ASSIGN_OR_RAISE(auto file_key_id, GenerateKeyId()); + EncryptedKey file_key{.key_id = file_key_id, + .encrypted_key_metadata = ToString(encrypted), + .encrypted_by_id = kek_id}; + AddKey(file_key); + return ManifestListEncryptionKeys{.key_encryption_key = keys_.at(kek_id), + .file_key = std::move(file_key)}; +} + +std::vector StandardEncryptionManager::encryption_keys() const { + std::lock_guard lock(mutex_); + std::vector result; + result.reserve(key_order_.size()); + for (const auto& key_id : key_order_) { + result.push_back(keys_.at(key_id)); + } + return result; +} + +namespace encryption { + +void RegisterStandardEncryptionManager() { + EncryptionManagerRegistry::Register([](EncryptionManagerRegistry::Options options) + -> Result> { + ICEBERG_ASSIGN_OR_RAISE(auto manager, + StandardEncryptionManager::Make(std::move(options))); + return manager; + }); +} + +namespace { + +[[maybe_unused]] const bool kStandardEncryptionManagerRegistered = []() { + RegisterStandardEncryptionManager(); + return true; +}(); + +} // namespace + +} // namespace encryption + +} // namespace iceberg diff --git a/src/iceberg/encryption/standard_encryption_manager.h b/src/iceberg/encryption/standard_encryption_manager.h new file mode 100644 index 0000000000..19a242dec1 --- /dev/null +++ b/src/iceberg/encryption/standard_encryption_manager.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/standard_encryption_manager.h +/// Envelope encryption manager, compatible with Java's StandardEncryptionManager. + +#include +#include +#include +#include +#include +#include + +#include "iceberg/encryption/encryption_manager.h" +#include "iceberg/iceberg_bundle_export.h" + +namespace iceberg { + +/// \brief Standard Iceberg envelope encryption. +/// +/// Key hierarchy: +/// - table master key: in the KMS, ID in table property `encryption.key-id`; +/// - key encryption keys (KEK): wrapped by the KMS and kept in `encryption-keys` +/// with `encrypted-by-id` = the table key ID and a KEY_TIMESTAMP property. A KEK is +/// used for 730 days (NIST SP 800-57), then a new one is created; +/// - manifest list key metadata: AES-GCM encrypted by a KEK with the KEK timestamp as +/// AAD, kept in `encryption-keys` and referenced by the snapshot `key-id`; +/// - manifest key metadata: in the manifest list `key_metadata` field; +/// - data and delete file key metadata: in the manifest `key_metadata` field. +/// +/// Unwrapped KEKs are cached for one hour. Thread-safe. +class ICEBERG_BUNDLE_EXPORT StandardEncryptionManager : public EncryptionManager { + public: + static constexpr std::string_view kKeyTimestamp = "KEY_TIMESTAMP"; + static constexpr int32_t kAadPrefixLength = 16; + static constexpr std::chrono::milliseconds kKekLifespan{730LL * 24 * 3600 * 1000}; + static constexpr std::chrono::milliseconds kUnwrappedKekCacheExpiry{3600LL * 1000}; + + static Result> Make( + EncryptionManagerRegistry::Options options); + + Result NewKeyMetadata() override; + Result> NewDecryptingInputFile( + std::unique_ptr encrypted, + std::span key_metadata) override; + Result> NewEncryptingOutputFile( + std::unique_ptr plain, + const StandardKeyMetadata& key_metadata) override; + Result> DecryptManifestListKeyMetadata( + std::string_view manifest_list_key_id) override; + Result RegisterManifestListKeyMetadata( + const StandardKeyMetadata& key_metadata) override; + + /// \brief All keys: those from table metadata and those created by this manager. + std::vector encryption_keys() const; + + const std::string& table_key_id() const { return table_key_id_; } + + /// \brief Shift the clock, for tests of KEK rotation and cache expiry. + void SetTestTimeShift(std::chrono::milliseconds shift); + + private: + struct CachedKek { + std::vector key; + int64_t unwrapped_at_ms; + }; + + explicit StandardEncryptionManager(EncryptionManagerRegistry::Options options); + + // The following require mutex_ to be held. + Result> UnwrapKek(const std::string& kek_id); + Result CurrentKekId(); + void AddKey(EncryptedKey key); + + int64_t NowMs() const; + + const std::string table_key_id_; + const int32_t data_key_length_; + const std::shared_ptr kms_; + + mutable std::mutex mutex_; + std::chrono::milliseconds time_shift_{0}; + std::vector key_order_; + std::unordered_map keys_; + std::unordered_map unwrapped_keks_; +}; + +namespace encryption { + +/// \brief Register StandardEncryptionManager in EncryptionManagerRegistry. +ICEBERG_BUNDLE_EXPORT void RegisterStandardEncryptionManager(); + +} // namespace encryption + +} // namespace iceberg diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 2513ff177e..919e6dd504 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -243,7 +243,11 @@ if(ICEBERG_BUILD_BUNDLE) endif() if(ICEBERG_ENCRYPTION) - add_iceberg_test(crypto_test USE_BUNDLE SOURCES aes_gcm_test.cc) + add_iceberg_test(crypto_test + USE_BUNDLE + SOURCES + aes_gcm_test.cc + standard_encryption_manager_test.cc) endif() add_iceberg_test(scan_test diff --git a/src/iceberg/test/resources/encryption/table/data/00001-1-86fba0db-c6f6-4ce2-a242-a0745cf50318-00001.puffin b/src/iceberg/test/resources/encryption/table/data/00001-1-86fba0db-c6f6-4ce2-a242-a0745cf50318-00001.puffin new file mode 100644 index 0000000000..9f6665dbf6 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/data/00001-1-86fba0db-c6f6-4ce2-a242-a0745cf50318-00001.puffin differ diff --git a/src/iceberg/test/resources/encryption/table/data/data-0.parquet b/src/iceberg/test/resources/encryption/table/data/data-0.parquet new file mode 100644 index 0000000000..cef1d5ea4e Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/data/data-0.parquet differ diff --git a/src/iceberg/test/resources/encryption/table/data/data-1.parquet b/src/iceberg/test/resources/encryption/table/data/data-1.parquet new file mode 100644 index 0000000000..e94835ef55 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/data/data-1.parquet differ diff --git a/src/iceberg/test/resources/encryption/table/data/data-2.avro b/src/iceberg/test/resources/encryption/table/data/data-2.avro new file mode 100644 index 0000000000..5542474566 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/data/data-2.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/462964d7-9cf9-4547-a01b-29e4eef4bc25-m0.avro b/src/iceberg/test/resources/encryption/table/metadata/462964d7-9cf9-4547-a01b-29e4eef4bc25-m0.avro new file mode 100644 index 0000000000..dbfde9f70f Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/462964d7-9cf9-4547-a01b-29e4eef4bc25-m0.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/511c33d0-d9b9-4c80-bce0-b6fe4679785b-m0.avro b/src/iceberg/test/resources/encryption/table/metadata/511c33d0-d9b9-4c80-bce0-b6fe4679785b-m0.avro new file mode 100644 index 0000000000..8be2d3aab9 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/511c33d0-d9b9-4c80-bce0-b6fe4679785b-m0.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/95f19655-198b-4c2a-a7e5-e3cf75d5ab54-m0.avro b/src/iceberg/test/resources/encryption/table/metadata/95f19655-198b-4c2a-a7e5-e3cf75d5ab54-m0.avro new file mode 100644 index 0000000000..7e81d8faf6 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/95f19655-198b-4c2a-a7e5-e3cf75d5ab54-m0.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/d1977369-b04a-4cf3-a491-6090bc1572fc-m0.avro b/src/iceberg/test/resources/encryption/table/metadata/d1977369-b04a-4cf3-a491-6090bc1572fc-m0.avro new file mode 100644 index 0000000000..06d0cca85e Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/d1977369-b04a-4cf3-a491-6090bc1572fc-m0.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/snap-1122461367364127429-1-462964d7-9cf9-4547-a01b-29e4eef4bc25.avro b/src/iceberg/test/resources/encryption/table/metadata/snap-1122461367364127429-1-462964d7-9cf9-4547-a01b-29e4eef4bc25.avro new file mode 100644 index 0000000000..c45ab3ca34 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/snap-1122461367364127429-1-462964d7-9cf9-4547-a01b-29e4eef4bc25.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/snap-5112456484007752895-1-95f19655-198b-4c2a-a7e5-e3cf75d5ab54.avro b/src/iceberg/test/resources/encryption/table/metadata/snap-5112456484007752895-1-95f19655-198b-4c2a-a7e5-e3cf75d5ab54.avro new file mode 100644 index 0000000000..195c5055d9 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/snap-5112456484007752895-1-95f19655-198b-4c2a-a7e5-e3cf75d5ab54.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/snap-5555854819025587960-1-d1977369-b04a-4cf3-a491-6090bc1572fc.avro b/src/iceberg/test/resources/encryption/table/metadata/snap-5555854819025587960-1-d1977369-b04a-4cf3-a491-6090bc1572fc.avro new file mode 100644 index 0000000000..eb6658a6f4 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/snap-5555854819025587960-1-d1977369-b04a-4cf3-a491-6090bc1572fc.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro b/src/iceberg/test/resources/encryption/table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro new file mode 100644 index 0000000000..bfc39890f1 Binary files /dev/null and b/src/iceberg/test/resources/encryption/table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro differ diff --git a/src/iceberg/test/resources/encryption/table/metadata/v1.metadata.json b/src/iceberg/test/resources/encryption/table/metadata/v1.metadata.json new file mode 100644 index 0000000000..274102950f --- /dev/null +++ b/src/iceberg/test/resources/encryption/table/metadata/v1.metadata.json @@ -0,0 +1 @@ +{"format-version":3,"table-uuid":"2091a278-2727-4bc0-8427-ffb04a2b7e6d","location":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table","last-sequence-number":0,"last-updated-ms":1791460592557,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"encryption.key-id":"keyA","write.parquet.compression-codec":"zstd"},"current-snapshot-id":null,"next-row-id":0,"refs":{},"snapshots":[],"statistics":[],"partition-statistics":[],"snapshot-log":[],"metadata-log":[]} diff --git a/src/iceberg/test/resources/encryption/table/metadata/v2.metadata.json b/src/iceberg/test/resources/encryption/table/metadata/v2.metadata.json new file mode 100644 index 0000000000..0ffe050c5d --- /dev/null +++ b/src/iceberg/test/resources/encryption/table/metadata/v2.metadata.json @@ -0,0 +1 @@ +{"format-version":3,"table-uuid":"2091a278-2727-4bc0-8427-ffb04a2b7e6d","location":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table","last-sequence-number":1,"last-updated-ms":1791460593555,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"encryption.key-id":"keyA","write.parquet.compression-codec":"zstd"},"current-snapshot-id":564307663132197943,"next-row-id":5,"encryption-keys":[{"key-id":"6kMyWaTMFqk5FVecTafB8w==","encrypted-key-metadata":"BA3fkOhBo//37HfxBA1lUEbSrgrv3k06ovUGyi70AINLvvCU9iCUdhwR9lM=","encrypted-by-id":"keyA","properties":{"KEY_TIMESTAMP":"1791460593549"}},{"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ==","encrypted-key-metadata":"UTlv697ZHJpwrMe7rfBpOBNsbrSKKZuHLJm2i/V0DiJmtTR3RxxBhYlxxBaDH6K9kpFRZ1YrkRCYLhgrY5pAyvj9KA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="}],"refs":{"main":{"snapshot-id":564307663132197943,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":564307663132197943,"timestamp-ms":1791460593552,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1791","changed-partition-count":"1","total-records":"5","total-files-size":"1791","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro","schema-id":0,"first-row-id":0,"added-rows":5,"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ=="}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1791460593552,"snapshot-id":564307663132197943}],"metadata-log":[{"timestamp-ms":1791460592557,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v1.metadata.json"}]} diff --git a/src/iceberg/test/resources/encryption/table/metadata/v3.metadata.json b/src/iceberg/test/resources/encryption/table/metadata/v3.metadata.json new file mode 100644 index 0000000000..e724cd20fc --- /dev/null +++ b/src/iceberg/test/resources/encryption/table/metadata/v3.metadata.json @@ -0,0 +1 @@ +{"format-version":3,"table-uuid":"2091a278-2727-4bc0-8427-ffb04a2b7e6d","location":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table","last-sequence-number":2,"last-updated-ms":1791460593713,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"encryption.key-id":"keyA","write.parquet.compression-codec":"zstd"},"current-snapshot-id":1122461367364127429,"next-row-id":10,"encryption-keys":[{"key-id":"6kMyWaTMFqk5FVecTafB8w==","encrypted-key-metadata":"BA3fkOhBo//37HfxBA1lUEbSrgrv3k06ovUGyi70AINLvvCU9iCUdhwR9lM=","encrypted-by-id":"keyA","properties":{"KEY_TIMESTAMP":"1791460593549"}},{"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ==","encrypted-key-metadata":"UTlv697ZHJpwrMe7rfBpOBNsbrSKKZuHLJm2i/V0DiJmtTR3RxxBhYlxxBaDH6K9kpFRZ1YrkRCYLhgrY5pAyvj9KA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="},{"key-id":"LU90Mju0jbgvcsbifDCF0A==","encrypted-key-metadata":"2VLA6waTLBagemORlfjUdPRjmo3M3fPei7WkwX/q4S8aEM8CnAW8gllBDpEPXa/70jzMCvlVXkdg5x0CCmzoBKaSkA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="}],"refs":{"main":{"snapshot-id":1122461367364127429,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":564307663132197943,"timestamp-ms":1791460593552,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1791","changed-partition-count":"1","total-records":"5","total-files-size":"1791","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro","schema-id":0,"first-row-id":0,"added-rows":5,"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ=="},{"sequence-number":2,"snapshot-id":1122461367364127429,"parent-snapshot-id":564307663132197943,"timestamp-ms":1791460593713,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"1","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1793","changed-partition-count":"1","total-records":"10","total-files-size":"3584","total-data-files":"2","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-1122461367364127429-1-462964d7-9cf9-4547-a01b-29e4eef4bc25.avro","schema-id":0,"first-row-id":5,"added-rows":5,"key-id":"LU90Mju0jbgvcsbifDCF0A=="}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1791460593552,"snapshot-id":564307663132197943},{"timestamp-ms":1791460593713,"snapshot-id":1122461367364127429}],"metadata-log":[{"timestamp-ms":1791460592557,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v1.metadata.json"},{"timestamp-ms":1791460593555,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v2.metadata.json"}]} diff --git a/src/iceberg/test/resources/encryption/table/metadata/v4.metadata.json b/src/iceberg/test/resources/encryption/table/metadata/v4.metadata.json new file mode 100644 index 0000000000..2426069f2b --- /dev/null +++ b/src/iceberg/test/resources/encryption/table/metadata/v4.metadata.json @@ -0,0 +1 @@ +{"format-version":3,"table-uuid":"2091a278-2727-4bc0-8427-ffb04a2b7e6d","location":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table","last-sequence-number":3,"last-updated-ms":1791460593828,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"encryption.key-id":"keyA","write.parquet.compression-codec":"zstd"},"current-snapshot-id":5112456484007752895,"next-row-id":13,"encryption-keys":[{"key-id":"6kMyWaTMFqk5FVecTafB8w==","encrypted-key-metadata":"BA3fkOhBo//37HfxBA1lUEbSrgrv3k06ovUGyi70AINLvvCU9iCUdhwR9lM=","encrypted-by-id":"keyA","properties":{"KEY_TIMESTAMP":"1791460593549"}},{"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ==","encrypted-key-metadata":"UTlv697ZHJpwrMe7rfBpOBNsbrSKKZuHLJm2i/V0DiJmtTR3RxxBhYlxxBaDH6K9kpFRZ1YrkRCYLhgrY5pAyvj9KA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="},{"key-id":"LU90Mju0jbgvcsbifDCF0A==","encrypted-key-metadata":"2VLA6waTLBagemORlfjUdPRjmo3M3fPei7WkwX/q4S8aEM8CnAW8gllBDpEPXa/70jzMCvlVXkdg5x0CCmzoBKaSkA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="},{"key-id":"ZkXVnuHz4Qlemc4wNaI8Eg==","encrypted-key-metadata":"qz9eUQScTKPymZ8tV+csNKXRhBF2hDHU7s0d7+AbbG10T6Zgqc/BVKXfL5FjYJ2KNQdCxpp4CAm3HlXpFbxR5HA+Kw==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="}],"refs":{"main":{"snapshot-id":5112456484007752895,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":564307663132197943,"timestamp-ms":1791460593552,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1791","changed-partition-count":"1","total-records":"5","total-files-size":"1791","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro","schema-id":0,"first-row-id":0,"added-rows":5,"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ=="},{"sequence-number":2,"snapshot-id":1122461367364127429,"parent-snapshot-id":564307663132197943,"timestamp-ms":1791460593713,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"1","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1793","changed-partition-count":"1","total-records":"10","total-files-size":"3584","total-data-files":"2","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-1122461367364127429-1-462964d7-9cf9-4547-a01b-29e4eef4bc25.avro","schema-id":0,"first-row-id":5,"added-rows":5,"key-id":"LU90Mju0jbgvcsbifDCF0A=="},{"sequence-number":3,"snapshot-id":5112456484007752895,"parent-snapshot-id":1122461367364127429,"timestamp-ms":1791460593828,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"2","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"445","changed-partition-count":"1","total-records":"13","total-files-size":"4029","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-5112456484007752895-1-95f19655-198b-4c2a-a7e5-e3cf75d5ab54.avro","schema-id":0,"first-row-id":10,"added-rows":3,"key-id":"ZkXVnuHz4Qlemc4wNaI8Eg=="}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1791460593552,"snapshot-id":564307663132197943},{"timestamp-ms":1791460593713,"snapshot-id":1122461367364127429},{"timestamp-ms":1791460593828,"snapshot-id":5112456484007752895}],"metadata-log":[{"timestamp-ms":1791460592557,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v1.metadata.json"},{"timestamp-ms":1791460593555,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v2.metadata.json"},{"timestamp-ms":1791460593713,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v3.metadata.json"}]} diff --git a/src/iceberg/test/resources/encryption/table/metadata/v5.metadata.json b/src/iceberg/test/resources/encryption/table/metadata/v5.metadata.json new file mode 100644 index 0000000000..241ea609c3 --- /dev/null +++ b/src/iceberg/test/resources/encryption/table/metadata/v5.metadata.json @@ -0,0 +1 @@ +{"format-version":3,"table-uuid":"2091a278-2727-4bc0-8427-ffb04a2b7e6d","location":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table","last-sequence-number":4,"last-updated-ms":1791460593995,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"encryption.key-id":"keyA","write.parquet.compression-codec":"zstd"},"current-snapshot-id":5555854819025587960,"next-row-id":13,"encryption-keys":[{"key-id":"6kMyWaTMFqk5FVecTafB8w==","encrypted-key-metadata":"BA3fkOhBo//37HfxBA1lUEbSrgrv3k06ovUGyi70AINLvvCU9iCUdhwR9lM=","encrypted-by-id":"keyA","properties":{"KEY_TIMESTAMP":"1791460593549"}},{"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ==","encrypted-key-metadata":"UTlv697ZHJpwrMe7rfBpOBNsbrSKKZuHLJm2i/V0DiJmtTR3RxxBhYlxxBaDH6K9kpFRZ1YrkRCYLhgrY5pAyvj9KA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="},{"key-id":"LU90Mju0jbgvcsbifDCF0A==","encrypted-key-metadata":"2VLA6waTLBagemORlfjUdPRjmo3M3fPei7WkwX/q4S8aEM8CnAW8gllBDpEPXa/70jzMCvlVXkdg5x0CCmzoBKaSkA==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="},{"key-id":"ZkXVnuHz4Qlemc4wNaI8Eg==","encrypted-key-metadata":"qz9eUQScTKPymZ8tV+csNKXRhBF2hDHU7s0d7+AbbG10T6Zgqc/BVKXfL5FjYJ2KNQdCxpp4CAm3HlXpFbxR5HA+Kw==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="},{"key-id":"VTWjHxdPbZFKJheX7XsR/Q==","encrypted-key-metadata":"EgZwxa5QZHjIWGAyrWOZiTfZEzOuQ1elzfuKke1DqxTQmGSHUTSNSxE0piZMRisSN/Ijp1EF7jbohWPi9OEr0BuBjQ==","encrypted-by-id":"6kMyWaTMFqk5FVecTafB8w=="}],"refs":{"main":{"snapshot-id":5555854819025587960,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":564307663132197943,"timestamp-ms":1791460593552,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1791","changed-partition-count":"1","total-records":"5","total-files-size":"1791","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-564307663132197943-1-511c33d0-d9b9-4c80-bce0-b6fe4679785b.avro","schema-id":0,"first-row-id":0,"added-rows":5,"key-id":"9Dpe+fdlQ+4lMH5jW0q1YQ=="},{"sequence-number":2,"snapshot-id":1122461367364127429,"parent-snapshot-id":564307663132197943,"timestamp-ms":1791460593713,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"1","manifests-replaced":"0","added-data-files":"1","added-records":"5","added-files-size":"1793","changed-partition-count":"1","total-records":"10","total-files-size":"3584","total-data-files":"2","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-1122461367364127429-1-462964d7-9cf9-4547-a01b-29e4eef4bc25.avro","schema-id":0,"first-row-id":5,"added-rows":5,"key-id":"LU90Mju0jbgvcsbifDCF0A=="},{"sequence-number":3,"snapshot-id":5112456484007752895,"parent-snapshot-id":1122461367364127429,"timestamp-ms":1791460593828,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"2","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"445","changed-partition-count":"1","total-records":"13","total-files-size":"4029","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-5112456484007752895-1-95f19655-198b-4c2a-a7e5-e3cf75d5ab54.avro","schema-id":0,"first-row-id":10,"added-rows":3,"key-id":"ZkXVnuHz4Qlemc4wNaI8Eg=="},{"sequence-number":4,"snapshot-id":5555854819025587960,"parent-snapshot-id":5112456484007752895,"timestamp-ms":1791460593995,"summary":{"operation":"delete","manifests-created":"1","manifests-kept":"3","manifests-replaced":"0","added-delete-files":"1","added-dvs":"1","added-files-size":"42","added-position-deletes":"1","changed-partition-count":"1","total-records":"13","total-files-size":"4071","total-data-files":"3","total-delete-files":"1","total-position-deletes":"1","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.13.0-SNAPSHOT (commit 4d5811f2c701b15be4379686b17144503ed1e685)"},"manifest-list":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/snap-5555854819025587960-1-d1977369-b04a-4cf3-a491-6090bc1572fc.avro","schema-id":0,"first-row-id":13,"added-rows":0,"key-id":"VTWjHxdPbZFKJheX7XsR/Q=="}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1791460593552,"snapshot-id":564307663132197943},{"timestamp-ms":1791460593713,"snapshot-id":1122461367364127429},{"timestamp-ms":1791460593828,"snapshot-id":5112456484007752895},{"timestamp-ms":1791460593995,"snapshot-id":5555854819025587960}],"metadata-log":[{"timestamp-ms":1791460592557,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v1.metadata.json"},{"timestamp-ms":1791460593555,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v2.metadata.json"},{"timestamp-ms":1791460593713,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v3.metadata.json"},{"timestamp-ms":1791460593828,"metadata-file":"/tmp/iceberg-cpp-encryption-fixture/encrypted_table/metadata/v4.metadata.json"}]} diff --git a/src/iceberg/test/resources/encryption/table/metadata/version-hint.text b/src/iceberg/test/resources/encryption/table/metadata/version-hint.text new file mode 100644 index 0000000000..7ed6ff82de --- /dev/null +++ b/src/iceberg/test/resources/encryption/table/metadata/version-hint.text @@ -0,0 +1 @@ +5 diff --git a/src/iceberg/test/standard_encryption_manager_test.cc b/src/iceberg/test/standard_encryption_manager_test.cc new file mode 100644 index 0000000000..9bc7fe9479 --- /dev/null +++ b/src/iceberg/test/standard_encryption_manager_test.cc @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/standard_encryption_manager.h" + +#include + +#include + +#include "iceberg/encryption/encryption_register.h" +#include "iceberg/encryption/in_memory_kms.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_metadata.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/test_resource.h" + +namespace iceberg { + +namespace { + +/// \brief Deterministic test bytes: byte i is (i * 31 + seed). +std::vector VectorBytes(size_t length, int seed) { + std::vector bytes(length); + for (size_t i = 0; i < length; ++i) { + bytes[i] = static_cast(static_cast(i) * 31 + seed); + } + return bytes; +} + +// The master key for "keyA" used by the Java fixture under +// test/resources/encryption/table. +const std::vector kMasterKeyA = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '0', '1', '2', '3', '4', '5'}; + +/// Counts calls to the KMS. +class CountingKms : public InMemoryKms { + public: + using InMemoryKms::InMemoryKms; + Result> WrapKey(std::span key, + std::string_view id) override { + ++wraps; + return InMemoryKms::WrapKey(key, id); + } + Result> UnwrapKey(std::span wrapped, + std::string_view id) override { + ++unwraps; + return InMemoryKms::UnwrapKey(wrapped, id); + } + int wraps = 0; + int unwraps = 0; +}; + +std::shared_ptr MakeKms() { + return std::make_shared( + std::unordered_map>{{"keyA", kMasterKeyA}}); +} + +} // namespace + +TEST(StandardEncryptionManagerTest, DecryptsJavaManifestListKeys) { + ICEBERG_UNWRAP_OR_FAIL( + auto metadata, + ReadTableMetadataFromResource("encryption/table/metadata/v5.metadata.json")); + ASSERT_EQ(metadata->snapshots.size(), 4); + ASSERT_EQ(metadata->encryption_keys.size(), 5); + + // Through the registry, as catalogs do + encryption::RegisterAll(); + auto kms = MakeKms(); + ICEBERG_UNWRAP_OR_FAIL(auto manager, EncryptionManagerRegistry::Make( + {.table_key_id = "keyA", + .encryption_keys = metadata->encryption_keys, + .kms = kms})); + + for (const auto& snapshot : metadata->snapshots) { + ASSERT_TRUE(snapshot->key_id.has_value()); + ICEBERG_UNWRAP_OR_FAIL(auto decrypted, + manager->DecryptManifestListKeyMetadata(*snapshot->key_id)); + ICEBERG_UNWRAP_OR_FAIL(auto key_metadata, StandardKeyMetadata::Parse(decrypted)); + EXPECT_EQ(key_metadata.encryption_key.size(), 16); + EXPECT_EQ(key_metadata.aad_prefix->size(), 16); + // The trusted length is the stored (encrypted) size of the manifest list file + auto name = std::filesystem::path(snapshot->manifest_list).filename().string(); + EXPECT_EQ( + key_metadata.file_length, + std::filesystem::file_size(GetResourcePath("encryption/table/metadata/" + name))); + } + // One key encryption key for all snapshots, unwrapped once + EXPECT_EQ(kms->unwraps, 1); +} + +TEST(StandardEncryptionManagerTest, RegisterAndDecryptManifestListKeyMetadata) { + auto kms = MakeKms(); + ICEBERG_UNWRAP_OR_FAIL(auto manager, StandardEncryptionManager::Make( + {.table_key_id = "keyA", .kms = kms})); + + ICEBERG_UNWRAP_OR_FAIL(auto key_metadata, manager->NewKeyMetadata()); + EXPECT_EQ(key_metadata.encryption_key.size(), 16); + EXPECT_EQ(key_metadata.aad_prefix->size(), 16); + EXPECT_FALSE(key_metadata.file_length.has_value()); + // The manifest list length is required + EXPECT_THAT(manager->RegisterManifestListKeyMetadata(key_metadata), + IsError(ErrorKind::kInvalidArgument)); + key_metadata.file_length = 999; + + ICEBERG_UNWRAP_OR_FAIL(auto keys, + manager->RegisterManifestListKeyMetadata(key_metadata)); + EXPECT_EQ(keys.key_encryption_key.encrypted_by_id, "keyA"); + EXPECT_TRUE(keys.key_encryption_key.properties.contains("KEY_TIMESTAMP")); + EXPECT_EQ(keys.file_key.encrypted_by_id, keys.key_encryption_key.key_id); + EXPECT_EQ(kms->wraps, 1); + EXPECT_EQ(manager->encryption_keys().size(), 2); + + // A reader that only has the committed keys and the KMS can decrypt it + ICEBERG_UNWRAP_OR_FAIL(auto reader, + StandardEncryptionManager::Make( + {.table_key_id = "keyA", + .encryption_keys = {keys.key_encryption_key, keys.file_key}, + .kms = kms})); + ICEBERG_UNWRAP_OR_FAIL(auto decrypted, + reader->DecryptManifestListKeyMetadata(keys.file_key.key_id)); + EXPECT_EQ(decrypted, key_metadata.Serialize()); + + // Key encryption keys and unknown IDs are rejected + EXPECT_THAT(reader->DecryptManifestListKeyMetadata(keys.key_encryption_key.key_id), + IsError(ErrorKind::kInvalid)); + EXPECT_THAT(reader->DecryptManifestListKeyMetadata("missing"), + IsError(ErrorKind::kInvalid)); +} + +TEST(StandardEncryptionManagerTest, RotatesKeyEncryptionKeys) { + auto kms = MakeKms(); + ICEBERG_UNWRAP_OR_FAIL(auto manager, StandardEncryptionManager::Make( + {.table_key_id = "keyA", .kms = kms})); + StandardKeyMetadata key_metadata{.encryption_key = VectorBytes(16, 1), + .aad_prefix = VectorBytes(16, 2), + .file_length = 100}; + + ICEBERG_UNWRAP_OR_FAIL(auto first, + manager->RegisterManifestListKeyMetadata(key_metadata)); + ICEBERG_UNWRAP_OR_FAIL(auto second, + manager->RegisterManifestListKeyMetadata(key_metadata)); + EXPECT_EQ(second.key_encryption_key.key_id, first.key_encryption_key.key_id); + EXPECT_NE(second.file_key.key_id, first.file_key.key_id); + + manager->SetTestTimeShift(StandardEncryptionManager::kKekLifespan); + ICEBERG_UNWRAP_OR_FAIL(auto third, + manager->RegisterManifestListKeyMetadata(key_metadata)); + EXPECT_NE(third.key_encryption_key.key_id, first.key_encryption_key.key_id); + EXPECT_EQ(kms->wraps, 2); + + // Manifest lists encrypted with the old key encryption key stay readable + ICEBERG_UNWRAP_OR_FAIL(auto decrypted, + manager->DecryptManifestListKeyMetadata(first.file_key.key_id)); + EXPECT_EQ(decrypted, key_metadata.Serialize()); +} + +TEST(StandardEncryptionManagerTest, IgnoresOutOfRangeKeyTimestamp) { + auto kms = MakeKms(); + ICEBERG_UNWRAP_OR_FAIL(auto manager, StandardEncryptionManager::Make( + {.table_key_id = "keyA", .kms = kms})); + StandardKeyMetadata key_metadata{.encryption_key = VectorBytes(16, 1), + .aad_prefix = VectorBytes(16, 2), + .file_length = 100}; + ICEBERG_UNWRAP_OR_FAIL(auto first, + manager->RegisterManifestListKeyMetadata(key_metadata)); + EXPECT_EQ(kms->wraps, 1); + + // A KEK whose KEY_TIMESTAMP is corrupted to INT64_MIN must never reach the + // "now - timestamp" subtraction, which would overflow. It should instead be + // treated as unparseable, forcing a fresh key encryption key to be created. + std::vector keys = manager->encryption_keys(); + for (auto& key : keys) { + if (key.encrypted_by_id == "keyA") { + key.properties[std::string(StandardEncryptionManager::kKeyTimestamp)] = + "-9223372036854775808"; + } + } + ICEBERG_UNWRAP_OR_FAIL( + auto reader, StandardEncryptionManager::Make( + {.table_key_id = "keyA", .encryption_keys = keys, .kms = kms})); + ICEBERG_UNWRAP_OR_FAIL(auto second, + reader->RegisterManifestListKeyMetadata(key_metadata)); + EXPECT_NE(second.key_encryption_key.key_id, first.key_encryption_key.key_id); + EXPECT_EQ(kms->wraps, 2); +} + +TEST(StandardEncryptionManagerTest, UnwrappedKeyCacheExpires) { + ICEBERG_UNWRAP_OR_FAIL( + auto metadata, + ReadTableMetadataFromResource("encryption/table/metadata/v5.metadata.json")); + auto kms = MakeKms(); + ICEBERG_UNWRAP_OR_FAIL(auto manager, StandardEncryptionManager::Make( + {.table_key_id = "keyA", + .encryption_keys = metadata->encryption_keys, + .kms = kms})); + const auto& key_id = *metadata->snapshots.front()->key_id; + ASSERT_THAT(manager->DecryptManifestListKeyMetadata(key_id), IsOk()); + ASSERT_THAT(manager->DecryptManifestListKeyMetadata(key_id), IsOk()); + EXPECT_EQ(kms->unwraps, 1); + + manager->SetTestTimeShift(StandardEncryptionManager::kUnwrappedKekCacheExpiry); + ASSERT_THAT(manager->DecryptManifestListKeyMetadata(key_id), IsOk()); + EXPECT_EQ(kms->unwraps, 2); +} + +TEST(StandardEncryptionManagerTest, DecryptingFileRequiresLength) { + auto kms = MakeKms(); + ICEBERG_UNWRAP_OR_FAIL(auto manager, StandardEncryptionManager::Make( + {.table_key_id = "keyA", .kms = kms})); + StandardKeyMetadata no_length{.encryption_key = VectorBytes(16, 1)}; + + class NullInputFile : public InputFile { + public: + std::string_view location() const override { return "file"; } + Result Size() const override { return 0; } + Result> Open() override { + return NotImplemented(""); + } + }; + EXPECT_THAT(manager->NewDecryptingInputFile(std::make_unique(), + no_length.Serialize()), + IsError(ErrorKind::kInvalid)); +} + +TEST(EncryptionManagerRegistryTest, ValidatesOptions) { + encryption::RegisterAll(); + EXPECT_TRUE(encryption::IsEncryptionAvailable()); + EXPECT_THAT(EncryptionManagerRegistry::Make({.table_key_id = "keyA"}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(EncryptionManagerRegistry::Make({.table_key_id = "", .kms = MakeKms()}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(EncryptionManagerRegistry::Make( + {.table_key_id = "keyA", .data_key_length = 20, .kms = MakeKms()}), + IsError(ErrorKind::kInvalidArgument)); +} + +} // namespace iceberg