From bb4dc1bf03c0ddcdb902d36c9865a087ae452fa7 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Sun, 1 Nov 2026 15:32:42 +0800 Subject: [PATCH] fix(row): make Arrow wrapper validation configurable --- src/iceberg/data/delete_filter.cc | 30 ++++--- src/iceberg/data/delete_filter.h | 14 +-- src/iceberg/data/delete_loader.cc | 6 +- src/iceberg/data/delete_loader.h | 4 +- src/iceberg/data/file_scan_task_reader.cc | 3 +- src/iceberg/row/arrow_array_wrapper.cc | 76 ++++++++++++---- src/iceberg/row/arrow_array_wrapper.h | 9 +- src/iceberg/test/delete_filter_test.cc | 27 ++++++ src/iceberg/test/struct_like_test.cc | 102 ++++++++++++++++++++++ 9 files changed, 231 insertions(+), 40 deletions(-) diff --git a/src/iceberg/data/delete_filter.cc b/src/iceberg/data/delete_filter.cc index 5f21a32de1..1f023ad46b 100644 --- a/src/iceberg/data/delete_filter.cc +++ b/src/iceberg/data/delete_filter.cc @@ -460,50 +460,50 @@ Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, bool need_row_pos_col, - std::shared_ptr counter) { + std::shared_ptr counter, bool validate_batch) { ICEBERG_ASSIGN_OR_RAISE(auto field_lookup, MakeFieldLookup(table_schema)); return Make(std::move(file_path), delete_files, std::move(requested_schema), std::move(io), std::move(field_lookup), need_row_pos_col, - std::move(counter)); + std::move(counter), validate_batch); } Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr table_metadata, std::shared_ptr requested_schema, std::shared_ptr io, - bool need_row_pos_col, std::shared_ptr counter) { + bool need_row_pos_col, std::shared_ptr counter, bool validate_batch) { ICEBERG_PRECHECK(table_metadata != nullptr, "Table metadata must not be null"); ICEBERG_ASSIGN_OR_RAISE(auto field_lookup, MakeFieldLookup(std::move(table_metadata))); return Make(std::move(file_path), delete_files, std::move(requested_schema), std::move(io), std::move(field_lookup), need_row_pos_col, - std::move(counter)); + std::move(counter), validate_batch); } Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, std::span> schemas, - bool need_row_pos_col, std::shared_ptr counter) { + bool need_row_pos_col, std::shared_ptr counter, bool validate_batch) { ICEBERG_ASSIGN_OR_RAISE(auto field_lookup, MakeFieldLookup(table_schema, schemas)); return Make(std::move(file_path), delete_files, std::move(requested_schema), std::move(io), std::move(field_lookup), need_row_pos_col, - std::move(counter)); + std::move(counter), validate_batch); } Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, bool need_row_pos_col, - std::shared_ptr counter) { + std::shared_ptr counter, bool validate_batch) { ICEBERG_PRECHECK(requested_schema != nullptr, "Requested schema must not be null"); ICEBERG_PRECHECK(field_lookup != nullptr, "Field lookup must not be null"); ICEBERG_PRECHECK(delete_files.empty() || io != nullptr, "FileIO must not be null when delete files are present"); - auto filter = std::unique_ptr( - new DeleteFilter(std::move(file_path), std::move(requested_schema), std::move(io), - std::move(field_lookup), need_row_pos_col, std::move(counter))); + auto filter = std::unique_ptr(new DeleteFilter( + std::move(file_path), std::move(requested_schema), std::move(io), + std::move(field_lookup), need_row_pos_col, std::move(counter), validate_batch)); ICEBERG_RETURN_UNEXPECTED(filter->Init(delete_files)); return filter; } @@ -511,13 +511,15 @@ Result> DeleteFilter::Make( DeleteFilter::DeleteFilter(std::string file_path, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, - bool need_row_pos_col, std::shared_ptr counter) + bool need_row_pos_col, std::shared_ptr counter, + bool validate_batch) : file_path_(std::move(file_path)), requested_schema_(std::move(requested_schema)), field_lookup_(std::move(field_lookup)), need_row_pos_col_(need_row_pos_col), + validate_batch_(validate_batch), counter_(std::move(counter)), - delete_loader_(std::move(io)) {} + delete_loader_(std::move(io), validate_batch) {} DeleteFilter::~DeleteFilter() = default; @@ -750,7 +752,9 @@ Result DeleteFilter::ComputeAliveRows(const ArrowSchema& batc batch.length <= static_cast(std::numeric_limits::max()), "Batch length {} exceeds int32_t row index capacity", batch.length); result.indices.reserve(static_cast(batch.length)); - ICEBERG_ASSIGN_OR_RAISE(auto row, ArrowArrayStructLike::Make(batch_schema, batch)); + ICEBERG_ASSIGN_OR_RAISE( + auto row, + ArrowArrayStructLike::Make(batch_schema, batch, /*row_index=*/0, validate_batch_)); for (int64_t i = 0; i < batch.length; ++i) { if (i > 0) { diff --git a/src/iceberg/data/delete_filter.h b/src/iceberg/data/delete_filter.h index 4cb9bace66..527290c150 100644 --- a/src/iceberg/data/delete_filter.h +++ b/src/iceberg/data/delete_filter.h @@ -110,28 +110,30 @@ class ICEBERG_DATA_EXPORT DeleteFilter { std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, bool need_row_pos_col = true, - std::shared_ptr counter = nullptr); + std::shared_ptr counter = nullptr, bool validate_batch = true); /// \brief Create a DeleteFilter using table metadata for schema-aware field lookup. static Result> Make( std::string file_path, std::span> delete_files, std::shared_ptr table_metadata, std::shared_ptr requested_schema, std::shared_ptr io, - bool need_row_pos_col = true, std::shared_ptr counter = nullptr); + bool need_row_pos_col = true, std::shared_ptr counter = nullptr, + bool validate_batch = true); /// \brief Create a DeleteFilter with table schemas for dropped equality fields. static Result> Make( std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, std::span> schemas, - bool need_row_pos_col = true, std::shared_ptr counter = nullptr); + bool need_row_pos_col = true, std::shared_ptr counter = nullptr, + bool validate_batch = true); /// \brief Create a DeleteFilter with a custom field lookup. static Result> Make( std::string file_path, std::span> delete_files, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, bool need_row_pos_col = true, - std::shared_ptr counter = nullptr); + std::shared_ptr counter = nullptr, bool validate_batch = true); ~DeleteFilter(); @@ -191,7 +193,8 @@ class ICEBERG_DATA_EXPORT DeleteFilter { DeleteFilter(std::string file_path, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, - bool need_row_pos_col, std::shared_ptr counter); + bool need_row_pos_col, std::shared_ptr counter, + bool validate_batch); Status Init(std::span> delete_files); Result> ComputeRequiredSchema() const; @@ -207,6 +210,7 @@ class ICEBERG_DATA_EXPORT DeleteFilter { FieldLookup field_lookup_; const bool need_row_pos_col_; + const bool validate_batch_; // Position of `_pos` in required_schema_ when existent std::optional pos_field_position_; std::shared_ptr counter_; diff --git a/src/iceberg/data/delete_loader.cc b/src/iceberg/data/delete_loader.cc index fa020c09e2..73242032ce 100644 --- a/src/iceberg/data/delete_loader.cc +++ b/src/iceberg/data/delete_loader.cc @@ -106,7 +106,8 @@ Status ValidateDV(const DataFile& dv, std::string_view data_file_path) { } // namespace -DeleteLoader::DeleteLoader(std::shared_ptr io) : io_(std::move(io)) {} +DeleteLoader::DeleteLoader(std::shared_ptr io, bool validate_batch) + : io_(std::move(io)), validate_batch_(validate_batch) {} DeleteLoader::~DeleteLoader() = default; @@ -255,7 +256,8 @@ Result> DeleteLoader::LoadEqualityDelete internal::ArrowArrayGuard batch_guard(&batch); ICEBERG_ASSIGN_OR_RAISE( - auto row, ArrowArrayStructLike::Make(arrow_schema, batch, /*row_index=*/0)); + auto row, ArrowArrayStructLike::Make(arrow_schema, batch, /*row_index=*/0, + validate_batch_)); for (int64_t i = 0; i < batch.length; ++i) { if (i > 0) { diff --git a/src/iceberg/data/delete_loader.h b/src/iceberg/data/delete_loader.h index 360914e855..df3da5736d 100644 --- a/src/iceberg/data/delete_loader.h +++ b/src/iceberg/data/delete_loader.h @@ -37,7 +37,8 @@ class ICEBERG_DATA_EXPORT DeleteLoader { public: /// \brief Create a DeleteLoader. /// \param io FileIO instance for reading delete files - explicit DeleteLoader(std::shared_ptr io); + /// \param validate_batch Whether equality-delete Arrow batches are fully validated + explicit DeleteLoader(std::shared_ptr io, bool validate_batch = true); ~DeleteLoader(); @@ -71,6 +72,7 @@ class ICEBERG_DATA_EXPORT DeleteLoader { std::string_view data_file_path) const; std::shared_ptr io_; + const bool validate_batch_; }; } // namespace iceberg diff --git a/src/iceberg/data/file_scan_task_reader.cc b/src/iceberg/data/file_scan_task_reader.cc index 8d3c73281a..cd874457fa 100644 --- a/src/iceberg/data/file_scan_task_reader.cc +++ b/src/iceberg/data/file_scan_task_reader.cc @@ -208,7 +208,8 @@ class FileScanTaskReader::Impl { ICEBERG_ASSIGN_OR_RAISE( auto delete_filter, DeleteFilter::Make(data_file->file_path, task.delete_files(), projected_schema_, - io_, field_lookup_, has_position_deletes, delete_counter_)); + io_, field_lookup_, has_position_deletes, delete_counter_, + properties_.Get(ReaderProperties::kArrowValidateBatch))); auto required_schema = delete_filter->RequiredSchema(); auto project_batch_function = ProjectionContext::ResolveProjectBatchFunction(); diff --git a/src/iceberg/row/arrow_array_wrapper.cc b/src/iceberg/row/arrow_array_wrapper.cc index 5a9cb0894d..9e48282eaf 100644 --- a/src/iceberg/row/arrow_array_wrapper.cc +++ b/src/iceberg/row/arrow_array_wrapper.cc @@ -85,18 +85,22 @@ Result ExtractValue(const ArrowSchema* schema, const ArrowArray* array, return Decimal(int_value); } case NANOARROW_TYPE_STRUCT: { + // The parent view validates nested children recursively; avoid rescanning them. ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr struct_like, - ArrowArrayStructLike::Make(*schema, *array, index)); + ArrowArrayStructLike::Make(*schema, *array, index, + /*validate_batch=*/false)); return struct_like; } case NANOARROW_TYPE_LIST: { ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr array_like, - ArrowArrayArrayLike::Make(*schema, *array, index)); + ArrowArrayArrayLike::Make(*schema, *array, index, + /*validate_batch=*/false)); return array_like; } case NANOARROW_TYPE_MAP: { ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr map_like, - ArrowArrayMapLike::Make(*schema, *array, index)); + ArrowArrayMapLike::Make(*schema, *array, index, + /*validate_batch=*/false)); return map_like; } case NANOARROW_TYPE_EXTENSION: @@ -113,8 +117,12 @@ Result ExtractValue(const ArrowSchema* schema, const ArrowArray* array, class ArrowArrayStructLike::Impl { public: - Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) - : schema_(schema), array_(std::cref(array)), row_index_(row_index) {} + Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) + : schema_(schema), + array_(std::cref(array)), + row_index_(row_index), + validate_batch_(validate_batch) {} ~Impl() = default; @@ -147,6 +155,10 @@ class ArrowArrayStructLike::Impl { ArrowError error; NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return {}; } @@ -161,6 +173,10 @@ class ArrowArrayStructLike::Impl { ArrowArrayViewInitFromSchema(&array_view_, &schema_, &error)); NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return {}; } @@ -171,11 +187,13 @@ class ArrowArrayStructLike::Impl { const ArrowSchema& schema_; std::reference_wrapper array_; int64_t row_index_; + bool validate_batch_; }; Result> ArrowArrayStructLike::Make( - const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) { - auto impl = std::make_unique(schema, array, row_index); + const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) { + auto impl = std::make_unique(schema, array, row_index, validate_batch); ICEBERG_RETURN_UNEXPECTED(impl->Init()); return std::unique_ptr(new ArrowArrayStructLike(std::move(impl))); } @@ -201,8 +219,12 @@ Status ArrowArrayStructLike::Reset(const ArrowArray& array, int64_t row_index) { class ArrowArrayArrayLike::Impl { public: - Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) - : schema_(schema), array_(std::cref(array)), row_index_(row_index) {} + Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) + : schema_(schema), + array_(std::cref(array)), + row_index_(row_index), + validate_batch_(validate_batch) {} ~Impl() = default; @@ -229,6 +251,10 @@ class ArrowArrayArrayLike::Impl { ArrowError error; NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -243,6 +269,10 @@ class ArrowArrayArrayLike::Impl { ArrowArrayViewInitFromSchema(&array_view_, &schema_, &error)); NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -268,11 +298,13 @@ class ArrowArrayArrayLike::Impl { int64_t offset_ = 0; int64_t length_ = 0; + bool validate_batch_; }; Result> ArrowArrayArrayLike::Make( - const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) { - auto impl = std::make_unique(schema, array, row_index); + const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) { + auto impl = std::make_unique(schema, array, row_index, validate_batch); ICEBERG_RETURN_UNEXPECTED(impl->Init()); return std::unique_ptr(new ArrowArrayArrayLike(std::move(impl))); } @@ -298,8 +330,12 @@ Status ArrowArrayArrayLike::Reset(const ArrowArray& array, int64_t row_index) { class ArrowArrayMapLike::Impl { public: - Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) - : schema_(schema), array_(std::cref(array)), row_index_(row_index) {} + Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) + : schema_(schema), + array_(std::cref(array)), + row_index_(row_index), + validate_batch_(validate_batch) {} ~Impl() = default; @@ -340,6 +376,10 @@ class ArrowArrayMapLike::Impl { ArrowError error; NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -354,6 +394,10 @@ class ArrowArrayMapLike::Impl { ArrowArrayViewInitFromSchema(&array_view_, &schema_, &error)); NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -387,11 +431,13 @@ class ArrowArrayMapLike::Impl { int64_t offset_ = 0; int64_t length_ = 0; + bool validate_batch_; }; Result> ArrowArrayMapLike::Make( - const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) { - auto impl = std::make_unique(schema, array, row_index); + const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) { + auto impl = std::make_unique(schema, array, row_index, validate_batch); ICEBERG_RETURN_UNEXPECTED(impl->Init()); return std::unique_ptr(new ArrowArrayMapLike(std::move(impl))); } diff --git a/src/iceberg/row/arrow_array_wrapper.h b/src/iceberg/row/arrow_array_wrapper.h index 7316baddf5..9af7781f1f 100644 --- a/src/iceberg/row/arrow_array_wrapper.h +++ b/src/iceberg/row/arrow_array_wrapper.h @@ -43,7 +43,8 @@ class ICEBERG_EXPORT ArrowArrayStructLike : public StructLike { static Result> Make(const ArrowSchema& schema, const ArrowArray& array, - int64_t row_index = 0); + int64_t row_index = 0, + bool validate_batch = true); ArrowArrayStructLike(const ArrowArrayStructLike&) = delete; ArrowArrayStructLike& operator=(const ArrowArrayStructLike&) = delete; @@ -70,7 +71,8 @@ class ICEBERG_EXPORT ArrowArrayArrayLike : public ArrayLike { static Result> Make(const ArrowSchema& schema, const ArrowArray& array, - int64_t row_index = 0); + int64_t row_index = 0, + bool validate_batch = true); ArrowArrayArrayLike(const ArrowArrayArrayLike& other) = delete; ArrowArrayArrayLike& operator=(const ArrowArrayArrayLike& other) = delete; @@ -99,7 +101,8 @@ class ICEBERG_EXPORT ArrowArrayMapLike : public MapLike { static Result> Make(const ArrowSchema& schema, const ArrowArray& array, - int64_t row_index = 0); + int64_t row_index = 0, + bool validate_batch = true); ArrowArrayMapLike(const ArrowArrayMapLike& other) = delete; ArrowArrayMapLike& operator=(const ArrowArrayMapLike& other) = delete; diff --git a/src/iceberg/test/delete_filter_test.cc b/src/iceberg/test/delete_filter_test.cc index cfacf1a2c1..18a15aa17b 100644 --- a/src/iceberg/test/delete_filter_test.cc +++ b/src/iceberg/test/delete_filter_test.cc @@ -1362,6 +1362,33 @@ TEST_F(DeleteFilterTest, NullPosInBatchErrors) { EXPECT_THAT(alive, IsError(ErrorKind::kInvalidArrowData)); } +TEST_F(DeleteFilterTest, ConfiguresBatchValidation) { + auto requested_schema = Project({2}); + std::vector> delete_files; + + for (bool validate_batch : {true, false}) { + ICEBERG_UNWRAP_OR_FAIL( + auto filter, + DeleteFilter::Make(std::string(kDataPath), delete_files, table_schema_, + requested_schema, file_io_, true, nullptr, validate_batch)); + ICEBERG_UNWRAP_OR_FAIL( + auto batch, + MakeBatch(*filter->RequiredSchema(), R"([["first"], ["second"], ["third"]])")); + + auto* offsets = const_cast( + reinterpret_cast(batch.array.children[0]->buffers[1])); + offsets[2] = offsets[1] - 1; + + auto alive = filter->ComputeAliveRows(batch.schema, batch.array); + if (validate_batch) { + EXPECT_THAT(alive, IsError(ErrorKind::kInvalidArrowData)); + } else { + ASSERT_THAT(alive, IsOk()); + EXPECT_THAT(alive.value().indices, ::testing::ElementsAre(0, 1, 2)); + } + } +} + TEST_F(DeleteFilterTest, ExpectedSchemaIsRequestedSchema) { auto requested_schema = Project({1}); auto eq_by_name = std::make_shared(DataFile{ diff --git a/src/iceberg/test/struct_like_test.cc b/src/iceberg/test/struct_like_test.cc index e04d2c089b..0e84a7cc43 100644 --- a/src/iceberg/test/struct_like_test.cc +++ b/src/iceberg/test/struct_like_test.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include "iceberg/arrow_c_data_guard_internal.h" #include "iceberg/expression/literal.h" @@ -618,6 +619,107 @@ TEST(ArrowArrayStructLike, PrimitiveMap) { } } +TEST(ArrowArrayStructLike, ConfigurableFullValidation) { + { + auto type = ::arrow::struct_({::arrow::field("id", ::arrow::int64(), false), + ::arrow::field("name", ::arrow::utf8())}); + auto array_result = ::arrow::json::ArrayFromJSONString( + type, R"([[1,"first"], [2,"second"], [3,"third"]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = const_cast( + reinterpret_cast(c_array.children[1]->buffers[1])); + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayStructLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + EXPECT_THAT(ArrowArrayStructLike::Make(c_schema, c_array, 0, false), IsOk()); + } + + { + auto type = ::arrow::list(::arrow::field("item", ::arrow::int32())); + auto array_result = ::arrow::json::ArrayFromJSONString(type, R"([[1, 2], [3], [4]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = + const_cast(reinterpret_cast(c_array.buffers[1])); + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayArrayLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + EXPECT_THAT(ArrowArrayArrayLike::Make(c_schema, c_array, 0, false), IsOk()); + } + + { + auto type = std::make_shared<::arrow::MapType>( + ::arrow::field("key", ::arrow::utf8(), false), + ::arrow::field("value", ::arrow::int32(), false)); + auto array_result = ::arrow::json::ArrayFromJSONString( + type, R"([[ ["a", 1], ["b", 2] ], [ ["c", 3] ], []])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = + const_cast(reinterpret_cast(c_array.buffers[1])); + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayMapLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + EXPECT_THAT(ArrowArrayMapLike::Make(c_schema, c_array, 0, false), IsOk()); + } +} + +TEST(ArrowArrayStructLike, NestedValidationPolicy) { + auto type = ::arrow::struct_({::arrow::field( + "nested", ::arrow::struct_({::arrow::field("value", ::arrow::utf8())}))}); + auto array_result = ::arrow::json::ArrayFromJSONString( + type, R"([[{"value":"first"}], [{"value":"second"}]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = const_cast( + reinterpret_cast(c_array.children[0]->children[0]->buffers[1])); + offsets[1] = offsets[0] + 1; + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayStructLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + ICEBERG_UNWRAP_OR_FAIL( + auto struct_like, + ArrowArrayStructLike::Make(c_schema, c_array, 0, /*validate_batch=*/false)); + EXPECT_THAT(struct_like->GetField(0), IsOk()); +} + TEST(ArrowArrayStructLike, Accessor) { Schema schema{std::vector{ SchemaField::MakeOptional(1, "c1", int32()),