From 07bf4c36db4bccb64352deed778ffe5a179a523c Mon Sep 17 00:00:00 2001 From: Metin Hatipogullari Date: Fri, 20 Nov 2026 16:32:18 +0100 Subject: [PATCH] ORC-2238: Do not skip row groups for IS_NULL on fields nested in a struct --- .../org/apache/orc/impl/RecordReaderImpl.java | 23 ++++++++++++ .../apache/orc/impl/TestRecordReaderImpl.java | 37 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java index 34ffe45204..3e4ad4ce19 100644 --- a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java +++ b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java @@ -1115,6 +1115,8 @@ public static class SargApplier { private final SearchArgument sarg; private final List sargLeaves; private final int[] filterColumns; + // true for leaves whose column is nested in a struct that may be null + private final boolean[] hasStructAncestor; private final long rowIndexStride; // same as the above array, but indices are set to true private final SchemaEvolution evolution; @@ -1149,12 +1151,28 @@ public SargApplier(SearchArgument sarg, this.convertToProlepticGregorian = convertToProlepticGregorian; filterColumns = mapSargColumnsToOrcInternalColIdx(sargLeaves, evolution); + hasStructAncestor = new boolean[filterColumns.length]; + for (int i = 0; i < filterColumns.length; ++i) { + hasStructAncestor[i] = evolution != null && filterColumns[i] > 0 && + hasStructAncestor(evolution.getFileSchema().findSubtype(filterColumns[i])); + } this.rowIndexStride = rowIndexStride; this.evolution = evolution; exceptionCount = new long[sargLeaves.size()]; this.useUTCTimestamp = useUTCTimestamp; } + private static boolean hasStructAncestor(TypeDescription type) { + for (TypeDescription parent = type.getParent(); + parent != null && parent.getParent() != null; + parent = parent.getParent()) { + if (parent.getCategory() == TypeDescription.Category.STRUCT) { + return true; + } + } + return false; + } + public void setRowIndexCols(boolean[] rowIndexCols) { // included will not be null, row options will fill the array with // trues if null @@ -1195,6 +1213,11 @@ public boolean[] pickRowGroups(StripeInformation stripe, leafValues[pred] = TruthValue.YES_NO_NULL; } else if (exceptionAnswer[pred] != null) { leafValues[pred] = exceptionAnswer[pred]; + } else if (hasStructAncestor[pred] && + sargLeaves.get(pred).getOperator() == PredicateLeaf.Operator.IS_NULL) { + // The column statistics only cover rows where every ancestor struct is present, + // so they cannot show that a row group has no nulls for this field. + leafValues[pred] = TruthValue.YES_NO_NULL; } else { if (indexes[columnIx] == null) { LOG.warn("Index is not populated for " + columnIx); diff --git a/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java b/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java index 5c47111db8..dcf96bec24 100644 --- a/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java +++ b/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java @@ -1971,6 +1971,43 @@ public void testCloseWithException() throws Exception { Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); + @Test + public void testIsNullOnFieldUnderNullStruct() throws Exception { + Configuration conf = new Configuration(); + Path path = new Path(workDir, "isNullUnderNullStruct.orc"); + TypeDescription schema = TypeDescription.fromString("struct>"); + try (Writer writer = OrcFile.createWriter(path, + OrcFile.writerOptions(conf).setSchema(schema).overwrite(true))) { + VectorizedRowBatch batch = schema.createRowBatch(); + LongColumnVector id = (LongColumnVector) batch.cols[0]; + StructColumnVector s = (StructColumnVector) batch.cols[1]; + LongColumnVector x = (LongColumnVector) s.fields[0]; + id.vector[0] = 1; + s.noNulls = false; + s.isNull[0] = true; + x.noNulls = false; + x.isNull[0] = true; + id.vector[1] = 2; + x.vector[1] = 5; + batch.size = 2; + writer.addRowBatch(batch); + } + + Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf)); + SearchArgument sarg = SearchArgumentFactory.newBuilder() + .isNull("s.x", PredicateLeaf.Type.LONG) + .build(); + VectorizedRowBatch batch = reader.getSchema().createRowBatch(); + long rowCount = 0; + try (RecordReader rows = reader.rows(reader.options().searchArgument(sarg, null))) { + while (rows.nextBatch(batch)) { + rowCount += batch.size; + } + } + // s is null in the first row, so s.x IS NULL matches it and the row group must be read + assertEquals(2, rowCount); + } + private void closeMockedRecordReader(DataReader mockedDataReader) throws IOException { Path path = new Path(workDir, "empty.orc"); FileSystem.get(conf).delete(path, true);