Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -1115,6 +1115,8 @@ public static class SargApplier {
private final SearchArgument sarg;
private final List<PredicateLeaf> sargLeaves;
private final int[] filterColumns;
// true for leaves whose column is nested in a struct that may be null
private final boolean[] hasStructAncestor;
private final long rowIndexStride;
// same as the above array, but indices are set to true
private final SchemaEvolution evolution;
Expand Down Expand Up @@ -1149,12 +1151,28 @@ public SargApplier(SearchArgument sarg,
this.convertToProlepticGregorian = convertToProlepticGregorian;
filterColumns = mapSargColumnsToOrcInternalColIdx(sargLeaves,
evolution);
hasStructAncestor = new boolean[filterColumns.length];
for (int i = 0; i < filterColumns.length; ++i) {
hasStructAncestor[i] = evolution != null && filterColumns[i] > 0 &&
hasStructAncestor(evolution.getFileSchema().findSubtype(filterColumns[i]));
}
this.rowIndexStride = rowIndexStride;
this.evolution = evolution;
exceptionCount = new long[sargLeaves.size()];
this.useUTCTimestamp = useUTCTimestamp;
}

private static boolean hasStructAncestor(TypeDescription type) {
for (TypeDescription parent = type.getParent();
parent != null && parent.getParent() != null;
parent = parent.getParent()) {
if (parent.getCategory() == TypeDescription.Category.STRUCT) {
return true;
}
}
return false;
}

public void setRowIndexCols(boolean[] rowIndexCols) {
// included will not be null, row options will fill the array with
// trues if null
Expand Down Expand Up @@ -1195,6 +1213,11 @@ public boolean[] pickRowGroups(StripeInformation stripe,
leafValues[pred] = TruthValue.YES_NO_NULL;
} else if (exceptionAnswer[pred] != null) {
leafValues[pred] = exceptionAnswer[pred];
} else if (hasStructAncestor[pred] &&
sargLeaves.get(pred).getOperator() == PredicateLeaf.Operator.IS_NULL) {
// The column statistics only cover rows where every ancestor struct is present,
// so they cannot show that a row group has no nulls for this field.
leafValues[pred] = TruthValue.YES_NO_NULL;
} else {
if (indexes[columnIx] == null) {
LOG.warn("Index is not populated for " + columnIx);
Expand Down
37 changes: 37 additions & 0 deletions java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -1971,6 +1971,43 @@ public void testCloseWithException() throws Exception {
Path workDir = new Path(System.getProperty("test.tmp.dir",
"target" + File.separator + "test" + File.separator + "tmp"));

@Test
public void testIsNullOnFieldUnderNullStruct() throws Exception {
Configuration conf = new Configuration();
Path path = new Path(workDir, "isNullUnderNullStruct.orc");
TypeDescription schema = TypeDescription.fromString("struct<id:int,s:struct<x:int>>");
try (Writer writer = OrcFile.createWriter(path,
OrcFile.writerOptions(conf).setSchema(schema).overwrite(true))) {
VectorizedRowBatch batch = schema.createRowBatch();
LongColumnVector id = (LongColumnVector) batch.cols[0];
StructColumnVector s = (StructColumnVector) batch.cols[1];
LongColumnVector x = (LongColumnVector) s.fields[0];
id.vector[0] = 1;
s.noNulls = false;
s.isNull[0] = true;
x.noNulls = false;
x.isNull[0] = true;
id.vector[1] = 2;
x.vector[1] = 5;
batch.size = 2;
writer.addRowBatch(batch);
}

Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf));
SearchArgument sarg = SearchArgumentFactory.newBuilder()
.isNull("s.x", PredicateLeaf.Type.LONG)
.build();
VectorizedRowBatch batch = reader.getSchema().createRowBatch();
long rowCount = 0;
try (RecordReader rows = reader.rows(reader.options().searchArgument(sarg, null))) {
while (rows.nextBatch(batch)) {
rowCount += batch.size;
}
}
// s is null in the first row, so s.x IS NULL matches it and the row group must be read
assertEquals(2, rowCount);
}

private void closeMockedRecordReader(DataReader mockedDataReader) throws IOException {
Path path = new Path(workDir, "empty.orc");
FileSystem.get(conf).delete(path, true);
Expand Down