diff --git a/vector/src/main/codegen/templates/StructWriters.java b/vector/src/main/codegen/templates/StructWriters.java index 413f707c70..3b1c30d892 100644 --- a/vector/src/main/codegen/templates/StructWriters.java +++ b/vector/src/main/codegen/templates/StructWriters.java @@ -86,6 +86,12 @@ public class ${mode}StructWriter extends AbstractFieldWriter { case EXTENSIONTYPE: extension(child.getName(), child.getType()); break; + case NULL: { + // There is no writer for the Null type, but the child vector still has to exist so that + // the container matches its field (e.g. when it is created from a Field by a TransferPair). + container.addOrGet(child.getName(), child.getFieldType(), NullVector.class); + break; + } case UNION: FieldType fieldType = new FieldType(addVectorAsNullable, MinorType.UNION.getType(), null, null); UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), fieldType, UnionVector.class), getNullableStructWriterFactory()); diff --git a/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 6bfe540d23..345e8d7afa 100644 --- a/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -177,12 +177,12 @@ public int getValueCapacity() { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new TransferImpl(ref); + return new TransferImpl(new NullVector(ref, field.getFieldType())); } @Override public TransferPair getTransferPair(Field field, BufferAllocator allocator) { - return new TransferImpl(field.getName()); + return new TransferImpl(new NullVector(field)); } @Override @@ -337,10 +337,6 @@ public String getName() { private class TransferImpl implements TransferPair { NullVector to; - public TransferImpl(String ref) { - to = new NullVector(ref); - } - @Deprecated public TransferImpl() { to = new NullVector(); diff --git a/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 17503f98c8..674bb554ae 100644 --- a/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -164,7 +164,7 @@ public enum MinorType { @Override public FieldVector getNewVector( Field field, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullVector(field.getName()); + return new NullVector(field); } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index adf4eba10c..52df39d380 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -216,6 +216,22 @@ public void testWithNullVector() { // no allocations to clear for NullVector } + @Test + public void testNullVectorTransferPairKeepsField() { + Field field = + new Field("n", new FieldType(false, ArrowType.Null.INSTANCE, null, Map.of("k", "v")), null); + NullVector fromNullVector = new NullVector(field); + + assertEquals(field, fromNullVector.getTransferPair(allocator).getTo().getField()); + assertEquals( + new Field("renamed", field.getFieldType(), null), + fromNullVector.getTransferPair("renamed", allocator).getTo().getField()); + + Field targetField = new Field("target", FieldType.nullable(ArrowType.Null.INSTANCE), null); + assertEquals( + targetField, fromNullVector.getTransferPair(targetField, allocator).getTo().getField()); + } + @Test public void testWithZeroVector() { ZeroVector fromZeroVector = new ZeroVector("zeroVector"); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 8c8a45f588..ac82434fae 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -17,10 +17,12 @@ package org.apache.arrow.vector; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.HashMap; @@ -379,6 +381,70 @@ public void testStructVectorTransferPairWithExtensionType() { toVector.close(); } + @Test + public void testStructVectorTransferPairWithNullType() { + Field nullField = + new Field( + "null_child", + new FieldType(false, ArrowType.Null.INSTANCE, null, Map.of("k", "v")), + null); + Field structField = + new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(nullField)); + + try (StructVector s1 = (StructVector) structField.createVector(allocator)) { + NullVector nullVector = s1.addOrGet("null_child", nullField.getFieldType(), NullVector.class); + s1.allocateNew(); + s1.setIndexDefined(0); + s1.setNull(1); + s1.setIndexDefined(2); + nullVector.setValueCount(3); + s1.setValueCount(3); + + TransferPair tp = s1.getTransferPair(structField, allocator); + try (StructVector toVector = (StructVector) tp.getTo()) { + assertEquals(structField, toVector.getField()); + + tp.transfer(); + assertEquals(3, toVector.getValueCount()); + assertFalse(toVector.isNull(0)); + assertTrue(toVector.isNull(1)); + assertFalse(toVector.isNull(2)); + NullVector toChild = toVector.getChild("null_child", NullVector.class); + assertNotNull(toChild); + assertEquals(3, toChild.getValueCount()); + assertEquals(nullField, toChild.getField()); + } + } + } + + @Test + public void testStructVectorFromFieldWithNullChild() { + Field nullField = + new Field( + "null_child", + new FieldType(false, ArrowType.Null.INSTANCE, null, Map.of("k", "v")), + null); + Field intField = new Field("int_child", FieldType.nullable(MinorType.INT.getType()), null); + Field structField = + new Field( + "struct", FieldType.nullable(new ArrowType.Struct()), List.of(nullField, intField)); + + try (StructVector vector = new StructVector(structField, allocator, null)) { + assertEquals(structField, vector.getField()); + assertEquals(2, vector.size()); + assertNotNull(vector.getChild("null_child", NullVector.class)); + + // The writer still works for the other children. + NullableStructWriter writer = vector.getWriter(); + writer.setPosition(0); + writer.start(); + writer.integer("int_child").writeInt(42); + writer.end(); + vector.setValueCount(1); + assertEquals(42, vector.getChild("int_child", IntVector.class).get(0)); + } + } + private StructVector simpleStructVector(String name, BufferAllocator allocator) { final String INT_COL = "struct_int_child"; final String FLT_COL = "struct_flt_child"; diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java index cfc70c2227..c755a25b05 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -27,6 +28,10 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -155,4 +160,20 @@ public void testRootWithDifferentChildTypes() { assertEquals("Vector schema roots have different schemas.", exp.getMessage()); } } + + @Test + public void testAppendSliceWithNonNullableNullVector() { + Schema schema = + new Schema(List.of(new Field("n", FieldType.notNullable(ArrowType.Null.INSTANCE), null))); + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + VectorSchemaRoot target = VectorSchemaRoot.create(schema, allocator)) { + root.setRowCount(3); + try (VectorSchemaRoot slice = root.slice(0, 2)) { + assertEquals(schema, slice.getSchema()); + VectorSchemaRootAppender.append(target, slice); + } + assertEquals(2, target.getRowCount()); + assertEquals(2, target.getVector("n").getValueCount()); + } + } }