From 743b7d76e27b0d7f9f941d09b49c66f55ade6797 Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:05:51 +0800 Subject: [PATCH 1/4] GH-830: Create Null children when building a struct writer from a Field NullableStructWriter/SingleStructWriter threw "Unknown type: NULL" when the struct field had a Null-typed child, which broke e.g. StructVector#getTransferPair(Field, ...). Add the NullVector child instead. --- .../main/codegen/templates/StructWriters.java | 7 +++ .../apache/arrow/vector/TestStructVector.java | 59 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/vector/src/main/codegen/templates/StructWriters.java b/vector/src/main/codegen/templates/StructWriters.java index 413f707c70..541047443d 100644 --- a/vector/src/main/codegen/templates/StructWriters.java +++ b/vector/src/main/codegen/templates/StructWriters.java @@ -86,6 +86,13 @@ public class ${mode}StructWriter extends AbstractFieldWriter { case EXTENSIONTYPE: extension(child.getName(), child.getType()); break; + case NULL: { + // There is no writer for the Null type, but the child vector still has to exist so that + // the container matches its field (e.g. when it is created from a Field by a TransferPair). + FieldType fieldType = new FieldType(addVectorAsNullable, MinorType.NULL.getType(), null, null); + container.addOrGet(child.getName(), fieldType, NullVector.class); + break; + } case UNION: FieldType fieldType = new FieldType(addVectorAsNullable, MinorType.UNION.getType(), null, null); UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), fieldType, UnionVector.class), getNullableStructWriterFactory()); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 8c8a45f588..352a201cc6 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -17,10 +17,12 @@ package org.apache.arrow.vector; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.HashMap; @@ -379,6 +381,63 @@ public void testStructVectorTransferPairWithExtensionType() { toVector.close(); } + @Test + public void testStructVectorTransferPairWithNullType() { + Field nullField = new Field("null_child", FieldType.nullable(ArrowType.Null.INSTANCE), null); + Field structField = + new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(nullField)); + + try (StructVector s1 = (StructVector) structField.createVector(allocator)) { + NullVector nullVector = + s1.addOrGet("null_child", FieldType.nullable(ArrowType.Null.INSTANCE), NullVector.class); + s1.allocateNew(); + s1.setIndexDefined(0); + s1.setNull(1); + s1.setIndexDefined(2); + nullVector.setValueCount(3); + s1.setValueCount(3); + + TransferPair tp = s1.getTransferPair(structField, allocator); + try (StructVector toVector = (StructVector) tp.getTo()) { + assertEquals(s1.getField(), toVector.getField()); + + tp.transfer(); + assertEquals(3, toVector.getValueCount()); + assertFalse(toVector.isNull(0)); + assertTrue(toVector.isNull(1)); + assertFalse(toVector.isNull(2)); + NullVector toChild = toVector.getChild("null_child", NullVector.class); + assertNotNull(toChild); + assertEquals(3, toChild.getValueCount()); + assertTrue(toChild.isNull(0)); + } + } + } + + @Test + public void testStructVectorFromFieldWithNullChild() { + Field nullField = new Field("null_child", FieldType.nullable(ArrowType.Null.INSTANCE), null); + Field intField = new Field("int_child", FieldType.nullable(MinorType.INT.getType()), null); + Field structField = + new Field( + "struct", FieldType.nullable(new ArrowType.Struct()), List.of(nullField, intField)); + + try (StructVector vector = new StructVector(structField, allocator, null)) { + assertEquals(structField, vector.getField()); + assertEquals(2, vector.size()); + assertNotNull(vector.getChild("null_child", NullVector.class)); + + // The writer still works for the other children. + NullableStructWriter writer = vector.getWriter(); + writer.setPosition(0); + writer.start(); + writer.integer("int_child").writeInt(42); + writer.end(); + vector.setValueCount(1); + assertEquals(42, vector.getChild("int_child", IntVector.class).get(0)); + } + } + private StructVector simpleStructVector(String name, BufferAllocator allocator) { final String INT_COL = "struct_int_child"; final String FLT_COL = "struct_flt_child"; From edf274a329ef120793ab30bca86794d68a6a7c2c Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:21:08 +0800 Subject: [PATCH 2/4] GH-830: Preserve Null child FieldType in struct writers Address review: construct NullVector from the full Field in Types, pass child.getFieldType() in StructWriters, and assert non-nullable Null children with metadata round-trip through TransferPair. --- .../main/codegen/templates/StructWriters.java | 3 +-- .../org/apache/arrow/vector/types/Types.java | 2 +- .../apache/arrow/vector/TestStructVector.java | 18 +++++++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/vector/src/main/codegen/templates/StructWriters.java b/vector/src/main/codegen/templates/StructWriters.java index 541047443d..3b1c30d892 100644 --- a/vector/src/main/codegen/templates/StructWriters.java +++ b/vector/src/main/codegen/templates/StructWriters.java @@ -89,8 +89,7 @@ public class ${mode}StructWriter extends AbstractFieldWriter { case NULL: { // There is no writer for the Null type, but the child vector still has to exist so that // the container matches its field (e.g. when it is created from a Field by a TransferPair). - FieldType fieldType = new FieldType(addVectorAsNullable, MinorType.NULL.getType(), null, null); - container.addOrGet(child.getName(), fieldType, NullVector.class); + container.addOrGet(child.getName(), child.getFieldType(), NullVector.class); break; } case UNION: diff --git a/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 17503f98c8..674bb554ae 100644 --- a/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -164,7 +164,7 @@ public enum MinorType { @Override public FieldVector getNewVector( Field field, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullVector(field.getName()); + return new NullVector(field); } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 352a201cc6..c881531c74 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -383,13 +383,17 @@ public void testStructVectorTransferPairWithExtensionType() { @Test public void testStructVectorTransferPairWithNullType() { - Field nullField = new Field("null_child", FieldType.nullable(ArrowType.Null.INSTANCE), null); + Field nullField = + new Field( + "null_child", + new FieldType(false, ArrowType.Null.INSTANCE, null, Map.of("k", "v")), + null); Field structField = new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(nullField)); try (StructVector s1 = (StructVector) structField.createVector(allocator)) { NullVector nullVector = - s1.addOrGet("null_child", FieldType.nullable(ArrowType.Null.INSTANCE), NullVector.class); + s1.addOrGet("null_child", nullField.getFieldType(), NullVector.class); s1.allocateNew(); s1.setIndexDefined(0); s1.setNull(1); @@ -399,7 +403,7 @@ public void testStructVectorTransferPairWithNullType() { TransferPair tp = s1.getTransferPair(structField, allocator); try (StructVector toVector = (StructVector) tp.getTo()) { - assertEquals(s1.getField(), toVector.getField()); + assertEquals(structField, toVector.getField()); tp.transfer(); assertEquals(3, toVector.getValueCount()); @@ -409,14 +413,18 @@ public void testStructVectorTransferPairWithNullType() { NullVector toChild = toVector.getChild("null_child", NullVector.class); assertNotNull(toChild); assertEquals(3, toChild.getValueCount()); - assertTrue(toChild.isNull(0)); + assertEquals(nullField, toChild.getField()); } } } @Test public void testStructVectorFromFieldWithNullChild() { - Field nullField = new Field("null_child", FieldType.nullable(ArrowType.Null.INSTANCE), null); + Field nullField = + new Field( + "null_child", + new FieldType(false, ArrowType.Null.INSTANCE, null, Map.of("k", "v")), + null); Field intField = new Field("int_child", FieldType.nullable(MinorType.INT.getType()), null); Field structField = new Field( From 6e9d1a386bc4d437a7df80839cf4ce2649230e90 Mon Sep 17 00:00:00 2001 From: Vincent <1729065730@qq.com> Date: Fri, 6 Nov 2026 19:11:25 +0800 Subject: [PATCH 3/4] GH-830: Apply spotless formatting to TestStructVector --- .../test/java/org/apache/arrow/vector/TestStructVector.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index c881531c74..ac82434fae 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -392,8 +392,7 @@ public void testStructVectorTransferPairWithNullType() { new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(nullField)); try (StructVector s1 = (StructVector) structField.createVector(allocator)) { - NullVector nullVector = - s1.addOrGet("null_child", nullField.getFieldType(), NullVector.class); + NullVector nullVector = s1.addOrGet("null_child", nullField.getFieldType(), NullVector.class); s1.allocateNew(); s1.setIndexDefined(0); s1.setNull(1); From 8fb9a8054ef7982ca90bb95820372a6da062e420 Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:51:00 +0800 Subject: [PATCH 4/4] GH-830: Keep the Field in NullVector transfer pairs Address review: NullVector transfer pairs rebuilt the target from the name only, so a non-nullable Null field came back nullable and VectorSchemaRoot.slice() + VectorSchemaRootAppender failed with "different schemas". Build the target from the source FieldType (or the given Field) instead. --- .../org/apache/arrow/vector/NullVector.java | 8 ++----- .../arrow/vector/TestSplitAndTransfer.java | 16 ++++++++++++++ .../util/TestVectorSchemaRootAppender.java | 21 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 6bfe540d23..345e8d7afa 100644 --- a/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -177,12 +177,12 @@ public int getValueCapacity() { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new TransferImpl(ref); + return new TransferImpl(new NullVector(ref, field.getFieldType())); } @Override public TransferPair getTransferPair(Field field, BufferAllocator allocator) { - return new TransferImpl(field.getName()); + return new TransferImpl(new NullVector(field)); } @Override @@ -337,10 +337,6 @@ public String getName() { private class TransferImpl implements TransferPair { NullVector to; - public TransferImpl(String ref) { - to = new NullVector(ref); - } - @Deprecated public TransferImpl() { to = new NullVector(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index adf4eba10c..52df39d380 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -216,6 +216,22 @@ public void testWithNullVector() { // no allocations to clear for NullVector } + @Test + public void testNullVectorTransferPairKeepsField() { + Field field = + new Field("n", new FieldType(false, ArrowType.Null.INSTANCE, null, Map.of("k", "v")), null); + NullVector fromNullVector = new NullVector(field); + + assertEquals(field, fromNullVector.getTransferPair(allocator).getTo().getField()); + assertEquals( + new Field("renamed", field.getFieldType(), null), + fromNullVector.getTransferPair("renamed", allocator).getTo().getField()); + + Field targetField = new Field("target", FieldType.nullable(ArrowType.Null.INSTANCE), null); + assertEquals( + targetField, fromNullVector.getTransferPair(targetField, allocator).getTo().getField()); + } + @Test public void testWithZeroVector() { ZeroVector fromZeroVector = new ZeroVector("zeroVector"); diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java index cfc70c2227..c755a25b05 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -27,6 +28,10 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -155,4 +160,20 @@ public void testRootWithDifferentChildTypes() { assertEquals("Vector schema roots have different schemas.", exp.getMessage()); } } + + @Test + public void testAppendSliceWithNonNullableNullVector() { + Schema schema = + new Schema(List.of(new Field("n", FieldType.notNullable(ArrowType.Null.INSTANCE), null))); + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + VectorSchemaRoot target = VectorSchemaRoot.create(schema, allocator)) { + root.setRowCount(3); + try (VectorSchemaRoot slice = root.slice(0, 2)) { + assertEquals(schema, slice.getSchema()); + VectorSchemaRootAppender.append(target, slice); + } + assertEquals(2, target.getRowCount()); + assertEquals(2, target.getVector("n").getValueCount()); + } + } }