buffers, int partition) {
+ long numRecords = 0;
+ long keyValueBytes = 0;
+ for (WrappedBuffer buffer : buffers) {
+ numRecords += buffer.recordsPerPartition[partition];
+ keyValueBytes += buffer.sizePerPartition[partition];
+ }
+ return Writer.maxRawLength(numRecords, keyValueBytes);
+ }
+
private long writePartition(int pos, WrappedBuffer wrappedBuffer, Writer writer,
DataInputBuffer keyBuffer, DataInputBuffer valBuffer) throws IOException {
long numRecords = 0;
@@ -1094,7 +1112,8 @@ private void mergeAll() throws IOException {
sourceDestNameTrimmed + ": " + "Skipping partition: " + i + " in final merge since it has no records");
continue;
}
- writer = new Writer(keySerialization, valSerialization, out, keyClass, valClass, codec, null, null);
+ writer = new Writer(keySerialization, valSerialization, out, keyClass, valClass, codec,
+ codecBufferSize, mergedMaxRawLength(i), null, null);
try {
if (currentBuffer.nextPosition != 0
&& currentBuffer.partitionPositions[i] != WrappedBuffer.PARTITION_ABSENT_POSITION) {
@@ -1102,27 +1121,25 @@ private void mergeAll() throws IOException {
writePartition(currentBuffer.partitionPositions[i], currentBuffer, writer, keyBuffer,
valBuffer);
}
- synchronized (spillInfoList) {
- for (SpillInfo spillInfo : spillInfoList) {
- TezIndexRecord indexRecord = spillInfo.spillRecord.getIndex(i);
- if (indexRecord.getPartLength() == 0) {
- // Skip empty partitions within a spill
- continue;
- }
- FSDataInputStream in = rfs.open(spillInfo.outPath);
- in.seek(indexRecord.getStartOffset());
- IFile.Reader reader = new IFile.Reader(in, indexRecord.getPartLength(), codec, null,
- additionalSpillBytesReadCounter, ifileReadAhead, ifileReadAheadLength,
- ifileBufferSize);
- while (reader.nextRawKey(keyBufferIFile)) {
- // TODO Inefficient. If spills are not compressed, a direct copy should be possible
- // given the current IFile format. Also extremely inefficient for large records,
- // since the entire record will be read into memory.
- reader.nextRawValue(valBufferIFile);
- writer.append(keyBufferIFile, valBufferIFile);
- }
- reader.close();
+ for (SpillInfo spillInfo : spillInfoList) {
+ TezIndexRecord indexRecord = spillInfo.spillRecord.getIndex(i);
+ if (indexRecord.getPartLength() == 0) {
+ // Skip empty partitions within a spill
+ continue;
+ }
+ FSDataInputStream in = rfs.open(spillInfo.outPath);
+ in.seek(indexRecord.getStartOffset());
+ IFile.Reader reader = new IFile.Reader(in, indexRecord.getPartLength(), codec, null,
+ additionalSpillBytesReadCounter, ifileReadAhead, ifileReadAheadLength,
+ ifileBufferSize);
+ while (reader.nextRawKey(keyBufferIFile)) {
+ // TODO Inefficient. If spills are not compressed, a direct copy should be possible
+ // given the current IFile format. Also extremely inefficient for large records,
+ // since the entire record will be read into memory.
+ reader.nextRawValue(valBufferIFile);
+ writer.append(keyBufferIFile, valBufferIFile);
}
+ reader.close();
}
writer.close();
fileOutputBytesCounter.increment(writer.getCompressedLength());
@@ -1148,15 +1165,26 @@ private void mergeAll() throws IOException {
LOG.info(sourceDestNameTrimmed + ": " + "Finished final spill after merging : " + numSpills.get() + " spills");
}
+ /**
+ * Upper bound of the raw length of a partition merged from the current buffer and the spills:
+ * each part counts the header and EOF markers the merged stream writes once.
+ */
+ private long mergedMaxRawLength(int partition) {
+ long maxRawLength = Writer.maxRawLength(currentBuffer.recordsPerPartition[partition],
+ currentBuffer.sizePerPartition[partition]);
+ for (SpillInfo spillInfo : spillInfoList) {
+ maxRawLength += spillInfo.spillRecord.getIndex(partition).getRawLength();
+ }
+ return maxRawLength;
+ }
+
private void deleteIntermediateSpills() {
// Delete the intermediate spill files
- synchronized (spillInfoList) {
- for (SpillInfo spill : spillInfoList) {
- try {
- rfs.delete(spill.outPath, false);
- } catch (IOException e) {
- LOG.warn("Unable to delete intermediate spill " + spill.outPath, e);
- }
+ for (SpillInfo spill : spillInfoList) {
+ try {
+ rfs.delete(spill.outPath, false);
+ } catch (IOException e) {
+ LOG.warn("Unable to delete intermediate spill " + spill.outPath, e);
}
}
}
@@ -1185,7 +1213,8 @@ private void writeLargeRecord(final Object key, final Object value, final int pa
spilledRecordsCounter.increment(1);
Writer writer = null;
try {
- writer = new IFile.Writer(keySerialization, valSerialization, out, keyClass, valClass, codec, null, null);
+ writer = new IFile.Writer(keySerialization, valSerialization, out, keyClass, valClass, codec,
+ codecBufferSize, -1, null, null);
writer.append(key, value);
outputLargeRecordsCounter.increment(1);
numRecordsPerPartition[i]++;
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java
index 674de15612..c8f212b0ae 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.compress.BlockCompressorStream;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -160,6 +161,71 @@ public static CompressionOutputStream createOutputStream(CompressionCodec codec,
}
}
+ /**
+ * The buffer size of the codec's output streams. The codec reads it from its Configuration for
+ * every stream it creates; a writer of many streams reads it once and passes it to
+ * {@link #createOutputStream(CompressionCodec, OutputStream, Compressor, int, long)}.
+ *
+ * @return the buffer size, or -1 if the codec creates its streams itself
+ */
+ public static int getOutputBufferSize(CompressionCodec codec) {
+ if (codec == null || getBlockCompressionOverhead(codec.getClass().getName(), 0) < 0) {
+ return -1;
+ }
+ Configuration conf = ((Configurable) codec).getConf();
+ synchronized (conf) {
+ return conf.getInt(getBufferSizeProperty(codec), getDefaultBufferSize(conf, codec));
+ }
+ }
+
+ /**
+ * Creates the output stream of a Snappy or LZ4 codec without reading its Configuration, sized to
+ * the data when that fits in one block; leaves any other codec to create its own stream.
+ *
+ * These codecs write a {@link BlockCompressorStream}: the input in blocks of at most
+ * bufferSize - overhead bytes, each compressed into a buffer of bufferSize bytes. Any block size
+ * of at least dataLength writes a stream of at most dataLength bytes as a single block, so a
+ * block of dataLength bytes, compressed into a buffer of its compression bound (the overhead
+ * formulas are the codecs' bounds), writes the same bytes as the configured block size. A small
+ * per-partition stream of a spill then no longer allocates and zeroes the configured buffer
+ * (256 KiB by default). A block of 0 bytes would never take input, hence a positive dataLength.
+ *
+ * @param bufferSize the codec's {@link #getOutputBufferSize(CompressionCodec)}
+ * @param dataLength upper bound of the bytes written to the stream, or -1 if unknown
+ */
+ public static CompressionOutputStream createOutputStream(CompressionCodec codec,
+ OutputStream checksumOut, Compressor compressor, int bufferSize, long dataLength)
+ throws IOException {
+ if (bufferSize < 0) {
+ return createOutputStream(codec, checksumOut, compressor);
+ }
+ String codecClassName = codec.getClass().getName();
+ int overhead = getBlockCompressionOverhead(codecClassName, bufferSize);
+ if (dataLength > 0 && dataLength < bufferSize - overhead) {
+ int blockSize = (int) dataLength;
+ int blockOverhead = getBlockCompressionOverhead(codecClassName, blockSize);
+ return new BlockCompressorStream(checksumOut, compressor, blockSize + blockOverhead,
+ blockOverhead);
+ }
+ return new BlockCompressorStream(checksumOut, compressor, bufferSize, overhead);
+ }
+
+ /**
+ * The overhead SnappyCodec and Lz4Codec reserve in their BlockCompressorStream for a buffer of
+ * the given size, or -1 for any other codec.
+ */
+ @VisibleForTesting
+ static int getBlockCompressionOverhead(String codecClassName, int bufferSize) {
+ switch (codecClassName) {
+ case "org.apache.hadoop.io.compress.SnappyCodec":
+ return bufferSize / 6 + 32;
+ case "org.apache.hadoop.io.compress.Lz4Codec":
+ return bufferSize / 255 + 16;
+ default:
+ return -1;
+ }
+ }
+
public static String getBufferSizeProperty(CompressionCodec codec) {
return getBufferSizeProperty(codec.getClass().getName());
}
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
index 29704e27fa..572f1aa179 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
@@ -18,6 +18,7 @@
*/
package org.apache.tez.runtime.library.common.sort.impl;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
@@ -53,11 +54,13 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.lz4.Lz4Compressor;
import org.apache.hadoop.io.serializer.Deserializer;
import org.apache.hadoop.io.serializer.SerializationFactory;
import org.apache.hadoop.io.serializer.WritableSerialization;
import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
import org.apache.tez.runtime.library.common.shuffle.orderedgrouped.InMemoryReader;
@@ -821,6 +824,54 @@ public void testLz4CompressedDataIsLargerThanOriginal() throws IOException {
comp.compress(buf, 0, 32);
}
+ /**
+ * A writer given an upper bound of its raw length writes the bytes of an unsized writer, from a
+ * record to several codec blocks, and with values longer than the write buffer.
+ */
+ @Test
+ public void testSizedWriterWritesUnsizedBytes() throws IOException {
+ CompressionCodec snappy = ReflectionUtils.newInstance(SnappyCodec.class, new Configuration(false));
+ int codecBufferSize = CodecUtils.getOutputBufferSize(snappy);
+ Random random = new Random(0);
+ // {records, value length bound}: one record, one block, several blocks, values above the write buffer
+ for (int[] shape : new int[][] {{1, 50}, {100, 50}, {10_000, 50}, {5, 200_000}}) {
+ List records = new ArrayList<>();
+ long keyValueBytes = 0;
+ for (int i = 0; i < shape[0]; i++) {
+ byte[] key = new byte[1 + random.nextInt(20)];
+ byte[] value = new byte[random.nextInt(shape[1])];
+ random.nextBytes(key);
+ random.nextBytes(value);
+ records.add(new byte[][] {key, value});
+ keyValueBytes += key.length + value.length;
+ }
+ long maxRawLength = Writer.maxRawLength(records.size(), keyValueBytes);
+
+ ByteArrayOutputStream unsized = new ByteArrayOutputStream();
+ ByteArrayOutputStream sized = new ByteArrayOutputStream();
+ writeRecords(new Writer(null, null, new FSDataOutputStream(unsized, null), null, null, snappy, null, null),
+ records);
+ Writer writer = writeRecords(new Writer(null, null, new FSDataOutputStream(sized, null), null, null, snappy,
+ codecBufferSize, maxRawLength, null, null), records);
+
+ String what = Arrays.toString(shape);
+ assertTrue(writer.getRawLength() <= maxRawLength, what);
+ assertArrayEquals(unsized.toByteArray(), sized.toByteArray(), what);
+ }
+ }
+
+ private static Writer writeRecords(Writer writer, List records) throws IOException {
+ DataInputBuffer key = new DataInputBuffer();
+ DataInputBuffer value = new DataInputBuffer();
+ for (byte[][] record : records) {
+ key.reset(record[0], record[0].length);
+ value.reset(record[1], record[1].length);
+ writer.append(key, value);
+ }
+ writer.close();
+ return writer;
+ }
+
private void initBufWithNumbers(byte[] buf, int... args) {
for (int i = 0; i < args.length; i++) {
buf[i] = (byte) args[i];
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
index 8fb045bae8..89c67a23c3 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
@@ -18,6 +18,7 @@
*/
package org.apache.tez.runtime.library.common.writers;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
@@ -37,11 +38,14 @@
import static org.mockito.Mockito.verify;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.management.ManagementFactory;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
@@ -51,6 +55,7 @@
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@@ -59,6 +64,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
@@ -67,8 +73,10 @@
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
@@ -105,6 +113,7 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -1540,10 +1549,8 @@ private void baseTest(int numRecords, int numPartitions, Set skippedPar
"Incorrect index permissions (group)");
// verify no intermediate spill files have been left around
- synchronized (kvWriter.spillInfoList) {
- for (SpillInfo spill : kvWriter.spillInfoList) {
- assertFalse(localFs.exists(spill.outPath), "lingering intermediate spill file " + spill.outPath);
- }
+ for (SpillInfo spill : kvWriter.spillInfoList) {
+ assertFalse(localFs.exists(spill.outPath), "lingering intermediate spill file " + spill.outPath);
}
}
@@ -1601,6 +1608,130 @@ private static String createRandomString(int size) {
return sb.toString();
}
+ /**
+ * A spill sizes each partition's stream to the partition without reading the codec's buffer size
+ * per stream, and writes the bytes of an unsized writer: a final spill of many partitions, spills
+ * of several buffers each, and a merge of many spills.
+ */
+ @Test
+ public void testSpillStreamsSizedToPartitions() throws IOException, InterruptedException {
+ setupInit(true, ReportPartitionStats.DISABLED);
+ int numPartitions = 1009;
+ UnorderedPartitionedKVWriter kvWriter = createSnappyWriter(numPartitions, 16 << 20, -1, true);
+ write(kvWriter, 50_000);
+ ReadCountingConfiguration conf = (ReadCountingConfiguration) kvWriter.conf;
+ // The final spill runs on this thread; it borrows the pooled compressor instead of creating one.
+ CodecPool.returnCompressor(CodecPool.getCompressor(kvWriter.codec));
+ int reads = conf.reads.get();
+ long allocated = getCurrentThreadAllocatedBytes();
+ kvWriter.close();
+ allocated = getCurrentThreadAllocatedBytes() - allocated;
+ assertEquals(1, kvWriter.numSpills.get());
+ assertEquals(reads, conf.reads.get());
+ // The configured buffers would take 320 KiB per partition: 256 KiB codec + 64 KiB write buffer.
+ assertTrue(allocated < numPartitions * 32L * 1024, "allocated " + allocated);
+ assertPartitionsWrittenUnsized(kvWriter, kvWriter.finalOutPath, kvWriter.finalIndexPath, numPartitions);
+
+ numPartitions = 10;
+ kvWriter = createSnappyWriter(numPartitions, 10 << 10, 1 << 10, false);
+ write(kvWriter, 5000);
+ kvWriter.close();
+ assertTrue(kvWriter.spillLimit > 1);
+ for (int i = 0; i < kvWriter.numSpills.get(); i++) {
+ assertPartitionsWrittenUnsized(kvWriter, kvWriter.outputFileHandler.getSpillFileForWrite(i, 0),
+ kvWriter.outputFileHandler.getSpillIndexFileForWrite(i, 0), numPartitions);
+ }
+
+ kvWriter = createSnappyWriter(numPartitions, 2048, -1, true);
+ write(kvWriter, 5000);
+ kvWriter.close();
+ assertTrue(kvWriter.numSpills.get() > 100);
+ assertPartitionsWrittenUnsized(kvWriter, kvWriter.finalOutPath, kvWriter.finalIndexPath, numPartitions);
+ }
+
+ /** Spills of a writer without final merge take all its buffers. */
+ private UnorderedPartitionedKVWriter createSnappyWriter(int numPartitions, long availableMemory,
+ int maxSingleBufferSizeBytes, boolean finalMerge) throws IOException {
+ OutputContext outputContext = createMockOutputContext(new TezCounters(), ApplicationId.newInstance(10000000, 1),
+ UUID.randomUUID().toString(), defaultConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
+ TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
+ Configuration conf = new ReadCountingConfiguration(
+ createConfiguration(outputContext, IntWritable.class, LongWritable.class, true, maxSingleBufferSizeBytes),
+ CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY);
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, SnappyCodec.class.getName());
+ conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, finalMerge);
+ if (!finalMerge) {
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_PARTITIONED_KVWRITER_BUFFER_MERGE_PERCENT, 100);
+ }
+ return new UnorderedPartitionedKVWriterForTest(outputContext, conf, numPartitions, availableMemory);
+ }
+
+ private static void write(UnorderedPartitionedKVWriter kvWriter, int numRecords) throws IOException {
+ IntWritable key = new IntWritable();
+ LongWritable value = new LongWritable();
+ for (int i = 0; i < numRecords; i++) {
+ key.set(i);
+ value.set(i);
+ kvWriter.write(key, value);
+ }
+ }
+
+ private static long getCurrentThreadAllocatedBytes() {
+ return ((com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean())
+ .getCurrentThreadAllocatedBytes();
+ }
+
+ /** Each partition of the output holds the bytes an unsized writer writes for its records. */
+ private static void assertPartitionsWrittenUnsized(UnorderedPartitionedKVWriter kvWriter, Path outputPath,
+ Path indexPath, int numPartitions) throws IOException {
+ byte[] output = new byte[(int) localFs.getFileStatus(outputPath).getLen()];
+ try (FSDataInputStream in = localFs.open(outputPath)) {
+ in.readFully(output);
+ }
+ TezSpillRecord spillRecord = new TezSpillRecord(indexPath, kvWriter.conf);
+ DataInputBuffer key = new DataInputBuffer();
+ DataInputBuffer value = new DataInputBuffer();
+ for (int i = 0; i < numPartitions; i++) {
+ TezIndexRecord indexRecord = spillRecord.getIndex(i);
+ if (indexRecord.getPartLength() == 0) {
+ continue;
+ }
+ int start = (int) indexRecord.getStartOffset();
+ byte[] segment = Arrays.copyOfRange(output, start, start + (int) indexRecord.getPartLength());
+ Reader reader = new Reader(new ByteArrayInputStream(segment), segment.length, kvWriter.codec, null, null,
+ false, 0, -1);
+ ByteArrayOutputStream unsized = new ByteArrayOutputStream();
+ IFile.Writer writer = new IFile.Writer(null, null, new FSDataOutputStream(unsized, null), null, null,
+ kvWriter.codec, null, null);
+ while (reader.nextRawKey(key)) {
+ reader.nextRawValue(value);
+ writer.append(key, value);
+ }
+ reader.close();
+ writer.close();
+ assertArrayEquals(unsized.toByteArray(), segment, "partition " + i);
+ }
+ }
+
+ /** Counts the reads of one property. */
+ private static class ReadCountingConfiguration extends Configuration {
+ private final String property;
+ private final AtomicInteger reads = new AtomicInteger();
+
+ ReadCountingConfiguration(Configuration conf, String property) {
+ super(conf);
+ this.property = property;
+ }
+
+ @Override
+ public String get(String name) {
+ if (property.equals(name)) {
+ reads.incrementAndGet();
+ }
+ return super.get(name);
+ }
+ }
+
private OutputContext createMockOutputContext(TezCounters counters, ApplicationId appId,
String uniqueId, String auxiliaryService) {
OutputContext outputContext = mock(OutputContext.class);
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java
index 5be628819a..1065c7456c 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java
@@ -18,13 +18,17 @@
*/
package org.apache.tez.runtime.library.utils;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
+import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -35,6 +39,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -48,11 +53,14 @@
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.ZStandardCodec;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.shuffle.orderedgrouped.DummyCompressionCodec;
import org.apache.tez.runtime.library.common.sort.impl.IFileInputStream;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
public class TestCodecUtils {
@@ -203,6 +211,90 @@ public void testDefaultBufferSize() {
CodecUtils.getDefaultBufferSize(conf, new Lz4Codec()));
}
+ /**
+ * A stream sized to its data writes the bytes of the configured stream, from one byte to several
+ * configured blocks of compressible and of incompressible data.
+ */
+ @ParameterizedTest
+ @ValueSource(classes = {SnappyCodec.class, Lz4Codec.class})
+ public void testOutputStreamSizedToData(Class extends CompressionCodec> codecClass) throws IOException {
+ Configuration conf = new Configuration(false);
+ conf.setInt(CodecUtils.getBufferSizeProperty(codecClass.getName()), 64 * 1024);
+ CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);
+ int bufferSize = CodecUtils.getOutputBufferSize(codec);
+ assertEquals(64 * 1024, bufferSize);
+ int maxBlockSize = bufferSize - CodecUtils.getBlockCompressionOverhead(codecClass.getName(), bufferSize);
+ for (boolean compressible : new boolean[] {true, false}) {
+ for (int length : new int[] {1, 1000, maxBlockSize - 1, maxBlockSize, maxBlockSize + 1, 5 * bufferSize}) {
+ byte[] data = randomBytes(length, compressible);
+ ByteArrayOutputStream configured = new ByteArrayOutputStream();
+ ByteArrayOutputStream sized = new ByteArrayOutputStream();
+ compress(codec, data, configured, -1, -1);
+ compress(codec, data, sized, bufferSize, length);
+ assertArrayEquals(configured.toByteArray(), sized.toByteArray(),
+ "length " + length + ", compressible " + compressible);
+ }
+ }
+ }
+
+ /** A length that underestimates the data costs more blocks, not the data. */
+ @ParameterizedTest
+ @ValueSource(classes = {SnappyCodec.class, Lz4Codec.class})
+ public void testOutputStreamSizedToUnderestimatedLength(Class extends CompressionCodec> codecClass)
+ throws IOException {
+ CompressionCodec codec = ReflectionUtils.newInstance(codecClass, new Configuration(false));
+ byte[] data = randomBytes(100_000, false);
+ ByteArrayOutputStream sized = new ByteArrayOutputStream();
+ compress(codec, data, sized, CodecUtils.getOutputBufferSize(codec), 1000);
+ assertArrayEquals(data, decompress(codec, sized.toByteArray(), data.length));
+ }
+
+ private static byte[] randomBytes(int length, boolean compressible) {
+ byte[] data = new byte[length];
+ Random random = new Random(length);
+ random.nextBytes(data);
+ if (compressible) {
+ for (int i = 0; i < length; i++) {
+ data[i] &= 0x0f;
+ }
+ }
+ return data;
+ }
+
+ /**
+ * Writes the data in 64 KiB chunks as IFile.Writer does, through the codec's own stream when
+ * bufferSize is -1.
+ */
+ private static void compress(CompressionCodec codec, byte[] data, OutputStream out, int bufferSize,
+ long dataLength) throws IOException {
+ Compressor compressor = CodecUtils.getCompressor(codec);
+ try {
+ CompressionOutputStream stream =
+ CodecUtils.createOutputStream(codec, out, compressor, bufferSize, dataLength);
+ for (int off = 0; off < data.length; off += 64 * 1024) {
+ stream.write(data, off, Math.min(64 * 1024, data.length - off));
+ }
+ stream.finish();
+ } finally {
+ CodecPool.returnCompressor(compressor);
+ }
+ }
+
+ private static byte[] decompress(CompressionCodec codec, byte[] compressed, int length)
+ throws IOException {
+ Decompressor decompressor = CodecUtils.getDecompressor(codec);
+ try {
+ InputStream in = CodecUtils.createInputStream(codec, new ByteArrayInputStream(compressed),
+ decompressor);
+ byte[] data = new byte[length];
+ IOUtils.readFully(in, data, 0, length);
+ assertEquals(-1, in.read());
+ return data;
+ } finally {
+ CodecPool.returnDecompressor(decompressor);
+ }
+ }
+
private void waitForLatch(CountDownLatch latch) {
try {
latch.await();