diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index 755d90385d..87aa1d17c2 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -30,10 +30,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; -import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -219,7 +217,8 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT_DEFAULT); this.pipelinedShuffle = pipelinedShuffleConf && !isFinalMergeEnabled; - this.finalEvents = Lists.newLinkedList(); + // Spill callbacks append from the spill threads, several of which can finish at once. + this.finalEvents = Collections.synchronizedList(Lists.newLinkedList()); this.dataViaEventsEnabled = conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED, @@ -272,9 +271,11 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c int maxThreads = Math.max(2, numBuffers/2); //TODO: Make use of TezSharedExecutor later - ExecutorService executor = new ThreadPoolExecutor(1, maxThreads, + // availableSlots bounds the spills in flight; the queue only absorbs the moment between a spill callback + // releasing its slot and its thread returning to the pool, which a SynchronousQueue rejected. + ThreadPoolExecutor executor = new ThreadPoolExecutor(maxThreads, maxThreads, 60L, TimeUnit.SECONDS, - new SynchronousQueue(), + new LinkedBlockingQueue(), new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat( @@ -282,6 +283,7 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c outputContext.getDestinationVertexName()) + "} #%d") .build() ); + executor.allowCoreThreadTimeOut(true); // to restrict submission of more tasks than threads (e.g numBuffers > numThreads) // This is maxThreads - 1, to avoid race between callback thread releasing semaphore and the // thread calling tryAcquire. diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java index 8fb045bae8..214c6d0d13 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java @@ -50,6 +50,9 @@ import java.util.Random; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -105,6 +108,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1118,6 +1122,59 @@ public void testSkippedPartitions_WithFinalMergeDisabled(boolean shouldCompress, baseTestWithFinalMergeDisabled(200, 10, Sets.newHashSet(2, 5), shouldCompress); } + /** + * With final merge off, a spill's callback adds the spill's event to the events of close() on the spill thread. + * Many small buffers keep many spills in flight, so several callbacks finish at once: no spill may be rejected, + * and close() must return one event per spill. The writers run 4 at a time to make that more frequent. + */ + @Test + @Timeout(value = 120, unit = TimeUnit.SECONDS) + public void testConcurrentSpillsWithFinalMergeDisabled() throws Exception { + setupInit(false, ReportPartitionStats.DISABLED); + ExecutorService writers = Executors.newFixedThreadPool(4); + try { + List> results = new ArrayList<>(); + for (int writer = 0; writer < 40; writer++) { + int id = writer; + results.add(writers.submit(() -> { + writeWithConcurrentSpills(id); + return null; + })); + } + for (Future result : results) { + result.get(); + } + } finally { + writers.shutdownNow(); + } + } + + private void writeWithConcurrentSpills(int writer) throws IOException, InterruptedException { + String auxiliaryService = defaultConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, + TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT); + OutputContext outputContext = createMockOutputContext(new TezCounters(), ApplicationId.newInstance(10000000, 1), + UUID.randomUUID().toString(), auxiliaryService); + Configuration conf = createConfiguration(outputContext, IntWritable.class, LongWritable.class, false, 4096); + conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false); + conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, false); + // 64 buffers of 4 KiB: 32 spill threads + UnorderedPartitionedKVWriter kvWriter = new UnorderedPartitionedKVWriterForTest(outputContext, conf, 16, + 64 * 4096L); + IntWritable key = new IntWritable(); + LongWritable value = new LongWritable(); + for (int i = 0; i < 200_000; i++) { + key.set(i); + value.set(i); + kvWriter.write(key, value); + } + List events = kvWriter.close(); + + assertFalse(events.contains(null), "writer " + writer + ": a null event"); + assertEquals(kvWriter.numSpills.get(), events.stream().filter(CompositeDataMovementEvent.class::isInstance) + .count(), "writer " + writer + ": spill events"); + localFs.delete(new Path(outputContext.getWorkDirs()[0]), true); + } + @SuppressWarnings("unchecked") private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, Set skippedPartitions, boolean shouldCompress) throws IOException, InterruptedException {