diff --git a/lib/internal/streams/readable.js b/lib/internal/streams/readable.js index 9bd7ea636919..bb11624d6269 100644 --- a/lib/internal/streams/readable.js +++ b/lib/internal/streams/readable.js @@ -428,6 +428,10 @@ Readable.prototype.push = function(chunk, encoding) { state.length = 0; readableAddChunkPushByteMode(this, state, first); } + } else if (state.fastChunk != null) { + // A push() from flowSync()'s 'data' event goes behind the prefetched + // chunk, and flowSync() must see that the buffer is in use. + materializeFastChunk(state); } debug('push', chunk); @@ -441,6 +445,12 @@ Readable.prototype.push = function(chunk, encoding) { Readable.prototype.unshift = function(chunk, encoding) { debug('unshift', chunk); const state = this._readableState; + // An unshift() from flowSync()'s 'data' event or _read() goes in front of + // the prefetched chunk, and flowSync() must see that the buffer is in use. + if ((state[kState] & kFastPush) !== 0 || state.fastChunk != null) { + state[kState] &= ~kFastPush; + materializeFastChunk(state); + } return (state[kState] & kObjectMode) === 0 ? readableAddChunkUnshiftByteMode(this, state, chunk, encoding) : readableAddChunkUnshiftObjectMode(this, state, chunk); @@ -630,6 +640,8 @@ Readable.prototype.isPaused = function() { // Backwards compatibility. Readable.prototype.setEncoding = function(enc) { const state = this._readableState; + // A push() that follows a setEncoding() from _read() has to be decoded. + state[kState] &= ~kFastPush; if (state.fastChunk != null) materializeFastChunk(state); @@ -698,8 +710,8 @@ function howMuchToRead(n, state) { // You can override either this method, or the async _read(n) below. Readable.prototype.read = function(n) { - // A nested read() during flowSync()'s 'data' event must see the - // prefetched chunk. Null for every read that is not inside that loop. + // A nested read() during flowSync()'s 'data' event or _read() must see + // the prefetched chunk. Null for every read that is not inside that loop. if (this._readableState.fastChunk != null) materializeFastChunk(this._readableState); @@ -1472,7 +1484,8 @@ function flowSync(stream, state) { deliverFlowChunk(stream, state, current); } - // Nested read() moved fastChunk into the buffer and may have refilled. + // A nested read(), unshift() or push() moved fastChunk into the buffer + // and may have added to it. if (state.fastChunk == null && state.length !== 0) break; } @@ -1485,11 +1498,14 @@ function flowSync(stream, state) { } // Put fastChunk at the head of the buffer. state.length already counts it. +// The buffer is in use from here on, so a later push() from the same _read() +// must add to it instead of replacing fastChunk and state.length. function materializeFastChunk(state) { const chunk = state.fastChunk; if (chunk == null) return; state.fastChunk = null; + state[kState] &= ~kFastPush; if (state.bufferIndex > 0) { state.buffer[--state.bufferIndex] = chunk; } else { diff --git a/test/parallel/test-stream-readable-flow-sync-buffered.js b/test/parallel/test-stream-readable-flow-sync-buffered.js new file mode 100644 index 000000000000..8f66e4dcf4e3 --- /dev/null +++ b/test/parallel/test-stream-readable-flow-sync-buffered.js @@ -0,0 +1,139 @@ +'use strict'; + +// A flowing byte-mode Readable that pushes from _read() keeps the prefetched +// chunk off the buffer. Data put on the buffer by unshift(), push() or a +// nested read() while that chunk is pending must not be lost or reordered, +// and a setEncoding() from _read() must apply to the chunks pushed after it. + +const common = require('../common'); +const assert = require('assert'); +const { Readable } = require('stream'); + +function source(chunks, read) { + return new Readable({ + read: read ?? function() { + this.push(chunks.shift() ?? null); + }, + }); +} + +function collect(stream, ondata, expected) { + const received = []; + let first = true; + stream.on('data', (chunk) => { + if (first) { + first = false; + ondata(chunk, received); + } else { + received.push(chunk); + } + }); + stream.on('end', common.mustCall(() => { + assert.strictEqual(Buffer.concat(received).toString(), expected); + assert.strictEqual(stream.readableLength, 0); + })); +} + +// unshift() from the 'data' listener. +{ + const stream = source([Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')]); + collect(stream, common.mustCall((chunk, received) => { + received.push(chunk.subarray(0, 1)); + stream.unshift(chunk.subarray(1)); + assert.strictEqual(stream.readableLength, 3); + }), 'ABCDEF'); +} + +// push() from the 'data' listener. +{ + const stream = source([Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')]); + collect(stream, common.mustCall((chunk, received) => { + received.push(chunk); + stream.push(Buffer.from('X')); + assert.strictEqual(stream.readableLength, 3); + }), 'ABCDXEF'); +} + +// unshift() from _read(), before and after push(). Every chunk must be +// delivered. The order is not checked. +{ + const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')]; + let reads = 0; + const stream = source(chunks, function() { + reads++; + if (reads === 2) + this.unshift(Buffer.from('X')); + this.push(chunks.shift() ?? null); + if (reads === 3) + this.unshift(Buffer.from('Y')); + }); + const received = []; + stream.on('data', (chunk) => received.push(...chunk.toString())); + stream.on('end', common.mustCall(() => { + assert.strictEqual(received.sort().join(''), 'ABCDEFXY'); + assert.strictEqual(stream.readableLength, 0); + })); +} + +// read(0) from _read() after push(). The nested _read() adds to the buffer. +{ + const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF'), + Buffer.from('GH')]; + let reads = 0; + const stream = source(chunks, function() { + reads++; + this.push(chunks.shift() ?? null); + if (reads === 2) + this.read(0); + }); + const received = []; + stream.on('data', (chunk) => received.push(chunk)); + stream.on('end', common.mustCall(() => { + assert.strictEqual(Buffer.concat(received).toString(), 'ABCDEFGH'); + assert.strictEqual(stream.readableLength, 0); + })); +} + +// read(0) from _read() between two push() calls. +{ + const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF'), + Buffer.from('GH'), Buffer.from('IJ')]; + let reads = 0; + const stream = source(chunks, function() { + reads++; + this.push(chunks.shift() ?? null); + if (reads === 2) { + this.read(0); + this.push(chunks.shift() ?? null); + } + }); + const received = []; + stream.on('data', (chunk) => received.push(chunk)); + stream.on('end', common.mustCall(() => { + assert.strictEqual(Buffer.concat(received).toString(), 'ABCDEFGHIJ'); + assert.strictEqual(stream.readableLength, 0); + })); +} + +// setEncoding() from _read(), before and after push(). The chunk delivered +// by the 'data' event in progress was read before the call and stays a +// Buffer. Every later chunk is a decoded string. +for (const before of [true, false]) { + const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')]; + let reads = 0; + const stream = source(chunks, function() { + reads++; + if (reads === 2 && before) + this.setEncoding('hex'); + this.push(chunks.shift() ?? null); + if (reads === 2 && !before) + this.setEncoding('hex'); + }); + const received = []; + stream.on('data', (chunk) => received.push(chunk)); + stream.on('end', common.mustCall(() => { + assert.deepStrictEqual(received[0], Buffer.from('AB')); + assert.strictEqual(received.slice(1).join(''), '43444546'); + assert.strictEqual(stream.readableLength, 0); + })); +}