diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 926d2ad76faa..a99d2a3a3686 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -215,6 +215,16 @@ deallocation, this may lead to significant memory fragmentation. It is strongly recommended that the results of compression operations be cached to avoid duplication of effort. +Because the asynchronous `zlib` APIs run on the internal threadpool, the +input and output buffers are accessed by a worker thread through raw pointers +for the duration of the operation. The underlying `ArrayBuffer` of a buffer +passed to (or produced by) an asynchronous compress/decompress operation must +not be detached (for example with `ArrayBuffer.prototype.transfer()`), +transferred via a structured clone or `postMessage()`, or resized (with +`ResizableArrayBuffer.prototype.resize()`) until the operation has completed. +Doing so invalidates the memory the worker thread is reading or writing and +causes the operation to fail with an error. + ## Compressing HTTP requests and responses The `node:zlib` module can be used to implement support for the `gzip`, `deflate`, diff --git a/lib/zlib.js b/lib/zlib.js index a2d2c158e5df..45dcdaf94e8e 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -687,7 +687,18 @@ function processCallback() { const have = handle.availOutBefore - availOutAfter; let streamBufferIsFull = false; if (have > 0) { - const out = self._outBuffer.slice(self._outOffset, self._outOffset + have); + let out; + try { + out = self._outBuffer.slice(self._outOffset, self._outOffset + have); + } catch (err) { + // The output buffer was detached/resized while the operation ran, so the + // compressed data cannot be read back. Surface it as a stream error + // rather than an uncaught exception. + this.buffer = null; + self.destroy(err); + this.cb(err); + return; + } self._outOffset += have; if (self[kBufferResult] === true) zlibBufferAddChunk(self, out); @@ -719,19 +730,11 @@ function processCallback() { handle.inOff += inDelta; handle.availInBefore = availInAfter; - - if (!streamBufferIsFull) { - this.write(handle.flushFlag, - this.buffer, // in - handle.inOff, // in_off - handle.availInBefore, // in_len - self._outBuffer, // out - self._outOffset, // out_off - self._chunkSize); // out_len - } else { - const oldRead = self._read; - self._read = (n) => { - self._read = oldRead; + // If the input/output buffer was detached or resized while the operation + // was in flight, the native bindings throw ERR_OUT_OF_RANGE. That must be + // surfaced as a stream error rather than an uncaught exception. + const reprocess = () => { + try { this.write(handle.flushFlag, this.buffer, // in handle.inOff, // in_off @@ -739,6 +742,22 @@ function processCallback() { self._outBuffer, // out self._outOffset, // out_off self._chunkSize); // out_len + } catch (err) { + // The buffer backing store was invalidated while the worker ran; the + // stream cannot make further progress on it. + this.buffer = null; + self.destroy(err); + this.cb(err); + } + }; + + if (!streamBufferIsFull) { + reprocess(); + } else { + const oldRead = self._read; + self._read = (n) => { + self._read = oldRead; + reprocess(); self._read(n); }; } diff --git a/src/node_zlib.cc b/src/node_zlib.cc index 72947b8b7839..3878d3b3bb04 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -547,6 +548,7 @@ class CompressionStream : public AsyncWrap, uint32_t in_off, in_len, out_off, out_len, flush; const char* in; char* out; + Local in_buf; CHECK_EQ(false, args[0]->IsUndefined() && "must provide flush value"); if (!args[0]->Uint32Value(context).To(&flush)) return; @@ -567,7 +569,7 @@ class CompressionStream : public AsyncWrap, in_off = 0; } else { CHECK(Buffer::HasInstance(args[1])); - Local in_buf = args[1].As(); + in_buf = args[1].As(); if (!args[2]->Uint32Value(context).To(&in_off)) return; if (!args[3]->Uint32Value(context).To(&in_len)) return; @@ -589,6 +591,11 @@ class CompressionStream : public AsyncWrap, CompressionStream* ctx; ASSIGN_OR_RETURN_UNWRAP(&ctx, args.This()); + if constexpr (async) { + ctx->SetAsyncWriteBuffers( + in_buf, in_off, in_len, out_buf, out_off, out_len, &in, &out); + } + ctx->Write(flush, in, in_len, out, out_len); } @@ -625,6 +632,89 @@ class CompressionStream : public AsyncWrap, ScheduleWork(); } + // The thread-pool worker operates on raw pointers into the JavaScript-owned + // backing stores of the input/output buffers. For an asynchronous write, pin + // those backing stores so that detaching or garbage-collecting the + // ArrayBuffers cannot free the memory while the work is in flight. A + // resizable ArrayBuffer can still have its pages decommitted by resize(), so + // such buffers are copied into memory owned by this object instead, and the + // output is copied back once the work completes. On entry, `in` and `out` + // point into the JS buffers (or `nullptr` for a pure flush); for resizable + // buffers they are redirected to the engine-owned copies. + void SetAsyncWriteBuffers(Local in_buf, + uint32_t in_off, + uint32_t in_len, + Local out_buf, + uint32_t out_off, + uint32_t out_len, + const char** in, + char** out) { + ClearWriteState(); + + if (!in_buf.IsEmpty()) { + Local in_view = in_buf.As(); + std::shared_ptr in_store = + in_view->Buffer()->GetBackingStore(); + if (in_store->IsResizableByUserJavaScript()) { + input_copy_ = std::make_unique(in_len); + memcpy(input_copy_.get(), Buffer::Data(in_buf) + in_off, in_len); + *in = input_copy_.get(); + } else { + input_store_ = std::move(in_store); + } + } + + Local out_view = out_buf.As(); + std::shared_ptr out_store = + out_view->Buffer()->GetBackingStore(); + if (out_store->IsResizableByUserJavaScript()) { + output_copy_ = std::make_unique(out_len); + output_copy_len_ = out_len; + output_copy_out_off_ = out_off; + output_copy_view_.Reset(AsyncWrap::env()->isolate(), out_view); + *out = output_copy_.get(); + } else { + output_store_ = std::move(out_store); + } + } + + // Copy staged output (used for resizable output buffers) back into the + // JavaScript buffer. Only bytes that still fit are copied so that we never + // write to pages that have been decommitted by a resize() while the work was + // in flight. + void CopyBackOutput() { + if (output_copy_ == nullptr || output_copy_view_.IsEmpty()) return; + + Local out_view = + output_copy_view_.Get(AsyncWrap::env()->isolate()); + Local out_obj = out_view.As(); + const size_t length = Buffer::Length(out_obj); + if (output_copy_out_off_ >= length) return; + + // `write_result_[0]` holds the remaining output space of the staged buffer; + // it can never exceed what we provided, but guard against any unexpected + // value to avoid a size_t underflow. + if (write_result_[0] >= output_copy_len_) return; + + const size_t written = output_copy_len_ - write_result_[0]; + const size_t to_copy = written < length - output_copy_out_off_ + ? written + : length - output_copy_out_off_; + memcpy(Buffer::Data(out_obj) + output_copy_out_off_, + output_copy_.get(), + to_copy); + } + + void ClearWriteState() { + input_store_.reset(); + output_store_.reset(); + input_copy_.reset(); + output_copy_.reset(); + output_copy_len_ = 0; + output_copy_out_off_ = 0; + output_copy_view_.Reset(); + } + void UpdateWriteResult() { ctx_.GetAfterWriteOffsets(&write_result_[1], &write_result_[0]); } @@ -655,6 +745,7 @@ class CompressionStream : public AsyncWrap, write_in_progress_ = false; if (status == UV_ECANCELED) { + ClearWriteState(); Close(); return; } @@ -665,11 +756,19 @@ class CompressionStream : public AsyncWrap, HandleScope handle_scope(env->isolate()); Context::Scope context_scope(env->context()); - if (!CheckError()) + if (!CheckError()) { + ClearWriteState(); return; + } UpdateWriteResult(); + // Copy any staged output (resizable output buffers) back into the JS + // buffer, and release the backing-store pins, before running the callback + // (which may reschedule the next chunk of the same write). + CopyBackOutput(); + ClearWriteState(); + // call the write() cb Local cb = object()->GetInternalField(kWriteJSCallback).template As(); @@ -779,6 +878,19 @@ class CompressionStream : public AsyncWrap, unsigned int refs_ = 0; uint32_t* write_result_ = nullptr; + // Backing-store pins and engine-owned copies for the input/output buffers of + // an asynchronous write. See SetAsyncWriteBuffers(). Holding a + // std::shared_ptr keeps the underlying memory alive even if + // the JS ArrayBuffer is detached or garbage-collected while the thread-pool + // worker reads/writes through a raw pointer. + std::shared_ptr input_store_; + std::shared_ptr output_store_; + std::unique_ptr input_copy_; + std::unique_ptr output_copy_; + size_t output_copy_len_ = 0; + uint32_t output_copy_out_off_ = 0; + v8::Global output_copy_view_; + CompressionContext ctx_; }; diff --git a/test/parallel/test-zlib-arraybuffer-detach.js b/test/parallel/test-zlib-arraybuffer-detach.js new file mode 100644 index 000000000000..2e9089c73d4e --- /dev/null +++ b/test/parallel/test-zlib-arraybuffer-detach.js @@ -0,0 +1,223 @@ +'use strict'; + +// Regression test for invalid memory access in the zlib bindings when the +// input/output ArrayBuffer of an in-flight asynchronous compression operation is +// detached (e.g. ArrayBuffer.prototype.transfer()) or resized +// (ResizableArrayBuffer.prototype.resize()) while the operation runs on the +// libuv threadpool. +// +// The worker thread reads/writes through raw pointers into JS-owned backing +// stores. Those pointers must remain valid for the whole operation even if the +// buffer is inspected, transferred or resized immediately after the work is +// scheduled. Before the fix, detaching or resizing the backing store while the +// worker ran caused invalid memory access and a deterministic crash. +// +// Each scenario uses a chunkSize large enough that the whole input is consumed +// in a single pass, so the operation (with the buffer safely pinned/copied) +// completes cleanly and produces the correct compressed output. + +const common = require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); + +// Deterministic, incompressible-ish byte pattern (period 256). +function makePattern(size) { + const data = Buffer.allocUnsafe(size); + for (let i = 0; i < size; i++) data[i] = (i * 31) & 0xff; + return data; +} + +// Compress `input` asynchronously, invalidating its backing store right after +// the work is scheduled, and return the compressed output. The operation must +// complete without corrupting native memory and must still produce the output +// that corresponds to the (original) input contents. +function compressAndInvalidate(name, factory, input, invalidate) { + return new Promise((resolve, reject) => { + const stream = factory(); + const chunks = []; + + const timer = setTimeout(() => { + reject(new Error(`timed out: ${name}`)); + }, 60_000); + + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('error', (err) => { + clearTimeout(timer); + reject(err); + }); + stream.on('end', () => { + clearTimeout(timer); + resolve(Buffer.concat(chunks)); + }); + + stream.write(input); + // Invalidate while the worker thread is (very likely) mid-operation. + invalidate(input); + stream.end(); + }); +} + +// For output-buffer invalidation the compressed data cannot be read back, so +// this only asserts that the stream reaches a terminal state (end/close/error) +// without corrupting native memory. Invalid memory access would crash instead. +// `setup` (optional) configures the stream before the write and returns the +// invalidator. +function runNoCrash(name, factory, input, setup) { + return new Promise((resolve, reject) => { + const stream = factory(); + stream.resume(); + + const timer = setTimeout(() => { + reject(new Error(`timed out: ${name}`)); + }, 60_000); + + stream.on('error', () => { + clearTimeout(timer); + resolve(name); + }); + stream.on('end', () => { + clearTimeout(timer); + resolve(name); + }); + stream.on('close', () => { + clearTimeout(timer); + resolve(name); + }); + + let invalidateOutput = () => {}; + if (setup) invalidateOutput = setup(stream); + + stream.write(input); + try { + invalidateOutput(); + } catch { + // Invalidating the stream's own (possibly non-resizable) output buffer may + // throw in the test; that is not a native crash. + clearTimeout(timer); + resolve(name); + return; + } + stream.end(); + }); +} + +async function main() { + // Sanity check: the common non-adversarial path still round-trips. + const original = Buffer.from( + 'The quick brown fox jumps over the lazy dog.'.repeat(1000)); + for (const [compress, decompress] of [ + [zlib.deflate, zlib.inflateSync], + [zlib.gzip, zlib.gunzipSync], + [zlib.brotliCompress, zlib.brotliDecompressSync], + [zlib.zstdCompress, zlib.zstdDecompressSync], + ]) { + const out = await new Promise((resolve, reject) => { + compress(original, (err, b) => { + if (err) reject(err); + else resolve(b); + }); + }); + assert.deepStrictEqual(decompress(out), original); + } + + // 8MB input with a 32MB output buffer: the worker consumes the whole input in + // one pass (the pattern is highly compressible), so there is no + // re-processing of a possibly-invalidated buffer. The buffer is large enough + // that the worker is still mid-operation when the invalidation runs. + const size = 8 * 1024 * 1024; + const chunkSize = 32 * 1024 * 1024; + + const scenarios = [ + { + name: 'deflate input detach', + factory: () => zlib.createDeflate({ chunkSize }), + decompress: zlib.inflateSync, + expected: () => makePattern(size), + input: () => makePattern(size), + invalidate: (input) => input.buffer.transfer(0), + }, + { + name: 'gzip input detach', + factory: () => zlib.createGzip({ chunkSize }), + decompress: zlib.gunzipSync, + expected: () => makePattern(size), + input: () => makePattern(size), + invalidate: (input) => input.buffer.transfer(0), + }, + { + name: 'brotli input detach', + factory: () => zlib.createBrotliCompress({ chunkSize }), + decompress: zlib.brotliDecompressSync, + expected: () => makePattern(size), + input: () => makePattern(size), + invalidate: (input) => input.buffer.transfer(0), + }, + { + name: 'zstd input detach', + factory: () => zlib.createZstdCompress({ chunkSize }), + decompress: zlib.zstdDecompressSync, + expected: () => makePattern(size), + input: () => makePattern(size), + invalidate: (input) => input.buffer.transfer(0), + }, + { + // Resizing a ResizableArrayBuffer decommits pages, which is the most + // reliable trigger: the engine must copy this input rather than hold a + // raw pointer into JS-owned memory. + name: 'deflate input resizable shrink', + factory: () => zlib.createDeflate({ chunkSize }), + decompress: zlib.inflateSync, + expected: () => makePattern(size), + input: () => { + const rab = new ArrayBuffer(size, { maxByteLength: size }); + const data = Buffer.from(rab); + for (let i = 0; i < size; i++) data[i] = (i * 31) & 0xff; + return data; + }, + invalidate: (input) => input.buffer.resize(4096), + }, + ]; + + for (const scenario of scenarios) { + const compressed = await compressAndInvalidate( + scenario.name, scenario.factory, scenario.input(), scenario.invalidate); + assert.deepStrictEqual( + scenario.decompress(compressed), + scenario.expected(), + `incorrect compressed output for ${scenario.name}`, + ); + } + + // Invalidating the output buffer must not corrupt native memory either. The + // default output buffer is non-resizable, but the user can replace it; the + // engine must pin it (or stage writes to its own memory) while the worker + // runs. + const outputScenarios = [ + { + name: 'deflate output detach', + factory: () => zlib.createDeflate({ chunkSize }), + setup: (stream) => { + const outBuf = stream._outBuffer; + return () => outBuf.buffer.transfer(0); + }, + }, + { + // Resizing a ResizableArrayBuffer decommits pages; the engine must stage + // output in its own memory for resizable output buffers. + name: 'deflate output resizable shrink', + factory: () => zlib.createDeflate({ chunkSize }), + setup: (stream) => { + const rab = new ArrayBuffer(chunkSize, { maxByteLength: chunkSize }); + stream._outBuffer = Buffer.from(rab); + return () => rab.resize(4096); + }, + }, + ]; + + for (const scenario of outputScenarios) { + await runNoCrash( + scenario.name, scenario.factory, makePattern(size), scenario.setup); + } +} + +main().then(common.mustCall());