stream: bump default highWaterMark

ronag · jcbhmr · commit 29dce7f5fd28 · 2024-05-15T21:01:53.000Z
This should give a performance boost accross the board. Given that the old limit is a decod old and memory capacity has doubled many times since I think it is appropriate to slightly bump the default limit. PR-URL: nodejs#52037 Refs: nodejs#46608 Refs: nodejs#50120 Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Yagiz Nizipli <yagiz.nizipli@sentry.io> Reviewed-By: Chengzhong Wu <legendecas@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Mohammed Keyvanzadeh <mohammadkeyvanzade94@gmail.com> Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>
diff --git a/benchmark/net/net-c2s.js b/benchmark/net/net-c2s.js
@@ -6,7 +6,7 @@ const net = require('net');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  len: [64, 102400, 1024 * 1024 * 16],
+  len: [64, 102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
diff --git a/benchmark/net/net-pipe.js b/benchmark/net/net-pipe.js
@@ -6,7 +6,7 @@ const net = require('net');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  len: [2, 64, 102400, 1024 * 1024 * 16],
+  len: [2, 64, 102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
diff --git a/benchmark/net/net-s2c.js b/benchmark/net/net-s2c.js
@@ -5,7 +5,7 @@ const common = require('../common.js');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 1024 * 1024],
+  sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 64 * 1024],
   type: ['utf', 'asc', 'buf'],
   recvbuflen: [0, 64 * 1024, 1024 * 1024],
   recvbufgenfn: ['true', 'false'],
diff --git a/benchmark/net/net-wrap-js-stream-passthrough.js b/benchmark/net/net-wrap-js-stream-passthrough.js
@@ -5,7 +5,7 @@ const common = require('../common.js');
 const { PassThrough } = require('stream');
 
 const bench = common.createBenchmark(main, {
-  len: [64, 102400, 1024 * 1024 * 16],
+  len: [64, 102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
diff --git a/benchmark/net/tcp-raw-c2s.js b/benchmark/net/tcp-raw-c2s.js
@@ -9,7 +9,7 @@ const util = require('util');
 // run the function with those settings.
 // if not, then queue up a bunch of child processes.
 const bench = common.createBenchmark(main, {
-  len: [102400, 1024 * 1024 * 16],
+  len: [102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
diff --git a/benchmark/net/tcp-raw-pipe.js b/benchmark/net/tcp-raw-pipe.js
@@ -9,7 +9,7 @@ const util = require('util');
 // run the function with those settings.
 // if not, then queue up a bunch of child processes.
 const bench = common.createBenchmark(main, {
-  len: [102400, 1024 * 1024 * 16],
+  len: [102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
diff --git a/benchmark/net/tcp-raw-s2c.js b/benchmark/net/tcp-raw-s2c.js
@@ -9,7 +9,7 @@ const util = require('util');
 // run the function with those settings.
 // If not, then queue up a bunch of child processes.
 const bench = common.createBenchmark(main, {
-  len: [102400, 1024 * 1024 * 16],
+  len: [102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
diff --git a/doc/api/stream.md b/doc/api/stream.md
@@ -3479,6 +3479,9 @@ method.
 
 <!-- YAML
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/52037
+    description: bump default highWaterMark.
   - version: v15.5.0
     pr-url: https://github.com/nodejs/node/pull/36431
     description: support passing in an AbortSignal.
@@ -3500,7 +3503,7 @@ changes:
 * `options` {Object}
   * `highWaterMark` {number} Buffer level when
     [`stream.write()`][stream-write] starts returning `false`. **Default:**
-    `16384` (16 KiB), or `16` for `objectMode` streams.
+    `65536` (64 KiB), or `16` for `objectMode` streams.
   * `decodeStrings` {boolean} Whether to encode `string`s passed to
     [`stream.write()`][stream-write] to `Buffer`s (with the encoding
     specified in the [`stream.write()`][stream-write] call) before passing
@@ -3856,6 +3859,9 @@ constructor and implement the [`readable._read()`][] method.
 
 <!-- YAML
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/52037
+    description: bump default highWaterMark.
   - version: v15.5.0
     pr-url: https://github.com/nodejs/node/pull/36431
     description: support passing in an AbortSignal.
@@ -3873,7 +3879,7 @@ changes:
 * `options` {Object}
   * `highWaterMark` {number} The maximum [number of bytes][hwm-gotcha] to store
     in the internal buffer before ceasing to read from the underlying resource.
-    **Default:** `16384` (16 KiB), or `16` for `objectMode` streams.
+    **Default:** `65536` (64 KiB), or `16` for `objectMode` streams.
   * `encoding` {string} If specified, then buffers will be decoded to
     strings using the specified encoding. **Default:** `null`.
   * `objectMode` {boolean} Whether this stream should behave
diff --git a/lib/internal/streams/state.js b/lib/internal/streams/state.js
@@ -8,7 +8,8 @@ const { validateInteger } = require('internal/validators');
 
 const { ERR_INVALID_ARG_VALUE } = require('internal/errors').codes;
 
-let defaultHighWaterMarkBytes = 16 * 1024;
+// TODO (fix): For some reason Windows CI fails with bigger hwm.
+let defaultHighWaterMarkBytes = process.platform === 'win32' ? 16 * 1024 : 64 * 1024;
 let defaultHighWaterMarkObjectMode = 16;
 
 function highWaterMarkFrom(options, isDuplex, duplexKey) {
diff --git a/test/parallel/test-http-no-read-no-dump.js b/test/parallel/test-http-no-read-no-dump.js
@@ -34,7 +34,7 @@ const server = http.createServer((req, res) => {
   }, common.mustCall((res) => {
     res.resume();
 
-    post.write(Buffer.alloc(16 * 1024).fill('X'));
+    post.write(Buffer.alloc(64 * 1024).fill('X'));
     onPause = () => {
       post.end('something');
     };
diff --git a/test/parallel/test-https-hwm.js b/test/parallel/test-https-hwm.js
@@ -62,5 +62,5 @@ const httpsServer = https.createServer({
     port: this.address().port,
     rejectUnauthorized: false,
     highWaterMark: undefined,
-  }, loadCallback(16 * 1024)).on('error', common.mustNotCall()).end();
+  }, loadCallback(process.platform === 'win32' ? 16 * 1024 : 64 * 1024)).on('error', common.mustNotCall()).end();
 }));
diff --git a/test/parallel/test-stream-duplex-readable-end.js b/test/parallel/test-stream-duplex-readable-end.js
@@ -7,13 +7,15 @@ const stream = require('stream');
 let loops = 5;
 
 const src = new stream.Readable({
+  highWaterMark: 16 * 1024,
   read() {
     if (loops--)
       this.push(Buffer.alloc(20000));
   }
 });
 
 const dst = new stream.Transform({
+  highWaterMark: 16 * 1024,
   transform(chunk, output, fn) {
     this.push(null);
     fn();
diff --git a/test/parallel/test-stream-pipe-await-drain-push-while-write.js b/test/parallel/test-stream-pipe-await-drain-push-while-write.js
@@ -4,6 +4,7 @@ const stream = require('stream');
 const assert = require('assert');
 
 const writable = new stream.Writable({
+  highWaterMark: 16 * 1024,
   write: common.mustCall(function(chunk, encoding, cb) {
     assert.strictEqual(
       readable._readableState.awaitDrainWriters,
@@ -26,6 +27,7 @@ const writable = new stream.Writable({
 // A readable stream which produces two buffers.
 const bufs = [Buffer.alloc(32 * 1024), Buffer.alloc(33 * 1024)]; // above hwm
 const readable = new stream.Readable({
+  highWaterMark: 16 * 1024,
   read: function() {
     while (bufs.length > 0) {
       this.push(bufs.shift());
diff --git a/test/parallel/test-stream-readable-infinite-read.js b/test/parallel/test-stream-readable-infinite-read.js
@@ -7,6 +7,7 @@ const { Readable } = require('stream');
 const buf = Buffer.alloc(8192);
 
 const readable = new Readable({
+  highWaterMark: 16 * 1024,
   read: common.mustCall(function() {
     this.push(buf);
   }, 31)
diff --git a/test/parallel/test-stream-transform-split-highwatermark.js b/test/parallel/test-stream-transform-split-highwatermark.js
@@ -2,9 +2,9 @@
 require('../common');
 const assert = require('assert');
 
-const { Transform, Readable, Writable } = require('stream');
+const { Transform, Readable, Writable, getDefaultHighWaterMark } = require('stream');
 
-const DEFAULT = 16 * 1024;
+const DEFAULT = getDefaultHighWaterMark();
 
 function testTransform(expectedReadableHwm, expectedWritableHwm, options) {
   const t = new Transform(options);
diff --git a/test/parallel/test-stream-transform-split-objectmode.js b/test/parallel/test-stream-transform-split-objectmode.js
@@ -25,12 +25,14 @@ const assert = require('assert');
 
 const Transform = require('stream').Transform;
 
-const parser = new Transform({ readableObjectMode: true });
+const parser = new Transform({
+  readableObjectMode: true
+});
 
 assert(parser._readableState.objectMode);
 assert(!parser._writableState.objectMode);
 assert.strictEqual(parser.readableHighWaterMark, 16);
-assert.strictEqual(parser.writableHighWaterMark, 16 * 1024);
+assert.strictEqual(parser.writableHighWaterMark, process.platform === 'win32' ? 16 * 1024 : 64 * 1024);
 assert.strictEqual(parser.readableHighWaterMark,
                    parser._readableState.highWaterMark);
 assert.strictEqual(parser.writableHighWaterMark,
@@ -57,7 +59,7 @@ const serializer = new Transform({ writableObjectMode: true });
 
 assert(!serializer._readableState.objectMode);
 assert(serializer._writableState.objectMode);
-assert.strictEqual(serializer.readableHighWaterMark, 16 * 1024);
+assert.strictEqual(serializer.readableHighWaterMark, process.platform === 'win32' ? 16 * 1024 : 64 * 1024);
 assert.strictEqual(serializer.writableHighWaterMark, 16);
 assert.strictEqual(parser.readableHighWaterMark,
                    parser._readableState.highWaterMark);
diff --git a/test/parallel/test-tls-connect-hwm-option.js b/test/parallel/test-tls-connect-hwm-option.js
@@ -37,7 +37,7 @@ server.listen(0, common.mustCall(() => {
     rejectUnauthorized: false,
     highWaterMark: undefined,
   }, common.mustCall(() => {
-    assert.strictEqual(defaultHighBob.readableHighWaterMark, 16 * 1024);
+    assert.strictEqual(defaultHighBob.readableHighWaterMark, process.platform === 'win32' ? 16 * 1024 : 64 * 1024);
     defaultHighBob.end();
   }));
 
diff --git a/test/parallel/test-zlib-brotli-16GB.js b/test/parallel/test-zlib-brotli-16GB.js
