zlib: add zstd support

Fixes #48412

Author: jkrems

benchmark/zlib/creation.js

@@ -5,7 +5,7 @@ const zlib = require('zlib');
 const bench = common.createBenchmark(main, {
   type: [
     'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
-    'BrotliCompress', 'BrotliDecompress',
+    'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress',
   ],
   options: ['true', 'false'],
   n: [5e5],

benchmark/zlib/pipe.js

@@ -7,22 +7,27 @@ const bench = common.createBenchmark(main, {
   inputLen: [1024],
   duration: [5],
   type: ['string', 'buffer'],
-  algorithm: ['gzip', 'brotli'],
+  algorithm: ['gzip', 'brotli', 'zstd'],
 }, {
   test: {
     inputLen: 1024,
     duration: 0.2,
   },
 });
 
+const algorithms = {
+  'gzip': [zlib.createGzip, zlib.createGunzip],
+  'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress],
+  'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress],
+};
+
 function main({ inputLen, duration, type, algorithm }) {
   const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
   const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
 
-  const input = algorithm === 'gzip' ?
-    zlib.createGzip() : zlib.createBrotliCompress();
-  const output = algorithm === 'gzip' ?
-    zlib.createGunzip() : zlib.createBrotliDecompress();
+  const [createCompress, createUncompress] = algorithms[algorithm];
+  const input = createCompress();
+  const output = createUncompress();
 
   let readFromOutput = 0;
   input.pipe(output);

doc/api/zlib.md

@@ -124,8 +124,8 @@ operations be cached to avoid duplication of effort.
 
 ## Compressing HTTP requests and responses
 
-The `node:zlib` module can be used to implement support for the `gzip`, `deflate`
-and `br` content-encoding mechanisms defined by
+The `node:zlib` module can be used to implement support for the `gzip`, `deflate`,
+`br` and `zstd` content-encoding mechanisms defined by
 [HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
 
 The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
@@ -148,7 +148,7 @@ const { pipeline } = require('node:stream');
 const request = http.get({ host: 'example.com',
                            path: '/',
                            port: 80,
-                           headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
+                           headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } });
 request.on('response', (response) => {
   const output = fs.createWriteStream('example.com_index.html');
 
@@ -170,6 +170,9 @@ request.on('response', (response) => {
     case 'deflate':
       pipeline(response, zlib.createInflate(), output, onError);
       break;
+    case 'zstd':
+      pipeline(response, zlib.createZstdDecompress(), output, onError);
+      break;
     default:
       pipeline(response, output, onError);
       break;
@@ -218,6 +221,9 @@ http.createServer((request, response) => {
   } else if (/\bbr\b/.test(acceptEncoding)) {
     response.writeHead(200, { 'Content-Encoding': 'br' });
     pipeline(raw, zlib.createBrotliCompress(), response, onError);
+  } else if (/\bzstd\b/.test(acceptEncoding)) {
+    response.writeHead(200, { 'Content-Encoding': 'zstd' });
+    pipeline(raw, zlib.createZstdCompress(), response, onError);
   } else {
     response.writeHead(200, {});
     pipeline(raw, response, onError);
@@ -238,6 +244,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64');
 zlib.unzip(
   buffer,
   // For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
+  // For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.
   { finishFlush: zlib.constants.Z_SYNC_FLUSH },
   (err, buffer) => {
     if (err) {
@@ -309,6 +316,16 @@ these options have different ranges than the zlib ones:
 
 See [below][Brotli parameters] for more details on Brotli-specific options.
 
+### For Zstd-based streams
+
+There are equivalents to the zlib options for Zstd-based streams, although
+these options have different ranges than the zlib ones:
+
+* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option.
+* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option.
+
+See [below][Zstd parameters] for more details on Zstd-specific options.
+
 ## Flushing
 
 Calling [`.flush()`][] on a compression stream will make `zlib` return as much
@@ -487,6 +504,44 @@ These advanced options are available for controlling decompression:
   * Boolean flag enabling “Large Window Brotli” mode (not compatible with the
     Brotli format as standardized in [RFC 7932][]).
 
+### Zstd constants
+
+<!-- YAML
+added: REPLACEME
+-->
+
+There are several options and other constants available for Zstd-based
+streams:
+
+#### Flush operations
+
+The following values are valid flush operations for Zstd-based streams:
+
+* `zlib.constants.ZSTD_e_continue` (default for all operations)
+* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`)
+* `zlib.constants.ZSTD_e_end` (default for the last chunk)
+
+#### Compressor options
+
+There are several options that can be set on Zstd encoders, affecting
+compression efficiency and speed. Both the keys and the values can be accessed
+as properties of the `zlib.constants` object.
+
+The most important options are:
+
+* `ZSTD_c_compressionLevel`
+  * Set compression parameters according to pre-defined cLevel table. Default
+    level is ZSTD_CLEVEL_DEFAULT==3.
+
+#### Decompressor options
+
+These advanced options are available for controlling decompression:
+
+* `ZSTD_d_windowLogMax`
+  * Select a size limit (in power of 2) beyond which the streaming API will
+    refuse to allocate memory buffer in order to protect the host from
+    unreasonable memory requirements.
+
 ## Class: `Options`
 
 <!-- YAML

lib/internal/errors.js

@@ -1917,3 +1917,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR',
 E('ERR_WORKER_UNSUPPORTED_OPERATION',
   '%s is not supported in workers', TypeError);
 E('ERR_ZLIB_INITIALIZATION_FAILED', 'Initialization failed', Error);
+E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError);

lib/zlib.js

@@ -49,6 +49,7 @@ const {
     ERR_INVALID_ARG_TYPE,
     ERR_OUT_OF_RANGE,
     ERR_ZLIB_INITIALIZATION_FAILED,
+    ERR_ZSTD_INVALID_PARAM,
   },
   genericNodeError,
   hideStackFrames,
@@ -88,9 +89,12 @@ const {
   // Node's compression stream modes (node_zlib_mode)
   DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
   BROTLI_DECODE, BROTLI_ENCODE,
+  ZSTD_COMPRESS, ZSTD_DECOMPRESS,
   // Brotli operations (~flush levels)
   BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
   BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA,
+  // Zstd end directives (~flush levels)
+  ZSTD_e_continue, ZSTD_e_flush, ZSTD_e_end,
 } = constants;
 
 // Translation table for return codes.
@@ -237,9 +241,11 @@ const checkRangesOrGetDefault = hideStackFrames(
 const FLUSH_BOUND = [
   [ Z_NO_FLUSH, Z_BLOCK ],
   [ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ],
+  [ ZSTD_e_continue, ZSTD_e_end ],
 ];
 const FLUSH_BOUND_IDX_NORMAL = 0;
 const FLUSH_BOUND_IDX_BROTLI = 1;
+const FLUSH_BOUND_IDX_ZSTD = 2;
 
 // The base class for all Zlib-style streams.
 function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
@@ -248,13 +254,15 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
   // The ZlibBase class is not exported to user land, the mode should only be
   // passed in by us.
   assert(typeof mode === 'number');
-  assert(mode >= DEFLATE && mode <= BROTLI_ENCODE);
+  assert(mode >= DEFLATE && mode <= ZSTD_DECOMPRESS);
 
   let flushBoundIdx;
-  if (mode !== BROTLI_ENCODE && mode !== BROTLI_DECODE) {
-    flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
-  } else {
+  if (mode === BROTLI_ENCODE || mode === BROTLI_DECODE) {
     flushBoundIdx = FLUSH_BOUND_IDX_BROTLI;
+  } else if (mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS) {
+    flushBoundIdx = FLUSH_BOUND_IDX_ZSTD;
+  } else {
+    flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
   }
 
   if (opts) {
@@ -888,6 +896,77 @@ ObjectSetPrototypeOf(BrotliDecompress.prototype, Brotli.prototype);
 ObjectSetPrototypeOf(BrotliDecompress, Brotli);
 
 
+const kMaxZstdParam = MathMaxApply(ArrayPrototypeMap(
+  ObjectKeys(constants),
+  (key) => (StringPrototypeStartsWith(key, 'ZSTD_c_') ?
+    constants[key] :
+    0),
+));
+
+const zstdInitParamsArray = new Uint32Array(kMaxZstdParam + 1);
+
+const zstdDefaultOpts = {
+  flush: ZSTD_e_continue,
+  finishFlush: ZSTD_e_end,
+  fullFlush: ZSTD_e_flush,
+};
+function Zstd(opts, mode) {
+  assert(mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS);
+
+  TypedArrayPrototypeFill(zstdInitParamsArray, -1);
+  if (opts?.params) {
+    ArrayPrototypeForEach(ObjectKeys(opts.params), (origKey) => {
+      const key = +origKey;
+      if (NumberIsNaN(key) || key < 0 || key > kMaxZstdParam ||
+          (zstdInitParamsArray[key] | 0) !== -1) {
+        throw new ERR_ZSTD_INVALID_PARAM(origKey);
+      }
+
+      const value = opts.params[origKey];
+      if (typeof value !== 'number' && typeof value !== 'boolean') {
+        throw new ERR_INVALID_ARG_TYPE('options.params[key]',
+                                       'number', opts.params[origKey]);
+      }
+      zstdInitParamsArray[key] = value;
+    });
+  }
+
+  const handle = mode === ZSTD_COMPRESS ?
+      new binding.ZstdCompress() : new binding.ZstdDecompress();
+
+  this._writeState = new Uint32Array(2);
+  if (!handle.init(zstdInitParamsArray,
+                   this._writeState,
+                   processCallback)) {
+    throw new ERR_ZLIB_INITIALIZATION_FAILED();
+  }
+
+  ReflectApply(ZlibBase, this, [opts, mode, handle, zstdDefaultOpts]);
+}
+ObjectSetPrototypeOf(Zstd.prototype, ZlibBase.prototype);
+ObjectSetPrototypeOf(Zstd, ZlibBase);
+
+
+function ZstdCompress(opts) {
+  if (!(this instanceof ZstdCompress))
+    return new ZstdCompress(opts);
+
+  ReflectApply(Zstd, this, [opts, ZSTD_COMPRESS]);
+}
+ObjectSetPrototypeOf(ZstdCompress.prototype, Zstd.prototype);
+ObjectSetPrototypeOf(ZstdCompress, Zstd);
+
+
+function ZstdDecompress(opts) {
+  if (!(this instanceof ZstdDecompress))
+    return new ZstdDecompress(opts);
+
+  ReflectApply(Zstd, this, [opts, ZSTD_DECOMPRESS]);
+}
+ObjectSetPrototypeOf(ZstdDecompress.prototype, Zstd.prototype);
+ObjectSetPrototypeOf(ZstdDecompress, Zstd);
+
+
 function createProperty(ctor) {
   return {
     __proto__: null,
@@ -917,6 +996,8 @@ module.exports = {
   Unzip,
   BrotliCompress,
   BrotliDecompress,
+  ZstdCompress,
+  ZstdDecompress,
 
   // Convenience methods.
   // compress/decompress a string or buffer in one step.
@@ -938,6 +1019,10 @@ module.exports = {
   brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
   brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
   brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
+  zstdCompress: createConvenienceMethod(ZstdCompress, false),
+  zstdCompressSync: createConvenienceMethod(ZstdCompress, true),
+  zstdDecompress: createConvenienceMethod(ZstdDecompress, false),
+  zstdDecompressSync: createConvenienceMethod(ZstdDecompress, true),
 };
 
 ObjectDefineProperties(module.exports, {
@@ -950,6 +1035,8 @@ ObjectDefineProperties(module.exports, {
   createUnzip: createProperty(Unzip),
   createBrotliCompress: createProperty(BrotliCompress),
   createBrotliDecompress: createProperty(BrotliDecompress),
+  createZstdCompress: createProperty(ZstdCompress),
+  createZstdDecompress: createProperty(ZstdDecompress),
   constants: {
     __proto__: null,
     configurable: false,