src: add encodeInto to TextEncoder

Add function encodeInto to TextEncoder, fix bug MessageChannel is not
defined in encodeInto.any.js.

Fixes: nodejs#28851
Refs: nodejs#26904

Author: AtticusYang

doc/api/util.md

@@ -1073,6 +1073,25 @@ The `TextEncoder` class is also available on the global object.
 UTF-8 encodes the `input` string and returns a `Uint8Array` containing the
 encoded bytes.
 
+```js
+const encoder = new TextEncoder();
+const src = 'this is some data';
+const dest = new Uint8Array(10);
+const ret = encoder.encodeInto(src, dest);
+```
+
+### textEncoder.encodeInto(src, dest)
+
+
+* `src` {string} The text to encode.
+* `dest` {Uint8Array} the array to hold the encode result.
+* Returns: {Object}
+  * `read` {number} The read Unicode code units of src.
+  * `written` {number} The written UTF-8 bytes of dest.
+
+UTF-8 encodes the `src` string to `dest` Unit8Array and returns an object
+containing the read Unicode code units and written UTF-8 bytes.
+
 ### textEncoder.encoding
 
 * {string}

lib/internal/encoding.js

@@ -25,10 +25,14 @@ const {
 
 const {
   isArrayBuffer,
-  isArrayBufferView
+  isArrayBufferView,
+  isUint8Array
 } = require('internal/util/types');
 
+const { validateString } = require('internal/validators');
+
 const {
+  encodeInto,
   encodeUtf8String
 } = internalBinding('buffer');
 
@@ -319,6 +323,14 @@ class TextEncoder {
     return encodeUtf8String(`${input}`);
   }
 
+  encodeInto(src, dest) {
+    validateEncoder(this);
+    validateString(src, 'src');
+    if (!dest || !isUint8Array(dest))
+      throw new ERR_INVALID_ARG_TYPE('dest', 'Uint8Array', dest);
+    return encodeInto(src, dest);
+  }
+
   [inspect](depth, opts) {
     validateEncoder(this);
     if (typeof depth === 'number' && depth < 0)
@@ -336,6 +348,7 @@ class TextEncoder {
 Object.defineProperties(
   TextEncoder.prototype, {
     'encode': { enumerable: true },
+    'encodeInto': { enumerable: true },
     'encoding': { enumerable: true },
     [Symbol.toStringTag]: {
       configurable: true,

src/env.h

@@ -214,7 +214,9 @@ constexpr size_t kFsStatsBufferLength =
   V(dns_txt_string, "TXT")                                                     \
   V(duration_string, "duration")                                               \
   V(emit_warning_string, "emitWarning")                                        \
+  V(encoding_read_string, "read")                                              \
   V(encoding_string, "encoding")                                               \
+  V(encoding_written_string, "written")                                        \
   V(entries_string, "entries")                                                 \
   V(entry_type_string, "entryType")                                            \
   V(env_pairs_string, "envPairs")                                              \

src/node_buffer.cc

@@ -31,6 +31,8 @@
 #include "v8-profiler.h"
 #include "v8.h"
 
+#include <unicode/unistr.h>
+
 #include <cstring>
 #include <climits>
 
@@ -56,6 +58,7 @@
 namespace node {
 namespace Buffer {
 
+using v8::Array;
 using v8::ArrayBuffer;
 using v8::ArrayBufferCreationMode;
 using v8::ArrayBufferView;
@@ -1051,6 +1054,117 @@ static void EncodeUtf8String(const FunctionCallbackInfo<Value>& args) {
 }
 
 
+static void EncodeInto(const FunctionCallbackInfo<Value>& args) {
+  Environment* env = Environment::GetCurrent(args);
+  Isolate* isolate = env->isolate();
+  Local<Context> context = env->context();
+  CHECK_GE(args.Length(), 2);
+  CHECK(args[0]->IsString());
+  CHECK(args[1]->IsUint8Array());
+
+  size_t read = 0;
+  size_t written = 0;
+
+  Utf8Value src(isolate, args[0]);
+  const char* p = *src;
+
+  Local<Uint8Array> dest = args[1].As<Uint8Array>();
+  Local<ArrayBuffer> buf = dest->Buffer();
+  char* write_result =
+      static_cast<char*>(buf->GetContents().Data()) + dest->ByteOffset();
+  size_t dest_length = dest->ByteLength();
+
+  for (size_t i = 0; i < src.length(); ) {
+    uint32_t code = 0;
+
+    if ((p[i] & 0x80) == 0) {
+      code = p[i];
+      i += 1;
+    } else if ((p[i] & 0xE0) == 0xC0 && (i + 1 < src.length())) {
+      code = (p[i] & 0x1F) << 6;
+      code |= (p[i+1] & 0x3F);
+      i += 2;
+    } else if ((p[i] & 0xF0) == 0xE0 && (i + 2 < src.length())) {
+      code = (p[i] & 0xF) << 12;
+      code |= (p[i+1] & 0x3F) << 6;
+      code |= (p[i+2] & 0x3F);
+      i += 3;
+    } else if ((p[i] & 0xF8) == 0xF0 && (i + 3 < src.length())) {
+      code = (p[i] & 0x7) << 18;
+      code |= (p[i+1] & 0x3F) << 12;
+      code |= (p[i+2] & 0x3F) << 6;
+      code |= (p[i+3] & 0x3F);
+      i += 4;
+    } else if ((p[i] & 0xFC) == 0xF8 && (i + 4 < src.length())) {
+      code = (p[i] & 0x3) << 24;
+      code |= (p[i+1] & 0x3F) << 18;
+      code |= (p[i+2] & 0x3F) << 12;
+      code |= (p[i+3] & 0x3F) << 6;
+      code |= (p[i+4] & 0x3F);
+      i += 5;
+    } else if ((p[i] & 0xFE) == 0xFC && (i + 5 < src.length())) {
+      code = (p[i] & 0x1) << 30;
+      code |= (p[i+1] & 0x3F) << 24;
+      code |= (p[i+2] & 0x3F) << 18;
+      code |= (p[i+3] & 0x3F) << 12;
+      code |= (p[i+4] & 0x3F) << 6;
+      code |= (p[i+5] & 0x3F);
+      i += 6;
+    }
+
+    if (code <= 0x7F) {
+      if (dest_length < 1) break;
+
+      *write_result++ = static_cast<char>(code);
+      read += 1;
+      written += 1;
+      dest_length -= 1;
+    } else if (code <= 0x7FF) {
+      if (dest_length < 2) break;
+
+      *write_result++ = (0xC0 | (code >> 6));
+      *write_result++ = (0x80 | (code & 0x3F));
+      read += 1;
+      written += 2;
+      dest_length -= 2;
+    } else if (code <= 0xFFFF) {
+      if (dest_length < 3) break;
+
+      *write_result++ = (0xE0 | (code >> 12));
+      *write_result++ = (0x80 | ((code >> 6) & 0x3F));
+      *write_result++ = (0x80 | (code & 0x3F));
+      read += 1;
+      written += 3;
+      dest_length -= 3;
+    } else if (code <= 0x1FFFFF) {
+      if (dest_length < 4) break;
+
+      *write_result++ = (0xF0 | (code >> 18));
+      *write_result++ = (0x80 | ((code >> 12) & 0x3F));
+      *write_result++ = (0x80 | ((code >> 6) & 0x3F));
+      *write_result++ = (0x80 | (code & 0x3F));
+      read += 2;
+      written += 4;
+      dest_length -= 4;
+    } else {
+      // invalid unicode
+    }
+  }
+
+  Local<Object> result = Object::New(isolate);
+  if (result->Set(context,
+                  env->encoding_read_string(),
+                  Integer::New(isolate, read)).IsNothing() ||
+      result->Set(context,
+                  env->encoding_written_string(),
+                  Integer::New(isolate, written)).IsNothing()) {
+    return;
+  }
+
+  args.GetReturnValue().Set(result);
+}
+
+
 void SetBufferPrototype(const FunctionCallbackInfo<Value>& args) {
   Environment* env = Environment::GetCurrent(args);
 
@@ -1082,6 +1196,7 @@ void Initialize(Local<Object> target,
   env->SetMethod(target, "swap32", Swap32);
   env->SetMethod(target, "swap64", Swap64);
 
+  env->SetMethod(target, "encodeInto", EncodeInto);
   env->SetMethodNoSideEffect(target, "encodeUtf8String", EncodeUtf8String);
 
   target->Set(env->context(),

test/fixtures/wpt/LICENSE.md

@@ -1,33 +1,11 @@
-# Dual-License for W3C Test Suites
+# The 3-Clause BSD License
 
-All documents in this Repository are licensed by contributors to be distributed under both the [W3C Test Suite License](#w3c-test-suite-license) and the [W3C 3-clause BSD License](#w3c-3-clause-bsd-license), reproduced below. The choice of license is up to the licensee. For more information, see [Licenses for W3C Test Suites](https://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html)
-
-# W3C Test Suite License
-
-This document, Test Suites and other documents that link to this statement are provided by the copyright holders under the following license: By using and/or copying this document, or the W3C document from which this statement is linked, you (the licensee) agree that you have read, understood, and will comply with the following terms and conditions:
-
-Permission to copy, and distribute the contents of this document, or the W3C document from which this statement is linked, in any medium for any purpose and without fee or royalty is hereby granted, provided that you include the following on ALL copies of the document, or portions thereof, that you use:
-
-*    A link or URL to the original W3C document.
-*    The pre-existing copyright notice of the original author, or if it doesn't exist, a notice (hypertext is preferred, but a textual representation is permitted) of the form: "Copyright © [$date-of-document] World Wide Web Consortium, (MIT, ERCIM, Keio, Beihang) and others. All Rights Reserved. http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html"
-*    If it exists, the STATUS of the W3C document.
-
-When space permits, inclusion of the full text of this NOTICE should be provided. We request that authorship attribution be provided in any software, documents, or other items or products that you create pursuant to the implementation of the contents of this document, or any portion thereof.
-
-No right to create modifications or derivatives of W3C documents is granted pursuant to this license. However, if additional requirements (documented in the Copyright FAQ) are satisfied, the right to create modifications or derivatives is sometimes granted by the W3C to individuals complying with those requirements.
-
-If a Test Suite distinguishes the test harness (or, framework for navigation) and the actual tests, permission is given to remove or alter the harness or navigation if the Test Suite in question allows to do so. The tests themselves shall NOT be changed in any way.
-
-The name and trademarks of W3C and other copyright holders may NOT be used in advertising or publicity pertaining to this document or other documents that link to this statement without specific, written prior permission. Title to copyright in this document will at all times remain with copyright holders. Permission is given to use the trademarked string "W3C" within claims of performance concerning W3C Specifications or features described therein, and there only, if the test suite so authorizes.
-
-THIS WORK IS PROVIDED BY W3C, MIT, ERCIM, KEIO, BEIHANG, THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL W3C, MIT, ERCIM, KEIO, BEIHANG, THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# W3C 3-clause BSD License
+Copyright 2019 web-platform-tests contributors
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
-*    Redistributions of works must retain the original copyright notice, this list of conditions and the following disclaimer.
-*    Redistributions in binary form must reproduce the original copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-*    Neither the name of the W3C nor the names of its contributors may be used to endorse or promote products derived from this work without specific prior written permission.
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

test/fixtures/wpt/README.md

@@ -11,7 +11,7 @@ See [test/wpt](../../wpt/README.md) for information on how these tests are run.
 Last update:
 
 - console: https://github.com/web-platform-tests/wpt/tree/9786a4b131/console
-- encoding: https://github.com/web-platform-tests/wpt/tree/7287608f90/encoding
+- encoding: https://github.com/web-platform-tests/wpt/tree/5059d2c777/encoding
 - url: https://github.com/web-platform-tests/wpt/tree/418f7fabeb/url
 - resources: https://github.com/web-platform-tests/wpt/tree/e1fddfbf80/resources
 - interfaces: https://github.com/web-platform-tests/wpt/tree/712c9f275e/interfaces

test/fixtures/wpt/encoding/encodeInto.any.js

@@ -126,7 +126,7 @@
  Float64Array].forEach(view => {
   test(() => {
     assert_throws(new TypeError(), () => new TextEncoder().encodeInto("", new view(new ArrayBuffer(0))));
-  }, "Invalid encodeInto() destination: " + view);
+  }, "Invalid encodeInto() destination: " + view.name);
  });
 
 test(() => {

test/fixtures/wpt/versions.json

@@ -4,7 +4,7 @@
     "path": "console"
   },
   "encoding": {
-    "commit": "7287608f90f6b9530635d10086fd2ab386faab38",
+    "commit": "5059d2c77703d67d2f76931b44e6d2437526b6e9",
     "path": "encoding"
   },
   "url": {

test/wpt/status/encoding.json

@@ -52,6 +52,5 @@
     "fail": "No implementation of TextDecoderStream and TextEncoderStream"
   },
   "encodeInto.any.js": {
-    "fail": "TextEncoder.prototype.encodeInto not implemented"
   }
 }

test/wpt/status/url.json

@@ -12,4 +12,4 @@
   "idlharness.any.js": {
     "fail": "getter/setter names are wrong, etc."
   }
-}
+}

test/wpt/test-encoding.js

@@ -3,10 +3,17 @@
 // Flags: --expose-internals
 
 require('../common');
+const { MessageChannel } = require('worker_threads');
 const { WPTRunner } = require('../common/wpt');
 const runner = new WPTRunner('encoding');
 
 // Copy global descriptors from the global object
 runner.copyGlobalsFromObject(global, ['TextDecoder', 'TextEncoder']);
 
+runner.defineGlobal('MessageChannel', {
+  get() {
+    return MessageChannel;
+  }
+});
+
 runner.runJsTests();