tools: port js2c.py to C++

This makes it easier to use third-party dependencies in this tool
(e.g. adding compression using algorithms not available in Python).
It is also much faster - locally js2c.py takes ~1.5s to generate the
output whereas this version takes ~0.1s - and consumes less memory
(~110MB v.s. 66MB).

This also modifies the js2c.py a bit to simplify the output, making
it easier to compare with one generated by the C++ version. Locally
the output from the two are identical. We'll remove js2c.py in a
subsequent commit when the C++ version is used by default.

PR-URL: #46997
Reviewed-By: Yagiz Nizipli <[email protected]>

Author: joyeecheung

Makefile

@@ -1409,6 +1409,7 @@ LINT_CPP_FILES = $(filter-out $(LINT_CPP_EXCLUDE), $(wildcard \
 	test/fixtures/*.c \
 	test/js-native-api/*/*.cc \
 	test/node-api/*/*.cc \
+	tools/js2c.cc \
 	tools/icu/*.cc \
 	tools/icu/*.h \
 	tools/code_cache/*.cc \

node.gyp

@@ -1176,6 +1176,35 @@
         }],
       ]
     }, # overlapped-checker
+    {
+      'target_name': 'node_js2c',
+      'type': 'executable',
+      'dependencies': [
+        'deps/simdutf/simdutf.gyp:simdutf',
+      ],
+      'include_dirs': [
+        'tools'
+      ],
+      'sources': [
+        'tools/js2c.cc',
+        'tools/executable_wrapper.h'
+      ],
+      'conditions': [
+        [ 'node_shared_libuv=="false"', {
+          'dependencies': [ 'deps/uv/uv.gyp:libuv' ],
+        }],
+        [ 'debug_node=="true"', {
+          'cflags!': [ '-O3' ],
+          'cflags': [ '-g', '-O0' ],
+          'defines': [ 'DEBUG' ],
+          'xcode_settings': {
+            'OTHER_CFLAGS': [
+              '-g', '-O0'
+            ],
+          },
+        }],
+      ]
+    },
     {
       'target_name': 'node_mksnapshot',
       'type': 'executable',

tools/executable_wrapper.h

@@ -0,0 +1,55 @@
+#ifndef TOOLS_EXECUTABLE_WRAPPER_H_
+#define TOOLS_EXECUTABLE_WRAPPER_H_
+
+// TODO(joyeecheung): reuse this in mksnapshot.
+#include "uv.h"
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+namespace node {
+#ifdef _WIN32
+using argv_type = wchar_t*;
+#define NODE_MAIN int wmain
+
+void FixupMain(int argc, argv_type raw_argv[], char*** argv) {
+  // Convert argv to UTF8.
+  *argv = new char*[argc + 1];
+  for (int i = 0; i < argc; i++) {
+    // Compute the size of the required buffer
+    DWORD size = WideCharToMultiByte(
+        CP_UTF8, 0, raw_argv[i], -1, nullptr, 0, nullptr, nullptr);
+    if (size == 0) {
+      // This should never happen.
+      fprintf(stderr, "Could not convert arguments to utf8.");
+      exit(1);
+    }
+    // Do the actual conversion
+    (*argv)[i] = new char[size];
+    DWORD result = WideCharToMultiByte(
+        CP_UTF8, 0, raw_argv[i], -1, (*argv)[i], size, nullptr, nullptr);
+    if (result == 0) {
+      // This should never happen.
+      fprintf(stderr, "Could not convert arguments to utf8.");
+      exit(1);
+    }
+  }
+  (*argv)[argc] = nullptr;
+}
+#else
+
+using argv_type = char*;
+#define NODE_MAIN int main
+
+void FixupMain(int argc, argv_type raw_argv[], char*** argv) {
+  *argv = uv_setup_args(argc, raw_argv);
+  // Disable stdio buffering, it interacts poorly with printf()
+  // calls elsewhere in the program (e.g., any logging from V8.)
+  setvbuf(stdout, nullptr, _IONBF, 0);
+  setvbuf(stderr, nullptr, _IONBF, 0);
+}
+#endif
+
+}  // namespace node
+
+#endif  // TOOLS_EXECUTABLE_WRAPPER_H_

tools/js2c.cc

@@ -55,22 +55,22 @@ def ReadFile(filename):
 namespace node {{
 
 namespace builtins {{
-
 {0}
-
 namespace {{
 const ThreadsafeCopyOnWrite<BuiltinSourceMap> global_source_map {{
-  BuiltinSourceMap{{ {1} }}
-}};
-}}
+  BuiltinSourceMap {{
+{1}
+  }}  // BuiltinSourceMap
+}};  // ThreadsafeCopyOnWrite
+}}  // anonymous namespace
 
 void BuiltinLoader::LoadJavaScriptSource() {{
   source_ = global_source_map;
 }}
 
 void RegisterExternalReferencesForInternalizedBuiltinCode(
   ExternalReferenceRegistry* registry) {{
-  {2}
+{2}
 }}
 
 UnionBytes BuiltinLoader::GetConfig() {{
@@ -98,9 +98,9 @@ def ReadFile(filename):
 static StaticExternalTwoByteResource {2}({0}, {3}, nullptr);
 """
 
-INITIALIZER = '{{"{0}", UnionBytes(&{1}) }},'
+INITIALIZER = '    {{"{0}", UnionBytes(&{1}) }},'
 
-REGISTRATION = 'registry->Register(&{0});'
+REGISTRATION = '  registry->Register(&{0});'
 
 CONFIG_GYPI_ID = 'config_raw'
 
@@ -110,7 +110,7 @@ def ReadFile(filename):
 
 is_verbose = False
 
-def GetDefinition(var, source, resource_var, step=30):
+def GetDefinition(var, source, resource_var):
   template = ONE_BYTE_STRING
   code_points = [ord(c) for c in source]
   if any(c > 127 for c in code_points):
@@ -122,12 +122,8 @@ def GetDefinition(var, source, resource_var, step=30):
       for i in range(0, len(encoded_source), 2)
     ]
 
-  # For easier debugging, align to the common 3 char for code-points.
-  elements_s = ['%3s' % x for x in code_points]
-  # Put no more then `step` code-points in a line.
-  slices = [elements_s[i:i + step] for i in range(0, len(elements_s), step)]
-  lines = [','.join(s) for s in slices]
-  array_content = ',\n'.join(lines)
+  elements_s = ['%s' % x for x in code_points]
+  array_content = ','.join(elements_s) + ','
   length = len(code_points)
   definition = template.format(var, array_content, resource_var, length)
 
@@ -174,8 +170,8 @@ def JS2C(source_files, target):
 
   # Emit result
   definitions = ''.join(definitions)
-  initializers = '\n  '.join(initializers)
-  registrations = '\n  '.join(registrations)
+  initializers = '\n'.join(initializers)
+  registrations = '\n'.join(registrations)
   out = TEMPLATE.format(definitions, initializers,
                         registrations, CONFIG_GYPI_RESOURCE_ID)
   write_if_chaged(out, target)
@@ -263,8 +259,10 @@ def main():
   assert len(source_files['.gypi']) == 1
   assert os.path.basename(source_files['.gypi'][0]) == 'config.gypi'
   source_files['config.gypi'] = source_files.pop('.gypi')[0]
-  JS2C(source_files, options.target)
+  source_files['.js'] = sorted(source_files['.js'])
+  source_files['.mjs'] = sorted(source_files['.mjs'])
 
+  JS2C(source_files, options.target)
 
 if __name__ == "__main__":
   main()