From 208d0472a9edd0523407e8e1b6e8be7353b2b05d Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 11 Jun 2021 11:28:27 +0800 Subject: [PATCH 1/8] bootstrap: refresh options in pre-execution Refresh the options map during pre-execution to pave the way for user land snapshots which may need to access run-time options at snapshot-building time. The default embedded bootstrap snapshot is still prevented from accessing them at snapshot building time since it serves a wider audience and is ignorant of application states, while the user-land snapshots are meant to be application-specific and so it makes sense for them to access runtime states as long as the snapshotted-code works with re-initialized runtime states. --- lib/internal/bootstrap/pre_execution.js | 8 ++++++++ lib/internal/options.js | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index 98428f02b43817..5cc6e5f5314502 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -14,6 +14,7 @@ const { const { getOptionValue, getEmbedderOptions, + refreshOptions, } = require('internal/options'); const { reconnectZeroFillToggle } = require('internal/buffer'); const { @@ -27,6 +28,8 @@ const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes; const assert = require('internal/assert'); function prepareMainThreadExecution(expandArgv1 = false) { + refreshRuntimeOptions(); + // TODO(joyeecheung): this is also necessary for workers when they deserialize // this toggle from the snapshot. reconnectZeroFillToggle(); @@ -87,6 +90,10 @@ function prepareMainThreadExecution(expandArgv1 = false) { initializeFrozenIntrinsics(); } +function refreshRuntimeOptions() { + refreshOptions(); +} + function patchProcessObject(expandArgv1) { const binding = internalBinding('process_methods'); binding.patchProcessObject(process); @@ -556,6 +563,7 @@ function loadPreloadModules() { } module.exports = { + refreshRuntimeOptions, patchProcessObject, setupCoverageHooks, setupWarningHandler, diff --git a/lib/internal/options.js b/lib/internal/options.js index 01b334d4ec5614..4d92ad681a1207 100644 --- a/lib/internal/options.js +++ b/lib/internal/options.js @@ -36,6 +36,11 @@ function getEmbedderOptions() { return embedderOptions; } +function refreshOptions() { + optionsMap = undefined; + aliasesMap = undefined; +} + function getOptionValue(optionName) { const options = getCLIOptionsFromBinding(); if (optionName.startsWith('--no-')) { @@ -68,5 +73,6 @@ module.exports = { }, getOptionValue, getAllowUnauthorized, - getEmbedderOptions + getEmbedderOptions, + refreshOptions }; From 4d59a74bd3293b566870e5a02812583ece2e7cfa Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 24 Mar 2022 21:37:08 +0800 Subject: [PATCH 2/8] build: add --node-snapshot-main configure option This adds a --build-snapshot runtime option which is currently only supported by the node_mksnapshot binary, and a --node-snapshot-main configure option that makes use it to run a custom script when building the embedded snapshot. The idea is to have this experimental feature in core as a configure-time feature for now, and investigate the renaming V8 bugs before we make it available to more users via making it a runtime option. --- configure.py | 19 ++++ lib/internal/bootstrap/pre_execution.js | 12 +- lib/internal/main/mksnapshot.js | 143 ++++++++++++++++++++++++ node.gyp | 53 ++++++--- src/node.cc | 10 ++ src/node_binding.cc | 1 + src/node_external_reference.h | 1 + src/node_options.cc | 5 + src/node_options.h | 1 + src/node_snapshotable.cc | 81 +++++++++++++- src/node_snapshotable.h | 7 +- tools/snapshot/node_mksnapshot.cc | 35 ++++-- 12 files changed, 339 insertions(+), 29 deletions(-) create mode 100644 lib/internal/main/mksnapshot.js diff --git a/configure.py b/configure.py index 8f74784cc0cfad..8bae2df6405eea 100755 --- a/configure.py +++ b/configure.py @@ -788,6 +788,13 @@ default=False, help='node will load builtin modules from disk instead of from binary') +parser.add_argument('--node-snapshot-main', + action='store', + dest='node_snapshot_main', + default=None, + help='Run a file when building the embedded snapshot. Currently ' + + 'experimental.') + # Create compile_commands.json in out/Debug and out/Release. parser.add_argument('-C', action='store_true', @@ -1216,6 +1223,18 @@ def configure_node(o): o['variables']['want_separate_host_toolset'] = int(cross_compiling) + if options.node_snapshot_main is not None: + if options.shared: + # This should be possible to fix, but we will need to refactor the + # libnode target to avoid building it twice. + error('--node-snapshot-main is incompatible with --shared') + if options.without_node_snapshot: + error('--node-snapshot-main is incompatible with ' + + '--without-node-snapshot') + if cross_compiling: + error('--node-snapshot-main is incompatible with cross compilation') + o['variables']['node_snapshot_main'] = options.node_snapshot_main + if options.without_node_snapshot or options.node_builtin_modules_path: o['variables']['node_use_node_snapshot'] = 'false' else: diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index 5cc6e5f5314502..4a5db2fab13d07 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -27,7 +27,8 @@ const { Buffer } = require('buffer'); const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes; const assert = require('internal/assert'); -function prepareMainThreadExecution(expandArgv1 = false) { +function prepareMainThreadExecution(expandArgv1 = false, + initialzeModules = true) { refreshRuntimeOptions(); // TODO(joyeecheung): this is also necessary for workers when they deserialize @@ -81,9 +82,13 @@ function prepareMainThreadExecution(expandArgv1 = false) { initializeSourceMapsHandlers(); initializeDeprecations(); initializeWASI(); + + if (!initialzeModules) { + return; + } + initializeCJSLoader(); initializeESMLoader(); - const CJSLoader = require('internal/modules/cjs/loader'); assert(!CJSLoader.hasLoadedAnyUserCJSModule); loadPreloadModules(); @@ -102,7 +107,8 @@ function patchProcessObject(expandArgv1) { ObjectDefineProperty(process, 'argv0', { enumerable: true, - configurable: false, + // Only set it to true during snapshot building. + configurable: getOptionValue('--build-snapshot'), value: process.argv[0] }); process.argv[0] = process.execPath; diff --git a/lib/internal/main/mksnapshot.js b/lib/internal/main/mksnapshot.js new file mode 100644 index 00000000000000..29ffa925f43e85 --- /dev/null +++ b/lib/internal/main/mksnapshot.js @@ -0,0 +1,143 @@ +'use strict'; + +const { + Error, + SafeSet, + SafeArrayIterator +} = primordials; + +const binding = internalBinding('mksnapshot'); +const { NativeModule } = require('internal/bootstrap/loaders'); +const { + compileSnapshotMain, +} = binding; + +const { + getOptionValue +} = require('internal/options'); + +const { + readFileSync +} = require('fs'); + +const supportedModules = new SafeSet(new SafeArrayIterator([ + // '_http_agent', + // '_http_client', + // '_http_common', + // '_http_incoming', + // '_http_outgoing', + // '_http_server', + '_stream_duplex', + '_stream_passthrough', + '_stream_readable', + '_stream_transform', + '_stream_wrap', + '_stream_writable', + // '_tls_common', + // '_tls_wrap', + 'assert', + 'assert/strict', + // 'async_hooks', + 'buffer', + // 'child_process', + // 'cluster', + 'console', + 'constants', + 'crypto', + // 'dgram', + // 'diagnostics_channel', + // 'dns', + // 'dns/promises', + // 'domain', + 'events', + 'fs', + 'fs/promises', + // 'http', + // 'http2', + // 'https', + // 'inspector', + // 'module', + // 'net', + 'os', + 'path', + 'path/posix', + 'path/win32', + // 'perf_hooks', + 'process', + 'punycode', + 'querystring', + // 'readline', + // 'repl', + 'stream', + 'stream/promises', + 'string_decoder', + 'sys', + 'timers', + 'timers/promises', + // 'tls', + // 'trace_events', + // 'tty', + 'url', + 'util', + 'util/types', + 'v8', + // 'vm', + // 'worker_threads', + // 'zlib', +])); + +const warnedModules = new SafeSet(); +function supportedInUserSnapshot(id) { + return supportedModules.has(id); +} + +function requireForUserSnapshot(id) { + if (!NativeModule.canBeRequiredByUsers(id)) { + // eslint-disable-next-line no-restricted-syntax + const err = new Error( + `Cannot find module '${id}'. ` + ); + err.code = 'MODULE_NOT_FOUND'; + throw err; + } + if (!supportedInUserSnapshot(id)) { + if (!warnedModules.has(id)) { + process.emitWarning( + `built-in module ${id} is not yet supported in user snapshots`); + warnedModules.add(id); + } + } + + return require(id); +} + +function main() { + const { + prepareMainThreadExecution + } = require('internal/bootstrap/pre_execution'); + + prepareMainThreadExecution(true, false); + process.on('beforeExit', function runCleanups() { + for (const cleanup of binding.cleanups) { + cleanup(); + } + process.off('beforeExit', runCleanups); + }); + + const file = process.argv[1]; + const path = require('path'); + const filename = path.resolve(file); + const dirname = path.dirname(filename); + const source = readFileSync(file, 'utf-8'); + const snapshotMainFunction = compileSnapshotMain(filename, source); + + if (getOptionValue('--inspect-brk')) { + internalBinding('inspector').callAndPauseOnStart( + snapshotMainFunction, undefined, + requireForUserSnapshot, filename, dirname); + } else { + snapshotMainFunction(requireForUserSnapshot, filename, dirname); + } +} + +main(); diff --git a/node.gyp b/node.gyp index 9f45d44e05444d..0a6eca012daad1 100644 --- a/node.gyp +++ b/node.gyp @@ -7,6 +7,7 @@ 'node_use_dtrace%': 'false', 'node_use_etw%': 'false', 'node_no_browser_globals%': 'false', + 'node_snapshot_main%': '', 'node_use_node_snapshot%': 'false', 'node_use_v8_platform%': 'true', 'node_use_bundled_v8%': 'true', @@ -315,23 +316,47 @@ 'dependencies': [ 'node_mksnapshot', ], - 'actions': [ - { - 'action_name': 'node_mksnapshot', - 'process_outputs_as_sources': 1, - 'inputs': [ - '<(node_mksnapshot_exec)', - ], - 'outputs': [ - '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + 'conditions': [ + ['node_snapshot_main!=""', { + 'actions': [ + { + 'action_name': 'node_mksnapshot', + 'process_outputs_as_sources': 1, + 'inputs': [ + '<(node_mksnapshot_exec)', + '<(node_snapshot_main)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + ], + 'action': [ + '<(node_mksnapshot_exec)', + '--build-snapshot', + '<(node_snapshot_main)', + '<@(_outputs)', + ], + }, ], - 'action': [ - '<@(_inputs)', - '<@(_outputs)', + }, { + 'actions': [ + { + 'action_name': 'node_mksnapshot', + 'process_outputs_as_sources': 1, + 'inputs': [ + '<(node_mksnapshot_exec)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + ], + 'action': [ + '<@(_inputs)', + '<@(_outputs)', + ], + }, ], - }, + }], ], - }, { + }, { 'sources': [ 'src/node_snapshot_stub.cc' ], diff --git a/src/node.cc b/src/node.cc index e503fd9f5b423c..5ba03b75407a1e 100644 --- a/src/node.cc +++ b/src/node.cc @@ -495,6 +495,10 @@ MaybeLocal StartExecution(Environment* env, StartExecutionCallback cb) { return StartExecution(env, "internal/main/inspect"); } + if (per_process::cli_options->build_snapshot) { + return StartExecution(env, "internal/main/mksnapshot"); + } + if (per_process::cli_options->print_help) { return StartExecution(env, "internal/main/print_help"); } @@ -1153,6 +1157,12 @@ int Start(int argc, char** argv) { return result.exit_code; } + if (per_process::cli_options->build_snapshot) { + fprintf(stderr, + "--build-snapshot is not yet supported in the node binary\n"); + return 1; + } + { bool use_node_snapshot = per_process::cli_options->per_isolate->node_snapshot; diff --git a/src/node_binding.cc b/src/node_binding.cc index 050c5dff0ad5fc..ed8ab9237fd3b3 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -59,6 +59,7 @@ V(js_udp_wrap) \ V(messaging) \ V(module_wrap) \ + V(mksnapshot) \ V(native_module) \ V(options) \ V(os) \ diff --git a/src/node_external_reference.h b/src/node_external_reference.h index c57a01ff39c20e..306c726631a214 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -66,6 +66,7 @@ class ExternalReferenceRegistry { V(handle_wrap) \ V(heap_utils) \ V(messaging) \ + V(mksnapshot) \ V(native_module) \ V(options) \ V(os) \ diff --git a/src/node_options.cc b/src/node_options.cc index 35ee54d223176a..bb568ce458a19a 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -725,6 +725,11 @@ PerProcessOptionsParser::PerProcessOptionsParser( "disable Object.prototype.__proto__", &PerProcessOptions::disable_proto, kAllowedInEnvironment); + AddOption("--build-snapshot", + "Generate a snapshot blob when the process exits." + "Currently only supported in the node_mksnapshot binary.", + &PerProcessOptions::build_snapshot, + kDisallowedInEnvironment); // 12.x renamed this inadvertently, so alias it for consistency within the // release line, while using the original name for consistency with older diff --git a/src/node_options.h b/src/node_options.h index 3335c12b8cdcf7..7a73e6e5987967 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -229,6 +229,7 @@ class PerProcessOptions : public Options { bool zero_fill_all_buffers = false; bool debug_arraybuffer_allocations = false; std::string disable_proto; + bool build_snapshot; std::vector security_reverts; bool print_bash_completion = false; diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 71b025df81eead..998f65f61f2db6 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -18,12 +18,18 @@ namespace node { using v8::Context; +using v8::Function; +using v8::FunctionCallbackInfo; using v8::HandleScope; using v8::Isolate; using v8::Local; +using v8::MaybeLocal; using v8::Object; +using v8::ScriptCompiler; +using v8::ScriptOrigin; using v8::SnapshotCreator; using v8::StartupData; +using v8::String; using v8::TryCatch; using v8::Value; @@ -82,6 +88,7 @@ const SnapshotData* NodeMainInstance::GetEmbeddedSnapshotData() { } void SnapshotBuilder::Generate(SnapshotData* out, + const std::string& entry_file, const std::vector args, const std::vector exec_args) { Isolate* isolate = Isolate::Allocate(); @@ -130,16 +137,32 @@ void SnapshotBuilder::Generate(SnapshotData* out, nullptr, node::EnvironmentFlags::kDefaultFlags, {}); + // TODO(joyeecheung): run env->InitializeInspector({}) here. // Run scripts in lib/internal/bootstrap/ { TryCatch bootstrapCatch(isolate); - v8::MaybeLocal result = env->RunBootstrapping(); + MaybeLocal result = env->RunBootstrapping(); if (bootstrapCatch.HasCaught()) { PrintCaughtException(isolate, context, bootstrapCatch); } result.ToLocalChecked(); } + // Run the entry point file + if (!entry_file.empty()) { + TryCatch bootstrapCatch(isolate); + // TODO(joyee): we could use the result for something special, like + // setting up initializers that should be invoked at snapshot + // dehydration. + MaybeLocal result = + LoadEnvironment(env, StartExecutionCallback{}); + if (bootstrapCatch.HasCaught()) { + PrintCaughtException(isolate, context, bootstrapCatch); + } + result.ToLocalChecked(); + // TODO(joyeecheung): run SpinEventLoop here. + } + if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { env->PrintAllBaseObjects(); printf("Environment = %p\n", env); @@ -182,10 +205,11 @@ void SnapshotBuilder::Generate(SnapshotData* out, } std::string SnapshotBuilder::Generate( + const std::string& entry_file, const std::vector args, const std::vector exec_args) { SnapshotData data; - Generate(&data, args, exec_args); + Generate(&data, entry_file, args, exec_args); std::string result = FormatBlob(&data); delete[] data.blob.data; return result; @@ -309,4 +333,57 @@ void SerializeBindingData(Environment* env, }); } +namespace mksnapshot { + +static void CompileSnapshotMain(const FunctionCallbackInfo& args) { + CHECK(args[0]->IsString()); + Local filename = args[0].As(); + Local source = args[1].As(); + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + ScriptOrigin origin(isolate, filename, 0, 0, true); + // TODO(joyee): do we need all of these? Maybe we would want a less + // internal version of them. + std::vector> parameters = { + FIXED_ONE_BYTE_STRING(isolate, "require"), + FIXED_ONE_BYTE_STRING(isolate, "__filename"), + FIXED_ONE_BYTE_STRING(isolate, "__dirname"), + }; + ScriptCompiler::Source script_source(source, origin); + Local fn; + if (ScriptCompiler::CompileFunctionInContext(context, + &script_source, + parameters.size(), + parameters.data(), + 0, + nullptr, + ScriptCompiler::kEagerCompile) + .ToLocal(&fn)) { + args.GetReturnValue().Set(fn); + } +} + +static void Initialize(Local target, + Local unused, + Local context, + void* priv) { + Environment* env = Environment::GetCurrent(context); + Isolate* isolate = context->GetIsolate(); + env->SetMethod(target, "compileSnapshotMain", CompileSnapshotMain); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "cleanups"), + v8::Array::New(isolate)) + .Check(); +} + +static void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(CompileSnapshotMain); + registry->Register(MarkBootstrapComplete); +} +} // namespace mksnapshot } // namespace node + +NODE_MODULE_CONTEXT_AWARE_INTERNAL(mksnapshot, node::mksnapshot::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(mksnapshot, + node::mksnapshot::RegisterExternalReferences) diff --git a/src/node_snapshotable.h b/src/node_snapshotable.h index 1ccd9a93226241..6aa3f5bfe0cdb1 100644 --- a/src/node_snapshotable.h +++ b/src/node_snapshotable.h @@ -125,9 +125,14 @@ bool IsSnapshotableType(FastStringKey key); class SnapshotBuilder { public: - static std::string Generate(const std::vector args, + static std::string Generate(const std::string& entry_file, + const std::vector args, const std::vector exec_args); + + // Generate the snapshot into out. + // entry_file should be the content of the UTF-8 encoded entry files. static void Generate(SnapshotData* out, + const std::string& entry_file, const std::vector args, const std::vector exec_args); }; diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index e591f64a2a0518..784009893b253e 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -24,13 +24,8 @@ int main(int argc, char* argv[]) { if (argc < 2) { std::cerr << "Usage: " << argv[0] << " \n"; - return 1; - } - - std::ofstream out; - out.open(argv[1], std::ios::out | std::ios::binary); - if (!out.is_open()) { - std::cerr << "Cannot open " << argv[1] << "\n"; + std::cerr << " " << argv[0] << " --build-snapshot " + << " \n"; return 1; } @@ -49,10 +44,32 @@ int main(int argc, char* argv[]) { CHECK(!result.early_return); CHECK_EQ(result.exit_code, 0); + std::string snapshot_main; + std::string out_path; + if (node::per_process::cli_options->build_snapshot) { + snapshot_main = result.args[1]; + out_path = result.args[2]; + } else { + out_path = result.args[1]; + } + + std::ofstream out(out_path, std::ios::out | std::ios::binary); + if (!out) { + std::cerr << "Cannot open " << out_path << "\n"; + return 1; + } + { - std::string snapshot = - node::SnapshotBuilder::Generate(result.args, result.exec_args); + std::string snapshot = node::SnapshotBuilder::Generate( + snapshot_main, result.args, result.exec_args); out << snapshot; + + if (!out) { + std::cerr << "Failed to write " << out_path << "\n"; + out.close(); + return 1; + } + out.close(); } From c812b986554b3146fcfc3b812e9fefc12ecd5a3c Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 25 Mar 2022 21:15:39 +0800 Subject: [PATCH 3/8] bootstrap: make I/O streams work with user-land snapshot Use the mksnapshot cleanup hooks to release the references to the native streams so that they can be destroyed right before the snapshot is taken, and move the initialization of the global console to pre-execution so that they can be re-initialized during snapshot dehydration. This makes it possible for user-land snapshots to use the I/O during snapshot building. --- lib/internal/bootstrap/pre_execution.js | 6 ++++ .../bootstrap/switches/is_main_thread.js | 34 +++++++++++++++++-- lib/internal/console/constructor.js | 6 ++++ lib/internal/console/global.js | 7 +--- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index 4a5db2fab13d07..3ef1bca0d4f35d 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -124,6 +124,12 @@ function patchProcessObject(expandArgv1) { } } + // We need to initialize the global console here again with process.stdout + // and friends for snapshot deserialization. + const globalConsole = require('internal/console/global'); + const { initializeGlobalConsole } = require('internal/console/constructor'); + initializeGlobalConsole(globalConsole); + // TODO(joyeecheung): most of these should be deprecated and removed, // except some that we need to be able to mutate during run time. addReadOnlyProcessAlias('_eval', '--eval'); diff --git a/lib/internal/bootstrap/switches/is_main_thread.js b/lib/internal/bootstrap/switches/is_main_thread.js index 606da95f21ca8f..b7bd79e09c4acf 100644 --- a/lib/internal/bootstrap/switches/is_main_thread.js +++ b/lib/internal/bootstrap/switches/is_main_thread.js @@ -122,15 +122,34 @@ let stdin; let stdout; let stderr; +let stdoutDestroy; +let stderrDestroy; + +function refreshStdoutOnSigWinch() { + stdout._refreshSize(); +} + +function refreshStderrOnSigWinch() { + stderr._refreshSize(); +} + function getStdout() { if (stdout) return stdout; stdout = createWritableStdioStream(1); stdout.destroySoon = stdout.destroy; // Override _destroy so that the fd is never actually closed. + stdoutDestroy = stdout._destroy; stdout._destroy = dummyDestroy; if (stdout.isTTY) { - process.on('SIGWINCH', () => stdout._refreshSize()); + process.on('SIGWINCH', refreshStdoutOnSigWinch); } + + internalBinding('mksnapshot').cleanups.push(function cleanupStdout() { + stdout._destroy = stdoutDestroy; + stdout.destroy(); + process.removeListener('SIGWINCH', refreshStdoutOnSigWinch); + stdout = undefined; + }); return stdout; } @@ -138,11 +157,18 @@ function getStderr() { if (stderr) return stderr; stderr = createWritableStdioStream(2); stderr.destroySoon = stderr.destroy; + stderrDestroy = stderr._destroy; // Override _destroy so that the fd is never actually closed. stderr._destroy = dummyDestroy; if (stderr.isTTY) { - process.on('SIGWINCH', () => stderr._refreshSize()); + process.on('SIGWINCH', refreshStderrOnSigWinch); } + internalBinding('mksnapshot').cleanups.push(function cleanupStderr() { + stderr._destroy = stderrDestroy; + stderr.destroy(); + process.removeListener('SIGWINCH', refreshStderrOnSigWinch); + stderr = undefined; + }); return stderr; } @@ -229,6 +255,10 @@ function getStdin() { } } + internalBinding('mksnapshot').cleanups.push(function cleanupStdin() { + stdin.destroy(); + stdin = undefined; + }); return stdin; } diff --git a/lib/internal/console/constructor.js b/lib/internal/console/constructor.js index 695a56164b7d84..5ad57be3bed6a8 100644 --- a/lib/internal/console/constructor.js +++ b/lib/internal/console/constructor.js @@ -669,9 +669,15 @@ Console.prototype.dirxml = Console.prototype.log; Console.prototype.error = Console.prototype.warn; Console.prototype.groupCollapsed = Console.prototype.group; +function initializeGlobalConsole(globalConsole) { + globalConsole[kBindStreamsLazy](process); + globalConsole[kBindProperties](true, 'auto'); +} + module.exports = { Console, kBindStreamsLazy, kBindProperties, + initializeGlobalConsole, formatTime // exported for tests }; diff --git a/lib/internal/console/global.js b/lib/internal/console/global.js index d6c0c24d529dcc..782a585957f746 100644 --- a/lib/internal/console/global.js +++ b/lib/internal/console/global.js @@ -21,9 +21,7 @@ const { } = primordials; const { - Console, - kBindStreamsLazy, - kBindProperties + Console } = require('internal/console/constructor'); const globalConsole = ObjectCreate({}); @@ -44,9 +42,6 @@ for (const prop of ReflectOwnKeys(Console.prototype)) { ReflectDefineProperty(globalConsole, prop, desc); } -globalConsole[kBindStreamsLazy](process); -globalConsole[kBindProperties](true, 'auto'); - // This is a legacy feature - the Console constructor is exposed on // the global console instance. globalConsole.Console = Console; From 39cd563e9abc95762033ed741d9effbdb2b693ca Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 24 Mar 2022 21:50:55 +0800 Subject: [PATCH 4/8] bootstrap: run inspector and event loop in snapshot builder This makes --inspect and stdin/out functional in the embedded snapshot, currently it's not guaranteed that these work perfectly, the plan is to investigate any out-of-sync states that might appear in user land snapshots with this while the it is a configure-time feature. --- src/node_snapshotable.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 998f65f61f2db6..0c8fa2027f649d 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -15,6 +15,10 @@ #include "node_v8.h" #include "node_v8_platform-inl.h" +#if HAVE_INSPECTOR +#include "inspector/worker_inspector.h" // ParentInspectorHandle +#endif + namespace node { using v8::Context; @@ -137,7 +141,7 @@ void SnapshotBuilder::Generate(SnapshotData* out, nullptr, node::EnvironmentFlags::kDefaultFlags, {}); - // TODO(joyeecheung): run env->InitializeInspector({}) here. + // Run scripts in lib/internal/bootstrap/ { TryCatch bootstrapCatch(isolate); @@ -150,6 +154,10 @@ void SnapshotBuilder::Generate(SnapshotData* out, // Run the entry point file if (!entry_file.empty()) { +#if HAVE_INSPECTOR + env->InitializeInspector({}); +#endif + TryCatch bootstrapCatch(isolate); // TODO(joyee): we could use the result for something special, like // setting up initializers that should be invoked at snapshot @@ -160,7 +168,15 @@ void SnapshotBuilder::Generate(SnapshotData* out, PrintCaughtException(isolate, context, bootstrapCatch); } result.ToLocalChecked(); - // TODO(joyeecheung): run SpinEventLoop here. + // FIXME(joyee): right now running the loop in the snapshot builder + // seems to intrudoces inconsistencies in JS land that need to be + // synchronized again after snapshot restoration. + int exit_code = SpinEventLoop(env).FromMaybe(1); + CHECK_EQ(exit_code, 0); + if (bootstrapCatch.HasCaught()) { + PrintCaughtException(isolate, context, bootstrapCatch); + abort(); + } } if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { From da8c19f7f5be99c93196fdecfa207b147f000ac5 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sat, 26 Mar 2022 20:40:54 +0800 Subject: [PATCH 5/8] fixup! build: add --node-snapshot-main configure option --- tools/snapshot/node_mksnapshot.cc | 58 +++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index 784009893b253e..c06cd9a48feeba 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -11,17 +11,64 @@ #include "util-inl.h" #include "v8.h" +int BuildSnapshot(int argc, char* argv[]); + #ifdef _WIN32 #include -int wmain(int argc, wchar_t* argv[]) { +int wmain(int argc, wchar_t* wargv[]) { + // Windows needs conversion from wchar_t to char. + + // Convert argv to UTF8. + char** argv = new char*[argc + 1]; + for (int i = 0; i < argc; i++) { + // Compute the size of the required buffer + DWORD size = WideCharToMultiByte(CP_UTF8, + 0, + wargv[i], + -1, + nullptr, + 0, + nullptr, + nullptr); + if (size == 0) { + // This should never happen. + fprintf(stderr, "Could not convert arguments to utf8."); + exit(1); + } + // Do the actual conversion + argv[i] = new char[size]; + DWORD result = WideCharToMultiByte(CP_UTF8, + 0, + wargv[i], + -1, + argv[i], + size, + nullptr, + nullptr); + if (result == 0) { + // This should never happen. + fprintf(stderr, "Could not convert arguments to utf8."); + exit(1); + } + } + argv[argc] = nullptr; #else // UNIX int main(int argc, char* argv[]) { argv = uv_setup_args(argc, argv); + + // Disable stdio buffering, it interacts poorly with printf() + // calls elsewhere in the program (e.g., any logging from V8.) + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); #endif // _WIN32 v8::V8::SetFlagsFromString("--random_seed=42"); + v8::V8::SetFlagsFromString("--harmony-import-assertions"); + return BuildSnapshot(argc, argv); +} +int BuildSnapshot(int argc, char* argv[]) { if (argc < 2) { std::cerr << "Usage: " << argv[0] << " \n"; std::cerr << " " << argv[0] << " --build-snapshot " @@ -29,17 +76,8 @@ int main(int argc, char* argv[]) { return 1; } -// Windows needs conversion from wchar_t to char. See node_main.cc -#ifdef _WIN32 - int node_argc = 1; - char argv0[] = "node"; - char* node_argv[] = {argv0, nullptr}; - node::InitializationResult result = - node::InitializeOncePerProcess(node_argc, node_argv); -#else node::InitializationResult result = node::InitializeOncePerProcess(argc, argv); -#endif CHECK(!result.early_return); CHECK_EQ(result.exit_code, 0); From b1f934f478b776a1e91c78d26f9d2b1338ef7e9e Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sat, 26 Mar 2022 21:04:41 +0800 Subject: [PATCH 6/8] bootsrap: reset process._exit and process.exitCode in pre-execution --- lib/internal/bootstrap/pre_execution.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index 3ef1bca0d4f35d..b64dfaf980fd92 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -111,6 +111,9 @@ function patchProcessObject(expandArgv1) { configurable: getOptionValue('--build-snapshot'), value: process.argv[0] }); + + process.exitCode = undefined; + process._exiting = false; process.argv[0] = process.execPath; if (expandArgv1 && process.argv[1] && From 4d0500d074d87671f888e93f77dde4c3a0b16f85 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 28 Mar 2022 23:28:27 +0800 Subject: [PATCH 7/8] fixup! build: add --node-snapshot-main configure option --- lib/internal/main/mksnapshot.js | 3 +-- src/node_snapshotable.cc | 18 ++++++++++-------- src/node_snapshotable.h | 5 +---- tools/snapshot/node_mksnapshot.cc | 29 ++++++----------------------- 4 files changed, 18 insertions(+), 37 deletions(-) diff --git a/lib/internal/main/mksnapshot.js b/lib/internal/main/mksnapshot.js index 29ffa925f43e85..3e1515a2d2e05e 100644 --- a/lib/internal/main/mksnapshot.js +++ b/lib/internal/main/mksnapshot.js @@ -117,11 +117,10 @@ function main() { } = require('internal/bootstrap/pre_execution'); prepareMainThreadExecution(true, false); - process.on('beforeExit', function runCleanups() { + process.once('beforeExit', function runCleanups() { for (const cleanup of binding.cleanups) { cleanup(); } - process.off('beforeExit', runCleanups); }); const file = process.argv[1]; diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 0c8fa2027f649d..79bc12f15cee1c 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -92,7 +92,6 @@ const SnapshotData* NodeMainInstance::GetEmbeddedSnapshotData() { } void SnapshotBuilder::Generate(SnapshotData* out, - const std::string& entry_file, const std::vector args, const std::vector exec_args) { Isolate* isolate = Isolate::Allocate(); @@ -152,15 +151,19 @@ void SnapshotBuilder::Generate(SnapshotData* out, result.ToLocalChecked(); } - // Run the entry point file - if (!entry_file.empty()) { + // If --build-snapshot is true, lib/internal/main/mksnapshot.js would be + // loaded via LoadEnvironment() to execute process.argv[1] as the entry + // point (we currently only support this kind of entry point, but we + // could also explore snapshotting other kinds of execution modes + // in the future). + if (per_process::cli_options->build_snapshot) { #if HAVE_INSPECTOR env->InitializeInspector({}); #endif TryCatch bootstrapCatch(isolate); - // TODO(joyee): we could use the result for something special, like - // setting up initializers that should be invoked at snapshot + // TODO(joyeecheung): we could use the result for something special, + // like setting up initializers that should be invoked at snapshot // dehydration. MaybeLocal result = LoadEnvironment(env, StartExecutionCallback{}); @@ -221,11 +224,10 @@ void SnapshotBuilder::Generate(SnapshotData* out, } std::string SnapshotBuilder::Generate( - const std::string& entry_file, const std::vector args, const std::vector exec_args) { SnapshotData data; - Generate(&data, entry_file, args, exec_args); + Generate(&data, args, exec_args); std::string result = FormatBlob(&data); delete[] data.blob.data; return result; @@ -358,7 +360,7 @@ static void CompileSnapshotMain(const FunctionCallbackInfo& args) { Isolate* isolate = args.GetIsolate(); Local context = isolate->GetCurrentContext(); ScriptOrigin origin(isolate, filename, 0, 0, true); - // TODO(joyee): do we need all of these? Maybe we would want a less + // TODO(joyeecheung): do we need all of these? Maybe we would want a less // internal version of them. std::vector> parameters = { FIXED_ONE_BYTE_STRING(isolate, "require"), diff --git a/src/node_snapshotable.h b/src/node_snapshotable.h index 6aa3f5bfe0cdb1..30b684eb68a2d6 100644 --- a/src/node_snapshotable.h +++ b/src/node_snapshotable.h @@ -125,14 +125,11 @@ bool IsSnapshotableType(FastStringKey key); class SnapshotBuilder { public: - static std::string Generate(const std::string& entry_file, - const std::vector args, + static std::string Generate(const std::vector args, const std::vector exec_args); // Generate the snapshot into out. - // entry_file should be the content of the UTF-8 encoded entry files. static void Generate(SnapshotData* out, - const std::string& entry_file, const std::vector args, const std::vector exec_args); }; diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index c06cd9a48feeba..60062854327868 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -23,14 +23,8 @@ int wmain(int argc, wchar_t* wargv[]) { char** argv = new char*[argc + 1]; for (int i = 0; i < argc; i++) { // Compute the size of the required buffer - DWORD size = WideCharToMultiByte(CP_UTF8, - 0, - wargv[i], - -1, - nullptr, - 0, - nullptr, - nullptr); + DWORD size = WideCharToMultiByte( + CP_UTF8, 0, wargv[i], -1, nullptr, 0, nullptr, nullptr); if (size == 0) { // This should never happen. fprintf(stderr, "Could not convert arguments to utf8."); @@ -38,14 +32,8 @@ int wmain(int argc, wchar_t* wargv[]) { } // Do the actual conversion argv[i] = new char[size]; - DWORD result = WideCharToMultiByte(CP_UTF8, - 0, - wargv[i], - -1, - argv[i], - size, - nullptr, - nullptr); + DWORD result = WideCharToMultiByte( + CP_UTF8, 0, wargv[i], -1, argv[i], size, nullptr, nullptr); if (result == 0) { // This should never happen. fprintf(stderr, "Could not convert arguments to utf8."); @@ -82,10 +70,8 @@ int BuildSnapshot(int argc, char* argv[]) { CHECK(!result.early_return); CHECK_EQ(result.exit_code, 0); - std::string snapshot_main; std::string out_path; if (node::per_process::cli_options->build_snapshot) { - snapshot_main = result.args[1]; out_path = result.args[2]; } else { out_path = result.args[1]; @@ -98,17 +84,14 @@ int BuildSnapshot(int argc, char* argv[]) { } { - std::string snapshot = node::SnapshotBuilder::Generate( - snapshot_main, result.args, result.exec_args); + std::string snapshot = + node::SnapshotBuilder::Generate(result.args, result.exec_args); out << snapshot; if (!out) { std::cerr << "Failed to write " << out_path << "\n"; - out.close(); return 1; } - - out.close(); } node::TearDownOncePerProcess(); From 25f4c8dedd18704b3ad8ee2a620570be7d7b3bc3 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 28 Mar 2022 23:30:24 +0800 Subject: [PATCH 8/8] fixup! bootstrap: run inspector and event loop in snapshot builder --- src/node_snapshotable.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 79bc12f15cee1c..962fb5f3b536d2 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -171,9 +171,9 @@ void SnapshotBuilder::Generate(SnapshotData* out, PrintCaughtException(isolate, context, bootstrapCatch); } result.ToLocalChecked(); - // FIXME(joyee): right now running the loop in the snapshot builder - // seems to intrudoces inconsistencies in JS land that need to be - // synchronized again after snapshot restoration. + // FIXME(joyeecheung): right now running the loop in the snapshot + // builder seems to introduces inconsistencies in JS land that need to + // be synchronized again after snapshot restoration. int exit_code = SpinEventLoop(env).FromMaybe(1); CHECK_EQ(exit_code, 0); if (bootstrapCatch.HasCaught()) {