diff --git a/configure.py b/configure.py index 8f74784cc0cfad..8bae2df6405eea 100755 --- a/configure.py +++ b/configure.py @@ -788,6 +788,13 @@ default=False, help='node will load builtin modules from disk instead of from binary') +parser.add_argument('--node-snapshot-main', + action='store', + dest='node_snapshot_main', + default=None, + help='Run a file when building the embedded snapshot. Currently ' + + 'experimental.') + # Create compile_commands.json in out/Debug and out/Release. parser.add_argument('-C', action='store_true', @@ -1216,6 +1223,18 @@ def configure_node(o): o['variables']['want_separate_host_toolset'] = int(cross_compiling) + if options.node_snapshot_main is not None: + if options.shared: + # This should be possible to fix, but we will need to refactor the + # libnode target to avoid building it twice. + error('--node-snapshot-main is incompatible with --shared') + if options.without_node_snapshot: + error('--node-snapshot-main is incompatible with ' + + '--without-node-snapshot') + if cross_compiling: + error('--node-snapshot-main is incompatible with cross compilation') + o['variables']['node_snapshot_main'] = options.node_snapshot_main + if options.without_node_snapshot or options.node_builtin_modules_path: o['variables']['node_use_node_snapshot'] = 'false' else: diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index 98428f02b43817..b64dfaf980fd92 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -14,6 +14,7 @@ const { const { getOptionValue, getEmbedderOptions, + refreshOptions, } = require('internal/options'); const { reconnectZeroFillToggle } = require('internal/buffer'); const { @@ -26,7 +27,10 @@ const { Buffer } = require('buffer'); const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes; const assert = require('internal/assert'); -function prepareMainThreadExecution(expandArgv1 = false) { +function prepareMainThreadExecution(expandArgv1 = false, + initialzeModules = true) { + refreshRuntimeOptions(); + // TODO(joyeecheung): this is also necessary for workers when they deserialize // this toggle from the snapshot. reconnectZeroFillToggle(); @@ -78,15 +82,23 @@ function prepareMainThreadExecution(expandArgv1 = false) { initializeSourceMapsHandlers(); initializeDeprecations(); initializeWASI(); + + if (!initialzeModules) { + return; + } + initializeCJSLoader(); initializeESMLoader(); - const CJSLoader = require('internal/modules/cjs/loader'); assert(!CJSLoader.hasLoadedAnyUserCJSModule); loadPreloadModules(); initializeFrozenIntrinsics(); } +function refreshRuntimeOptions() { + refreshOptions(); +} + function patchProcessObject(expandArgv1) { const binding = internalBinding('process_methods'); binding.patchProcessObject(process); @@ -95,9 +107,13 @@ function patchProcessObject(expandArgv1) { ObjectDefineProperty(process, 'argv0', { enumerable: true, - configurable: false, + // Only set it to true during snapshot building. + configurable: getOptionValue('--build-snapshot'), value: process.argv[0] }); + + process.exitCode = undefined; + process._exiting = false; process.argv[0] = process.execPath; if (expandArgv1 && process.argv[1] && @@ -111,6 +127,12 @@ function patchProcessObject(expandArgv1) { } } + // We need to initialize the global console here again with process.stdout + // and friends for snapshot deserialization. + const globalConsole = require('internal/console/global'); + const { initializeGlobalConsole } = require('internal/console/constructor'); + initializeGlobalConsole(globalConsole); + // TODO(joyeecheung): most of these should be deprecated and removed, // except some that we need to be able to mutate during run time. addReadOnlyProcessAlias('_eval', '--eval'); @@ -556,6 +578,7 @@ function loadPreloadModules() { } module.exports = { + refreshRuntimeOptions, patchProcessObject, setupCoverageHooks, setupWarningHandler, diff --git a/lib/internal/bootstrap/switches/is_main_thread.js b/lib/internal/bootstrap/switches/is_main_thread.js index 606da95f21ca8f..b7bd79e09c4acf 100644 --- a/lib/internal/bootstrap/switches/is_main_thread.js +++ b/lib/internal/bootstrap/switches/is_main_thread.js @@ -122,15 +122,34 @@ let stdin; let stdout; let stderr; +let stdoutDestroy; +let stderrDestroy; + +function refreshStdoutOnSigWinch() { + stdout._refreshSize(); +} + +function refreshStderrOnSigWinch() { + stderr._refreshSize(); +} + function getStdout() { if (stdout) return stdout; stdout = createWritableStdioStream(1); stdout.destroySoon = stdout.destroy; // Override _destroy so that the fd is never actually closed. + stdoutDestroy = stdout._destroy; stdout._destroy = dummyDestroy; if (stdout.isTTY) { - process.on('SIGWINCH', () => stdout._refreshSize()); + process.on('SIGWINCH', refreshStdoutOnSigWinch); } + + internalBinding('mksnapshot').cleanups.push(function cleanupStdout() { + stdout._destroy = stdoutDestroy; + stdout.destroy(); + process.removeListener('SIGWINCH', refreshStdoutOnSigWinch); + stdout = undefined; + }); return stdout; } @@ -138,11 +157,18 @@ function getStderr() { if (stderr) return stderr; stderr = createWritableStdioStream(2); stderr.destroySoon = stderr.destroy; + stderrDestroy = stderr._destroy; // Override _destroy so that the fd is never actually closed. stderr._destroy = dummyDestroy; if (stderr.isTTY) { - process.on('SIGWINCH', () => stderr._refreshSize()); + process.on('SIGWINCH', refreshStderrOnSigWinch); } + internalBinding('mksnapshot').cleanups.push(function cleanupStderr() { + stderr._destroy = stderrDestroy; + stderr.destroy(); + process.removeListener('SIGWINCH', refreshStderrOnSigWinch); + stderr = undefined; + }); return stderr; } @@ -229,6 +255,10 @@ function getStdin() { } } + internalBinding('mksnapshot').cleanups.push(function cleanupStdin() { + stdin.destroy(); + stdin = undefined; + }); return stdin; } diff --git a/lib/internal/console/constructor.js b/lib/internal/console/constructor.js index 695a56164b7d84..5ad57be3bed6a8 100644 --- a/lib/internal/console/constructor.js +++ b/lib/internal/console/constructor.js @@ -669,9 +669,15 @@ Console.prototype.dirxml = Console.prototype.log; Console.prototype.error = Console.prototype.warn; Console.prototype.groupCollapsed = Console.prototype.group; +function initializeGlobalConsole(globalConsole) { + globalConsole[kBindStreamsLazy](process); + globalConsole[kBindProperties](true, 'auto'); +} + module.exports = { Console, kBindStreamsLazy, kBindProperties, + initializeGlobalConsole, formatTime // exported for tests }; diff --git a/lib/internal/console/global.js b/lib/internal/console/global.js index d6c0c24d529dcc..782a585957f746 100644 --- a/lib/internal/console/global.js +++ b/lib/internal/console/global.js @@ -21,9 +21,7 @@ const { } = primordials; const { - Console, - kBindStreamsLazy, - kBindProperties + Console } = require('internal/console/constructor'); const globalConsole = ObjectCreate({}); @@ -44,9 +42,6 @@ for (const prop of ReflectOwnKeys(Console.prototype)) { ReflectDefineProperty(globalConsole, prop, desc); } -globalConsole[kBindStreamsLazy](process); -globalConsole[kBindProperties](true, 'auto'); - // This is a legacy feature - the Console constructor is exposed on // the global console instance. globalConsole.Console = Console; diff --git a/lib/internal/main/mksnapshot.js b/lib/internal/main/mksnapshot.js new file mode 100644 index 00000000000000..3e1515a2d2e05e --- /dev/null +++ b/lib/internal/main/mksnapshot.js @@ -0,0 +1,142 @@ +'use strict'; + +const { + Error, + SafeSet, + SafeArrayIterator +} = primordials; + +const binding = internalBinding('mksnapshot'); +const { NativeModule } = require('internal/bootstrap/loaders'); +const { + compileSnapshotMain, +} = binding; + +const { + getOptionValue +} = require('internal/options'); + +const { + readFileSync +} = require('fs'); + +const supportedModules = new SafeSet(new SafeArrayIterator([ + // '_http_agent', + // '_http_client', + // '_http_common', + // '_http_incoming', + // '_http_outgoing', + // '_http_server', + '_stream_duplex', + '_stream_passthrough', + '_stream_readable', + '_stream_transform', + '_stream_wrap', + '_stream_writable', + // '_tls_common', + // '_tls_wrap', + 'assert', + 'assert/strict', + // 'async_hooks', + 'buffer', + // 'child_process', + // 'cluster', + 'console', + 'constants', + 'crypto', + // 'dgram', + // 'diagnostics_channel', + // 'dns', + // 'dns/promises', + // 'domain', + 'events', + 'fs', + 'fs/promises', + // 'http', + // 'http2', + // 'https', + // 'inspector', + // 'module', + // 'net', + 'os', + 'path', + 'path/posix', + 'path/win32', + // 'perf_hooks', + 'process', + 'punycode', + 'querystring', + // 'readline', + // 'repl', + 'stream', + 'stream/promises', + 'string_decoder', + 'sys', + 'timers', + 'timers/promises', + // 'tls', + // 'trace_events', + // 'tty', + 'url', + 'util', + 'util/types', + 'v8', + // 'vm', + // 'worker_threads', + // 'zlib', +])); + +const warnedModules = new SafeSet(); +function supportedInUserSnapshot(id) { + return supportedModules.has(id); +} + +function requireForUserSnapshot(id) { + if (!NativeModule.canBeRequiredByUsers(id)) { + // eslint-disable-next-line no-restricted-syntax + const err = new Error( + `Cannot find module '${id}'. ` + ); + err.code = 'MODULE_NOT_FOUND'; + throw err; + } + if (!supportedInUserSnapshot(id)) { + if (!warnedModules.has(id)) { + process.emitWarning( + `built-in module ${id} is not yet supported in user snapshots`); + warnedModules.add(id); + } + } + + return require(id); +} + +function main() { + const { + prepareMainThreadExecution + } = require('internal/bootstrap/pre_execution'); + + prepareMainThreadExecution(true, false); + process.once('beforeExit', function runCleanups() { + for (const cleanup of binding.cleanups) { + cleanup(); + } + }); + + const file = process.argv[1]; + const path = require('path'); + const filename = path.resolve(file); + const dirname = path.dirname(filename); + const source = readFileSync(file, 'utf-8'); + const snapshotMainFunction = compileSnapshotMain(filename, source); + + if (getOptionValue('--inspect-brk')) { + internalBinding('inspector').callAndPauseOnStart( + snapshotMainFunction, undefined, + requireForUserSnapshot, filename, dirname); + } else { + snapshotMainFunction(requireForUserSnapshot, filename, dirname); + } +} + +main(); diff --git a/lib/internal/options.js b/lib/internal/options.js index 01b334d4ec5614..4d92ad681a1207 100644 --- a/lib/internal/options.js +++ b/lib/internal/options.js @@ -36,6 +36,11 @@ function getEmbedderOptions() { return embedderOptions; } +function refreshOptions() { + optionsMap = undefined; + aliasesMap = undefined; +} + function getOptionValue(optionName) { const options = getCLIOptionsFromBinding(); if (optionName.startsWith('--no-')) { @@ -68,5 +73,6 @@ module.exports = { }, getOptionValue, getAllowUnauthorized, - getEmbedderOptions + getEmbedderOptions, + refreshOptions }; diff --git a/node.gyp b/node.gyp index 9f45d44e05444d..0a6eca012daad1 100644 --- a/node.gyp +++ b/node.gyp @@ -7,6 +7,7 @@ 'node_use_dtrace%': 'false', 'node_use_etw%': 'false', 'node_no_browser_globals%': 'false', + 'node_snapshot_main%': '', 'node_use_node_snapshot%': 'false', 'node_use_v8_platform%': 'true', 'node_use_bundled_v8%': 'true', @@ -315,23 +316,47 @@ 'dependencies': [ 'node_mksnapshot', ], - 'actions': [ - { - 'action_name': 'node_mksnapshot', - 'process_outputs_as_sources': 1, - 'inputs': [ - '<(node_mksnapshot_exec)', - ], - 'outputs': [ - '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + 'conditions': [ + ['node_snapshot_main!=""', { + 'actions': [ + { + 'action_name': 'node_mksnapshot', + 'process_outputs_as_sources': 1, + 'inputs': [ + '<(node_mksnapshot_exec)', + '<(node_snapshot_main)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + ], + 'action': [ + '<(node_mksnapshot_exec)', + '--build-snapshot', + '<(node_snapshot_main)', + '<@(_outputs)', + ], + }, ], - 'action': [ - '<@(_inputs)', - '<@(_outputs)', + }, { + 'actions': [ + { + 'action_name': 'node_mksnapshot', + 'process_outputs_as_sources': 1, + 'inputs': [ + '<(node_mksnapshot_exec)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + ], + 'action': [ + '<@(_inputs)', + '<@(_outputs)', + ], + }, ], - }, + }], ], - }, { + }, { 'sources': [ 'src/node_snapshot_stub.cc' ], diff --git a/src/node.cc b/src/node.cc index e503fd9f5b423c..5ba03b75407a1e 100644 --- a/src/node.cc +++ b/src/node.cc @@ -495,6 +495,10 @@ MaybeLocal StartExecution(Environment* env, StartExecutionCallback cb) { return StartExecution(env, "internal/main/inspect"); } + if (per_process::cli_options->build_snapshot) { + return StartExecution(env, "internal/main/mksnapshot"); + } + if (per_process::cli_options->print_help) { return StartExecution(env, "internal/main/print_help"); } @@ -1153,6 +1157,12 @@ int Start(int argc, char** argv) { return result.exit_code; } + if (per_process::cli_options->build_snapshot) { + fprintf(stderr, + "--build-snapshot is not yet supported in the node binary\n"); + return 1; + } + { bool use_node_snapshot = per_process::cli_options->per_isolate->node_snapshot; diff --git a/src/node_binding.cc b/src/node_binding.cc index 050c5dff0ad5fc..ed8ab9237fd3b3 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -59,6 +59,7 @@ V(js_udp_wrap) \ V(messaging) \ V(module_wrap) \ + V(mksnapshot) \ V(native_module) \ V(options) \ V(os) \ diff --git a/src/node_external_reference.h b/src/node_external_reference.h index c57a01ff39c20e..306c726631a214 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -66,6 +66,7 @@ class ExternalReferenceRegistry { V(handle_wrap) \ V(heap_utils) \ V(messaging) \ + V(mksnapshot) \ V(native_module) \ V(options) \ V(os) \ diff --git a/src/node_options.cc b/src/node_options.cc index 35ee54d223176a..bb568ce458a19a 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -725,6 +725,11 @@ PerProcessOptionsParser::PerProcessOptionsParser( "disable Object.prototype.__proto__", &PerProcessOptions::disable_proto, kAllowedInEnvironment); + AddOption("--build-snapshot", + "Generate a snapshot blob when the process exits." + "Currently only supported in the node_mksnapshot binary.", + &PerProcessOptions::build_snapshot, + kDisallowedInEnvironment); // 12.x renamed this inadvertently, so alias it for consistency within the // release line, while using the original name for consistency with older diff --git a/src/node_options.h b/src/node_options.h index 3335c12b8cdcf7..7a73e6e5987967 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -229,6 +229,7 @@ class PerProcessOptions : public Options { bool zero_fill_all_buffers = false; bool debug_arraybuffer_allocations = false; std::string disable_proto; + bool build_snapshot; std::vector security_reverts; bool print_bash_completion = false; diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 71b025df81eead..962fb5f3b536d2 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -15,15 +15,25 @@ #include "node_v8.h" #include "node_v8_platform-inl.h" +#if HAVE_INSPECTOR +#include "inspector/worker_inspector.h" // ParentInspectorHandle +#endif + namespace node { using v8::Context; +using v8::Function; +using v8::FunctionCallbackInfo; using v8::HandleScope; using v8::Isolate; using v8::Local; +using v8::MaybeLocal; using v8::Object; +using v8::ScriptCompiler; +using v8::ScriptOrigin; using v8::SnapshotCreator; using v8::StartupData; +using v8::String; using v8::TryCatch; using v8::Value; @@ -130,16 +140,48 @@ void SnapshotBuilder::Generate(SnapshotData* out, nullptr, node::EnvironmentFlags::kDefaultFlags, {}); + // Run scripts in lib/internal/bootstrap/ { TryCatch bootstrapCatch(isolate); - v8::MaybeLocal result = env->RunBootstrapping(); + MaybeLocal result = env->RunBootstrapping(); if (bootstrapCatch.HasCaught()) { PrintCaughtException(isolate, context, bootstrapCatch); } result.ToLocalChecked(); } + // If --build-snapshot is true, lib/internal/main/mksnapshot.js would be + // loaded via LoadEnvironment() to execute process.argv[1] as the entry + // point (we currently only support this kind of entry point, but we + // could also explore snapshotting other kinds of execution modes + // in the future). + if (per_process::cli_options->build_snapshot) { +#if HAVE_INSPECTOR + env->InitializeInspector({}); +#endif + + TryCatch bootstrapCatch(isolate); + // TODO(joyeecheung): we could use the result for something special, + // like setting up initializers that should be invoked at snapshot + // dehydration. + MaybeLocal result = + LoadEnvironment(env, StartExecutionCallback{}); + if (bootstrapCatch.HasCaught()) { + PrintCaughtException(isolate, context, bootstrapCatch); + } + result.ToLocalChecked(); + // FIXME(joyeecheung): right now running the loop in the snapshot + // builder seems to introduces inconsistencies in JS land that need to + // be synchronized again after snapshot restoration. + int exit_code = SpinEventLoop(env).FromMaybe(1); + CHECK_EQ(exit_code, 0); + if (bootstrapCatch.HasCaught()) { + PrintCaughtException(isolate, context, bootstrapCatch); + abort(); + } + } + if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { env->PrintAllBaseObjects(); printf("Environment = %p\n", env); @@ -309,4 +351,57 @@ void SerializeBindingData(Environment* env, }); } +namespace mksnapshot { + +static void CompileSnapshotMain(const FunctionCallbackInfo& args) { + CHECK(args[0]->IsString()); + Local filename = args[0].As(); + Local source = args[1].As(); + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + ScriptOrigin origin(isolate, filename, 0, 0, true); + // TODO(joyeecheung): do we need all of these? Maybe we would want a less + // internal version of them. + std::vector> parameters = { + FIXED_ONE_BYTE_STRING(isolate, "require"), + FIXED_ONE_BYTE_STRING(isolate, "__filename"), + FIXED_ONE_BYTE_STRING(isolate, "__dirname"), + }; + ScriptCompiler::Source script_source(source, origin); + Local fn; + if (ScriptCompiler::CompileFunctionInContext(context, + &script_source, + parameters.size(), + parameters.data(), + 0, + nullptr, + ScriptCompiler::kEagerCompile) + .ToLocal(&fn)) { + args.GetReturnValue().Set(fn); + } +} + +static void Initialize(Local target, + Local unused, + Local context, + void* priv) { + Environment* env = Environment::GetCurrent(context); + Isolate* isolate = context->GetIsolate(); + env->SetMethod(target, "compileSnapshotMain", CompileSnapshotMain); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "cleanups"), + v8::Array::New(isolate)) + .Check(); +} + +static void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(CompileSnapshotMain); + registry->Register(MarkBootstrapComplete); +} +} // namespace mksnapshot } // namespace node + +NODE_MODULE_CONTEXT_AWARE_INTERNAL(mksnapshot, node::mksnapshot::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(mksnapshot, + node::mksnapshot::RegisterExternalReferences) diff --git a/src/node_snapshotable.h b/src/node_snapshotable.h index 1ccd9a93226241..30b684eb68a2d6 100644 --- a/src/node_snapshotable.h +++ b/src/node_snapshotable.h @@ -127,6 +127,8 @@ class SnapshotBuilder { public: static std::string Generate(const std::vector args, const std::vector exec_args); + + // Generate the snapshot into out. static void Generate(SnapshotData* out, const std::vector args, const std::vector exec_args); diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index e591f64a2a0518..60062854327868 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -11,49 +11,87 @@ #include "util-inl.h" #include "v8.h" +int BuildSnapshot(int argc, char* argv[]); + #ifdef _WIN32 #include -int wmain(int argc, wchar_t* argv[]) { +int wmain(int argc, wchar_t* wargv[]) { + // Windows needs conversion from wchar_t to char. + + // Convert argv to UTF8. + char** argv = new char*[argc + 1]; + for (int i = 0; i < argc; i++) { + // Compute the size of the required buffer + DWORD size = WideCharToMultiByte( + CP_UTF8, 0, wargv[i], -1, nullptr, 0, nullptr, nullptr); + if (size == 0) { + // This should never happen. + fprintf(stderr, "Could not convert arguments to utf8."); + exit(1); + } + // Do the actual conversion + argv[i] = new char[size]; + DWORD result = WideCharToMultiByte( + CP_UTF8, 0, wargv[i], -1, argv[i], size, nullptr, nullptr); + if (result == 0) { + // This should never happen. + fprintf(stderr, "Could not convert arguments to utf8."); + exit(1); + } + } + argv[argc] = nullptr; #else // UNIX int main(int argc, char* argv[]) { argv = uv_setup_args(argc, argv); + + // Disable stdio buffering, it interacts poorly with printf() + // calls elsewhere in the program (e.g., any logging from V8.) + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); #endif // _WIN32 v8::V8::SetFlagsFromString("--random_seed=42"); + v8::V8::SetFlagsFromString("--harmony-import-assertions"); + return BuildSnapshot(argc, argv); +} +int BuildSnapshot(int argc, char* argv[]) { if (argc < 2) { std::cerr << "Usage: " << argv[0] << " \n"; + std::cerr << " " << argv[0] << " --build-snapshot " + << " \n"; return 1; } - std::ofstream out; - out.open(argv[1], std::ios::out | std::ios::binary); - if (!out.is_open()) { - std::cerr << "Cannot open " << argv[1] << "\n"; - return 1; - } - -// Windows needs conversion from wchar_t to char. See node_main.cc -#ifdef _WIN32 - int node_argc = 1; - char argv0[] = "node"; - char* node_argv[] = {argv0, nullptr}; - node::InitializationResult result = - node::InitializeOncePerProcess(node_argc, node_argv); -#else node::InitializationResult result = node::InitializeOncePerProcess(argc, argv); -#endif CHECK(!result.early_return); CHECK_EQ(result.exit_code, 0); + std::string out_path; + if (node::per_process::cli_options->build_snapshot) { + out_path = result.args[2]; + } else { + out_path = result.args[1]; + } + + std::ofstream out(out_path, std::ios::out | std::ios::binary); + if (!out) { + std::cerr << "Cannot open " << out_path << "\n"; + return 1; + } + { std::string snapshot = node::SnapshotBuilder::Generate(result.args, result.exec_args); out << snapshot; - out.close(); + + if (!out) { + std::cerr << "Failed to write " << out_path << "\n"; + return 1; + } } node::TearDownOncePerProcess();