Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bootstrap: support configure-time user-land snapshot #42466

Closed
19 changes: 19 additions & 0 deletions configure.py
Original file line number Diff line number Diff line change
@@ -788,6 +788,13 @@
default=False,
help='node will load builtin modules from disk instead of from binary')

parser.add_argument('--node-snapshot-main',
action='store',
dest='node_snapshot_main',
default=None,
help='Run a file when building the embedded snapshot. Currently ' +
'experimental.')

# Create compile_commands.json in out/Debug and out/Release.
parser.add_argument('-C',
action='store_true',
@@ -1216,6 +1223,18 @@ def configure_node(o):

o['variables']['want_separate_host_toolset'] = int(cross_compiling)

if options.node_snapshot_main is not None:
if options.shared:
# This should be possible to fix, but we will need to refactor the
# libnode target to avoid building it twice.
error('--node-snapshot-main is incompatible with --shared')
if options.without_node_snapshot:
error('--node-snapshot-main is incompatible with ' +
'--without-node-snapshot')
if cross_compiling:
error('--node-snapshot-main is incompatible with cross compilation')
o['variables']['node_snapshot_main'] = options.node_snapshot_main

if options.without_node_snapshot or options.node_builtin_modules_path:
o['variables']['node_use_node_snapshot'] = 'false'
else:
29 changes: 26 additions & 3 deletions lib/internal/bootstrap/pre_execution.js
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ const {
const {
getOptionValue,
getEmbedderOptions,
refreshOptions,
} = require('internal/options');
const { reconnectZeroFillToggle } = require('internal/buffer');
const {
@@ -26,7 +27,10 @@ const { Buffer } = require('buffer');
const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes;
const assert = require('internal/assert');

function prepareMainThreadExecution(expandArgv1 = false) {
function prepareMainThreadExecution(expandArgv1 = false,
initialzeModules = true) {
refreshRuntimeOptions();

// TODO(joyeecheung): this is also necessary for workers when they deserialize
// this toggle from the snapshot.
reconnectZeroFillToggle();
@@ -78,15 +82,23 @@ function prepareMainThreadExecution(expandArgv1 = false) {
initializeSourceMapsHandlers();
initializeDeprecations();
initializeWASI();

if (!initialzeModules) {
return;
}

initializeCJSLoader();
initializeESMLoader();

const CJSLoader = require('internal/modules/cjs/loader');
assert(!CJSLoader.hasLoadedAnyUserCJSModule);
loadPreloadModules();
initializeFrozenIntrinsics();
}

function refreshRuntimeOptions() {
refreshOptions();
}

function patchProcessObject(expandArgv1) {
const binding = internalBinding('process_methods');
binding.patchProcessObject(process);
@@ -95,9 +107,13 @@ function patchProcessObject(expandArgv1) {

ObjectDefineProperty(process, 'argv0', {
enumerable: true,
configurable: false,
// Only set it to true during snapshot building.
configurable: getOptionValue('--build-snapshot'),
value: process.argv[0]
});

process.exitCode = undefined;
process._exiting = false;
process.argv[0] = process.execPath;

if (expandArgv1 && process.argv[1] &&
@@ -111,6 +127,12 @@ function patchProcessObject(expandArgv1) {
}
}

// We need to initialize the global console here again with process.stdout
// and friends for snapshot deserialization.
const globalConsole = require('internal/console/global');
const { initializeGlobalConsole } = require('internal/console/constructor');
initializeGlobalConsole(globalConsole);

// TODO(joyeecheung): most of these should be deprecated and removed,
// except some that we need to be able to mutate during run time.
addReadOnlyProcessAlias('_eval', '--eval');
@@ -556,6 +578,7 @@ function loadPreloadModules() {
}

module.exports = {
refreshRuntimeOptions,
patchProcessObject,
setupCoverageHooks,
setupWarningHandler,
34 changes: 32 additions & 2 deletions lib/internal/bootstrap/switches/is_main_thread.js
Original file line number Diff line number Diff line change
@@ -122,27 +122,53 @@ let stdin;
let stdout;
let stderr;

let stdoutDestroy;
let stderrDestroy;

function refreshStdoutOnSigWinch() {
stdout._refreshSize();
}

function refreshStderrOnSigWinch() {
stderr._refreshSize();
}

function getStdout() {
if (stdout) return stdout;
stdout = createWritableStdioStream(1);
stdout.destroySoon = stdout.destroy;
// Override _destroy so that the fd is never actually closed.
stdoutDestroy = stdout._destroy;
stdout._destroy = dummyDestroy;
if (stdout.isTTY) {
process.on('SIGWINCH', () => stdout._refreshSize());
process.on('SIGWINCH', refreshStdoutOnSigWinch);
}

internalBinding('mksnapshot').cleanups.push(function cleanupStdout() {
stdout._destroy = stdoutDestroy;
stdout.destroy();
process.removeListener('SIGWINCH', refreshStdoutOnSigWinch);
stdout = undefined;
});
return stdout;
}

function getStderr() {
if (stderr) return stderr;
stderr = createWritableStdioStream(2);
stderr.destroySoon = stderr.destroy;
stderrDestroy = stderr._destroy;
// Override _destroy so that the fd is never actually closed.
stderr._destroy = dummyDestroy;
if (stderr.isTTY) {
process.on('SIGWINCH', () => stderr._refreshSize());
process.on('SIGWINCH', refreshStderrOnSigWinch);
}
internalBinding('mksnapshot').cleanups.push(function cleanupStderr() {
stderr._destroy = stderrDestroy;
stderr.destroy();
process.removeListener('SIGWINCH', refreshStderrOnSigWinch);
stderr = undefined;
});
return stderr;
}

@@ -229,6 +255,10 @@ function getStdin() {
}
}

internalBinding('mksnapshot').cleanups.push(function cleanupStdin() {
stdin.destroy();
stdin = undefined;
});
return stdin;
}

6 changes: 6 additions & 0 deletions lib/internal/console/constructor.js
Original file line number Diff line number Diff line change
@@ -669,9 +669,15 @@ Console.prototype.dirxml = Console.prototype.log;
Console.prototype.error = Console.prototype.warn;
Console.prototype.groupCollapsed = Console.prototype.group;

function initializeGlobalConsole(globalConsole) {
globalConsole[kBindStreamsLazy](process);
globalConsole[kBindProperties](true, 'auto');
}

module.exports = {
Console,
kBindStreamsLazy,
kBindProperties,
initializeGlobalConsole,
formatTime // exported for tests
};
7 changes: 1 addition & 6 deletions lib/internal/console/global.js
Original file line number Diff line number Diff line change
@@ -21,9 +21,7 @@ const {
} = primordials;

const {
Console,
kBindStreamsLazy,
kBindProperties
Console
} = require('internal/console/constructor');

const globalConsole = ObjectCreate({});
@@ -44,9 +42,6 @@ for (const prop of ReflectOwnKeys(Console.prototype)) {
ReflectDefineProperty(globalConsole, prop, desc);
}

globalConsole[kBindStreamsLazy](process);
globalConsole[kBindProperties](true, 'auto');

// This is a legacy feature - the Console constructor is exposed on
// the global console instance.
globalConsole.Console = Console;
142 changes: 142 additions & 0 deletions lib/internal/main/mksnapshot.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
'use strict';

const {
Error,
SafeSet,
SafeArrayIterator
} = primordials;

const binding = internalBinding('mksnapshot');
const { NativeModule } = require('internal/bootstrap/loaders');
const {
compileSnapshotMain,
} = binding;

const {
getOptionValue
} = require('internal/options');

const {
readFileSync
} = require('fs');

const supportedModules = new SafeSet(new SafeArrayIterator([
// '_http_agent',
// '_http_client',
// '_http_common',
// '_http_incoming',
// '_http_outgoing',
// '_http_server',
'_stream_duplex',
'_stream_passthrough',
'_stream_readable',
'_stream_transform',
'_stream_wrap',
'_stream_writable',
// '_tls_common',
// '_tls_wrap',
'assert',
'assert/strict',
// 'async_hooks',
'buffer',
// 'child_process',
// 'cluster',
'console',
'constants',
'crypto',
// 'dgram',
// 'diagnostics_channel',
// 'dns',
// 'dns/promises',
// 'domain',
'events',
'fs',
'fs/promises',
// 'http',
// 'http2',
// 'https',
// 'inspector',
// 'module',
// 'net',
'os',
'path',
'path/posix',
'path/win32',
// 'perf_hooks',
'process',
'punycode',
'querystring',
// 'readline',
// 'repl',
'stream',
'stream/promises',
'string_decoder',
'sys',
'timers',
'timers/promises',
// 'tls',
// 'trace_events',
// 'tty',
'url',
'util',
'util/types',
'v8',
// 'vm',
// 'worker_threads',
// 'zlib',
]));

const warnedModules = new SafeSet();
function supportedInUserSnapshot(id) {
return supportedModules.has(id);
}

function requireForUserSnapshot(id) {
if (!NativeModule.canBeRequiredByUsers(id)) {
// eslint-disable-next-line no-restricted-syntax
const err = new Error(
`Cannot find module '${id}'. `
);
err.code = 'MODULE_NOT_FOUND';
throw err;
}
if (!supportedInUserSnapshot(id)) {
if (!warnedModules.has(id)) {
process.emitWarning(
`built-in module ${id} is not yet supported in user snapshots`);
warnedModules.add(id);
}
}

return require(id);
}

function main() {
const {
prepareMainThreadExecution
} = require('internal/bootstrap/pre_execution');

prepareMainThreadExecution(true, false);
process.once('beforeExit', function runCleanups() {
for (const cleanup of binding.cleanups) {
cleanup();
}
});

const file = process.argv[1];
const path = require('path');
const filename = path.resolve(file);
const dirname = path.dirname(filename);
const source = readFileSync(file, 'utf-8');
const snapshotMainFunction = compileSnapshotMain(filename, source);

if (getOptionValue('--inspect-brk')) {
internalBinding('inspector').callAndPauseOnStart(
snapshotMainFunction, undefined,
requireForUserSnapshot, filename, dirname);
} else {
snapshotMainFunction(requireForUserSnapshot, filename, dirname);
}
}

main();
8 changes: 7 additions & 1 deletion lib/internal/options.js
Original file line number Diff line number Diff line change
@@ -36,6 +36,11 @@ function getEmbedderOptions() {
return embedderOptions;
}

function refreshOptions() {
optionsMap = undefined;
aliasesMap = undefined;
}

function getOptionValue(optionName) {
const options = getCLIOptionsFromBinding();
if (optionName.startsWith('--no-')) {
@@ -68,5 +73,6 @@ module.exports = {
},
getOptionValue,
getAllowUnauthorized,
getEmbedderOptions
getEmbedderOptions,
refreshOptions
};
53 changes: 39 additions & 14 deletions node.gyp
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
'node_use_dtrace%': 'false',
'node_use_etw%': 'false',
'node_no_browser_globals%': 'false',
'node_snapshot_main%': '',
'node_use_node_snapshot%': 'false',
'node_use_v8_platform%': 'true',
'node_use_bundled_v8%': 'true',
@@ -315,23 +316,47 @@
'dependencies': [
'node_mksnapshot',
],
'actions': [
{
'action_name': 'node_mksnapshot',
'process_outputs_as_sources': 1,
'inputs': [
'<(node_mksnapshot_exec)',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
'conditions': [
['node_snapshot_main!=""', {
'actions': [
{
'action_name': 'node_mksnapshot',
'process_outputs_as_sources': 1,
'inputs': [
'<(node_mksnapshot_exec)',
'<(node_snapshot_main)',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
],
'action': [
'<(node_mksnapshot_exec)',
'--build-snapshot',
'<(node_snapshot_main)',
'<@(_outputs)',
],
},
],
'action': [
'<@(_inputs)',
'<@(_outputs)',
}, {
'actions': [
{
'action_name': 'node_mksnapshot',
'process_outputs_as_sources': 1,
'inputs': [
'<(node_mksnapshot_exec)',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc',
],
'action': [
'<@(_inputs)',
'<@(_outputs)',
],
},
],
},
}],
],
}, {
}, {
'sources': [
'src/node_snapshot_stub.cc'
],
10 changes: 10 additions & 0 deletions src/node.cc
Original file line number Diff line number Diff line change
@@ -495,6 +495,10 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
return StartExecution(env, "internal/main/inspect");
}

if (per_process::cli_options->build_snapshot) {
return StartExecution(env, "internal/main/mksnapshot");
}

if (per_process::cli_options->print_help) {
return StartExecution(env, "internal/main/print_help");
}
@@ -1153,6 +1157,12 @@ int Start(int argc, char** argv) {
return result.exit_code;
}

if (per_process::cli_options->build_snapshot) {
fprintf(stderr,
"--build-snapshot is not yet supported in the node binary\n");
return 1;
}

{
bool use_node_snapshot =
per_process::cli_options->per_isolate->node_snapshot;
1 change: 1 addition & 0 deletions src/node_binding.cc
Original file line number Diff line number Diff line change
@@ -59,6 +59,7 @@
V(js_udp_wrap) \
V(messaging) \
V(module_wrap) \
V(mksnapshot) \
V(native_module) \
V(options) \
V(os) \
1 change: 1 addition & 0 deletions src/node_external_reference.h
Original file line number Diff line number Diff line change
@@ -66,6 +66,7 @@ class ExternalReferenceRegistry {
V(handle_wrap) \
V(heap_utils) \
V(messaging) \
V(mksnapshot) \
V(native_module) \
V(options) \
V(os) \
5 changes: 5 additions & 0 deletions src/node_options.cc
Original file line number Diff line number Diff line change
@@ -725,6 +725,11 @@ PerProcessOptionsParser::PerProcessOptionsParser(
"disable Object.prototype.__proto__",
&PerProcessOptions::disable_proto,
kAllowedInEnvironment);
AddOption("--build-snapshot",
"Generate a snapshot blob when the process exits."
"Currently only supported in the node_mksnapshot binary.",
&PerProcessOptions::build_snapshot,
kDisallowedInEnvironment);

// 12.x renamed this inadvertently, so alias it for consistency within the
// release line, while using the original name for consistency with older
1 change: 1 addition & 0 deletions src/node_options.h
Original file line number Diff line number Diff line change
@@ -229,6 +229,7 @@ class PerProcessOptions : public Options {
bool zero_fill_all_buffers = false;
bool debug_arraybuffer_allocations = false;
std::string disable_proto;
bool build_snapshot;

std::vector<std::string> security_reverts;
bool print_bash_completion = false;
97 changes: 96 additions & 1 deletion src/node_snapshotable.cc
Original file line number Diff line number Diff line change
@@ -15,15 +15,25 @@
#include "node_v8.h"
#include "node_v8_platform-inl.h"

#if HAVE_INSPECTOR
#include "inspector/worker_inspector.h" // ParentInspectorHandle
#endif

namespace node {

using v8::Context;
using v8::Function;
using v8::FunctionCallbackInfo;
using v8::HandleScope;
using v8::Isolate;
using v8::Local;
using v8::MaybeLocal;
using v8::Object;
using v8::ScriptCompiler;
using v8::ScriptOrigin;
using v8::SnapshotCreator;
using v8::StartupData;
using v8::String;
using v8::TryCatch;
using v8::Value;

@@ -130,16 +140,48 @@ void SnapshotBuilder::Generate(SnapshotData* out,
nullptr,
node::EnvironmentFlags::kDefaultFlags,
{});

// Run scripts in lib/internal/bootstrap/
{
TryCatch bootstrapCatch(isolate);
v8::MaybeLocal<Value> result = env->RunBootstrapping();
MaybeLocal<Value> result = env->RunBootstrapping();
if (bootstrapCatch.HasCaught()) {
PrintCaughtException(isolate, context, bootstrapCatch);
}
result.ToLocalChecked();
}

// If --build-snapshot is true, lib/internal/main/mksnapshot.js would be
// loaded via LoadEnvironment() to execute process.argv[1] as the entry
// point (we currently only support this kind of entry point, but we
// could also explore snapshotting other kinds of execution modes
// in the future).
if (per_process::cli_options->build_snapshot) {
#if HAVE_INSPECTOR
env->InitializeInspector({});
#endif

TryCatch bootstrapCatch(isolate);
// TODO(joyeecheung): we could use the result for something special,
// like setting up initializers that should be invoked at snapshot
// dehydration.
MaybeLocal<Value> result =
LoadEnvironment(env, StartExecutionCallback{});
if (bootstrapCatch.HasCaught()) {
PrintCaughtException(isolate, context, bootstrapCatch);
}
result.ToLocalChecked();
// FIXME(joyeecheung): right now running the loop in the snapshot
// builder seems to introduces inconsistencies in JS land that need to
// be synchronized again after snapshot restoration.
int exit_code = SpinEventLoop(env).FromMaybe(1);
CHECK_EQ(exit_code, 0);
if (bootstrapCatch.HasCaught()) {
PrintCaughtException(isolate, context, bootstrapCatch);
abort();
}
}

if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) {
env->PrintAllBaseObjects();
printf("Environment = %p\n", env);
@@ -309,4 +351,57 @@ void SerializeBindingData(Environment* env,
});
}

namespace mksnapshot {

static void CompileSnapshotMain(const FunctionCallbackInfo<Value>& args) {
CHECK(args[0]->IsString());
Local<String> filename = args[0].As<String>();
Local<String> source = args[1].As<String>();
Isolate* isolate = args.GetIsolate();
Local<Context> context = isolate->GetCurrentContext();
ScriptOrigin origin(isolate, filename, 0, 0, true);
// TODO(joyeecheung): do we need all of these? Maybe we would want a less
// internal version of them.
std::vector<Local<String>> parameters = {
FIXED_ONE_BYTE_STRING(isolate, "require"),
FIXED_ONE_BYTE_STRING(isolate, "__filename"),
FIXED_ONE_BYTE_STRING(isolate, "__dirname"),
};
ScriptCompiler::Source script_source(source, origin);
Local<Function> fn;
if (ScriptCompiler::CompileFunctionInContext(context,
&script_source,
parameters.size(),
parameters.data(),
0,
nullptr,
ScriptCompiler::kEagerCompile)
.ToLocal(&fn)) {
args.GetReturnValue().Set(fn);
}
}

static void Initialize(Local<Object> target,
Local<Value> unused,
Local<Context> context,
void* priv) {
Environment* env = Environment::GetCurrent(context);
Isolate* isolate = context->GetIsolate();
env->SetMethod(target, "compileSnapshotMain", CompileSnapshotMain);
target
->Set(context,
FIXED_ONE_BYTE_STRING(isolate, "cleanups"),
v8::Array::New(isolate))
.Check();
}

static void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
registry->Register(CompileSnapshotMain);
registry->Register(MarkBootstrapComplete);
}
} // namespace mksnapshot
} // namespace node

NODE_MODULE_CONTEXT_AWARE_INTERNAL(mksnapshot, node::mksnapshot::Initialize)
NODE_MODULE_EXTERNAL_REFERENCE(mksnapshot,
node::mksnapshot::RegisterExternalReferences)
2 changes: 2 additions & 0 deletions src/node_snapshotable.h
Original file line number Diff line number Diff line change
@@ -127,6 +127,8 @@ class SnapshotBuilder {
public:
static std::string Generate(const std::vector<std::string> args,
const std::vector<std::string> exec_args);

// Generate the snapshot into out.
static void Generate(SnapshotData* out,
const std::vector<std::string> args,
const std::vector<std::string> exec_args);
74 changes: 56 additions & 18 deletions tools/snapshot/node_mksnapshot.cc
Original file line number Diff line number Diff line change
@@ -11,49 +11,87 @@
#include "util-inl.h"
#include "v8.h"

int BuildSnapshot(int argc, char* argv[]);

#ifdef _WIN32
#include <windows.h>

int wmain(int argc, wchar_t* argv[]) {
int wmain(int argc, wchar_t* wargv[]) {
// Windows needs conversion from wchar_t to char.

// Convert argv to UTF8.
char** argv = new char*[argc + 1];
for (int i = 0; i < argc; i++) {
// Compute the size of the required buffer
DWORD size = WideCharToMultiByte(
CP_UTF8, 0, wargv[i], -1, nullptr, 0, nullptr, nullptr);
if (size == 0) {
// This should never happen.
fprintf(stderr, "Could not convert arguments to utf8.");
exit(1);
}
// Do the actual conversion
argv[i] = new char[size];
DWORD result = WideCharToMultiByte(
CP_UTF8, 0, wargv[i], -1, argv[i], size, nullptr, nullptr);
if (result == 0) {
// This should never happen.
fprintf(stderr, "Could not convert arguments to utf8.");
exit(1);
}
}
argv[argc] = nullptr;
#else // UNIX
int main(int argc, char* argv[]) {
argv = uv_setup_args(argc, argv);

// Disable stdio buffering, it interacts poorly with printf()
// calls elsewhere in the program (e.g., any logging from V8.)
setvbuf(stdout, nullptr, _IONBF, 0);
setvbuf(stderr, nullptr, _IONBF, 0);
#endif // _WIN32

v8::V8::SetFlagsFromString("--random_seed=42");
v8::V8::SetFlagsFromString("--harmony-import-assertions");
return BuildSnapshot(argc, argv);
}

int BuildSnapshot(int argc, char* argv[]) {
if (argc < 2) {
std::cerr << "Usage: " << argv[0] << " <path/to/output.cc>\n";
std::cerr << " " << argv[0] << " --build-snapshot "
<< "<path/to/script.js> <path/to/output.cc>\n";
return 1;
}

std::ofstream out;
out.open(argv[1], std::ios::out | std::ios::binary);
if (!out.is_open()) {
std::cerr << "Cannot open " << argv[1] << "\n";
return 1;
}

// Windows needs conversion from wchar_t to char. See node_main.cc
#ifdef _WIN32
int node_argc = 1;
char argv0[] = "node";
char* node_argv[] = {argv0, nullptr};
node::InitializationResult result =
node::InitializeOncePerProcess(node_argc, node_argv);
#else
node::InitializationResult result =
node::InitializeOncePerProcess(argc, argv);
#endif

CHECK(!result.early_return);
CHECK_EQ(result.exit_code, 0);

std::string out_path;
if (node::per_process::cli_options->build_snapshot) {
out_path = result.args[2];
} else {
out_path = result.args[1];
}

std::ofstream out(out_path, std::ios::out | std::ios::binary);
if (!out) {
std::cerr << "Cannot open " << out_path << "\n";
return 1;
}

{
std::string snapshot =
node::SnapshotBuilder::Generate(result.args, result.exec_args);
out << snapshot;
out.close();

if (!out) {
std::cerr << "Failed to write " << out_path << "\n";
return 1;
}
}

node::TearDownOncePerProcess();