vm: add dynamic import support

PR-URL: #22381
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Guy Bedford <[email protected]>
Reviewed-By: Tiancheng "Timothy" Gu <[email protected]>

Author: devsnek

Diff for: doc/api/errors.md

@@ -1779,6 +1779,11 @@ The V8 `BreakIterator` API was used but the full ICU data set is not installed.
 While using the Performance Timing API (`perf_hooks`), no valid performance
 entry types were found.
 
+<a id="ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING"></a>
+### ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING
+
+A dynamic import callback was not specified.
+
 <a id="ERR_VM_MODULE_ALREADY_LINKED"></a>
 ### ERR_VM_MODULE_ALREADY_LINKED
 

Diff for: doc/api/vm.md

@@ -167,10 +167,19 @@ const contextifiedSandbox = vm.createContext({ secret: 42 });
     in stack traces produced by this `Module`.
   * `columnOffset` {integer} Specifies the column number offset that is
     displayed in stack traces produced by this `Module`.
-  * `initalizeImportMeta` {Function} Called during evaluation of this `Module`
+  * `initializeImportMeta` {Function} Called during evaluation of this `Module`
     to initialize the `import.meta`. This function has the signature `(meta,
     module)`, where `meta` is the `import.meta` object in the `Module`, and
     `module` is this `vm.SourceTextModule` object.
+  * `importModuleDynamically` {Function} Called during evaluation of this
+    module when `import()` is called. This function has the signature
+    `(specifier, module)` where `specifier` is the specifier passed to
+    `import()` and `module` is this `vm.SourceTextModule`. If this option is
+    not specified, calls to `import()` will reject with
+    [`ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING`][]. This method can return a
+    [Module Namespace Object][], but returning a `vm.SourceTextModule` is
+    recommended in order to take advantage of error tracking, and to avoid
+    issues with namespaces that contain `then` function exports.
 
 Creates a new ES `Module` object.
 
@@ -436,6 +445,15 @@ changes:
     The `cachedDataProduced` value will be set to either `true` or `false`
     depending on whether code cache data is produced successfully.
     This option is deprecated in favor of `script.createCachedData()`.
+  * `importModuleDynamically` {Function} Called during evaluation of this
+    module when `import()` is called. This function has the signature
+    `(specifier, module)` where `specifier` is the specifier passed to
+    `import()` and `module` is this `vm.SourceTextModule`. If this option is
+    not specified, calls to `import()` will reject with
+    [`ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING`][]. This method can return a
+    [Module Namespace Object][], but returning a `vm.SourceTextModule` is
+    recommended in order to take advantage of error tracking, and to avoid
+    issues with namespaces that contain `then` function exports.
 
 Creating a new `vm.Script` object compiles `code` but does not run it. The
 compiled `vm.Script` can be run later multiple times. The `code` is not bound to
@@ -945,6 +963,7 @@ associating it with the `sandbox` object is what this document refers to as
 "contextifying" the `sandbox`.
 
 [`Error`]: errors.html#errors_class_error
+[`ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING`]: errors.html#ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING
 [`URL`]: url.html#url_class_url
 [`eval()`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/eval
 [`script.runInContext()`]: #vm_script_runincontext_contextifiedsandbox_options
@@ -954,6 +973,7 @@ associating it with the `sandbox` object is what this document refers to as
 [`vm.runInContext()`]: #vm_vm_runincontext_code_contextifiedsandbox_options
 [`vm.runInThisContext()`]: #vm_vm_runinthiscontext_code_options
 [GetModuleNamespace]: https://tc39.github.io/ecma262/#sec-getmodulenamespace
+[Module Namespace Object]: https://tc39.github.io/ecma262/#sec-module-namespace-exotic-objects
 [ECMAScript Module Loader]: esm.html#esm_ecmascript_modules
 [Evaluate() concrete method]: https://tc39.github.io/ecma262/#sec-moduleevaluation
 [HostResolveImportedModule]: https://tc39.github.io/ecma262/#sec-hostresolveimportedmodule

Diff for: lib/internal/bootstrap/loaders.js

@@ -107,6 +107,8 @@
     };
   }
 
+  // Create this WeakMap in js-land because V8 has no C++ API for WeakMap
+  internalBinding('module_wrap').callbackMap = new WeakMap();
   const { ContextifyScript } = internalBinding('contextify');
 
   // Set up NativeModule

Diff for: lib/internal/errors.js

@@ -873,6 +873,8 @@ E('ERR_V8BREAKITERATOR',
 // This should probably be a `TypeError`.
 E('ERR_VALID_PERFORMANCE_ENTRY_TYPE',
   'At least one valid performance entry type is required', Error);
+E('ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING',
+  'A dynamic import callback was not specified.', TypeError);
 E('ERR_VM_MODULE_ALREADY_LINKED', 'Module has already been linked', Error);
 E('ERR_VM_MODULE_DIFFERENT_CONTEXT',
   'Linked modules must use the same context', Error);

Diff for: lib/internal/modules/cjs/loader.js

@@ -29,6 +29,7 @@ const assert = require('assert').ok;
 const fs = require('fs');
 const internalFS = require('internal/fs/utils');
 const path = require('path');
+const { URL } = require('url');
 const {
   internalModuleReadJSON,
   internalModuleStat
@@ -656,6 +657,13 @@ Module.prototype.require = function(id) {
 // (needed for setting breakpoint when called with --inspect-brk)
 var resolvedArgv;
 
+function normalizeReferrerURL(referrer) {
+  if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
+    return pathToFileURL(referrer).href;
+  }
+  return new URL(referrer).href;
+}
+
 
 // Run the file contents in the correct scope or sandbox. Expose
 // the correct helper variables (require, module, exports) to
@@ -671,7 +679,12 @@ Module.prototype._compile = function(content, filename) {
   var compiledWrapper = vm.runInThisContext(wrapper, {
     filename: filename,
     lineOffset: 0,
-    displayErrors: true
+    displayErrors: true,
+    importModuleDynamically: experimentalModules ? async (specifier) => {
+      if (asyncESM === undefined) lazyLoadESM();
+      const loader = await asyncESM.loaderPromise;
+      return loader.import(specifier, normalizeReferrerURL(filename));
+    } : undefined,
   });
 
   var inspectorWrapper = null;

Diff for: lib/internal/modules/esm/translators.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { NativeModule } = require('internal/bootstrap/loaders');
-const { ModuleWrap } = internalBinding('module_wrap');
+const { ModuleWrap, callbackMap } = internalBinding('module_wrap');
 const {
   stripShebang,
   stripBOM
@@ -15,6 +15,8 @@ const { _makeLong } = require('path');
 const { SafeMap } = require('internal/safe_globals');
 const { URL } = require('url');
 const { debuglog, promisify } = require('util');
+const esmLoader = require('internal/process/esm_loader');
+
 const readFileAsync = promisify(fs.readFile);
 const readFileSync = fs.readFileSync;
 const StringReplace = Function.call.bind(String.prototype.replace);
@@ -25,13 +27,27 @@ const debug = debuglog('esm');
 const translators = new SafeMap();
 module.exports = translators;
 
+function initializeImportMeta(meta, { url }) {
+  meta.url = url;
+}
+
+async function importModuleDynamically(specifier, { url }) {
+  const loader = await esmLoader.loaderPromise;
+  return loader.import(specifier, url);
+}
+
 // Strategy for loading a standard JavaScript module
 translators.set('esm', async (url) => {
   const source = `${await readFileAsync(new URL(url))}`;
   debug(`Translating StandardModule ${url}`);
+  const module = new ModuleWrap(stripShebang(source), url);
+  callbackMap.set(module, {
+    initializeImportMeta,
+    importModuleDynamically,
+  });
   return {
-    module: new ModuleWrap(stripShebang(source), url),
-    reflect: undefined
+    module,
+    reflect: undefined,
   };
 });
 

Diff for: lib/internal/process/esm_loader.js

@@ -2,40 +2,42 @@
 
 const {
   setImportModuleDynamicallyCallback,
-  setInitializeImportMetaObjectCallback
+  setInitializeImportMetaObjectCallback,
+  callbackMap,
 } = internalBinding('module_wrap');
 
 const { pathToFileURL } = require('internal/url');
 const Loader = require('internal/modules/esm/loader');
-const path = require('path');
-const { URL } = require('url');
 const {
-  initImportMetaMap,
-  wrapToModuleMap
+  wrapToModuleMap,
 } = require('internal/vm/source_text_module');
+const {
+  ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING,
+} = require('internal/errors').codes;
 
-function normalizeReferrerURL(referrer) {
-  if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
-    return pathToFileURL(referrer).href;
+function initializeImportMetaObject(wrap, meta) {
+  if (callbackMap.has(wrap)) {
+    const { initializeImportMeta } = callbackMap.get(wrap);
+    if (initializeImportMeta !== undefined) {
+      initializeImportMeta(meta, wrapToModuleMap.get(wrap) || wrap);
+    }
   }
-  return new URL(referrer).href;
 }
 
-function initializeImportMetaObject(wrap, meta) {
-  const vmModule = wrapToModuleMap.get(wrap);
-  if (vmModule === undefined) {
-    // This ModuleWrap belongs to the Loader.
-    meta.url = wrap.url;
-  } else {
-    const initializeImportMeta = initImportMetaMap.get(vmModule);
-    if (initializeImportMeta !== undefined) {
-      // This ModuleWrap belongs to vm.SourceTextModule,
-      // initializer callback was provided.
-      initializeImportMeta(meta, vmModule);
+async function importModuleDynamicallyCallback(wrap, specifier) {
+  if (callbackMap.has(wrap)) {
+    const { importModuleDynamically } = callbackMap.get(wrap);
+    if (importModuleDynamically !== undefined) {
+      return importModuleDynamically(
+        specifier, wrapToModuleMap.get(wrap) || wrap);
     }
   }
+  throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING();
 }
 
+setInitializeImportMetaObjectCallback(initializeImportMetaObject);
+setImportModuleDynamicallyCallback(importModuleDynamicallyCallback);
+
 let loaderResolve;
 exports.loaderPromise = new Promise((resolve, reject) => {
   loaderResolve = resolve;
@@ -44,8 +46,6 @@ exports.loaderPromise = new Promise((resolve, reject) => {
 exports.ESMLoader = undefined;
 
 exports.setup = function() {
-  setInitializeImportMetaObjectCallback(initializeImportMetaObject);
-
   let ESMLoader = new Loader();
   const loaderPromise = (async () => {
     const userLoader = process.binding('config').userLoader;
@@ -60,10 +60,5 @@ exports.setup = function() {
   })();
   loaderResolve(loaderPromise);
 
-  setImportModuleDynamicallyCallback(async (referrer, specifier) => {
-    const loader = await loaderPromise;
-    return loader.import(specifier, normalizeReferrerURL(referrer));
-  });
-
   exports.ESMLoader = ESMLoader;
 };

Diff for: lib/internal/vm/source_text_module.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const { isModuleNamespaceObject } = require('util').types;
 const { URL } = require('internal/url');
 const { isContext } = internalBinding('contextify');
 const {
@@ -9,7 +10,7 @@ const {
   ERR_VM_MODULE_LINKING_ERRORED,
   ERR_VM_MODULE_NOT_LINKED,
   ERR_VM_MODULE_NOT_MODULE,
-  ERR_VM_MODULE_STATUS
+  ERR_VM_MODULE_STATUS,
 } = require('internal/errors').codes;
 const {
   getConstructorOf,
@@ -21,6 +22,7 @@ const { validateInt32, validateUint32 } = require('internal/validators');
 
 const {
   ModuleWrap,
+  callbackMap,
   kUninstantiated,
   kInstantiating,
   kInstantiated,
@@ -43,8 +45,6 @@ const perContextModuleId = new WeakMap();
 const wrapMap = new WeakMap();
 const dependencyCacheMap = new WeakMap();
 const linkingStatusMap = new WeakMap();
-// vm.SourceTextModule -> function
-const initImportMetaMap = new WeakMap();
 // ModuleWrap -> vm.SourceTextModule
 const wrapToModuleMap = new WeakMap();
 const defaultModuleName = 'vm:module';
@@ -63,7 +63,8 @@ class SourceTextModule {
       context,
       lineOffset = 0,
       columnOffset = 0,
-      initializeImportMeta
+      initializeImportMeta,
+      importModuleDynamically,
     } = options;
 
     if (context !== undefined) {
@@ -96,20 +97,39 @@ class SourceTextModule {
     validateInt32(lineOffset, 'options.lineOffset');
     validateInt32(columnOffset, 'options.columnOffset');
 
-    if (initializeImportMeta !== undefined) {
-      if (typeof initializeImportMeta === 'function') {
-        initImportMetaMap.set(this, initializeImportMeta);
-      } else {
-        throw new ERR_INVALID_ARG_TYPE(
-          'options.initializeImportMeta', 'function', initializeImportMeta);
-      }
+    if (initializeImportMeta !== undefined &&
+        typeof initializeImportMeta !== 'function') {
+      throw new ERR_INVALID_ARG_TYPE(
+        'options.initializeImportMeta', 'function', initializeImportMeta);
+    }
+
+    if (importModuleDynamically !== undefined &&
+        typeof importModuleDynamically !== 'function') {
+      throw new ERR_INVALID_ARG_TYPE(
+        'options.importModuleDynamically', 'function', importModuleDynamically);
     }
 
     const wrap = new ModuleWrap(src, url, context, lineOffset, columnOffset);
     wrapMap.set(this, wrap);
     linkingStatusMap.set(this, 'unlinked');
     wrapToModuleMap.set(wrap, this);
 
+    callbackMap.set(wrap, {
+      initializeImportMeta,
+      importModuleDynamically: importModuleDynamically ? async (...args) => {
+        const m = await importModuleDynamically(...args);
+        if (isModuleNamespaceObject(m)) {
+          return m;
+        }
+        if (!m || !wrapMap.has(m))
+          throw new ERR_VM_MODULE_NOT_MODULE();
+        const childLinkingStatus = linkingStatusMap.get(m);
+        if (childLinkingStatus === 'errored')
+          throw m.error;
+        return m.namespace;
+      } : undefined,
+    });
+
     Object.defineProperties(this, {
       url: { value: url, enumerable: true },
       context: { value: context, enumerable: true },
@@ -245,6 +265,7 @@ class SourceTextModule {
 
 module.exports = {
   SourceTextModule,
-  initImportMetaMap,
-  wrapToModuleMap
+  wrapToModuleMap,
+  wrapMap,
+  linkingStatusMap,
 };