Skip to content

Commit 34c0a7b

Browse files
committed
Create proof of concept module loader implementation
1 parent 2a0d2e0 commit 34c0a7b

File tree

4 files changed

+293
-2
lines changed

4 files changed

+293
-2
lines changed

packages/plugin-pnp/sources/PnpLinker.ts

+13-1
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ import {Linker, LinkOptions, MinimalLinkOptions, Manifest, MessageName, Dependen
22
import {FetchResult, Ident, Locator, Package, BuildDirective, BuildType} from '@yarnpkg/core';
33
import {miscUtils, structUtils} from '@yarnpkg/core';
44
import {CwdFS, FakeFS, PortablePath, npath, ppath, toFilename, xfs} from '@yarnpkg/fslib';
5-
import {generateInlinedScript, generateSplitScript, PnpSettings} from '@yarnpkg/pnp';
5+
import {generateInlinedScript, generateModuleLoader, generateSplitScript, PnpSettings} from '@yarnpkg/pnp';
66
import {UsageError} from 'clipanion';
7+
import {basename} from 'path';
78

89
import {AbstractPnpInstaller} from './AbstractPnpInstaller';
910
import {getPnpPath} from './index';
@@ -118,9 +119,11 @@ export class PnpInstaller extends AbstractPnpInstaller {
118119
const pnpDataPath = this.opts.project.configuration.get(`pnpDataPath`);
119120

120121
await xfs.removePromise(pnpPath.other);
122+
await xfs.removePromise(pnpPath.otherLoader);
121123

122124
if (this.opts.project.configuration.get(`nodeLinker`) !== `pnp`) {
123125
await xfs.removePromise(pnpPath.main);
126+
await xfs.removePromise(pnpPath.mainLoader);
124127
await xfs.removePromise(pnpDataPath);
125128

126129
return;
@@ -152,6 +155,15 @@ export class PnpInstaller extends AbstractPnpInstaller {
152155
await xfs.chmodPromise(pnpDataPath, 0o644);
153156
}
154157

158+
if (this.opts.project.configuration.get(`pnpEnableExperimentalLoader`)) {
159+
const loaderFile = generateModuleLoader(basename(pnpPath.main));
160+
161+
await xfs.changeFilePromise(pnpPath.mainLoader, loaderFile, {automaticNewlines: true});
162+
await xfs.chmodPromise(pnpPath.mainLoader, 0o755);
163+
} else {
164+
await xfs.removePromise(pnpPath.mainLoader);
165+
}
166+
155167
const pnpUnpluggedFolder = this.opts.project.configuration.get(`pnpUnpluggedFolder`);
156168
if (this.unpluggedPaths.size === 0) {
157169
await xfs.removePromise(pnpUnpluggedFolder);

packages/plugin-pnp/sources/index.ts

+29-1
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,28 @@ export const getPnpPath = (project: Project) => {
1111
let mainFilename;
1212
let otherFilename;
1313

14+
let mainLoaderFilename;
15+
let otherLoaderFilename;
16+
1417
if (project.topLevelWorkspace.manifest.type === `module`) {
1518
mainFilename = `.pnp.cjs`;
1619
otherFilename = `.pnp.js`;
20+
21+
mainLoaderFilename = '.pnp-loader.js';
22+
otherLoaderFilename = '.pnp-loader.mjs';
1723
} else {
1824
mainFilename = `.pnp.js`;
1925
otherFilename = `.pnp.cjs`;
26+
27+
mainLoaderFilename = '.pnp-loader.mjs';
28+
otherLoaderFilename = '.pnp-loader.js';
2029
}
2130

2231
return {
2332
main: ppath.join(project.cwd, mainFilename as Filename),
2433
other: ppath.join(project.cwd, otherFilename as Filename),
34+
mainLoader: ppath.join(project.cwd, mainLoaderFilename as Filename),
35+
otherLoader: ppath.join(project.cwd, otherLoaderFilename as Filename),
2536
};
2637
};
2738

@@ -30,8 +41,9 @@ export const quotePathIfNeeded = (path: string) => {
3041
};
3142

3243
async function setupScriptEnvironment(project: Project, env: {[key: string]: string}, makePathWrapper: (name: string, argv0: string, args: Array<string>) => Promise<void>) {
33-
const pnpPath: PortablePath = getPnpPath(project).main;
44+
const {main: pnpPath, mainLoader: pnpLoaderPath} = getPnpPath(project);
3445
const pnpRequire = `--require ${quotePathIfNeeded(npath.fromPortablePath(pnpPath))}`;
46+
const enablePnpLoader = `--experimental-loader ${quotePathIfNeeded(npath.fromPortablePath(pnpLoaderPath))}`;
3547

3648
if (pnpPath.includes(' ') && semver.lt(process.versions.node, '12.0.0'))
3749
throw new Error(`Expected the build location to not include spaces when using Node < 12.0.0 (${process.versions.node})`);
@@ -44,11 +56,22 @@ async function setupScriptEnvironment(project: Project, env: {[key: string]: str
4456

4557
env.NODE_OPTIONS = nodeOptions;
4658
}
59+
60+
if (project.configuration.get(`pnpEnableExperimentalLoader`) && xfs.existsSync(pnpLoaderPath)) {
61+
let nodeOptions = env.NODE_OPTIONS || ``;
62+
63+
nodeOptions = nodeOptions.replace(/\s*--experimental-loader\s+\S*\.pnp-loader\.m?js\s*/g, ` `).trim();
64+
nodeOptions = nodeOptions ? `${enablePnpLoader} ${nodeOptions}` : enablePnpLoader;
65+
66+
env.NODE_OPTIONS = nodeOptions;
67+
}
4768
}
4869

4970
async function populateYarnPaths(project: Project, definePath: (path: PortablePath | null) => void) {
5071
definePath(getPnpPath(project).main);
5172
definePath(getPnpPath(project).other);
73+
definePath(getPnpPath(project).mainLoader);
74+
definePath(getPnpPath(project).otherLoader);
5275

5376
definePath(project.configuration.get(`pnpDataPath`));
5477
definePath(project.configuration.get(`pnpUnpluggedFolder`));
@@ -101,6 +124,11 @@ const plugin: Plugin<CoreHooks & StageHooks> = {
101124
type: SettingsType.ABSOLUTE_PATH,
102125
default: `./.pnp.data.json`,
103126
},
127+
pnpEnableExperimentalLoader: {
128+
description: `Enables experimental support for ESM loaders`,
129+
type: SettingsType.BOOLEAN,
130+
default: false,
131+
},
104132
},
105133
linkers: [
106134
PnpLinker,
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
export function generateModuleLoader(pnpFilename: string) {
2+
return `import {createRequire, builtinModules} from 'module';
3+
import {URL, pathToFileURL, fileURLToPath} from 'url';
4+
import {join, extname} from 'path';
5+
6+
let require = createRequire(import.meta.url);
7+
8+
const pnpapi = require('./${pnpFilename}');
9+
pnpapi.setup();
10+
11+
/**
12+
* Node's fs module, patched by pnp
13+
*
14+
* @type {typeof import('fs')}
15+
*/
16+
const fs = require('fs');
17+
18+
/**
19+
* Read the given file, can read into zipfiles of the yarn cache
20+
*
21+
* @param {string} path
22+
* @returns {Promise<Buffer>}
23+
*/
24+
function readFile(path) {
25+
return new Promise((resolve, reject) => {
26+
fs.readFile(path, (err, content) => err ? reject(err) : resolve(content));
27+
});
28+
}
29+
30+
/**
31+
* Check whether the given string is a valid URL
32+
*
33+
* @param {string} str
34+
* @returns {boolean}
35+
*/
36+
function isValidURL(str) {
37+
try {
38+
new URL(str);
39+
return true;
40+
} catch {
41+
return false;
42+
}
43+
}
44+
45+
/**
46+
* Check whether the given identifier is a raw identifier, or absolute/relative
47+
*
48+
* @param {string} identifier
49+
* @returns {boolean} true if the identifier is raw, false if it's absolute or relative
50+
*/
51+
function isRawIdentifier(identifier) {
52+
return !identifier.startsWith('/') && !identifier.startsWith('./') && !identifier.startsWith('../');
53+
}
54+
55+
/**
56+
* Resolve the given identifier to a URL
57+
*
58+
* This loader adds two special types of URL:
59+
*
60+
* Certain node builtins are replaced with patched versions (fs). For those identifiers this function
61+
* returns a URL with a custom \`yarn-builtin:\` protocol.
62+
*
63+
* Files that are resolved to be coming from the yarn cache get a \`?yarn-cache\` query parameter added
64+
* to the actual \`file:\` URL.
65+
*
66+
* @param {string} identifier
67+
* @param {object} context
68+
* @param {string} context.parentURL
69+
* @param {string[]} context.conditions
70+
* @param {typeof resolve} defaultResolve
71+
* @returns {Promise<{url: string}>}
72+
*/
73+
export async function resolve(identifier, context, defaultResolve) {
74+
if (identifier === 'fs' || identifier === 'fs/promises') {
75+
const url = new URL(\`yarn-builtin:///\${identifier}\`);
76+
url.searchParams.set('actual', defaultResolve(identifier, context, defaultResolve).url);
77+
78+
return {
79+
url: url.href,
80+
};
81+
}
82+
83+
// We only handle raw identifiers, so identifiers that are
84+
// - builtins (apart from fs handled above)
85+
// - URLs (probably only file URLs, but who knows)
86+
// - relative/absolute
87+
// are handled by the default resolve function instead
88+
//
89+
// For relative identifiers we make an additional pass: if the parent comes from
90+
// the yarn cache, we assume the new file will need to come from the yarn cache
91+
// as well.
92+
// This works out because a non-yarn cache file can be loaded using the fs we use
93+
// to read from the yarn cache.
94+
95+
if (builtinModules.includes(identifier) || isValidURL(identifier)) {
96+
return defaultResolve(identifier, context, defaultResolve);
97+
}
98+
99+
const {parentURL} = context;
100+
101+
if (!isRawIdentifier(identifier)) {
102+
const result = await defaultResolve(identifier, context, defaultResolve);
103+
104+
if (parentURL && (new URL(parentURL)).searchParams.has('yarn-cache')) {
105+
const url = new URL(result.url);
106+
url.searchParams.set('yarn-cache', '');
107+
result.url = url.href;
108+
}
109+
110+
return result;
111+
}
112+
113+
const parentPath = parentURL ? fileURLToPath(parentURL) : undefined;
114+
const unqualified = pnpapi.resolveToUnqualified(identifier, parentPath);
115+
const qualified = pnpapi.resolveUnqualified(unqualified);
116+
117+
const url = pathToFileURL(qualified);
118+
url.searchParams.set('yarn-cache', '');
119+
120+
return {
121+
url: url.href,
122+
};
123+
}
124+
125+
/**
126+
* Return the format of the module defined by the given URL
127+
*
128+
* @param {string} urlString URL of the module
129+
* @param {object} context (currently empty)
130+
* @param {typeof getFormat} defaultGetFormat
131+
* @returns {Promise<{format: string}>}
132+
*/
133+
export async function getFormat(urlString, context, defaultGetFormat) {
134+
const url = new URL(urlString);
135+
136+
if (url.protocol === 'yarn-builtin:') {
137+
return {format: 'dynamic'};
138+
}
139+
140+
if (!url.searchParams.has('yarn-cache')) {
141+
return defaultGetFormat(url, context, defaultGetFormat);
142+
}
143+
144+
const qualified = fileURLToPath(url);
145+
146+
// We cannot return the commonjs format here, because we can't hook into how
147+
// the module loader loads commonjs modules. In other words, that will use the
148+
// actual builtin filesystem, which fails to load files from the yarn cache.
149+
switch (extname(qualified)) {
150+
case '.mjs': return {format: 'module'};
151+
case '.cjs': return {format: 'dynamic'};
152+
case '.json': return {format: 'dynamic'};
153+
case '.js': {
154+
const {packageLocation} = pnpapi.getPackageInformation(
155+
pnpapi.findPackageLocator(qualified)
156+
);
157+
158+
const manifest = JSON.parse(await readFile(join(packageLocation, 'package.json')));
159+
const isModulePackage = manifest.type === 'module';
160+
161+
return {format: isModulePackage ? 'module' : 'dynamic'};
162+
}
163+
default:
164+
throw new Error(\`Can't define format for file with extension \${extanme(qualified)}\`);
165+
}
166+
}
167+
168+
/**
169+
* Read the source file defined by the given URL
170+
*
171+
* @param {string} urlString
172+
* @param {object} context
173+
* @param {string} context.format
174+
* @param {typeof getSource} defaultGetSource
175+
* @returns {Promise<{source: string|Buffer}>} response
176+
*/
177+
export async function getSource(urlString, context, defaultGetSource) {
178+
const url = new URL(urlString);
179+
180+
if (!url.searchParams.has('yarn-cache')) {
181+
return defaultGetSource(url, context, defaultGetSource);
182+
}
183+
184+
return {
185+
source: await readFile(fileURLToPath(url)),
186+
};
187+
}
188+
189+
/**
190+
* Instantiate dynamic modules defined by this loader
191+
*
192+
* @param {string} urlString
193+
* @returns {Promise<{exports: string[], execute: Function}>}
194+
*/
195+
export async function dynamicInstantiate(urlString) {
196+
const url = new URL(urlString);
197+
198+
if (url.protocol === 'yarn-builtin:') {
199+
const identifier = url.pathname.slice(1);
200+
const builtinModule = await import(url.searchParams.get('actual'));
201+
const keys = Object.getOwnPropertyNames(builtinModule);
202+
203+
return {
204+
exports: keys,
205+
execute: exports => {
206+
const actualModule = require(identifier);
207+
208+
for (const key of keys) {
209+
if (key === 'default') {
210+
exports[key].set(actualModule);
211+
} else if (typeof builtinModule[key] !== 'function') {
212+
exports[key].set(actualModule[key]);
213+
} else {
214+
const fn = function (...args) {
215+
return actualModule[key](...args);
216+
};
217+
218+
Object.defineProperties(fn, {
219+
name: {
220+
configurable: true,
221+
value: builtinModule[key].name,
222+
},
223+
length: {
224+
configurable: true,
225+
value: builtinModule[key].length,
226+
},
227+
});
228+
229+
exports[key].set(fn);
230+
}
231+
}
232+
}
233+
};
234+
}
235+
236+
if (url.searchParams.has('yarn-cache')) {
237+
const path = fileURLToPath(url);
238+
239+
return {
240+
exports: ['default'],
241+
execute: exports => {
242+
exports.default.set(require(path));
243+
},
244+
};
245+
}
246+
247+
throw new Error(\`Unable to dynamically instantiate URL \${urlString}\`);
248+
}
249+
`;
250+
}

packages/yarnpkg-pnp/sources/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export * from './types';
2+
export * from './generateModuleLoader';
23
export * from './generatePnpScript';
34
export * from './hydratePnpApi';
45
export * from './makeRuntimeApi';

0 commit comments

Comments
 (0)