_worker.js/ directory support in Pages (#2966)

* `_worker.js/` directory support in Pages

* Refactor some of the shared no bundle logic and address misc PR comments

Author: GregBrimble

.changeset/cuddly-rules-rest.md

@@ -0,0 +1,5 @@
+---
+"wrangler": minor
+---
+
+feat: Add support for the undocumented `_worker.js/` directory in Pages

fixtures/local-mode-tests/package.json

@@ -9,25 +9,8 @@
 	"main": "index.js",
 	"scripts": {
 		"check:type": "tsc && tsc -p tests/tsconfig.json",
-		"test": "cross-env NODE_ENV=local-testing NODE_OPTIONS=--experimental-vm-modules npx jest --forceExit",
-		"test:ci": "cross-env NODE_ENV=local-testing NODE_OPTIONS=--experimental-vm-modules npx jest --forceExit"
-	},
-	"jest": {
-		"restoreMocks": true,
-		"testRegex": ".*.(test|spec)\\.[jt]sx?$",
-		"testTimeout": 30000,
-		"transform": {
-			"^.+\\.c?(t|j)sx?$": [
-				"esbuild-jest",
-				{
-					"sourcemap": true
-				}
-			]
-		},
-		"transformIgnorePatterns": [
-			"node_modules/(?!find-up|locate-path|p-locate|p-limit|p-timeout|p-queue|yocto-queue|path-exists|execa|strip-final-newline|npm-run-path|path-key|onetime|mimic-fn|human-signals|is-stream|get-port|supports-color|pretty-bytes)",
-			"wrangler-dist/cli.js"
-		]
+		"test": "npx vitest",
+		"test:ci": "npx vitest"
 	},
 	"devDependencies": {
 		"@cloudflare/workers-types": "^4.20221111.1",

fixtures/pages-workerjs-directory/package.json

@@ -0,0 +1,18 @@
+{
+	"name": "pages-workerjs-directory",
+	"version": "0.0.0",
+	"private": true,
+	"sideEffects": false,
+	"scripts": {
+		"check:type": "tsc",
+		"dev": "npx wrangler pages dev public --port 8794",
+		"test": "npx vitest",
+		"test:ci": "npx vitest"
+	},
+	"devDependencies": {
+		"undici": "^5.9.1"
+	},
+	"engines": {
+		"node": ">=16.13"
+	}
+}

fixtures/pages-workerjs-directory/public/_worker.js/add.wasm

@@ -0,0 +1,23 @@
+import staticMod from "./static.js";
+import add from "./add.wasm";
+
+export default {
+	async fetch(request, env) {
+		const { pathname } = new URL(request.url);
+
+		if (pathname === "/wasm") {
+			const addModule = await WebAssembly.instantiate(add);
+			return new Response(addModule.exports.add(1, 2).toString());
+		}
+
+		if (pathname === "/static") {
+			return new Response(staticMod);
+		}
+
+		if (pathname !== "/") {
+			return new Response((await import(`./${pathname.slice(1)}`)).default);
+		}
+
+		return env.ASSETS.fetch(request);
+	},
+};

fixtures/pages-workerjs-directory/public/_worker.js/index.js

@@ -0,0 +1 @@
+export default "test";

fixtures/pages-workerjs-directory/public/_worker.js/other-script.js

@@ -0,0 +1 @@
+export default "static";

fixtures/pages-workerjs-directory/public/_worker.js/static.js

@@ -0,0 +1 @@
+<h1>Hello, world!</h1>

fixtures/pages-workerjs-directory/public/index.html

@@ -0,0 +1,45 @@
+import { execSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path, { join, resolve } from "node:path";
+import { fetch } from "undici";
+import { describe, it } from "vitest";
+import { runWranglerPagesDev } from "../../shared/src/run-wrangler-long-lived";
+
+describe.concurrent("Pages _worker.js/ directory", () => {
+	it("should support non-bundling with 'dev'", async ({ expect }) => {
+		const { ip, port, stop } = await runWranglerPagesDev(
+			resolve(__dirname, ".."),
+			"public",
+			["--port=0"]
+		);
+		await expect(
+			fetch(`http://${ip}:${port}/`).then((resp) => resp.text())
+		).resolves.toContain("Hello, world!");
+		await expect(
+			fetch(`http://${ip}:${port}/wasm`).then((resp) => resp.text())
+		).resolves.toContain("3");
+		await expect(
+			fetch(`http://${ip}:${port}/static`).then((resp) => resp.text())
+		).resolves.toContain("static");
+		await expect(
+			fetch(`http://${ip}:${port}/other-script`).then((resp) => resp.text())
+		).resolves.toContain("test");
+		await stop();
+	});
+
+	it("should bundle", async ({ expect }) => {
+		const dir = tmpdir();
+		const file = join(dir, "./_worker.bundle");
+
+		execSync(
+			`npx wrangler pages functions build --build-output-directory public --outfile ${file} --bindings="{\\"d1_databases\\":{\\"FOO\\":{}}}"`,
+			{
+				cwd: path.resolve(__dirname, ".."),
+			}
+		);
+
+		expect(readFileSync(file, "utf-8")).toContain("D1_ERROR");
+		expect(readFileSync(file, "utf-8")).toContain('"static"');
+	});
+});

fixtures/pages-workerjs-directory/tests/index.test.ts

@@ -0,0 +1,12 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"esModuleInterop": true,
+		"module": "CommonJS",
+		"lib": ["ES2020"],
+		"types": ["node"],
+		"moduleResolution": "node",
+		"noEmit": true
+	},
+	"include": ["tests", "../../node-types.d.ts"]
+}

fixtures/pages-workerjs-directory/tsconfig.json

@@ -449,4 +449,74 @@ export default {
 		hello.js:2:36: ERROR: Could not resolve \\"node:async_hooks\\""
 	`);
 	});
+
+	it("should compile a _worker.js/ directory", async () => {
+		mkdirSync("public");
+		mkdirSync("public/_worker.js");
+		writeFileSync(
+			"public/_worker.js/index.js",
+			`
+import { cat } from "./cat.js";
+
+export default {
+  async fetch(request, env) {
+		return new Response("Hello from _worker.js/index.js" + cat);
+	},
+};`
+		);
+		writeFileSync(
+			"public/_worker.js/cat.js",
+			`
+export const cat = "cat";`
+		);
+
+		await runWrangler(`pages functions build --outfile=public/_worker.bundle`);
+
+		expect(existsSync("public/_worker.bundle")).toBe(true);
+		expect(std.out).toMatchInlineSnapshot(`
+		"🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
+		✨ Compiled Worker successfully"
+	`);
+
+		const workerBundleContents = readFileSync("public/_worker.bundle", "utf-8");
+		const workerBundleWithConstantData = replaceRandomWithConstantData(
+			workerBundleContents,
+			[
+				[/------formdata-undici-0.[0-9]*/g, "------formdata-undici-0.test"],
+				[/bundledWorker-0.[0-9]*.mjs/g, "bundledWorker-0.test.mjs"],
+				[/bundledWorker-0.[0-9]*.map/g, "bundledWorker-0.test.map"],
+			]
+		);
+
+		expect(workerBundleWithConstantData).toMatchInlineSnapshot(`
+		"------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"metadata\\"
+
+		{\\"main_module\\":\\"bundledWorker-0.test.mjs\\"}
+		------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"bundledWorker-0.test.mjs\\"; filename=\\"bundledWorker-0.test.mjs\\"
+		Content-Type: application/javascript+module
+
+		import { cat } from \\"./cat.js\\";
+		var worker_default = {
+		  async fetch(request, env) {
+		    return new Response(\\"Hello from _worker.js/index.js\\" + cat);
+		  }
+		};
+		export {
+		  worker_default as default
+		};
+		//# sourceMappingURL=bundledWorker-0.test.mjs.map
+
+		------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"cat.js\\"; filename=\\"cat.js\\"
+		Content-Type: application/javascript+module
+
+
+		export const cat = \\"cat\\";
+		------formdata-undici-0.test--"
+	`);
+
+		expect(std.err).toMatchInlineSnapshot(`""`);
+	});
 });

packages/wrangler/src/__tests__/pages/functions-build.test.ts

@@ -3,6 +3,7 @@ import { startApiDev, startDev } from "../dev";
 import { logger } from "../logger";
 
 import type { Environment } from "../config";
+import type { Rule } from "../config/environment";
 import type { EnablePagesAssetsServiceBindingOptions } from "../miniflare-cli/types";
 import type { RequestInit, Response, RequestInfo } from "undici";
 
@@ -42,6 +43,9 @@ export interface UnstableDevOptions {
 		bucket_name: string;
 		preview_bucket_name?: string;
 	}[];
+	processEntrypoint?: boolean;
+	moduleRoot?: string;
+	rules?: Rule[];
 	logLevel?: "none" | "info" | "error" | "log" | "warn" | "debug"; // Specify logging level  [choices: "debug", "info", "log", "warn", "error", "none"] [default: "log"]
 	inspect?: boolean;
 	local?: boolean;
@@ -150,6 +154,7 @@ export async function unstable_dev(
 					},
 					config: options?.config,
 					env: options?.env,
+					processEntrypoint: !!options?.processEntrypoint,
 					bundle: options?.bundle,
 					compatibilityDate: options?.compatibilityDate,
 					compatibilityFlags: options?.compatibilityFlags,

packages/wrangler/src/api/dev.ts

@@ -1,4 +1,4 @@
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve as resolvePath } from "node:path";
 import { cwd } from "node:process";
@@ -14,6 +14,7 @@ import {
 import {
 	buildRawWorker,
 	checkRawWorker,
+	traverseAndBuildWorkerJSDirectory,
 } from "../../pages/functions/buildWorker";
 import { validateRoutes } from "../../pages/functions/routes-validation";
 import { upload } from "../../pages/upload";
@@ -65,7 +66,7 @@ interface PagesPublishOptions {
 
 	/**
 	 * Whether to run bundling on `_worker.js` before deploying.
-	 * Default: false
+	 * Default: true
 	 */
 	bundle?: boolean;
 
@@ -95,9 +96,12 @@ export async function publish({
 		_redirects: string | undefined,
 		_routesGenerated: string | undefined,
 		_routesCustom: string | undefined,
+		_workerJSIsDirectory = false,
 		_workerJS: string | undefined;
 
-	const workerScriptPath = resolvePath(directory, "_worker.js");
+	bundle = bundle ?? true;
+
+	const _workerPath = resolvePath(directory, "_worker.js");
 
 	try {
 		_headers = readFileSync(join(directory, "_headers"), "utf-8");
@@ -116,7 +120,10 @@ export async function publish({
 	} catch {}
 
 	try {
-		_workerJS = readFileSync(workerScriptPath, "utf-8");
+		_workerJSIsDirectory = lstatSync(_workerPath).isDirectory();
+		if (!_workerJSIsDirectory) {
+			_workerJS = readFileSync(_workerPath, "utf-8");
+		}
 	} catch {}
 
 	// Grab the bindings from the API, we need these for shims and other such hacky inserts
@@ -240,16 +247,23 @@ export async function publish({
 	 * Advanced Mode
 	 * https://developers.cloudflare.com/pages/platform/functions/#advanced-mode
 	 *
-	 * When using a _worker.js file, the entire /functions directory is ignored
+	 * When using a _worker.js file or _worker.js/ directory, the entire /functions directory is ignored
 	 * – this includes its routing and middleware characteristics.
 	 */
-	if (_workerJS) {
+	if (_workerJSIsDirectory) {
+		workerBundle = await traverseAndBuildWorkerJSDirectory({
+			workerJSDirectory: _workerPath,
+			buildOutputDirectory: directory,
+			d1Databases,
+			nodejsCompat,
+		});
+	} else if (_workerJS) {
 		if (bundle) {
 			const outfile = join(tmpdir(), `./bundledWorker-${Math.random()}.mjs`);
 			workerBundle = await buildRawWorker({
-				workerScriptPath,
+				workerScriptPath: _workerPath,
 				outfile,
-				directory: directory ?? ".",
+				directory,
 				local: false,
 				sourcemap: true,
 				watch: false,
@@ -258,17 +272,19 @@ export async function publish({
 				nodejsCompat,
 			});
 		} else {
-			await checkRawWorker(workerScriptPath, () => {});
-			// TODO: Replace this with the cool new no-bundle stuff when that lands: https://github.com/cloudflare/workers-sdk/pull/2769
+			await checkRawWorker(_workerPath, () => {});
+			// TODO: Let users configure this in the future.
 			workerBundle = {
 				modules: [],
 				dependencies: {},
 				stop: undefined,
-				resolvedEntryPointPath: workerScriptPath,
+				resolvedEntryPointPath: _workerPath,
 				bundleType: "esm",
 			};
 		}
+	}
 
+	if (_workerJS || _workerJSIsDirectory) {
 		const workerBundleContents = await createUploadWorkerBundleContents(
 			workerBundle as BundleResult
 		);
@@ -302,7 +318,7 @@ export async function publish({
 	 * Pages Functions
 	 * https://developers.cloudflare.com/pages/platform/functions/
 	 */
-	if (builtFunctions && !_workerJS) {
+	if (builtFunctions && !_workerJS && !_workerJSIsDirectory) {
 		const workerBundleContents = await createUploadWorkerBundleContents(
 			workerBundle as BundleResult
 		);

packages/wrangler/src/api/pages/publish.tsx

@@ -117,6 +117,8 @@ export async function bundleWorker(
 	entry: Entry,
 	destination: string,
 	options: {
+		// When `bundle` is set to false, we apply shims to the Worker, but won't pull in any imports
+		bundle?: boolean;
 		serveAssetsFromWorker: boolean;
 		assets?: StaticAssetsConfig;
 		betaD1Shims?: string[];
@@ -149,6 +151,7 @@ export async function bundleWorker(
 	}
 ): Promise<BundleResult> {
 	const {
+		bundle = true,
 		serveAssetsFromWorker,
 		betaD1Shims,
 		doBindings,
@@ -350,7 +353,7 @@ export async function bundleWorker(
 
 	const buildOptions: esbuild.BuildOptions & { metafile: true } = {
 		entryPoints: [inputEntry.file],
-		bundle: true,
+		bundle,
 		absWorkingDir: entry.directory,
 		outdir: destination,
 		entryNames: entryName || path.parse(entry.file).name,
@@ -362,7 +365,7 @@ export async function bundleWorker(
 			  }
 			: {}),
 		inject,
-		external: ["__STATIC_CONTENT_MANIFEST"],
+		external: bundle ? ["__STATIC_CONTENT_MANIFEST"] : undefined,
 		format: entry.format === "modules" ? "esm" : "iife",
 		target: COMMON_ESBUILD_OPTIONS.target,
 		sourcemap: sourcemap ?? true, // this needs to use ?? to accept false