bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)

Makefile.pre.in

@@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 	else true; \
 	fi
 
+# wasm32-emscripten build
+# wasm assets directory is relative to current build dir, e.g. "./usr/local".
+# --preload-file turns a relative asset path into an absolute path.
+WASM_ASSETS_DIR=".$(prefix)"
+WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
+
+$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
+                pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
+	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
+	    --builddir . --prefix $(prefix)
+
+python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
+	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
+	    $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
+	    -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
+
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
@@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \
 	$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
 				-s Modules \
 				Modules/Setup.local \
+				@MODULES_SETUP_STDLIB@ \
 				$(srcdir)/Modules/Setup.bootstrap \
 				$(srcdir)/Modules/Setup
 	@mv config.c Modules
@@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
 	-rm -f _bootstrap_python
+	-rm -f python.html python.js python.data
 	-rm -f Programs/_testembed Programs/_freeze_module
 	-rm -f Python/deepfreeze/*.[co]
 	-rm -f Python/frozen_modules/*.h

Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst

@@ -0,0 +1 @@
+A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README.

Tools/wasm/README.md

@@ -0,0 +1,55 @@
+# Python WebAssembly (WASM) build
+
+This directory contains configuration and helpers to facilitate cross
+compilation of CPython to WebAssembly (WASM).
+
+## wasm32-emscripten build
+
+Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
+tool chain and a build Python interpreter.
+All commands below are relative to a repository checkout.
+
+### Compile a build Python interpreter
+
+```shell
+mkdir -p builddir/build
+pushd builddir/build
+../../configure -C
+make -j$(nproc)
+popd
+```
+
+### Fetch and build additional emscripten ports
+
+```shell
+embuilder build zlib
+```
+
+### Cross compile to wasm32-emscripten
+
+```shell
+mkdir -p builddir/emscripten
+pushd builddir/emscripten
+
+CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
+  emconfigure ../../configure -C \
+    --host=wasm32-unknown-emscripten \
+    --build=$(../../config.guess) \
+    --with-build-python=$(pwd)/../build/python
+
+emmake make -j$(nproc) python.html
+```
+
+### Test in browser
+
+Serve `python.html` with a local webserver and open the file in a browser.
+
+```shell
+emrun python.html
+```
+
+or
+
+```shell
+python3 -m http.server
+```

Tools/wasm/config.site-wasm32-emscripten

@@ -0,0 +1,70 @@
+# config.site override for cross compiling to wasm32-emscripten platform
+#
+# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
+#     emconfigure ./configure --host=wasm32-unknown-emscripten --build=...
+#
+# Written by Christian Heimes <[email protected]>
+# Partly based on pyodide's pyconfig.undefs.h file.
+#
+
+# cannot be detected in cross builds
+ac_cv_buggy_getaddrinfo=no
+
+# Emscripten has no /dev/pt*
+ac_cv_file__dev_ptmx=no
+ac_cv_file__dev_ptc=no
+
+# dummy readelf, Emscripten build does not need readelf.
+ac_cv_prog_ac_ct_READELF=true
+
+# new undefined symbols / unsupported features
+ac_cv_func_posix_spawn=no
+ac_cv_func_posix_spawnp=no
+ac_cv_func_eventfd=no
+ac_cv_func_memfd_create=no
+ac_cv_func_prlimit=no
+
+# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
+ac_cv_func_shutdown=no
+
+# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
+ac_cv_lib_bz2_BZ2_bzCompress=no
+
+# The rest is based on pyodide
+# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
+
+ac_cv_func_epoll=no
+ac_cv_func_epoll_create1=no
+ac_cv_header_linux_vm_sockets_h=no
+ac_cv_func_socketpair=no
+ac_cv_func_utimensat=no
+ac_cv_func_sigaction=no
+
+# Untested syscalls in emscripten
+ac_cv_func_openat=no
+ac_cv_func_mkdirat=no
+ac_cv_func_fchownat=no
+ac_cv_func_renameat=no
+ac_cv_func_linkat=no
+ac_cv_func_symlinkat=no
+ac_cv_func_readlinkat=no
+ac_cv_func_fchmodat=no
+ac_cv_func_dup3=no
+
+# Syscalls not implemented in emscripten
+ac_cv_func_preadv2=no
+ac_cv_func_preadv=no
+ac_cv_func_pwritev2=no
+ac_cv_func_pwritev=no
+ac_cv_func_pipe2=no
+ac_cv_func_nice=no
+
+# Syscalls that resulted in a segfault
+ac_cv_func_utimensat=no
+ac_cv_header_sys_ioctl_h=no
+
+# sockets are supported, but only in non-blocking mode
+# ac_cv_header_sys_socket_h=no
+
+# Unsupported functionality
+#undef HAVE_PTHREAD_H

Tools/wasm/wasm_assets.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+"""Create a WASM asset bundle directory structure.
+
+The WASM asset bundles are pre-loaded by the final WASM build. The bundle
+contains:
+
+- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
+- os.py as marker module {PREFIX}/lib/python3.11/os.py
+- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
+"""
+
+import argparse
+import pathlib
+import shutil
+import sys
+import zipfile
+
+# source directory
+SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
+SRCDIR_LIB = SRCDIR / "Lib"
+
+# sysconfig data relative to build dir.
+SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
+
+# Library directory relative to $(prefix).
+WASM_LIB = pathlib.PurePath("lib")
+WASM_STDLIB_ZIP = (
+    WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
+)
+WASM_STDLIB = (
+    WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
+)
+WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
+
+
+# Don't ship large files / packages that are not particularly useful at
+# the moment.
+OMIT_FILES = (
+    # regression tests
+    "test/",
+    # user interfaces: TK, curses
+    "curses/",
+    "idlelib/",
+    "tkinter/",
+    "turtle.py",
+    "turtledemo/",
+    # package management
+    "ensurepip/",
+    "venv/",
+    # build system
+    "distutils/",
+    "lib2to3/",
+    # concurrency
+    "concurrent/",
+    "multiprocessing/",
+    # deprecated
+    "asyncore.py",
+    "asynchat.py",
+    # Synchronous network I/O and protocols are not supported; for example,
+    # socket.create_connection() raises an exception:
+    # "BlockingIOError: [Errno 26] Operation in progress".
+    "cgi.py",
+    "cgitb.py",
+    "email/",
+    "ftplib.py",
+    "http/",
+    "imaplib.py",
+    "nntplib.py",
+    "poplib.py",
+    "smtpd.py",
+    "smtplib.py",
+    "socketserver.py",
+    "telnetlib.py",
+    "urllib/",
+    "wsgiref/",
+    "xmlrpc/",
+    # dbm / gdbm
+    "dbm/",
+    # other platforms
+    "_aix_support.py",
+    "_bootsubprocess.py",
+    "_osx_support.py",
+    # webbrowser
+    "antigravity.py",
+    "webbrowser.py",
+    # ctypes
+    "ctypes/",
+    # Pure Python implementations of C extensions
+    "_pydecimal.py",
+    "_pyio.py",
+    # Misc unused or large files
+    "pydoc_data/",
+    "msilib/",
+)
+
+# regression test sub directories
+OMIT_SUBDIRS = (
+    "ctypes/test/",
+    "tkinter/test/",
+    "unittest/test/",
+)
+
+
+OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
+OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
+
+
+def filterfunc(name: str) -> bool:
+    return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
+
+
+def create_stdlib_zip(
+    args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
+) -> None:
+    sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
+    if not sysconfig_data:
+        raise ValueError("No sysconfigdata file found")
+
+    with zipfile.PyZipFile(
+        args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
+    ) as pzf:
+        for entry in sorted(args.srcdir_lib.iterdir()):
+            if entry.name == "__pycache__":
+                continue
+            if entry in OMIT_ABSOLUTE:
+                continue
+            if entry.name.endswith(".py") or entry.is_dir():
+                # writepy() writes .pyc files (bytecode).
+                pzf.writepy(entry, filterfunc=filterfunc)
+        for entry in sysconfig_data:
+            pzf.writepy(entry)
+
+
+def path(val: str) -> pathlib.Path:
+    return pathlib.Path(val).absolute()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--builddir",
+    help="absolute build directory",
+    default=pathlib.Path(".").absolute(),
+    type=path,
+)
+parser.add_argument(
+    "--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
+)
+
+
+def main():
+    args = parser.parse_args()
+
+    relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
+    args.srcdir = SRCDIR
+    args.srcdir_lib = SRCDIR_LIB
+    args.wasm_root = args.builddir / relative_prefix
+    args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
+    args.wasm_stdlib = args.wasm_root / WASM_STDLIB
+    args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
+
+    # Empty, unused directory for dynamic libs, but required for site initialization.
+    args.wasm_dynload.mkdir(parents=True, exist_ok=True)
+    marker = args.wasm_dynload / ".empty"
+    marker.touch()
+    # os.py is a marker for finding the correct lib directory.
+    shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
+    # The rest of stdlib that's useful in a WASM context.
+    create_stdlib_zip(args)
+    size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
+    parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
+
+
+if __name__ == "__main__":
+    main()