From 27af2ba4f1b65c3861876b5449485137be2d11b6 Mon Sep 17 00:00:00 2001 From: martinRenou Date: Mon, 2 Jan 2023 14:15:44 +0100 Subject: [PATCH 1/2] Get rid of the dockerfile and make a dev install in the docs This allows us to try out xeus-python directly from readthedocs for each PR --- .github/workflows/build.yml | 13 ---------- Dockerfile | 29 ---------------------- MANIFEST.in | 2 +- build.js | 49 +++++++++++++++++++++++++++++++++++++ build_xeus_python.js | 35 -------------------------- copy_output.sh | 15 ------------ docs/build-environment.yml | 6 ++++- package.json | 6 ++--- pyproject.toml | 2 +- 9 files changed, 59 insertions(+), 98 deletions(-) delete mode 100644 Dockerfile create mode 100644 build.js delete mode 100644 build_xeus_python.js delete mode 100755 copy_output.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f565c2b..69342a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,19 +47,6 @@ jobs: restore-keys: | yarn- - - name: Setup xeus-python build cache - uses: actions/cache@v2 - with: - path: | - src/python_data.data - src/python_data.js - src/web_worker_kernel.ts - src/worker.ts - src/xpython_wasm.js - src/xpython_wasm.wasm - src/xpython_wasm.hash - key: ${{ hashFiles('Dockerfile') }} - - name: Install dependencies run: python -m pip install -U jupyterlab~=3.1 check-manifest diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0431980..0000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM --platform=linux/amd64 mambaorg/micromamba:0.23.1 as build - -ARG MAMBA_DOCKERFILE_ACTIVATE=1 -ARG PYTHON_VERSION=3.10 - -RUN micromamba install --yes -c conda-forge \ - git pip python=$PYTHON_VERSION click typer \ - "empack>=2.0.2" - -################################################################## -# Create emscripten env and pack it -################################################################## - -RUN micromamba create -n xeus-python-kernel \ - --platform=emscripten-32 \ - --root-prefix=/tmp/xeus-python-kernel \ - -c https://repo.mamba.pm/emscripten-forge \ - -c https://repo.mamba.pm/conda-forge \ - --yes \ - python=$PYTHON_VERSION xeus-python xeus-lite - -RUN mkdir -p xeus-python-kernel && cd xeus-python-kernel && \ - cp /tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.js . && \ - cp /tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm . && \ - empack pack env --env-prefix /tmp/xeus-python-kernel/envs/xeus-python-kernel --outname python_data - -COPY copy_output.sh . - -ENTRYPOINT ["/bin/bash"] diff --git a/MANIFEST.in b/MANIFEST.in index be8ab04..f939d59 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,7 @@ include ts*.json include yarn.lock include webpack.config.js include worker.webpack.config.js -include Dockerfile +include build.js recursive-include tests *.py graft share diff --git a/build.js b/build.js new file mode 100644 index 0000000..d5db4fc --- /dev/null +++ b/build.js @@ -0,0 +1,49 @@ +const { execSync } = require('child_process'); +const which = require('which'); +const { copyFileSync } = require('fs'); + +const PYTHON_VERSION = '3.10'; + +let command = null; + +try { + command = which.sync('micromamba'); + console.log('[xeus-python] Creating emscripten env with micromamba'); +} catch (err) { + try { + command = which.sync('mamba'); + console.log('[xeus-python] Creating emscripten env with mamba'); + } catch (err) { + try { + command = which.sync('conda'); + console.log('[xeus-python] Creating emscripten env with conda'); + } catch (err) {} + } +} + +/** + * Create the Emscripten env and pack it. + */ +function build() { + // Create env + execSync(` + ${command} create -n xeus-python-kernel \ + --platform=emscripten-32 \ + --root-prefix=/tmp/xeus-python-kernel \ + -c https://repo.mamba.pm/emscripten-forge \ + -c https://repo.mamba.pm/conda-forge \ + --yes \ + python=${PYTHON_VERSION} xeus-python xeus-lite + `); + + // Copy xeus-python output + copyFileSync('/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.js', 'src'); + copyFileSync('/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm', 'src'); + + // Pack env + execSync('empack pack env --env-prefix /tmp/xeus-python-kernel/envs/xeus-python-kernel --outname src/python_data'); +} + +if (require.main === module) { + build(); +} diff --git a/build_xeus_python.js b/build_xeus_python.js deleted file mode 100644 index e7b8901..0000000 --- a/build_xeus_python.js +++ /dev/null @@ -1,35 +0,0 @@ -const { existsSync, readFileSync, writeFileSync } = require('fs'); -const crypto = require('crypto'); -const path = require('path'); -const { execSync } = require('child_process'); - -const VERSION_FILE_NAME = 'xpython_wasm.hash'; -const VERSION_FILE_PATH = path.join('src', VERSION_FILE_NAME); - -const hashSum = crypto.createHash('sha256'); - -hashSum.update(readFileSync('Dockerfile')); -const buildHash = hashSum.digest('hex'); - -let needsRebuild = true; - -if (existsSync(VERSION_FILE_PATH)) { - const currentHash = readFileSync(VERSION_FILE_PATH, { - encoding: 'utf8', - flag: 'r' - }); - - if (currentHash === buildHash) { - needsRebuild = false; - } -} - -if (needsRebuild) { - execSync('docker build -t jupyterlite-xeus-kernel .'); - - execSync( - 'docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) jupyterlite-xeus-kernel copy_output.sh' - ); - - writeFileSync(VERSION_FILE_PATH, buildHash); -} diff --git a/copy_output.sh b/copy_output.sh deleted file mode 100755 index dae26c7..0000000 --- a/copy_output.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -mkdir -p /src/src - -cd /tmp/xeus-python-kernel -ls -cp *python*.{js,wasm,data} /src/src - -cd /tmp/xeus-python-kernel/envs/xeus-python-kernel/share/xeus-lite -cp *.ts /src/src - -echo "=============================================" -echo "Compiling wasm bindings done" -echo "=============================================" diff --git a/docs/build-environment.yml b/docs/build-environment.yml index fc7b9df..0aa922e 100644 --- a/docs/build-environment.yml +++ b/docs/build-environment.yml @@ -4,10 +4,14 @@ channels: - conda-forge dependencies: + - python=3.10 + - click + - typer - mamba - pydata-sphinx-theme - pip - pip: - jupyterlite==0.1.0b16 - jupyterlite-sphinx - - jupyterlite-xeus-python==0.6.1 + - empack>=2.0.3 + - .. diff --git a/package.json b/package.json index 2b00821..22b459d 100644 --- a/package.json +++ b/package.json @@ -31,10 +31,10 @@ "scripts": { "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib", "build:worker": "webpack --config worker.webpack.config.js --mode=development", - "build": "jlpm run build:dockerimage && jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev", - "build:dockerimage": "node build_xeus_python.js", + "build": "jlpm run build:xeus-python && jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev", + "build:xeus-python": "node build.js", "build:worker:prod": "webpack --config worker.webpack.config.js --mode=production", - "build:prod": "jlpm run clean && jlpm run build build:dockerimage && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension", + "build:prod": "jlpm run clean && jlpm run build build:xeus-python && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension", "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", "build:lib": "jlpm run sed-worker && tsc", diff --git a/pyproject.toml b/pyproject.toml index cb5296c..4b2e924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,4 +14,4 @@ build_cmd = "build:prod" npm = ["jlpm"] [tool.check-manifest] -ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"] +ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"] From d888f2e95473f9a75b758f3f36705bc2612aea7e Mon Sep 17 00:00:00 2001 From: martinRenou Date: Wed, 4 Jan 2023 12:01:55 +0100 Subject: [PATCH 2/2] Split the Python package into the Addon and the Build part --- .github/workflows/build.yml | 2 +- build.js | 49 ---- docs/build-environment.yml | 7 +- environment.yml | 3 + jupyterlite_xeus_python/build.py | 291 +++++++++++++++++++++ jupyterlite_xeus_python/env_build_addon.py | 223 ++-------------- package.json | 10 +- pyproject.toml | 2 +- setup.py | 40 ++- 9 files changed, 345 insertions(+), 282 deletions(-) delete mode 100644 build.js create mode 100644 jupyterlite_xeus_python/build.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69342a7..9e22bc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: yarn- - name: Install dependencies - run: python -m pip install -U jupyterlab~=3.1 check-manifest + run: python -m pip install -U jupyterlab~=3.1 check-manifest typer empack - name: Build the extension run: | diff --git a/build.js b/build.js deleted file mode 100644 index d5db4fc..0000000 --- a/build.js +++ /dev/null @@ -1,49 +0,0 @@ -const { execSync } = require('child_process'); -const which = require('which'); -const { copyFileSync } = require('fs'); - -const PYTHON_VERSION = '3.10'; - -let command = null; - -try { - command = which.sync('micromamba'); - console.log('[xeus-python] Creating emscripten env with micromamba'); -} catch (err) { - try { - command = which.sync('mamba'); - console.log('[xeus-python] Creating emscripten env with mamba'); - } catch (err) { - try { - command = which.sync('conda'); - console.log('[xeus-python] Creating emscripten env with conda'); - } catch (err) {} - } -} - -/** - * Create the Emscripten env and pack it. - */ -function build() { - // Create env - execSync(` - ${command} create -n xeus-python-kernel \ - --platform=emscripten-32 \ - --root-prefix=/tmp/xeus-python-kernel \ - -c https://repo.mamba.pm/emscripten-forge \ - -c https://repo.mamba.pm/conda-forge \ - --yes \ - python=${PYTHON_VERSION} xeus-python xeus-lite - `); - - // Copy xeus-python output - copyFileSync('/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.js', 'src'); - copyFileSync('/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm', 'src'); - - // Pack env - execSync('empack pack env --env-prefix /tmp/xeus-python-kernel/envs/xeus-python-kernel --outname src/python_data'); -} - -if (require.main === module) { - build(); -} diff --git a/docs/build-environment.yml b/docs/build-environment.yml index 0aa922e..d5e660f 100644 --- a/docs/build-environment.yml +++ b/docs/build-environment.yml @@ -5,13 +5,14 @@ channels: dependencies: - python=3.10 + - pip - click - typer - - mamba - pydata-sphinx-theme - - pip + - yarn + - jupyterlab + - empack>=2.0.3 - pip: - jupyterlite==0.1.0b16 - jupyterlite-sphinx - - empack>=2.0.3 - .. diff --git a/environment.yml b/environment.yml index f9db93f..b00f745 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,9 @@ channels: dependencies: - python - pip + - traitlets + - requests + - typer - pytest - emsdk >=3.1.11 - empack >=2.0.2 diff --git a/jupyterlite_xeus_python/build.py b/jupyterlite_xeus_python/build.py new file mode 100644 index 0000000..3ab47c2 --- /dev/null +++ b/jupyterlite_xeus_python/build.py @@ -0,0 +1,291 @@ +import os +from copy import copy +from pathlib import Path +import requests +import shutil +from subprocess import check_call, run, DEVNULL +from typing import List +from urllib.parse import urlparse + +import yaml + +from empack.file_packager import pack_environment +from empack.file_patterns import PkgFileFilter, pkg_file_filter_from_yaml + +import typer + +try: + from mamba.api import create as mamba_create + + MAMBA_PYTHON_AVAILABLE = True +except ImportError: + MAMBA_PYTHON_AVAILABLE = False + +MAMBA_COMMAND = shutil.which("mamba") +MICROMAMBA_COMMAND = shutil.which("micromamba") +CONDA_COMMAND = shutil.which("conda") + +PYTHON_VERSION = "3.10" + +CHANNELS = [ + "https://repo.mamba.pm/emscripten-forge", + "https://repo.mamba.pm/conda-forge", +] + +PLATFORM = "emscripten-32" + + +def create_env( + env_name, + root_prefix, + specs, + channels, +): + """Create the emscripten environment with the given specs.""" + prefix_path = Path(root_prefix) / "envs" / env_name + + if MAMBA_PYTHON_AVAILABLE: + mamba_create( + env_name=env_name, + base_prefix=root_prefix, + specs=specs, + channels=channels, + target_platform=PLATFORM, + ) + return + + channels_args = [] + for channel in channels: + channels_args.extend(["-c", channel]) + + if MAMBA_COMMAND: + # Mamba needs the directory to exist already + prefix_path.mkdir(parents=True, exist_ok=True) + return _create_env_with_config(MAMBA_COMMAND, prefix_path, specs, channels_args) + + if MICROMAMBA_COMMAND: + run( + [ + MICROMAMBA_COMMAND, + "create", + "--yes", + "--root-prefix", + root_prefix, + "--name", + env_name, + f"--platform={PLATFORM}", + *channels_args, + *specs, + ], + check=True, + ) + return + + if CONDA_COMMAND: + return _create_env_with_config(CONDA_COMMAND, prefix_path, specs, channels_args) + + raise RuntimeError( + """Failed to create the virtual environment for xeus-python, + please make sure at least mamba, micromamba or conda is installed. + """ + ) + + +def _create_env_with_config(conda, prefix_path, specs, channels_args): + run( + [conda, "create", "--yes", "--prefix", prefix_path, *channels_args], + check=True, + ) + _create_config(prefix_path) + run( + [ + conda, + "install", + "--yes", + "--prefix", + prefix_path, + *channels_args, + *specs, + ], + check=True, + ) + + +def _create_config(prefix_path): + with open(prefix_path / ".condarc", "w") as fobj: + fobj.write(f"subdir: {PLATFORM}") + os.environ["CONDARC"] = str(prefix_path / ".condarc") + + +def build_and_pack_emscripten_env( + python_version: str = PYTHON_VERSION, + xeus_python_version: str = "", + packages: List[str] = [], + environment_file: str = "", + root_prefix: str = "/tmp/xeus-python-kernel", + env_name: str = "xeus-python-kernel", + empack_config: str = "", + output_path: str = ".", + build_worker: bool = False, + force: bool = False, +): + """Build a conda environment for the emscripten platform and pack it with empack.""" + channels = copy(CHANNELS) + specs = [ + f"python={python_version}", + "xeus-lite", + "xeus-python" + if not xeus_python_version + else f"xeus-python={xeus_python_version}", + *packages, + ] + bail_early = True + + if packages or xeus_python_version or environment_file: + bail_early = False + + # Process environment.yml file + if environment_file and Path(environment_file).exists(): + bail_early = False + + with open(Path(environment_file)) as f: + env_data = yaml.safe_load(f) + + if env_data.get("name") is not None: + env_name = env_data["name"] + + if env_data.get("channels") is not None: + channels.extend( + [channel for channel in env_data["channels"] if channel not in CHANNELS] + ) + + if env_data.get("dependencies") is not None: + dependencies = env_data["dependencies"] + + for dependency in dependencies: + if isinstance(dependency, str) and dependency not in specs: + specs.append(dependency) + elif isinstance(dependency, dict) and dependency.get("pip") is not None: + raise RuntimeError( + """Cannot install pip dependencies in the xeus-python Emscripten environment (yet?). + """ + ) + + # Bail early if there is nothing to do + if bail_early and not force: + return [] + + orig_config = os.environ.get("CONDARC") + + # Cleanup tmp dir in case it's not empty + shutil.rmtree(Path(root_prefix) / "envs", ignore_errors=True) + Path(root_prefix).mkdir(parents=True, exist_ok=True) + + output_path = Path(output_path).resolve() + output_path.mkdir(parents=True, exist_ok=True) + + prefix_path = Path(root_prefix) / "envs" / env_name + + try: + # Create emscripten env with the given packages + create_env(env_name, root_prefix, specs, channels) + + pack_kwargs = {} + + # Download env filter config + if empack_config: + empack_config_is_url = urlparse(empack_config).scheme in ("http", "https") + if empack_config_is_url: + empack_config_content = requests.get(empack_config).content + pack_kwargs["pkg_file_filter"] = PkgFileFilter.parse_obj( + yaml.safe_load(empack_config_content) + ) + else: + pack_kwargs["pkg_file_filter"] = pkg_file_filter_from_yaml( + empack_config + ) + + # Pack the environment + pack_environment( + env_prefix=prefix_path, + outname=Path(output_path) / "python_data", + export_name="globalThis.Module", + **pack_kwargs, + ) + + # Copy xeus-python output + for file in ["xpython_wasm.js", "xpython_wasm.wasm"]: + shutil.copyfile(prefix_path / "bin" / file, Path(output_path) / file) + + # Copy worker code and process it + if build_worker: + shutil.copytree( + prefix_path / "share" / "xeus-lite", + Path(output_path), + dirs_exist_ok=True, + ) + + with open(Path(output_path) / "worker.ts", "r") as fobj: + worker = fobj.read() + + worker = worker.replace("XEUS_KERNEL_FILE", "'xpython_wasm.js'") + worker = worker.replace("LANGUAGE_DATA_FILE", "'python_data.js'") + + with open(Path(output_path) / "worker.ts", "w") as fobj: + fobj.write(worker) + except Exception as e: + raise e + finally: + if orig_config is not None: + os.environ["CONDARC"] = orig_config + elif "CONDARC" in os.environ: + del os.environ["CONDARC"] + + return prefix_path + + +def main( + python_version: str = PYTHON_VERSION, + xeus_python_version: str = "", + packages: List[str] = typer.Option( + [], help="The list of packages you want to install" + ), + environment_file: str = typer.Option( + "", help="The path to the environment.yml file you want to use" + ), + root_prefix: str = "/tmp/xeus-python-kernel", + env_name: str = "xeus-python-kernel", + empack_config: str = typer.Option( + "", + help="The empack config file to use. If not provided, the default empack config will be used", + ), + output_path: str = typer.Option( + ".", + help="The directory where to output the packed environment", + ), + build_worker: bool = typer.Option( + False, + help="Whether or not to build the TypeScript worker code for using xeus-python in JupyterLite", + ), +): + """Build and pack an emscripten environment.""" + return build_and_pack_emscripten_env( + python_version, + xeus_python_version, + packages, + environment_file, + root_prefix, + env_name, + empack_config, + output_path, + build_worker, + force=True, + ) + + +def start(): + typer.run(main) + + +if __name__ == "__main__": + start() diff --git a/jupyterlite_xeus_python/env_build_addon.py b/jupyterlite_xeus_python/env_build_addon.py index 5667b89..3afb73e 100644 --- a/jupyterlite_xeus_python/env_build_addon.py +++ b/jupyterlite_xeus_python/env_build_addon.py @@ -27,31 +27,10 @@ ENV_EXTENSIONS, ) -JUPYTERLITE_XEUS_PYTHON = "@jupyterlite/xeus-python-kernel" - -# TODO Make this configurable -PYTHON_VERSION = "3.10" - -CHANNELS = [ - "https://repo.mamba.pm/emscripten-forge", - "https://repo.mamba.pm/conda-forge", -] -PLATFORM = "emscripten-32" - -SILENT = dict(stdout=DEVNULL, stderr=DEVNULL) - -try: - from mamba.api import create as mamba_create - - MAMBA_PYTHON_AVAILABLE = True -except ImportError: - MAMBA_PYTHON_AVAILABLE = False +from .build import build_and_pack_emscripten_env -MAMBA_COMMAND = shutil.which("mamba") - -MICROMAMBA_COMMAND = shutil.which("micromamba") +JUPYTERLITE_XEUS_PYTHON = "@jupyterlite/xeus-python-kernel" -CONDA_COMMAND = shutil.which("conda") class PackagesList(List): def from_string(self, s): @@ -81,34 +60,13 @@ class XeusPythonEnv(FederatedExtensionAddon): environment_file = Unicode( "environment.yml", config=True, - description="The path to the environment file. Defaults to \"environment.yml\"", + description='The path to the environment file. Defaults to "environment.yml"', ) - @property - def prefix_path(self): - """The environment prefix.""" - return Path(self.root_prefix) / "envs" / self.env_name - def __init__(self, *args, **kwargs): super(XeusPythonEnv, self).__init__(*args, **kwargs) self.cwd = TemporaryDirectory() - self.root_prefix = "/tmp/xeus-python-kernel" - self.env_name = "xeus-python-kernel" - self.channels = CHANNELS - self.specs = [ - f"python={PYTHON_VERSION}", - "xeus-python" - if not self.xeus_python_version - else f"xeus-python={self.xeus_python_version}", - *self.packages, - ] - - # Cleanup tmp dir in case it's not empty - shutil.rmtree(Path(self.root_prefix) / "envs", ignore_errors=True) - Path(self.root_prefix).mkdir(parents=True, exist_ok=True) - - self.orig_config = os.environ.get("CONDARC") def post_build(self, manager): """yield a doit task to create the emscripten-32 env and grab anything we need from it""" @@ -118,72 +76,16 @@ def post_build(self, manager): if pkg_data.get("name") == JUPYTERLITE_XEUS_PYTHON: yield from self.safe_copy_extension(pkg_json) - bail_early = True - if self.packages or self.xeus_python_version: - bail_early = False - - # Process environment.yml file - if (Path(self.manager.lite_dir) / self.environment_file).exists(): - bail_early = False - - with open(Path(self.manager.lite_dir) / self.environment_file) as f: - env_data = yaml.safe_load(f) - - if env_data.get("name") is not None: - self.env_name = env_data["name"] - - if env_data.get("channels") is not None: - channels = env_data["channels"] - - for channel in channels: - if channel not in self.channels: - self.channels.append(channel) - - if env_data.get("dependencies") is not None: - dependencies = env_data["dependencies"] - - for dependency in dependencies: - if isinstance(dependency, str) and dependency not in self.specs: - self.specs.append(dependency) - elif isinstance(dependency, dict) and dependency.get("pip") is not None: - raise RuntimeError( - """Cannot install pip dependencies in the xeus-python Emscripten environment (yet?). - """ - ) - - # Bail early if there is nothing to do - if bail_early: - return [] - - # Create emscripten env with the given packages - self.create_env() - - pack_kwargs = {} - - # Download env filter config - if self.empack_config is not None: - empack_config_is_url = urlparse(self.empack_config).scheme in ("http", "https") - if empack_config_is_url: - empack_config_content = requests.get(self.empack_config).content - pack_kwargs["pkg_file_filter"] = PkgFileFilter.parse_obj( - yaml.safe_load(empack_config_content) - ) - else: - pack_kwargs["pkg_file_filter"] = pkg_file_filter_from_yaml(self.empack_config) - - # Pack the environment - pack_environment( - env_prefix=self.prefix_path, - outname=Path(self.cwd.name) / "python_data", - export_name="globalThis.Module", - **pack_kwargs, + env_prefix = build_and_pack_emscripten_env( + xeus_python_version=self.xeus_python_version, + packages=self.packages, + environment_file=Path(self.manager.lite_dir) / self.environment_file, + empack_config=self.empack_config, + output_path=self.cwd.name, ) # Find the federated extensions in the emscripten-env and install them - root = self.prefix_path / SHARE_LABEXTENSIONS - - # Copy federated extensions found in the emscripten-env - for pkg_json in self.env_extensions(root): + for pkg_json in self.env_extensions(env_prefix / SHARE_LABEXTENSIONS): yield from self.safe_copy_extension(pkg_json) # TODO Currently we're shamelessly overwriting the @@ -192,26 +94,17 @@ def post_build(self, manager): # (make jupyterlite-xeus-python extension somewhat configurable?) dest = self.output_extensions / "@jupyterlite" / "xeus-python-kernel" / "static" - for file in ["python_data.js", "python_data.data"]: + for file in [ + "python_data.js", + "python_data.data", + "xpython_wasm.js", + "xpython_wasm.wasm", + ]: yield dict( name=f"xeus:copy:{file}", actions=[(self.copy_one, [Path(self.cwd.name) / file, dest / file])], ) - for file in ["xpython_wasm.js", "xpython_wasm.wasm"]: - yield dict( - name=f"xeus:copy:{file}", - actions=[ - ( - self.copy_one, - [ - self.prefix_path / "bin" / file, - dest / file, - ], - ) - ], - ) - jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON lab_extensions_root = manager.output_dir / LAB_EXTENSIONS lab_extensions = self.env_extensions(lab_extensions_root) @@ -223,81 +116,6 @@ def post_build(self, manager): actions=[(self.patch_jupyterlite_json, [jupyterlite_json])], ) - def create_env(self): - """Create the xeus-python emscripten-32 env with either mamba, micromamba or conda.""" - if MAMBA_PYTHON_AVAILABLE: - mamba_create( - env_name=self.env_name, - base_prefix=self.root_prefix, - specs=self.specs, - channels=self.channels, - target_platform=PLATFORM, - ) - return - - channels = [] - for channel in self.channels: - channels.extend(["-c", channel]) - - if MAMBA_COMMAND: - # Mamba needs the directory to exist already - self.prefix_path.mkdir(parents=True, exist_ok=True) - return self._create_env_with_config(MAMBA_COMMAND, channels) - - if MICROMAMBA_COMMAND: - run( - [ - MICROMAMBA_COMMAND, - "create", - "--yes", - "--root-prefix", - self.root_prefix, - "--name", - self.env_name, - f"--platform={PLATFORM}", - *channels, - *self.specs, - ], - cwd=self.cwd.name, - check=True, - ) - return - - if CONDA_COMMAND: - return self._create_env_with_config(CONDA_COMMAND, channels) - - raise RuntimeError( - """Failed to create the virtual environment for xeus-python, - please make sure at least mamba, micromamba or conda is installed. - """ - ) - - def _create_env_with_config(self, conda, channels): - run( - [conda, "create", "--yes", "--prefix", self.prefix_path, *channels], - cwd=self.cwd.name, - check=True, - ) - self._create_config() - run( - [ - conda, - "install", - "--yes", - "--prefix", - self.prefix_path, - *channels, - *self.specs, - ], - cwd=self.cwd.name, - check=True, - ) - - def _create_config(self): - with open(self.prefix_path / ".condarc", "w") as fobj: - fobj.write(f"subdir: {PLATFORM}") - os.environ["CONDARC"] = str(self.prefix_path / ".condarc") - def safe_copy_extension(self, pkg_json): """Copy a labextension, and overwrite it if it's already in the output @@ -329,12 +147,3 @@ def dedupe_federated_extensions(self, config): named[ext["name"]] = ext config[FEDERATED_EXTENSIONS] = sorted(named.values(), key=lambda x: x["name"]) - - def __del__(self): - # Cleanup - shutil.rmtree(Path(self.root_prefix) / "envs", ignore_errors=True) - - if self.orig_config is not None: - os.environ["CONDARC"] = self.orig_config - elif "CONDARC" in os.environ: - del os.environ["CONDARC"] diff --git a/package.json b/package.json index 22b459d..ba6e134 100644 --- a/package.json +++ b/package.json @@ -13,10 +13,7 @@ "url": "https://github.com/jupyterlite/xeus-python-kernel/issues" }, "license": "BSD-3-Clause", - "author": { - "name": "Thorsten Beier, JupyterLite Contributors", - "email": "" - }, + "author": "JupyterLite Contributors", "files": [ "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf,wasm}", "style/**/*.{css,js,eot,gif,html,jpg,json,png,svg,woff2,ttf}" @@ -32,13 +29,12 @@ "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib", "build:worker": "webpack --config worker.webpack.config.js --mode=development", "build": "jlpm run build:xeus-python && jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev", - "build:xeus-python": "node build.js", + "build:xeus-python": "python jupyterlite_xeus_python/build.py --output-path src --build-worker", "build:worker:prod": "webpack --config worker.webpack.config.js --mode=production", "build:prod": "jlpm run clean && jlpm run build build:xeus-python && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension", "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", - "build:lib": "jlpm run sed-worker && tsc", - "sed-worker": "shx sed -i \"s/XEUS_KERNEL_FILE/'.\\/xpython_wasm.js'/\" src/worker.ts && shx sed -i \"s/LANGUAGE_DATA_FILE/'.\\/python_data.js'/\" src/worker.ts ", + "build:lib": "tsc", "clean": "jlpm run clean:lib", "clean:lib": "rimraf lib tsconfig.tsbuildinfo", "clean:labextension": "rimraf jupyterlite_xeus_python/labextension", diff --git a/pyproject.toml b/pyproject.toml index 4b2e924..e14f8e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.1"] +requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.1", "empack"] build-backend = "jupyter_packaging.build_api" [tool.jupyter-packaging.options] diff --git a/setup.py b/setup.py index e80920d..fbc47c2 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,9 @@ # The name of the project name = "jupyterlite-xeus-python" -lab_path = (HERE / "share" / "jupyter" / "labextensions" / "@jupyterlite" / "xeus-python-kernel") +lab_path = ( + HERE / "share" / "jupyter" / "labextensions" / "@jupyterlite" / "xeus-python-kernel" +) # Representative files that should exist after a successful build ensured_targets = [ @@ -21,13 +23,17 @@ str(lab_path / "static" / "python_data.data"), str(lab_path / "static" / "xpython_wasm.js"), str(lab_path / "static" / "xpython_wasm.wasm"), - str(lab_path / "static" / "style.js") + str(lab_path / "static" / "style.js"), ] labext_name = "@jupyterlite/xeus-python-kernel" data_files_spec = [ - ("share/jupyter/labextensions/%s" % labext_name, str(lab_path.relative_to(HERE)), "**"), + ( + "share/jupyter/labextensions/%s" % labext_name, + str(lab_path.relative_to(HERE)), + "**", + ), ("share/jupyter/labextensions/%s" % labext_name, str("."), "install.json"), ] @@ -46,8 +52,7 @@ name=name, version=version, url=pkg_json["homepage"], - author=pkg_json["author"]["name"], - author_email=pkg_json["author"]["email"], + author=pkg_json["author"], description=pkg_json["description"], license=pkg_json["license"], long_description=long_description, @@ -58,6 +63,7 @@ "jupyterlite", "requests", "empack>=2.0.2,<3", + "typer", ], zip_safe=False, include_package_data=True, @@ -73,6 +79,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Framework :: Jupyter", "Framework :: Jupyter :: JupyterLab", "Framework :: Jupyter :: JupyterLab :: 3", @@ -82,25 +89,30 @@ entry_points={ "jupyterlite.addon.v0": [ "jupyterlite-xeus-python=jupyterlite_xeus_python.env_build_addon:XeusPythonEnv" - ] - } + ], + "console_scripts": [ + "jupyterlite-xeus-python-build=jupyterlite_xeus_python.build:start" + ], + }, ) try: - from jupyter_packaging import ( - wrap_installers, - npm_builder, - get_data_files - ) + from jupyter_packaging import wrap_installers, npm_builder, get_data_files + post_develop = npm_builder( build_cmd="build:prod", source_dir="src", build_dir=lab_path, npm=["jlpm"] ) - setup_args["cmdclass"] = wrap_installers(post_develop=post_develop, ensured_targets=ensured_targets) + setup_args["cmdclass"] = wrap_installers( + post_develop=post_develop, ensured_targets=ensured_targets + ) setup_args["data_files"] = get_data_files(data_files_spec) except ImportError as e: import logging + logging.basicConfig(format="%(levelname)s: %(message)s") - logging.warning("Build tool `jupyter-packaging` is missing. Install it with pip or conda.") + logging.warning( + "Build tool `jupyter-packaging` is missing. Install it with pip or conda." + ) if not ("--name" in sys.argv or "--version" in sys.argv): raise e