From a93b774df83066e24f612a59fb1058b57e6f11ab Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 7 Dec 2022 21:24:27 +0100 Subject: [PATCH] Upgraded to xeus-python 0.15 --- Dockerfile | 4 +- copy_output.sh | 3 + package.json | 6 +- pyproject.toml | 2 +- src/index.ts | 4 +- src/worker.ts | 199 ------------------------------------ src/xeus_server_kernel.ts | 206 -------------------------------------- 7 files changed, 12 insertions(+), 412 deletions(-) delete mode 100644 src/worker.ts delete mode 100644 src/xeus_server_kernel.ts diff --git a/Dockerfile b/Dockerfile index 2673436..2813adb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,12 +17,12 @@ RUN micromamba create -n xeus-python-kernel \ -c https://repo.mamba.pm/emscripten-forge \ -c https://repo.mamba.pm/conda-forge \ --yes \ - python=$PYTHON_VERSION xeus-python + python=$PYTHON_VERSION xeus-python xeus-lite RUN mkdir -p xeus-python-kernel && cd xeus-python-kernel && \ cp /tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.js . && \ cp /tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm . && \ - empack pack python core /tmp/xeus-python-kernel/envs/xeus-python-kernel --version=$PYTHON_VERSION + empack pack env --env-prefix /tmp/xeus-python-kernel/envs/xeus-python-kernel --outname python_data --config /opt/conda/share/empack/empack_config.yaml COPY copy_output.sh . diff --git a/copy_output.sh b/copy_output.sh index 3b75200..dae26c7 100755 --- a/copy_output.sh +++ b/copy_output.sh @@ -7,6 +7,9 @@ cd /tmp/xeus-python-kernel ls cp *python*.{js,wasm,data} /src/src +cd /tmp/xeus-python-kernel/envs/xeus-python-kernel/share/xeus-lite +cp *.ts /src/src + echo "=============================================" echo "Compiling wasm bindings done" echo "=============================================" diff --git a/package.json b/package.json index b444db3..c683360 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension", "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", - "build:lib": "tsc", + "build:lib": " jlpm run sed-worker && tsc", + "sed-worker": "shx sed -i \"s/XEUS_KERNEL_FILE/'.\\/xlua_wasm.js'/\" src/worker.ts && shx sed -i \"s/LANGUAGE_DATA_FILE/''/\" src/worker.ts ", "clean": "jlpm run clean:lib", "clean:lib": "rimraf lib tsconfig.tsbuildinfo", "clean:labextension": "rimraf jupyterlite_xeus_python/labextension", @@ -76,7 +77,8 @@ "prettier": "^2.1.1", "rimraf": "^3.0.2", "source-map-loader": "^4.0.0", - "typescript": "~4.2.3" + "typescript": "~4.2.3", + "shx": "^0.3.0" }, "sideEffects": [ "style/*.css", diff --git a/pyproject.toml b/pyproject.toml index cb27cd7..cb5296c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,4 +14,4 @@ build_cmd = "build:prod" npm = ["jlpm"] [tool.check-manifest] -ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"] +ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"] diff --git a/src/index.ts b/src/index.ts index d66eac4..9adaddb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { import { IKernel, IKernelSpecs } from '@jupyterlite/kernel'; -import { XeusServerKernel } from './xeus_server_kernel'; +import { WebWorkerKernel } from './web_worker_kernel'; import logo32 from '!!file-loader?context=.!../style/logos/python-logo-32x32.png'; import logo64 from '!!file-loader?context=.!../style/logos/python-logo-64x64.png'; @@ -36,7 +36,7 @@ const server_kernel: JupyterLiteServerPlugin = { } }, create: async (options: IKernel.IOptions): Promise => { - return new XeusServerKernel({ + return new WebWorkerKernel({ ...options, mountDrive: serviceWorkerRegistrationWrapper.enabled }); diff --git a/src/worker.ts b/src/worker.ts deleted file mode 100644 index 4aac1cd..0000000 --- a/src/worker.ts +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) Thorsten Beier -// Copyright (c) JupyterLite Contributors -// Distributed under the terms of the Modified BSD License. - -import { expose } from 'comlink'; - -import { - DriveFS, - DriveFSEmscriptenNodeOps, - IEmscriptenFSNode, - IStats -} from '@jupyterlite/contents'; - -declare function createXeusModule(options: any): any; - -globalThis.Module = {}; - -// TODO Remove this. This is to ensure we always perform node ops on Nodes and -// not Streams, but why is it needed??? Why do we get Streams and not Nodes from -// emscripten in the case of xeus-python??? -class StreamNodeOps extends DriveFSEmscriptenNodeOps { - private getNode(nodeOrStream: any) { - if (nodeOrStream['node']) { - return nodeOrStream['node']; - } - return nodeOrStream; - } - - lookup(parent: IEmscriptenFSNode, name: string): IEmscriptenFSNode { - return super.lookup(this.getNode(parent), name); - } - - getattr(node: IEmscriptenFSNode): IStats { - return super.getattr(this.getNode(node)); - } - - setattr(node: IEmscriptenFSNode, attr: IStats): void { - super.setattr(this.getNode(node), attr); - } - - mknod( - parent: IEmscriptenFSNode, - name: string, - mode: number, - dev: any - ): IEmscriptenFSNode { - return super.mknod(this.getNode(parent), name, mode, dev); - } - - rename( - oldNode: IEmscriptenFSNode, - newDir: IEmscriptenFSNode, - newName: string - ): void { - super.rename(this.getNode(oldNode), this.getNode(newDir), newName); - } - - rmdir(parent: IEmscriptenFSNode, name: string): void { - super.rmdir(this.getNode(parent), name); - } - - readdir(node: IEmscriptenFSNode): string[] { - return super.readdir(this.getNode(node)); - } -} - -// TODO Remove this when we don't need StreamNodeOps anymore -class LoggingDrive extends DriveFS { - constructor(options: DriveFS.IOptions) { - super(options); - - this.node_ops = new StreamNodeOps(this); - } -} - -// when a toplevel cell uses an await, the cell is implicitly -// wrapped in a async function. Since the webloop - eventloop -// implementation does not support `eventloop.run_until_complete(f)` -// we need to convert the toplevel future in a javascript Promise -// this `toplevel` promise is then awaited before we -// execute the next cell. After the promise is awaited we need -// to do some cleanup and delete the python proxy -// (ie a js-wrapped python object) to avoid memory leaks -globalThis.toplevel_promise = null; -globalThis.toplevel_promise_py_proxy = null; - -let resolveInputReply: any; - -async function get_stdin() { - const replyPromise = new Promise(resolve => { - resolveInputReply = resolve; - }); - return replyPromise; -} - -(self as any).get_stdin = get_stdin; - -class XeusPythonKernel { - constructor() { - this._ready = new Promise(resolve => { - this.initialize(resolve); - }); - } - - async ready(): Promise { - return await this._ready; - } - - mount(driveName: string, mountpoint: string, baseUrl: string): void { - const { FS, PATH, ERRNO_CODES } = globalThis.Module; - - this._drive = new LoggingDrive({ - FS, - PATH, - ERRNO_CODES, - baseUrl, - driveName, - mountpoint - }); - - FS.mkdir(mountpoint); - FS.mount(this._drive, {}, mountpoint); - FS.chdir(mountpoint); - } - - cd(path: string) { - if (!path) { - return; - } - - globalThis.Module.FS.chdir(path); - } - - async processMessage(event: any): Promise { - await this._ready; - - if ( - globalThis.toplevel_promise !== null && - globalThis.toplevel_promise_py_proxy !== null - ) { - await globalThis.toplevel_promise; - globalThis.toplevel_promise_py_proxy.delete(); - globalThis.toplevel_promise_py_proxy = null; - globalThis.toplevel_promise = null; - } - - const msg_type = event.msg.header.msg_type; - - if (msg_type === 'input_reply') { - resolveInputReply(event.msg); - } else { - this._raw_xserver.notify_listener(event.msg); - } - } - - private async initialize(resolve: () => void) { - importScripts('./xpython_wasm.js'); - - globalThis.Module = await createXeusModule({}); - - importScripts('./python_data.js'); - - await this.waitRunDependency(); - - this._raw_xkernel = new globalThis.Module.xkernel(); - this._raw_xserver = this._raw_xkernel.get_server(); - - if (!this._raw_xkernel) { - console.error('Failed to start kernel!'); - } - - this._raw_xkernel.start(); - - resolve(); - } - - private async waitRunDependency() { - const promise = new Promise(resolve => { - globalThis.Module.monitorRunDependencies = (n: number) => { - if (n === 0) { - resolve(); - } - }; - }); - // If there are no pending dependencies left, monitorRunDependencies will - // never be called. Since we can't check the number of dependencies, - // manually trigger a call. - globalThis.Module.addRunDependency('dummy'); - globalThis.Module.removeRunDependency('dummy'); - return promise; - } - - private _raw_xkernel: any; - private _raw_xserver: any; - private _drive: DriveFS | null = null; - private _ready: PromiseLike; -} - -expose(new XeusPythonKernel()); diff --git a/src/xeus_server_kernel.ts b/src/xeus_server_kernel.ts deleted file mode 100644 index f46e1b6..0000000 --- a/src/xeus_server_kernel.ts +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (c) Thorsten Beier -// Copyright (c) JupyterLite Contributors -// Distributed under the terms of the Modified BSD License. - -import { wrap } from 'comlink'; -import type { Remote } from 'comlink'; - -import { ISignal, Signal } from '@lumino/signaling'; -import { PromiseDelegate } from '@lumino/coreutils'; - -import { PageConfig } from '@jupyterlab/coreutils'; -import { KernelMessage } from '@jupyterlab/services'; - -import { IKernel } from '@jupyterlite/kernel'; - -interface IXeusKernel { - ready(): Promise; - - mount(driveName: string, mountpoint: string, baseUrl: string): Promise; - - cd(path: string): Promise; - - processMessage(msg: any): Promise; -} - -export class XeusServerKernel implements IKernel { - /** - * Instantiate a new XeusServerKernel - * - * @param options The instantiation options for a new XeusServerKernel - */ - constructor(options: XeusServerKernel.IOptions) { - const { id, name, sendMessage, location } = options; - this._id = id; - this._name = name; - this._location = location; - this._sendMessage = sendMessage; - this._worker = new Worker(new URL('./worker.js', import.meta.url), { - type: 'module' - }); - this._worker.onmessage = e => { - this._processWorkerMessage(e.data); - }; - this._remote = wrap(this._worker); - this.initFileSystem(options); - } - - async handleMessage(msg: KernelMessage.IMessage): Promise { - this._parent = msg; - this._parentHeader = msg.header; - await this._sendMessageToWorker(msg); - } - - private async _sendMessageToWorker(msg: any): Promise { - // TODO Remove this?? - if (msg.header.msg_type !== 'input_reply') { - this._executeDelegate = new PromiseDelegate(); - } - await this._remote.processMessage({ msg, parent: this.parent }); - if (msg.header.msg_type !== 'input_reply') { - return await this._executeDelegate.promise; - } - } - - /** - * Get the last parent header - */ - get parentHeader(): - | KernelMessage.IHeader - | undefined { - return this._parentHeader; - } - - /** - * Get the last parent message (mimick ipykernel's get_parent) - */ - get parent(): KernelMessage.IMessage | undefined { - return this._parent; - } - - /** - * Get the kernel location - */ - get location(): string { - return this._location; - } - - /** - * Process a message coming from the pyodide web worker. - * - * @param msg The worker message to process. - */ - private _processWorkerMessage(msg: any): void { - if (!msg.header) { - return; - } - - msg.header.session = this._parentHeader?.session ?? ''; - msg.session = this._parentHeader?.session ?? ''; - this._sendMessage(msg); - - // resolve promise - if ( - msg.header.msg_type === 'status' && - msg.content.execution_state === 'idle' - ) { - this._executeDelegate.resolve(); - } - } - - /** - * A promise that is fulfilled when the kernel is ready. - */ - get ready(): Promise { - return Promise.resolve(); - } - - /** - * Return whether the kernel is disposed. - */ - get isDisposed(): boolean { - return this._isDisposed; - } - - /** - * A signal emitted when the kernel is disposed. - */ - get disposed(): ISignal { - return this._disposed; - } - - /** - * Dispose the kernel. - */ - dispose(): void { - if (this.isDisposed) { - return; - } - this._worker.terminate(); - (this._worker as any) = null; - (this._remote as any) = null; - this._isDisposed = true; - this._disposed.emit(void 0); - } - - /** - * Get the kernel id - */ - get id(): string { - return this._id; - } - - /** - * Get the name of the kernel - */ - get name(): string { - return this._name; - } - - private async initFileSystem(options: XeusServerKernel.IOptions) { - let driveName: string; - let localPath: string; - - if (options.location.includes(':')) { - const parts = options.location.split(':'); - driveName = parts[0]; - localPath = parts[1]; - } else { - driveName = ''; - localPath = options.location; - } - - await this._remote.ready(); - - if (options.mountDrive) { - await this._remote.mount(driveName, '/drive', PageConfig.getBaseUrl()); - await this._remote.cd(localPath); - } - } - - private _id: string; - private _name: string; - private _location: string; - private _remote: Remote; - private _isDisposed = false; - private _disposed = new Signal(this); - private _worker: Worker; - private _sendMessage: IKernel.SendMessage; - private _executeDelegate = new PromiseDelegate(); - private _parentHeader: - | KernelMessage.IHeader - | undefined = undefined; - private _parent: KernelMessage.IMessage | undefined = undefined; -} - -/** - * A namespace for XeusServerKernel statics. - */ -export namespace XeusServerKernel { - /** - * The instantiation options for a Pyodide kernel - */ - export interface IOptions extends IKernel.IOptions { - mountDrive: boolean; - } -}