Update jupyterlite and enable file-system access

+ Use comlink

Author: martinRenou

.gitignore

@@ -117,7 +117,6 @@ python_data.js
 python_data.data
 xpython_wasm.js
 xpython_wasm.wasm
-xeus_python.worker.js
 
 # Labextension
 share

README.md

@@ -13,7 +13,7 @@ The [xeus-python](https://github.com/jupyter-xeus/xeus-python) Python kernel for
 
 ## Requirements
 
-- JupyterLite >= 0.1.0a14
+- JupyterLite >= 0.1.0b9
 
 ## Install
 

package.json

@@ -32,7 +32,7 @@
     "build:dockerimage": "docker build -t mydockerimage . ",
     "build:dockerimage_no_cache": "docker build -t mydockerimage --no-cache . ",
     "build:emscripten": "echo $(pwd) && docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) mydockerimage copy_output.sh",
-    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.worker.js src/xpython_wasm.js src/python_data.js src/python_data.data lib",
+    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib",
     "build:wasm": "jlpm run build:dockerimage && jlpm run build:emscripten",
     "build:wasm_no_cache": "jlpm run build:dockerimage_no_cache && jlpm run build:emscripten",
     "build": "jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension:dev",
@@ -56,7 +56,9 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@jupyterlite/server": "^0.1.0-alpha.14"
+    "@jupyterlite/contents": "^0.1.0-beta.9",
+    "@jupyterlite/server": "^0.1.0-beta.9",
+    "comlink": "^4.3.1"
   },
   "devDependencies": {
     "@jupyterlab/builder": "^3.4.1",

pyproject.toml

@@ -14,4 +14,4 @@ build_cmd = "build:prod"
 npm = ["jlpm"]
 
 [tool.check-manifest]
-ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xeus_python.worker.js", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"]
+ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"]

src/index.ts

@@ -3,6 +3,7 @@
 // Distributed under the terms of the Modified BSD License.
 
 import {
+  IServiceWorkerRegistrationWrapper,
   JupyterLiteServer,
   JupyterLiteServerPlugin
 } from '@jupyterlite/server';
@@ -18,8 +19,12 @@ import logo64 from '../style/logos/python-logo-64x64.png';
 const server_kernel: JupyterLiteServerPlugin<void> = {
   id: '@jupyterlite/xeus-kernel-extension:kernel',
   autoStart: true,
-  requires: [IKernelSpecs],
-  activate: (app: JupyterLiteServer, kernelspecs: IKernelSpecs) => {
+  requires: [IKernelSpecs, IServiceWorkerRegistrationWrapper],
+  activate: (
+    app: JupyterLiteServer,
+    kernelspecs: IKernelSpecs,
+    serviceWorkerRegistrationWrapper: IServiceWorkerRegistrationWrapper
+  ) => {
     kernelspecs.register({
       spec: {
         name: 'xeus-python',
@@ -41,7 +46,8 @@ const server_kernel: JupyterLiteServerPlugin<void> = {
       },
       create: async (options: IKernel.IOptions): Promise<IKernel> => {
         return new XeusServerKernel({
-          ...options
+          ...options,
+          mountDrive: serviceWorkerRegistrationWrapper.enabled
         });
       }
     });

src/worker.ts

@@ -2,6 +2,10 @@
 // Copyright (c) JupyterLite Contributors
 // Distributed under the terms of the Modified BSD License.
 
+import { expose } from 'comlink';
+
+import { DriveFS } from '@jupyterlite/contents';
+
 declare function createXeusModule(options: any): any;
 
 globalThis.Module = {};
@@ -17,27 +21,7 @@ globalThis.Module = {};
 globalThis.toplevel_promise = null;
 globalThis.toplevel_promise_py_proxy = null;
 
-// We alias self to ctx and give it our newly created type
-const ctx: Worker = self as any;
-let raw_xkernel: any;
-let raw_xserver: any;
-
-async function waitRunDependency() {
-  const promise = new Promise((r: any) => {
-    globalThis.Module.monitorRunDependencies = (n: number) => {
-      if (n === 0) {
-        console.log('all `RunDependencies` loaded');
-        r();
-      }
-    };
-  });
-  // If there are no pending dependencies left, monitorRunDependencies will
-  // never be called. Since we can't check the number of dependencies,
-  // manually trigger a call.
-  globalThis.Module.addRunDependency('dummy');
-  globalThis.Module.removeRunDependency('dummy');
-  return promise;
-}
+let resolveInputReply: any;
 
 async function get_stdin() {
   const replyPromise = new Promise(resolve => {
@@ -46,49 +30,106 @@ async function get_stdin() {
   return replyPromise;
 }
 
-// eslint-disable-next-line
-// @ts-ignore: breaks typedoc
-ctx.get_stdin = get_stdin;
+(self as any).get_stdin = get_stdin;
 
-let resolveInputReply: any;
+class XeusPythonKernel {
+  constructor() {
+    this._ready = new Promise(resolve => {
+      this.initialize(resolve);
+    });
+  }
 
-async function load() {
-  const options: any = {};
+  async ready(): Promise<void> {
+    return await this._ready;
+  }
 
-  importScripts('./xpython_wasm.js');
+  mount(driveName: string, mountpoint: string, baseUrl: string): void {
+    const { FS, PATH, ERRNO_CODES } = globalThis.Module;
+
+    this._drive = new DriveFS({
+      FS,
+      PATH,
+      ERRNO_CODES,
+      baseUrl,
+      driveName,
+      mountpoint
+    });
+
+    FS.mkdir(mountpoint);
+    FS.mount(this._drive, {}, mountpoint);
+    FS.chdir(mountpoint);
+  }
 
-  globalThis.Module = await createXeusModule(options);
+  cd(path: string) {
+    const { FS } = globalThis.Module;
 
-  importScripts('./python_data.js');
+    FS.chdir(path);
+  }
 
-  await waitRunDependency();
-  raw_xkernel = new globalThis.Module.xkernel();
-  raw_xserver = raw_xkernel.get_server();
-  raw_xkernel!.start();
-}
+  async processMessage(msg: any): Promise<void> {
+    await this._ready;
+
+    if (
+      globalThis.toplevel_promise !== null &&
+      globalThis.toplevel_promise_py_proxy !== null
+    ) {
+      await globalThis.toplevel_promise;
+      globalThis.toplevel_promise_py_proxy.delete();
+      globalThis.toplevel_promise_py_proxy = null;
+      globalThis.toplevel_promise = null;
+    }
+
+    const msg_type = msg.header.msg_type;
+
+    if (msg_type === 'input_reply') {
+      resolveInputReply(msg);
+    } else {
+      this._raw_xserver.notify_listener(msg);
+    }
+  }
 
-const loadCppModulePromise = load();
+  private async initialize(resolve: () => void) {
+    importScripts('./xpython_wasm.js');
 
-ctx.onmessage = async (event: MessageEvent): Promise<void> => {
-  await loadCppModulePromise;
+    globalThis.Module = await createXeusModule({});
 
-  if (
-    globalThis.toplevel_promise !== null &&
-    globalThis.toplevel_promise_py_proxy !== null
-  ) {
-    await globalThis.toplevel_promise;
-    globalThis.toplevel_promise_py_proxy.delete();
-    globalThis.toplevel_promise_py_proxy = null;
-    globalThis.toplevel_promise = null;
-  }
+    importScripts('./python_data.js');
+
+    await this.waitRunDependency();
+
+    this._raw_xkernel = new globalThis.Module.xkernel();
+    this._raw_xserver = this._raw_xkernel.get_server();
+
+    if (!this._raw_xkernel) {
+      console.error('Failed to start kernel!');
+    }
 
-  const data = event.data;
-  const msg = data.msg;
-  const msg_type = msg.header.msg_type;
+    this._raw_xkernel.start();
 
-  if (msg_type === 'input_reply') {
-    resolveInputReply(msg);
-  } else {
-    raw_xserver!.notify_listener(msg);
+    resolve();
   }
-};
+
+  private async waitRunDependency() {
+    const promise = new Promise((resolve: any) => {
+      globalThis.Module.monitorRunDependencies = (n: number) => {
+        if (n === 0) {
+          console.log('all `RunDependencies` loaded');
+          resolve();
+        }
+      };
+    });
+    // If there are no pending dependencies left, monitorRunDependencies will
+    // never be called. Since we can't check the number of dependencies,
+    // manually trigger a call.
+    globalThis.Module.addRunDependency('dummy');
+    globalThis.Module.removeRunDependency('dummy');
+    return promise;
+  }
+
+  private _raw_xkernel: any;
+  private _raw_xserver: any;
+  private _drive: DriveFS | null = null;
+  private _ready: PromiseLike<void>;
+}
+
+expose(new XeusPythonKernel());

src/xeus_server_kernel.ts

@@ -2,11 +2,26 @@
 // Copyright (c) JupyterLite Contributors
 // Distributed under the terms of the Modified BSD License.
 
+import { wrap } from 'comlink';
+import type { Remote } from 'comlink';
+
+import { ISignal, Signal } from '@lumino/signaling';
+import { PromiseDelegate } from '@lumino/coreutils';
+
+import { PageConfig } from '@jupyterlab/coreutils';
 import { KernelMessage } from '@jupyterlab/services';
+
 import { IKernel } from '@jupyterlite/kernel';
-import { ISignal, Signal } from '@lumino/signaling';
 
-import { PromiseDelegate } from '@lumino/coreutils';
+interface IXeusKernel {
+  ready(): Promise<void>;
+
+  mount(driveName: string, mountpoint: string, baseUrl: string): Promise<void>;
+
+  cd(path: string): Promise<void>;
+
+  processMessage(msg: any): Promise<void>;
+}
 
 export class XeusServerKernel implements IKernel {
   /**
@@ -17,26 +32,33 @@ export class XeusServerKernel implements IKernel {
 
   xeus_interpreter: any;
   constructor(options: XeusServerKernel.IOptions) {
-    const { id, name, sendMessage } = options;
+    const { id, name, sendMessage, location } = options;
     this._id = id;
     this._name = name;
+    this._location = location;
     this._sendMessage = sendMessage;
-    this._worker = new Worker(new URL('./worker.js', import.meta.url));
+    this._worker = new Worker(new URL('./worker.js', import.meta.url), {
+      type: 'module',
+    });
     this._worker.onmessage = e => {
       this._processWorkerMessage(e.data);
     };
+    this._remote = wrap(this._worker);
+    this.initFileSytem(options);
   }
+
   async handleMessage(msg: KernelMessage.IMessage): Promise<void> {
     this._parent = msg;
     this._parentHeader = msg.header;
     await this._sendMessageToWorker(msg);
   }
 
   private async _sendMessageToWorker(msg: any): Promise<void> {
+    // TODO Remove this??
     if (msg.header.msg_type !== 'input_reply') {
       this._executeDelegate = new PromiseDelegate<void>();
     }
-    this._worker.postMessage({ msg, parent: this.parent });
+    await this._remote.processMessage({ msg, parent: this.parent });
     if (msg.header.msg_type !== 'input_reply') {
       return await this._executeDelegate.promise;
     }
@@ -58,6 +80,13 @@ export class XeusServerKernel implements IKernel {
     return this._parent;
   }
 
+  /**
+   * Get the kernel location
+   */
+  get location(): string {
+    return this._location;
+  }
+
   /**
    * Process a message coming from the pyodide web worker.
    *
@@ -81,7 +110,7 @@ export class XeusServerKernel implements IKernel {
    * A promise that is fulfilled when the kernel is ready.
    */
   get ready(): Promise<void> {
-    return Promise.resolve();
+    return this._remote.ready();
   }
 
   /**
@@ -123,8 +152,31 @@ export class XeusServerKernel implements IKernel {
     return this._name;
   }
 
+  private async initFileSytem(options: XeusServerKernel.IOptions) {
+    let driveName: string;
+    let localPath: string;
+
+    if (options.location.includes(':')) {
+      const parts = options.location.split(':');
+      driveName = parts[0];
+      localPath = parts[1];
+    } else {
+      driveName = '';
+      localPath = options.location;
+    }
+
+    await this._remote.ready();
+
+    if (options.mountDrive) {
+      await this._remote.mount(driveName, '/drive', PageConfig.getBaseUrl());
+      await this._remote.cd(localPath);
+    }
+  }
+
   private _id: string;
   private _name: string;
+  private _location: string;
+  private _remote: Remote<IXeusKernel>;
   private _isDisposed = false;
   private _disposed = new Signal<this, void>(this);
   private _worker: Worker;
@@ -143,5 +195,7 @@ export namespace XeusServerKernel {
   /**
    * The instantiation options for a Pyodide kernel
    */
-  export interface IOptions extends IKernel.IOptions {}
+  export interface IOptions extends IKernel.IOptions {
+    mountDrive: boolean;
+  }
 }