Update jupyterlite and enable file-system access

.gitignore

@@ -117,7 +117,6 @@ python_data.js
 python_data.data
 xpython_wasm.js
 xpython_wasm.wasm
-xeus_python.worker.js
 
 # Labextension
 share

MANIFEST.in

@@ -7,6 +7,7 @@ include install.json
 include ts*.json
 include yarn.lock
 include webpack.config.js
+include worker.webpack.config.js
 include Dockerfile
 recursive-include tests *.py
 

README.md

@@ -13,7 +13,7 @@ The [xeus-python](https://github.com/jupyter-xeus/xeus-python) Python kernel for
 
 ## Requirements
 
-- JupyterLite >= 0.1.0a14
+- JupyterLite >= 0.1.0b9
 
 ## Install
 

package.json

@@ -29,14 +29,16 @@
     "url": "https://github.com/jupyterlite/xeus-python-kernel.git"
   },
   "scripts": {
-    "build:dockerimage": "docker build -t mydockerimage . ",
-    "build:dockerimage_no_cache": "docker build -t mydockerimage --no-cache . ",
-    "build:emscripten": "echo $(pwd) && docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) mydockerimage copy_output.sh",
-    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.worker.js src/xpython_wasm.js src/python_data.js src/python_data.data lib",
+    "build:dockerimage": "docker build -t jupyterlite-xeus-kernel . ",
+    "build:dockerimage_no_cache": "docker build -t jupyterlite-xeus-kernel --no-cache . ",
+    "build:emscripten": "echo $(pwd) && docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) jupyterlite-xeus-kernel copy_output.sh",
+    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib",
     "build:wasm": "jlpm run build:dockerimage && jlpm run build:emscripten",
     "build:wasm_no_cache": "jlpm run build:dockerimage_no_cache && jlpm run build:emscripten",
-    "build": "jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension:dev",
-    "build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension",
+    "build:worker": "webpack --config worker.webpack.config.js --mode=development",
+    "build:worker:prod": "webpack --config worker.webpack.config.js --mode=production",
+    "build": "jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev",
+    "build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension",
     "build:labextension": "jupyter labextension build .",
     "build:labextension:dev": "jupyter labextension build --development True .",
     "build:lib": "tsc",
@@ -56,7 +58,9 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@jupyterlite/server": "^0.1.0-alpha.14"
+    "@jupyterlite/contents": "^0.1.0-beta.9",
+    "@jupyterlite/server": "^0.1.0-beta.9",
+    "comlink": "^4.3.1"
   },
   "devDependencies": {
     "@jupyterlab/builder": "^3.4.1",
@@ -70,6 +74,7 @@
     "npm-run-all": "^4.1.5",
     "prettier": "^2.1.1",
     "rimraf": "^3.0.2",
+    "source-map-loader": "^4.0.0",
     "typescript": "~4.2.3"
   },
   "sideEffects": [

pyproject.toml

@@ -14,4 +14,4 @@ build_cmd = "build:prod"
 npm = ["jlpm"]
 
 [tool.check-manifest]
-ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xeus_python.worker.js", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"]
+ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"]

src/index.ts

@@ -3,6 +3,7 @@
 // Distributed under the terms of the Modified BSD License.
 
 import {
+  IServiceWorkerRegistrationWrapper,
   JupyterLiteServer,
   JupyterLiteServerPlugin
 } from '@jupyterlite/server';
@@ -18,8 +19,12 @@ import logo64 from '../style/logos/python-logo-64x64.png';
 const server_kernel: JupyterLiteServerPlugin<void> = {
   id: '@jupyterlite/xeus-kernel-extension:kernel',
   autoStart: true,
-  requires: [IKernelSpecs],
-  activate: (app: JupyterLiteServer, kernelspecs: IKernelSpecs) => {
+  requires: [IKernelSpecs, IServiceWorkerRegistrationWrapper],
+  activate: (
+    app: JupyterLiteServer,
+    kernelspecs: IKernelSpecs,
+    serviceWorkerRegistrationWrapper: IServiceWorkerRegistrationWrapper
+  ) => {
     kernelspecs.register({
       spec: {
         name: 'xeus-python',
@@ -41,7 +46,8 @@ const server_kernel: JupyterLiteServerPlugin<void> = {
       },
       create: async (options: IKernel.IOptions): Promise<IKernel> => {
         return new XeusServerKernel({
-          ...options
+          ...options,
+          mountDrive: serviceWorkerRegistrationWrapper.enabled
         });
       }
     });

src/worker.ts

@@ -2,10 +2,77 @@
 // Copyright (c) JupyterLite Contributors
 // Distributed under the terms of the Modified BSD License.
 
+import { expose } from 'comlink';
+
+import {
+  DriveFS,
+  DriveFSEmscriptenNodeOps,
+  IEmscriptenFSNode,
+  IStats
+} from '@jupyterlite/contents';
+
 declare function createXeusModule(options: any): any;
 
 globalThis.Module = {};
 
+// TODO Remove this. This is to ensure we always perform node ops on Nodes and
+// not Streams, but why is it needed??? Why do we get Streams and not Nodes from
+// emscripten in the case of xeus-python???
+class StreamNodeOps extends DriveFSEmscriptenNodeOps {
+  private getNode(nodeOrStream: any) {
+    if (nodeOrStream['node']) {
+      return nodeOrStream['node'];
+    }
+    return nodeOrStream;
+  }
+
+  lookup(parent: IEmscriptenFSNode, name: string): IEmscriptenFSNode {
+    return super.lookup(this.getNode(parent), name);
+  }
+
+  getattr(node: IEmscriptenFSNode): IStats {
+    return super.getattr(this.getNode(node));
+  }
+
+  setattr(node: IEmscriptenFSNode, attr: IStats): void {
+    super.setattr(this.getNode(node), attr);
+  }
+
+  mknod(
+    parent: IEmscriptenFSNode,
+    name: string,
+    mode: number,
+    dev: any
+  ): IEmscriptenFSNode {
+    return super.mknod(this.getNode(parent), name, mode, dev);
+  }
+
+  rename(
+    oldNode: IEmscriptenFSNode,
+    newDir: IEmscriptenFSNode,
+    newName: string
+  ): void {
+    super.rename(this.getNode(oldNode), this.getNode(newDir), newName);
+  }
+
+  rmdir(parent: IEmscriptenFSNode, name: string): void {
+    super.rmdir(this.getNode(parent), name);
+  }
+
+  readdir(node: IEmscriptenFSNode): string[] {
+    return super.readdir(this.getNode(node));
+  }
+}
+
+// TODO Remove this when we don't need StreamNodeOps anymore
+class LoggingDrive extends DriveFS {
+  constructor(options: DriveFS.IOptions) {
+    super(options);
+
+    this.node_ops = new StreamNodeOps(this);
+  }
+}
+
 // when a toplevel cell uses an await, the cell is implicitly
 // wrapped in a async function. Since the webloop - eventloop
 // implementation does not support `eventloop.run_until_complete(f)`
@@ -17,27 +84,7 @@ globalThis.Module = {};
 globalThis.toplevel_promise = null;
 globalThis.toplevel_promise_py_proxy = null;
 
-// We alias self to ctx and give it our newly created type
-const ctx: Worker = self as any;
-let raw_xkernel: any;
-let raw_xserver: any;
-
-async function waitRunDependency() {
-  const promise = new Promise((r: any) => {
-    globalThis.Module.monitorRunDependencies = (n: number) => {
-      if (n === 0) {
-        console.log('all `RunDependencies` loaded');
-        r();
-      }
-    };
-  });
-  // If there are no pending dependencies left, monitorRunDependencies will
-  // never be called. Since we can't check the number of dependencies,
-  // manually trigger a call.
-  globalThis.Module.addRunDependency('dummy');
-  globalThis.Module.removeRunDependency('dummy');
-  return promise;
-}
+let resolveInputReply: any;
 
 async function get_stdin() {
   const replyPromise = new Promise(resolve => {
@@ -46,49 +93,107 @@ async function get_stdin() {
   return replyPromise;
 }
 
-// eslint-disable-next-line
-// @ts-ignore: breaks typedoc
-ctx.get_stdin = get_stdin;
+(self as any).get_stdin = get_stdin;
 
-let resolveInputReply: any;
+class XeusPythonKernel {
+  constructor() {
+    this._ready = new Promise(resolve => {
+      this.initialize(resolve);
+    });
+  }
 
-async function load() {
-  const options: any = {};
+  async ready(): Promise<void> {
+    return await this._ready;
+  }
 
-  importScripts('./xpython_wasm.js');
+  mount(driveName: string, mountpoint: string, baseUrl: string): void {
+    const { FS, PATH, ERRNO_CODES } = globalThis.Module;
+
+    this._drive = new LoggingDrive({
+      FS,
+      PATH,
+      ERRNO_CODES,
+      baseUrl,
+      driveName,
+      mountpoint
+    });
+
+    FS.mkdir(mountpoint);
+    FS.mount(this._drive, {}, mountpoint);
+    FS.chdir(mountpoint);
+  }
 
-  globalThis.Module = await createXeusModule(options);
+  cd(path: string) {
+    if (!path) {
+      return;
+    }
 
-  importScripts('./python_data.js');
+    globalThis.Module.FS.chdir(path);
+  }
 
-  await waitRunDependency();
-  raw_xkernel = new globalThis.Module.xkernel();
-  raw_xserver = raw_xkernel.get_server();
-  raw_xkernel!.start();
-}
+  async processMessage(event: any): Promise<void> {
+    await this._ready;
+
+    if (
+      globalThis.toplevel_promise !== null &&
+      globalThis.toplevel_promise_py_proxy !== null
+    ) {
+      await globalThis.toplevel_promise;
+      globalThis.toplevel_promise_py_proxy.delete();
+      globalThis.toplevel_promise_py_proxy = null;
+      globalThis.toplevel_promise = null;
+    }
+
+    const msg_type = event.msg.header.msg_type;
+
+    if (msg_type === 'input_reply') {
+      resolveInputReply(event.msg);
+    } else {
+      this._raw_xserver.notify_listener(event.msg);
+    }
+  }
 
-const loadCppModulePromise = load();
+  private async initialize(resolve: () => void) {
+    importScripts('./xpython_wasm.js');
 
-ctx.onmessage = async (event: MessageEvent): Promise<void> => {
-  await loadCppModulePromise;
+    globalThis.Module = await createXeusModule({});
 
-  if (
-    globalThis.toplevel_promise !== null &&
-    globalThis.toplevel_promise_py_proxy !== null
-  ) {
-    await globalThis.toplevel_promise;
-    globalThis.toplevel_promise_py_proxy.delete();
-    globalThis.toplevel_promise_py_proxy = null;
-    globalThis.toplevel_promise = null;
-  }
+    importScripts('./python_data.js');
+
+    await this.waitRunDependency();
+
+    this._raw_xkernel = new globalThis.Module.xkernel();
+    this._raw_xserver = this._raw_xkernel.get_server();
+
+    if (!this._raw_xkernel) {
+      console.error('Failed to start kernel!');
+    }
 
-  const data = event.data;
-  const msg = data.msg;
-  const msg_type = msg.header.msg_type;
+    this._raw_xkernel.start();
 
-  if (msg_type === 'input_reply') {
-    resolveInputReply(msg);
-  } else {
-    raw_xserver!.notify_listener(msg);
+    resolve();
   }
-};
+
+  private async waitRunDependency() {
+    const promise = new Promise<void>(resolve => {
+      globalThis.Module.monitorRunDependencies = (n: number) => {
+        if (n === 0) {
+          resolve();
+        }
+      };
+    });
+    // If there are no pending dependencies left, monitorRunDependencies will
+    // never be called. Since we can't check the number of dependencies,
+    // manually trigger a call.
+    globalThis.Module.addRunDependency('dummy');
+    globalThis.Module.removeRunDependency('dummy');
+    return promise;
+  }
+
+  private _raw_xkernel: any;
+  private _raw_xserver: any;
+  private _drive: DriveFS | null = null;
+  private _ready: PromiseLike<void>;
+}
+
+expose(new XeusPythonKernel());