Node fetching code; README update

Author: tloen

.gitignore

@@ -1 +1,2 @@
-**/api_token.txt
+**/api_token.txt
+.vscode

README.md

@@ -2,4 +2,8 @@
 
 > The man who is without qualification good at deliberating is the man who is capable of aiming in accordance with calculation at the best for man of things attainable by action.
 
-Eudaemon is a system of productivity tools intended primarily for my own use.
+**Eudaemon** is a highly opinionated system of productivity tools based on the framework of Aristotle's _Nicomachean Ethics_.
+
+_Daemonist_ (δ) creates semi-responsive dashboards within [Dynalist](https://dynalist.io/) that pull data from the real world, surfacing information when it's relevant and moving it to logs when it isn't.
+
+_Teleion_ (τ) provides an interactive view for reasoning about long-term goals and subgoals in terms of directed acyclic graphs.

daemonist/babel.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  presets: ["@babel/preset-env"]
+};

daemonist/package-lock.json

@@ -21,7 +21,16 @@
   },
   "homepage": "https://github.com/tloen/eudaemon#readme",
   "dependencies": {
+    "lodash": "^4.17.15",
     "node-fetch": "^2.6.0",
     "typescript": "^3.5.3"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.5.5",
+    "@babel/core": "^7.5.5",
+    "@babel/node": "^7.5.5",
+    "@babel/preset-env": "^7.5.5",
+    "@types/lodash-es": "^4.17.3",
+    "@types/node-fetch": "^2.5.0"
   }
 }

daemonist/package.json

@@ -2,7 +2,7 @@ import { Dynalist } from "./dynalist";
 
 export const ENDPOINTS = {
   GET_FILES: "https://dynalist.io/api/v1/file/list",
-  READ_DOCUMENT: "https://dynalist.io/api/v1/doc/read"
+  GET_DOCUMENT: "https://dynalist.io/api/v1/doc/read"
 };
 
 type FailCode =
@@ -33,12 +33,22 @@ export interface GetFilesResponse extends SuccessfulResponse {
   files: Dynalist.File[];
 }
 
-export interface DocumentReadResponse extends SuccessfulResponse {
-  title: string;
-  nodes: Dynalist.Node[];
+export interface DocumentReadNode {
+  id: string;
+  content: string;
+  note: string;
+  created: number;
+  modified: number;
+  children: string[];
+
+  checked?: boolean;
+  checkbox?: boolean;
+  color?: Dynalist.Color;
+  heading?: Dynalist.HeadingLevel;
+  collapsed?: boolean;
 }
 
-export interface NodeKey {
-  fileID: string;
-  nodeID: string;
+export interface DocumentReadResponse extends SuccessfulResponse {
+  title: string;
+  nodes: DocumentReadNode[];
 }

daemonist/src/api.ts

@@ -0,0 +1,4 @@
+import { Dynalist } from "./dynalist";
+
+const א = /[🏺😈👿🤖]/;
+const hasא = (node: Dynalist.Node) => !!א.exec(node.note);

daemonist/src/daemon.ts

@@ -30,14 +30,18 @@ export namespace Dynalist {
     H3
   }
 
+  export interface NodeKey {
+    documentId: string;
+    nodeId: string;
+  }
   export interface Node {
-    id: string;
+    key: NodeKey;
     content: string;
     note: string;
-    created: number;
-    modified: number;
-    children: string[];
+    created: Date;
+    modified: Date;
 
+    children?: NodeKey[];
     checked?: boolean;
     checkbox?: boolean;
     color?: Color;

daemonist/src/dynalist.ts

@@ -1,6 +1,8 @@
 import { DynalistAPI } from "./session";
 
-const api = new DynalistAPI("<|key|>");
+const api = new DynalistAPI(process.env.DYNALIST_API_TOKEN);
 
 // api.getFileTree().then(console.log);
-api.readDocument("HP3Ov2goSBuIPEzMA2T3jRkk").then(console.log);
+// api.readDocument("HP3Ov2goSBuIPEzMA2T3jRkk").then(console.log);
+// api.getAllDocuments().then(console.log);
+api.getNodeForest().then(console.log);

daemonist/src/index.ts

@@ -2,7 +2,8 @@ import {
   GetFilesResponse,
   ENDPOINTS,
   DocumentReadResponse,
-  SuccessfulResponse
+  SuccessfulResponse,
+  DocumentReadNode
 } from "./api";
 import { Dynalist } from "./dynalist";
 import fetch from "node-fetch";
@@ -14,46 +15,81 @@ interface FetchRequest {
 
 export class DynalistAPI {
   private token: string;
-  // private fileTree: Dynalist.File[] | undefined;
-  // private cache; // TODO
+  private cache: Cache;
 
   constructor(token: string) {
     this.token = token;
   }
 
-  // Limit: 60 times per minute
-  public async readAllDocuments(): Promise<Dynalist.Document[]> {
-    return this.getFileTree()
-      .then(
-        (files): Promise<DocumentReadResponse[]> => {
-          const requests = files.map(
-            (file): FetchRequest => ({
-              url: ENDPOINTS.READ_DOCUMENT,
-              params: {
-                file_id: file.id
-              }
-            })
-          );
-          return this.batchPostFetch<DocumentReadResponse>(requests, 1500);
-        }
-      )
-      .then(responses =>
-        responses.map(response => {
-          return {
-            id: "abc",
-            title: response.title,
-            nodes: response.nodes
-          };
+  public getNodeForest(): Promise<Dynalist.Node[]> {
+    return this.getAllDocuments().then(files =>
+      files.map(file => file.nodes).flat()
+    );
+  }
+
+  // Limit: 60 reads per minute
+  public async getAllDocuments(): Promise<Dynalist.Document[]> {
+    return this.getFileTree().then(files => {
+      const documentFiles = files.filter(file => file.type === "document");
+
+      const requests = documentFiles.map(
+        (file): FetchRequest => ({
+          url: ENDPOINTS.GET_DOCUMENT,
+          params: {
+            file_id: file.id
+          }
         })
       );
+      return this.batchPostFetch<DocumentReadResponse>(requests, 1200).then(
+        responses =>
+          responses.map((response, i) => ({
+            id: documentFiles[i].id,
+            title: response.title,
+            nodes: response.nodes.map(rawNode =>
+              this.transformNode(rawNode, documentFiles[i].id)
+            )
+          }))
+      );
+    });
   }
 
-  public readDocument(documentId: string): Promise<Dynalist.Document> {
-    return this.postFetch(ENDPOINTS.READ_DOCUMENT, {
+  public getDocumentByTitle(title: string): Promise<Dynalist.Document> {
+    return this.getFileTree().then(files => {
+      const matchingFiles = files.filter(
+        file => file.type === "document" && file.title == title
+      );
+      if (matchingFiles.length == 0) return undefined;
+      else return this.getDocumentById(matchingFiles[0].id);
+    });
+  }
+
+  private transformNode(
+    rawNode: DocumentReadNode,
+    documentId: string
+  ): Dynalist.Node {
+    return {
+      ...rawNode,
+      key: {
+        documentId,
+        nodeId: rawNode.id
+      },
+      created: new Date(rawNode.created),
+      modified: new Date(rawNode.modified),
+      children: rawNode.children
+        ? rawNode.children.map(nodeId => ({
+            documentId,
+            nodeId
+          }))
+        : []
+    };
+  }
+
+  public getDocumentById(documentId: string): Promise<Dynalist.Document> {
+    return this.postFetch(ENDPOINTS.GET_DOCUMENT, {
       file_id: documentId
     }).then((body: DocumentReadResponse) => ({
       id: documentId,
-      nodes: body.nodes,
+      nodes: body.nodes.map(rawNode => this.transformNode(rawNode, documentId)),
       title: body.title
     }));
   }
@@ -64,11 +100,17 @@ export class DynalistAPI {
     );
   }
 
+  // TODO: merge this into getting all docs
   private async batchPostFetch<T extends SuccessfulResponse>(
     requests: FetchRequest[],
     waitMilliseconds: number
   ): Promise<T[]> {
     const results: T[] = [];
+    console.log(
+      `Got ${requests.length} requests with wait time ${waitMilliseconds /
+        1000} seconds. Expect at least ${(requests.length * waitMilliseconds) /
+        1000} seconds delay.`
+    );
     for (const request of requests) {
       results.push(await this.postFetch<T>(request.url, request.params));
       await new Promise(_ => setTimeout(_, waitMilliseconds));
@@ -80,6 +122,7 @@ export class DynalistAPI {
     url: string,
     params?: any
   ): Promise<T> {
+    console.log(`POST ${url} with params ${JSON.stringify(params)}`);
     return fetch(url, {
       headers: {
         "Content-Type": "application/json"

daemonist/src/session.ts

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "lib": ["es2015", "es6", "dom"],
+    "lib": ["es2015", "es6", "dom", "esnext"],
     "outDir": "./build",
     "target": "es5"
   },

daemonist/tsconfig.json

@@ -0,0 +1 @@
+This is not here yet.