Starts on #11, though tests not working yet

Author: robertleeplummerjr

.idea/jsLibraryMappings.xml

@@ -1,3 +1,3 @@
-exports.NeuralNetwork = require("./neuralnetwork").NeuralNetwork;
+exports.NeuralNetwork = require("./neural-network").NeuralNetwork;
 exports.crossValidate = require("./cross-validate");
 exports.likely = require("./likely");

index.html

@@ -1,76 +1,68 @@
-var _ = require("underscore")._;
+import { _ } from'underscore';
 
 function testPartition(classifierConst, opts, trainOpts, trainSet, testSet) {
-  var classifier = new classifierConst(opts);
-
-  var beginTrain = Date.now();
-
-  var trainingStats = classifier.train(trainSet, trainOpts);
-
-  var beginTest = Date.now();
-
-  var testStats = classifier.test(testSet);
-
-  var endTest = Date.now();
-
-  var stats = _(testStats).extend({
-    trainTime : beginTest - beginTrain,
-    testTime : endTest - beginTest,
-    iterations: trainingStats.iterations,
-    trainError: trainingStats.error,
-    learningRate: trainOpts.learningRate,
-    hidden: classifier.hiddenSizes,
-    network: classifier.toJSON()
-  });
+  let classifier = new classifierConst(opts)
+    , beginTrain = Date.now()
+    , trainingStats = classifier.train(trainSet, trainOpts)
+    , beginTest = Date.now()
+    , testStats = classifier.test(testSet)
+    , endTest = Date.now()
+    , stats = _(testStats).extend({
+        trainTime : beginTest - beginTrain,
+        testTime : endTest - beginTest,
+        iterations: trainingStats.iterations,
+        trainError: trainingStats.error,
+        learningRate: trainOpts.learningRate,
+        hidden: classifier.hiddenSizes,
+        network: classifier.toJSON()
+      })
+    ;
 
   return stats;
 }
 
-module.exports = function crossValidate(classifierConst, data, opts, trainOpts, k) {
-  k = k || 4;
-  var size = data.length / k;
+export default function crossValidate(classifierConst, data, opts, trainOpts, k = 4) {
+  let size = data.length / k;
 
   data = _(data).sortBy(function() {
     return Math.random();
   });
 
-  var avgs = {
-    error : 0,
-    trainTime : 0,
-    testTime : 0,
-    iterations: 0,
-    trainError: 0
-  };
-
-  var stats = {
-    truePos: 0,
-    trueNeg: 0,
-    falsePos: 0,
-    falseNeg: 0,
-    total: 0
-  };
-
-  var misclasses = [];
-
-  var results = _.range(k).map(function(i) {
-    var dclone = _(data).clone();
-    var testSet = dclone.splice(i * size, size);
-    var trainSet = dclone;
-
-    var result = testPartition(classifierConst, opts, trainOpts, trainSet, testSet);
-
-    _(avgs).each(function(sum, stat) {
-      avgs[stat] = sum + result[stat];
-    });
-
-    _(stats).each(function(sum, stat) {
-      stats[stat] = sum + result[stat];
-    })
-
-    misclasses.push(result.misclasses);
-
-    return result;
-  });
+  let avgs = {
+        error : 0,
+        trainTime : 0,
+        testTime : 0,
+        iterations: 0,
+        trainError: 0
+      }
+    , stats = {
+        truePos: 0,
+        trueNeg: 0,
+        falsePos: 0,
+        falseNeg: 0,
+        total: 0
+      }
+    , misclasses = []
+    , results = _.range(k).map(function(i) {
+        let dclone = _(data).clone()
+          , testSet = dclone.splice(i * size, size)
+          , trainSet = dclone
+          , result = testPartition(classifierConst, opts, trainOpts, trainSet, testSet)
+          ;
+
+        _(avgs).each(function(sum, stat) {
+          avgs[stat] = sum + result[stat];
+        });
+
+        _(stats).each(function(sum, stat) {
+          stats[stat] = sum + result[stat];
+        });
+
+        misclasses.push(result.misclasses);
+
+        return result;
+      })
+    ;
 
   _(avgs).each(function(sum, i) {
     avgs[i] = sum / k;

lib/brain.js

@@ -1,8 +1,10 @@
-module.exports = function (input, net) {
-  var output = net.run(input);
-  var maxProp = null;
-  var maxValue = -1;
-  for (var prop in output) {
+export default function (input, net) {
+  let output = net.run(input)
+    , maxProp = null
+    , maxValue = -1
+    ;
+
+  for (let prop in output) {
     if (output.hasOwnProperty(prop)) {
       var value = output[prop];
       if (value > maxValue) {
@@ -11,5 +13,6 @@ module.exports = function (input, net) {
       }
     }
   }
+  
   return maxProp;
-};
+};

lib/cross-validate.js

@@ -1,58 +1,55 @@
-var _ = require("underscore");
+import _ from 'underscore';
 
 /* Functions for turning sparse hashes into arrays and vice versa */
-
-function buildLookup(hashes) {
+export function buildLookup(hashes) {
   // [{a: 1}, {b: 6, c: 7}] -> {a: 0, b: 1, c: 2}
-  var hash = _(hashes).reduce(function(memo, hash) {
+  let hash = _(hashes).reduce(function(memo, hash) {
     return _(memo).extend(hash);
   }, {});
   return lookupFromHash(hash);
 }
 
-function lookupFromHash(hash) {
+export function lookupFromHash(hash) {
   // {a: 6, b: 7} -> {a: 0, b: 1}
-  var lookup = {};
-  var index = 0;
-  for (var i in hash) {
+  let lookup = {}
+    , index = 0
+    ;
+
+  for (let i in hash) {
     lookup[i] = index++;
   }
+
   return lookup;
 }
 
-function toArray(lookup, hash) {
+export function toArray(lookup, hash) {
   // {a: 0, b: 1}, {a: 6} -> [6, 0]
-  var array = [];
-  for (var i in lookup) {
+  let array = [];
+  for (let i in lookup) {
     array[lookup[i]] = hash[i] || 0;
   }
   return array;
 }
 
-function toHash(lookup, array) {
+export function toHash(lookup, array) {
   // {a: 0, b: 1}, [6, 7] -> {a: 6, b: 7}
-  var hash = {};
-  for (var i in lookup) {
+  let hash = {};
+  for (let i in lookup) {
     hash[i] = array[lookup[i]];
   }
   return hash;
 }
 
-function lookupFromArray(array) {
-  var lookup = {};
-  // super fast loop
-  var z = 0;
-  var i = array.length;
+export function lookupFromArray(array) {
+  let lookup = {}
+    // super fast loop
+    , z = 0
+    , i = array.length
+    ;
+
   while (i-- > 0) {
     lookup[array[i]] = z++;
-  };
+  }
+
   return lookup;
 }
-
-module.exports = {
-  buildLookup: buildLookup,
-  lookupFromHash: lookupFromHash,
-  toArray: toArray,
-  toHash: toHash,
-  lookupFromArray: lookupFromArray
-};

lib/likely.js

@@ -0,0 +1,387 @@
+var Matrix = require('./matrix'),
+  RandomMatrix = require('./matrix/random'),
+  add = require('./matrix/add'),
+  multiply = require('./matrix/multiply'),
+  multiplyElement = require('./matrix/multiply-element');
+
+// Transformer definitions
+function Graph(needs_backprop) {
+  if(typeof needs_backprop === 'undefined') { needs_backprop = true; }
+  this.needs_backprop = needs_backprop;
+
+  // this will store a list of functions that perform backprop,
+  // in their forward pass order. So in backprop we will go
+  // backwards and evoke each one
+  this.backprop = [];
+}
+Graph.prototype = {
+  backward: function() {
+    while(this.backprop.length > 1) {
+      this.backprop.pop()(); // tick!
+    }
+  },
+  rowPluck: function(m, ix) {
+    // pluck a row of m with index ix and return it as col vector
+    assert(ix >= 0 && ix < m.n);
+    var d = m.d;
+    var out = new Matrix(d, 1);
+    for(var i=0,n=d;i<n;i++){ out.w[i] = m.w[d * ix + i]; } // copy over the data
+
+    if(this.needs_backprop) {
+      this.backprop.push(function backward() {
+        for(var i=0,n=d;i<n;i++){ m.dw[d * ix + i] += out.dw[i]; }
+      });
+    }
+    return out;
+  },
+  tanh: function(m) {
+    // tanh nonlinearity
+    var out = new Matrix(m.n, m.d);
+    var n = m.w.length;
+    for(var i=0;i<n;i++) {
+      out.w[i] = Math.tanh(m.w[i]);
+    }
+
+    if(this.needs_backprop) {
+      this.backprop.push(function backward() {
+        for(var i=0;i<n;i++) {
+          // grad for z = tanh(x) is (1 - z^2)
+          var mwi = out.w[i];
+          m.dw[i] += (1.0 - mwi * mwi) * out.dw[i];
+        }
+      });
+    }
+    return out;
+  },
+  sigmoid: function(m) {
+    // sigmoid nonlinearity
+    var out = new Matrix(m.n, m.d);
+    var n = m.w.length;
+    for(var i=0;i<n;i++) {
+      out.w[i] = sig(m.w[i]);
+    }
+
+    if(this.needs_backprop) {
+      this.backprop.push(function backward() {
+        for(var i=0;i<n;i++) {
+          // grad for z = tanh(x) is (1 - z^2)
+          var mwi = out.w[i];
+          m.dw[i] += mwi * (1.0 - mwi) * out.dw[i];
+        }
+      });
+    }
+    return out;
+  },
+  relu: function(m) {
+    var out = new Matrix(m.n, m.d);
+    var n = m.w.length;
+    for(var i=0;i<n;i++) {
+      out.w[i] = Math.max(0, m.w[i]); // relu
+    }
+    if(this.needs_backprop) {
+      this.backprop.push(function backward() {
+        for(var i=0;i<n;i++) {
+          m.dw[i] += m.w[i] > 0 ? out.dw[i] : 0.0;
+        }
+      });
+    }
+    return out;
+  }
+};
+
+function softmax(m) {
+  var out = new Matrix(m.n, m.d); // probability volume
+  var maxval = -999999;
+  for(var i=0,n=m.w.length;i<n;i++) { if(m.w[i] > maxval) maxval = m.w[i]; }
+
+  var s = 0.0;
+  for(var i=0,n=m.w.length;i<n;i++) {
+    out.w[i] = Math.exp(m.w[i] - maxval);
+    s += out.w[i];
+  }
+  for(var i=0,n=m.w.length;i<n;i++) { out.w[i] /= s; }
+
+  // no backward pass here needed
+  // since we will use the computed probabilities outside
+  // to set gradients directly on m
+  return out;
+}
+
+function Solver() {
+  this.decay_rate = 0.999;
+  this.smooth_eps = 1e-8;
+  this.step_cache = {};
+}
+Solver.prototype = {
+  step: function(model, step_size, regc, clipval) {
+    // perform parameter update
+    var solver_stats = {};
+    var num_clipped = 0;
+    var num_tot = 0;
+    for(var k in model) {
+      if(model.hasOwnProperty(k)) {
+        var m = model[k]; // mat ref
+        if(!(k in this.step_cache)) { this.step_cache[k] = new Matrix(m.n, m.d); }
+        var s = this.step_cache[k];
+        for(var i=0,n=m.w.length;i<n;i++) {
+
+          // rmsprop adaptive learning rate
+          var mdwi = m.dw[i];
+          s.w[i] = s.w[i] * this.decay_rate + (1.0 - this.decay_rate) * mdwi * mdwi;
+
+          // gradient clip
+          if(mdwi > clipval) {
+            mdwi = clipval;
+            num_clipped++;
+          }
+          if(mdwi < -clipval) {
+            mdwi = -clipval;
+            num_clipped++;
+          }
+          num_tot++;
+
+          // update (and regularize)
+          m.w[i] += - step_size * mdwi / Math.sqrt(s.w[i] + this.smooth_eps) - regc * m.w[i];
+          m.dw[i] = 0; // reset gradients for next iteration
+        }
+      }
+    }
+    solver_stats['ratio_clipped'] = num_clipped*1.0/num_tot;
+    return solver_stats;
+  }
+};
+
+function LSTM(input_size, hidden_sizes, output_size) {
+  // hidden size should be a list
+
+  var model = [];
+  for(var d=0;d<hidden_sizes.length;d++) { // loop over depths
+    var prev_size = d === 0 ? input_size : hidden_sizes[d - 1];
+    var hidden_size = hidden_sizes[d];
+    model.push({
+      // gates parameters
+      wix: new RandomMatrix(hidden_size, prev_size , 0, 0.08),
+      wih: new RandomMatrix(hidden_size, hidden_size , 0, 0.08),
+      bi: new Matrix(hidden_size, 1),
+
+      wfx: new RandomMatrix(hidden_size, prev_size , 0, 0.08),
+      wfh: new RandomMatrix(hidden_size, hidden_size , 0, 0.08),
+      bf: new Matrix(hidden_size, 1),
+
+      wox: new RandomMatrix(hidden_size, prev_size , 0, 0.08),
+      woh: new RandomMatrix(hidden_size, hidden_size , 0, 0.08),
+      bo: new Matrix(hidden_size, 1),
+
+      // cell write params
+      wcx: new RandomMatrix(hidden_size, prev_size , 0, 0.08),
+      wch: new RandomMatrix(hidden_size, hidden_size , 0, 0.08),
+      bc: new Matrix(hidden_size, 1)
+    });
+  }
+  // decoder params
+  model.whd = new RandomMatrix(output_size, hidden_size, 0, 0.08);
+  model.bd = new Matrix(output_size, 1);
+  return model;
+}
+
+/**
+ *
+ * @param {Graph} graph
+ * @param model
+ * @param hidden_sizes
+ * @param x
+ * @param prev
+ * @returns {{h: Array, c: Array, o}}
+ */
+function forwardLSTM(graph, model, hidden_sizes, x, prev) {
+  // forward prop for a single tick of LSTM
+  // G is graph to append ops to
+  // model contains LSTM parameters
+  // x is 1D column vector with observation
+  // prev is a struct containing hidden and cell
+  // from previous iteration
+
+  var hidden_prevs,
+    cell_prevs;
+  if(typeof prev.h === 'undefined') {
+    hidden_prevs = [];
+    cell_prevs = [];
+    for(var d=0;d<hidden_sizes.length;d++) {
+      hidden_prevs.push(new Matrix(hidden_sizes[d],1));
+      cell_prevs.push(new Matrix(hidden_sizes[d],1));
+    }
+  } else {
+    hidden_prevs = prev.h;
+    cell_prevs = prev.c;
+  }
+
+  var hidden = [];
+  var cell = [];
+  for(var d=0;d<hidden_sizes.length;d++) {
+
+    var input_vector = d === 0 ? x : hidden[d-1];
+    var hidden_prev = hidden_prevs[d];
+    var cell_prev = cell_prevs[d];
+
+    // input gate
+    var h0 = multiply(model[d].wix, input_vector);
+    var h1 = multiply(model[d].wih, hidden_prev);
+    var input_gate = graph.sigmoid(add(add(h0,h1),model[d].bi));
+
+    // forget gate
+    var h2 = multiply(model[d].wfx, input_vector);
+    var h3 = multiply(model[d].wfh, hidden_prev);
+    var forget_gate = graph.sigmoid(add(add(h2, h3),model[d].bf));
+
+    // output gate
+    var h4 = multiply(model[d].wox, input_vector);
+    var h5 = multiply(model[d].woh, hidden_prev);
+    var output_gate = graph.sigmoid(add(add(h4, h5),model[d].bo));
+
+    // write operation on cells
+    var h6 = multiply(model[d].wcx, input_vector);
+    var h7 = multiply(model[d].wch, hidden_prev);
+    var cell_write = graph.tanh(add(add(h6, h7),model[d].bc));
+
+    // compute new cell activation
+    var retain_cell = multiplyElement(forget_gate, cell_prev); // what do we keep from cell
+    var write_cell = multiplyElement(input_gate, cell_write); // what do we write to cell
+    var cell_d = add(retain_cell, write_cell); // new cell contents
+
+    // compute hidden state as gated, saturated cell activations
+    var hidden_d = multiplyElement(output_gate, graph.tanh(cell_d));
+
+    hidden.push(hidden_d);
+    cell.push(cell_d);
+  }
+
+  // one decoder to outputs at end
+  var output = add(multiply(model.whd, hidden[hidden.length - 1]), model.bd);
+
+  // return cell memory, hidden representation and output
+  return {
+    h: hidden,
+    c: cell,
+    o: output
+  };
+}
+
+function RNN(input_size, hidden_sizes, output_size) {
+  // hidden size should be a list
+
+  var model = [];
+  for(var d=0;d<hidden_sizes.length;d++) { // loop over depths
+    var prev_size = d === 0 ? input_size : hidden_sizes[d - 1];
+    var hidden_size = hidden_sizes[d];
+    model.push({
+      wxh: new RandomMatrix(hidden_size, prev_size, 0, 0.08),
+      whh: new RandomMatrix(hidden_size, hidden_size, 0, 0.08),
+      bhh: new Matrix(hidden_size, 1)
+    });
+  }
+  // decoder params
+  model.whd = new RandomMatrix(output_size, hidden_size, 0, 0.08);
+  model.bd = new Matrix(output_size, 1);
+  return model;
+}
+
+/**
+ *
+ * @param {Graph} graph
+ * @param model
+ * @param hidden_sizes
+ * @param x
+ * @param prev
+ * @returns {{h: Array, o}}
+ */
+function forwardRNN(graph, model, hidden_sizes, x, prev) {
+  // forward prop for a single tick of RNN
+  // G is graph to append ops to
+  // model contains RNN parameters
+  // x is 1D column vector with observation
+  // prev is a struct containing hidden activations from last step
+  var hidden_prevs;
+  if(typeof prev.h === 'undefined') {
+    hidden_prevs = [];
+    for(var d=0;d<hidden_sizes.length;d++) {
+      hidden_prevs.push(new Matrix(hidden_sizes[d],1));
+    }
+  } else {
+    hidden_prevs = prev.h;
+  }
+
+  var hidden = [];
+  for(var d=0;d<hidden_sizes.length;d++) {
+
+    var input_vector = d === 0 ? x : hidden[d-1];
+    var hidden_prev = hidden_prevs[d];
+
+    var h0 = multiply(model[d].wxh, input_vector);
+    var h1 = multiply(model[d].whh, hidden_prev);
+    var hidden_d = graph.relu(add(add(h0, h1), model[d].bhh));
+
+    hidden.push(hidden_d);
+  }
+
+  // one decoder to outputs at end
+  var output = add(multiply(model.whd, hidden[hidden.length - 1]), model.bd);
+
+  // return cell memory, hidden representation and output
+  return {
+    h: hidden,
+    o: output
+  };
+}
+
+function sig(x) {
+  // helper function for computing sigmoid
+  return 1.0 / (1 + Math.exp(-x));
+}
+
+function maxi(w) {
+  // argmax of array w
+  var maxv = w[0];
+  var maxix = 0;
+  for(var i=1,n=w.length;i<n;i++) {
+    var v = w[i];
+    if(v > maxv) {
+      maxix = i;
+      maxv = v;
+    }
+  }
+  return maxix;
+}
+
+function samplei(w) {
+  // sample argmax from w, assuming w are
+  // probabilities that sum to one
+  var r = randf(0,1);
+  var x = 0.0;
+  var i = 0;
+  while(true) {
+    x += w[i];
+    if(x > r) { return i; }
+    i++;
+  }
+}
+
+// various utils
+module.exports = {
+  maxi: maxi,
+  samplei: samplei,
+  randi: randi,
+  softmax: softmax,
+
+  // classes
+  Matrix: Matrix,
+  RandomMatrix: RandomMatrix,
+
+  forwardLSTM: forwardLSTM,
+  LSTM: LSTM,
+  forwardRNN: forwardRNN,
+  RNN: RNN,
+
+// optimization
+  Solver: Solver,
+  Graph: Graph
+};

lib/lookup.js

@@ -0,0 +1,17 @@
+module.export = function add(m1, m2, backPropagateArray) {
+  if (m1.w.length !== m2.w.length) throw new Error('matrix addition dimensions misaligned');
+
+  var out = new Matrix(m1.n, m1.d);
+  for(var i=0,n=m1.w.length;i<n;i++) {
+    out.w[i] = m1.w[i] + m2.w[i];
+  }
+  if(typeof backPropagateArray !== 'undefined') {
+    backPropagateArray.push(function backward() {
+      for(var i=0,n=m1.w.length;i<n;i++) {
+        m1.dw[i] += out.dw[i];
+        m2.dw[i] += out.dw[i];
+      }
+    });
+  }
+  return out;
+};

lib/neural-network.js

@@ -0,0 +1,78 @@
+var zeros = require('./zeros');
+
+// Mat holds a matrix
+function Matrix(n, d) {
+  // n is number of rows d is number of columns
+  this.n = n;
+  this.d = d;
+  this.w = zeros(n * d);
+  this.dw = zeros(n * d);
+}
+
+Matrix.prototype = {
+  get: function(row, col) {
+    // slow but careful accessor function
+    // we want row-major order
+    var ix = (this.d * row) + col;
+    if (ix < 0 && ix >= this.w.length) throw new Error('get accessor is skewed');
+    return this.w[ix];
+  },
+  set: function(row, col, v) {
+    // slow but careful accessor function
+    var ix = (this.d * row) + col;
+    if (ix < 0 && ix >= this.w.length) throw new Error('set accessor is skewed');
+    this.w[ix] = v;
+  },
+  toJSON: function() {
+    return {
+      n: this.n,
+      d: this.d,
+      w: this.w
+    };
+  },
+  fromJSON: function(json) {
+    this.n = json.n;
+    this.d = json.d;
+    this.w = zeros(this.n * this.d);
+    this.dw = zeros(this.n * this.d);
+    for(var i=0,n=this.n * this.d;i<n;i++) {
+      this.w[i] = json.w[i]; // copy over weights
+    }
+  },
+
+  // fill matrix with random gaussian numbers
+  fillRandN: function(mu, std) {
+    for(var i=0,n=this.w.length;i<n;i++) {
+      this.w[i] = randn(mu, std);
+    }
+  },
+
+  // fill matrix with random gaussian numbers
+  fillRand: function(lo, hi) {
+    for(var i=0,n=this.w.length;i<n;i++) {
+      this.w[i] = randf(lo, hi);
+    }
+  }
+};
+
+function randn(mu, std){ return mu+gaussRandom()*std; }
+
+// Random numbers utils
+var return_v = false;
+var v_val = 0.0;
+function gaussRandom() {
+  if(return_v) {
+    return_v = false;
+    return v_val;
+  }
+  var u = 2*Math.random()-1;
+  var v = 2*Math.random()-1;
+  var r = u*u + v*v;
+  if(r == 0 || r > 1) return gaussRandom();
+  var c = Math.sqrt(-2*Math.log(r)/r);
+  v_val = v*c; // cache this
+  return_v = true;
+  return u*c;
+}
+
+module.export = Matrix;

lib/neuralnetwork.js

@@ -0,0 +1,18 @@
+module.exports = function elementMultiply(m1, m2, backPropagateArray) {
+  if (m1.w.length !== m2.w.length) throw new Error('matrix element multiplication dimensions misaligned');
+
+  var out = new Matrix(m1.n, m1.d);
+  for(var i=0,n=m1.w.length;i<n;i++) {
+    out.w[i] = m1.w[i] * m2.w[i];
+
+  }
+  if(typeof backPropagateArray !== 'undefined') {
+    backPropagateArray.push(function backward() {
+      for(var i=0,n=m1.w.length;i<n;i++) {
+        m1.dw[i] += m2.w[i] * out.dw[i];
+        m2.dw[i] += m1.w[i] * out.dw[i];
+      }
+    });
+  }
+  return out;
+};

lib/recurrent/index.js

@@ -0,0 +1,38 @@
+/**
+ * multiply matrices m1 * m2
+ * @param {Matrix} m1
+ * @param {Matrix} m2
+ * @param backPropagateArray
+ * @returns {Matrix}
+ */
+module.exports = function multiply(m1, m2, backPropagateArray) {
+  if (m1.d !== m2.n) throw new Error('matrix multiplication dimensions misaligned');
+
+  var n = m1.n;
+  var d = m2.d;
+  var out = new Matrix(n, d);
+  for(var i=0;i<m1.n;i++) { // loop over rows of m1
+    for(var j=0;j<m2.d;j++) { // loop over cols of m2
+      var dot = 0.0;
+      for(var k=0;k<m1.d;k++) { // dot product loop
+        dot += m1.w[m1.d*i+k] * m2.w[m2.d*k+j];
+      }
+      out.w[d*i+j] = dot;
+    }
+  }
+
+  if(typeof backPropagateArray !== 'undefined') {
+    backPropagateArray.push(function backward() {
+      for(var i=0;i<m1.n;i++) { // loop over rows of m1
+        for(var j=0;j<m2.d;j++) { // loop over cols of m2
+          for(var k=0;k<m1.d;k++) { // dot product loop
+            var b = out.dw[d*i+j];
+            m1.dw[m1.d*i+k] += m2.w[m2.d*k+j] * b;
+            m2.dw[m2.d*k+j] += m1.w[m1.d*i+k] * b;
+          }
+        }
+      }
+    });
+  }
+  return out;
+};

lib/recurrent/lstm.js

@@ -0,0 +1,12 @@
+var Matrix = require('./index');
+
+function RandomMatrixN(n,d,mu,std) {
+  this.n = n;
+  this.d = d;
+  this.w = zeros(n * d);
+  this.dw = zeros(n * d);
+  this.fillRandN(mu, std);
+}
+RandomMatrixN.prototype = Matrix.prototype;
+
+module.exports = RandomMatrixN;

lib/recurrent/matrix/add.js

@@ -0,0 +1,17 @@
+var Matrix = require('./index');
+
+/** return Mat but filled with random numbers from gaussian
+ * @param n
+ * @param d
+ * @param mu
+ * @param std
+ * @constructor
+ */
+function RandomMatrix(n,d,mu,std) {
+  this.w = zeros(n * d);
+  this.dw = zeros(n * d);
+  this.fillRand(-std, std); // kind of :P
+}
+RandomMatrix.prototype = Matrix.prototype;
+
+module.exports = RandomMatrix;

lib/recurrent/matrix/index.js

@@ -0,0 +1,12 @@
+// helper function returns array of zeros of length n
+// and uses typed arrays if available
+module.export = function zeros(n) {
+  if(typeof ArrayBuffer === 'undefined') {
+    // lacking browser support
+    var arr = new Array(n);
+    for(var i=0;i<n;i++) { arr[i] = 0; }
+    return arr;
+  } else {
+    return new Float64Array(n);
+  }
+};

lib/recurrent/matrix/multiply-element.js

@@ -0,0 +1,7 @@
+function randf(a, b) { return Math.random()*(b-a)+a; }
+function randi(a, b) { return Math.floor(Math.random()*(b-a)+a); }
+
+module.exports = {
+  f: randf,
+  i: randi
+};

lib/recurrent/matrix/multiply.js

@@ -0,0 +1,132 @@
+import { Writable } from 'stream';
+
+class TrainStream extends Writable {
+  constructor(opts = {}) {
+    super({ objectMode: true });
+
+    // require the neuralNetwork
+    if (!opts.neuralNetwork) {
+      throw new Error('no neural network specified');
+    }
+
+    this.neuralNetwork = opts.neuralNetwork;
+    this.dataFormatDetermined = false;
+
+    this.inputKeys = [];
+    this.outputKeys = []; // keeps track of keys seen
+    this.i = 0; // keep track of the for loop i variable that we got rid of
+    this.iterations = opts.iterations || 20000;
+    this.errorThresh = opts.errorThresh || 0.005;
+    this.log = opts.log ? (typeof opts.log === 'function' ? opts.log : console.log) : false;
+    this.logPeriod = opts.logPeriod || 10;
+    this.callback = opts.callback;
+    this.callbackPeriod = opts.callbackPeriod || 10;
+    this.floodCallback = opts.floodCallback;
+    this.doneTrainingCallback = opts.doneTrainingCallback;
+
+    this.size = 0;
+    this.count = 0;
+
+    this.sum = 0;
+
+    this.on('finish', this.finishStreamIteration);
+
+    return this;
+  }
+
+  /*
+   _write expects data to be in the form of a datum.
+   ie. {input: {a: 1 b: 0}, output: {z: 0}}
+   */
+  _write(chunk, enc, next) {
+    if (!chunk) { // check for the end of one interation of the stream
+      this.emit('finish');
+      return next();
+    }
+
+    if (!this.dataFormatDetermined) {
+      this.size++;
+      this.inputKeys = _.union(this.inputKeys, _.keys(chunk.input));
+      this.outputKeys = _.union(this.outputKeys, _.keys(chunk.output));
+      this.firstDatum = this.firstDatum || chunk;
+      return next();
+    }
+
+    this.count++;
+
+    let data = this.neuralNetwork.formatData(chunk);
+    this.trainDatum(data[0]);
+
+    // tell the Readable Stream that we are ready for more data
+    next();
+  }
+
+  trainDatum(datum) {
+    let err = this.neuralNetwork.trainPattern(datum.input, datum.output);
+    this.sum += err;
+  }
+
+  finishStreamIteration () {
+    if (this.dataFormatDetermined && this.size !== this.count) {
+      console.log("This iteration's data length was different from the first.");
+    }
+
+    if (!this.dataFormatDetermined) {
+      // create the lookup
+      this.neuralNetwork.inputLookup = lookup.lookupFromArray(this.inputKeys);
+      if(!_.isArray(this.firstDatum.output)){
+        this.neuralNetwork.outputLookup = lookup.lookupFromArray(this.outputKeys);
+      }
+
+      let data = this.neuralNetwork.formatData(this.firstDatum)
+        , inputSize = data[0].input.length
+        , outputSize = data[0].output.length
+        , hiddenSizes = this.hiddenSizes
+        ;
+
+      if (!hiddenSizes) {
+        hiddenSizes = [Math.max(3, Math.floor(inputSize / 2))];
+      }
+      let sizes = _([inputSize, hiddenSizes, outputSize]).flatten();
+      this.dataFormatDetermined = true;
+      this.neuralNetwork.initialize(sizes);
+
+      if (typeof this.floodCallback === 'function') {
+        this.floodCallback();
+      }
+      return;
+    }
+
+    let error = this.sum / this.size;
+
+    if (this.log && (this.i % this.logPeriod == 0)) {
+      this.log("iterations:", this.i, "training error:", error);
+    }
+    if (this.callback && (this.i % this.callbackPeriod == 0)) {
+      this.callback({
+        error: error,
+        iterations: this.i
+      });
+    }
+
+    this.sum = 0;
+    this.count = 0;
+    // update the iterations
+    this.i++;
+
+    // do a check here to see if we need the stream again
+    if (this.i < this.iterations && error > this.errorThresh) {
+      if (typeof this.floodCallback === 'function') {
+        return this.floodCallback();
+      }
+    } else {
+      // done training
+      if (typeof this.doneTrainingCallback === 'function') {
+        return this.doneTrainingCallback({
+          error: error,
+          iterations: this.i
+        });
+      }
+    }
+  }
+}

lib/recurrent/matrix/random-n.js

@@ -0,0 +1,8 @@
+export default function mse(errors) {
+  // mean squared error
+  let sum = 0;
+  for (let i = 0; i < errors.length; i++) {
+    sum += Math.pow(errors[i], 2);
+  }
+  return sum / errors.length;
+}

lib/recurrent/matrix/random.js

@@ -0,0 +1,3 @@
+export default function randomWeight() {
+  return Math.random() * 0.4 - 0.2;
+}

lib/recurrent/matrix/zeros.js

@@ -0,0 +1,9 @@
+import randomWeight from './utilities/random-weight';
+
+export default function randos(size) {
+  let array = new Array(size);
+  for (let i = 0; i < size; i++) {
+    array[i] = randomWeight();
+  }
+  return array;
+}

lib/recurrent/maxi.js

@@ -0,0 +1,7 @@
+export default function zeros(size) {
+  let array = new Array(size);
+  for (let i = 0; i < size; i++) {
+    array[i] = 0;
+  }
+  return array;
+}