#1. Refactoring code

Signed-off-by: Zhongkai Fu <[email protected]>

Author: zhongkaifu

RNNSharp/BiRNN.cs

@@ -46,6 +46,12 @@ public override void SetTrainingSet(DataSet train)
             backwardRNN.SetTrainingSet(train);
         }
 
+        public override void initWeights()
+        {
+            forwardRNN.initWeights();
+            backwardRNN.initWeights();
+
+        }
         public override void SetValidationSet(DataSet validation)
         {
             m_ValidationSet = validation;

RNNSharp/LSTMRNN.cs

@@ -57,9 +57,9 @@ public struct LSTMWeight
 
     public class LSTMRNN : RNN
     {
-        public new LSTMCell[] neuHidden;		//neurons in hidden layer
-        protected new LSTMWeight[][] mat_input2hidden;
-        protected new LSTMWeight[][] mat_feature2hidden;
+        public LSTMCell[] neuHidden;		//neurons in hidden layer
+        protected LSTMWeight[][] mat_input2hidden;
+        protected LSTMWeight[][] mat_feature2hidden;
 
         //for LSTM layer
         const bool NORMAL = true;
@@ -157,8 +157,7 @@ public override void loadNetBin(string filename)
             fea_size = br.ReadInt32();
 
             //Create cells of each layer
-            CreateCells();
-            CreateHiddenLayerCells(br);
+            CreateCell(br);
 
             //Load weight matrix between each two layer pairs
             //weight input->hidden
@@ -369,17 +368,34 @@ public void LSTMCellInit(bool type, LSTMCell c)
 
         public override void initMem()
         {
-            base.initMem();
+            CreateCell(null);
 
-            CreateHiddenLayerCells(null);
+            mat_hidden2output = new Matrix(L2, L1);
 
+            for (int i = 0; i < MAX_RNN_HIST; i++)
+            {
+                m_Diff[i] = new double[L2];
+            }
+
+            m_tagBigramTransition = new Matrix(L2, L2);
+            m_DeltaBigramLM = new Matrix(L2, L2);
+
+            Console.WriteLine("[TRACE] Initializing weights, random value is {0}", random(-1.0, 1.0));// yy debug
             initWeights();
         }
 
-        private void CreateHiddenLayerCells(BinaryReader br)
+        private void CreateCell(BinaryReader br)
         {
-            neuHidden = new LSTMCell[L1 + 1];
+            neuFeatures = new double[fea_size];
+            neuOutput = new neuron[L2];
+
+            for (int a = 0; a < L2; a++)
+            {
+                neuOutput[a].cellOutput = 0;
+                neuOutput[a].er = 0;
+            }
 
+            neuHidden = new LSTMCell[L1 + 1];
             for (int i = 0; i < L1; i++)
             {
                 neuHidden[i] = new LSTMCell();
@@ -429,14 +445,14 @@ public void matrixXvectorADD(neuron[] dest, LSTMCell[] srcvec, Matrix srcmatrix,
             });
         }
 
-        public void matrixXvectorADD(LSTMCell[] dest, neuron[] srcvec, LSTMWeight[][] srcmatrix, int from, int to, int from2, int to2)
+        public void matrixXvectorADD(LSTMCell[] dest, double[] srcvec, LSTMWeight[][] srcmatrix, int from, int to, int from2, int to2)
         {
             //ac mod
             Parallel.For(0, (to - from), parallelOption, i =>
             {
                 for (int j = 0; j < to2 - from2; j++)
                 {
-                    dest[i + from].netIn += srcvec[j + from2].cellOutput * srcmatrix[i][j].wInputInputGate;
+                    dest[i + from].netIn += srcvec[j + from2] * srcmatrix[i][j].wInputInputGate;
                 }
             });
         }
@@ -485,9 +501,9 @@ public override void learnNet(State state, int timeat, bool biRNN = false)
                     for (int j = 0; j < fea_size; j++)
                     {
                         LSTMWeight w = w_i[j];
-                        w_i[j].dSInputCell = w.dSInputCell * c.yForget + gPrime(c.netCellState) * c.yIn * neuFeatures[j].cellOutput;
-                        w_i[j].dSInputInputGate = w.dSInputInputGate * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * neuFeatures[j].cellOutput;
-                        w_i[j].dSInputForgetGate = w.dSInputForgetGate * c.yForget + c.previousCellState * fPrime(c.netForget) * neuFeatures[j].cellOutput;
+                        w_i[j].dSInputCell = w.dSInputCell * c.yForget + gPrime(c.netCellState) * c.yIn * neuFeatures[j];
+                        w_i[j].dSInputInputGate = w.dSInputInputGate * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * neuFeatures[j];
+                        w_i[j].dSInputForgetGate = w.dSInputForgetGate * c.yForget + c.previousCellState * fPrime(c.netForget) * neuFeatures[j];
 
                     }
                 }
@@ -560,14 +576,14 @@ public override void learnNet(State state, int timeat, bool biRNN = false)
                           w_i[j].wInputCell += alpha * cellStateError * w_i[j].dSInputCell - w_i[j].wInputCell * beta2;
                           w_i[j].wInputInputGate += alpha * cellStateError * w_i[j].dSInputInputGate - w_i[j].wInputInputGate * beta2;
                           w_i[j].wInputForgetGate += alpha * cellStateError * w_i[j].dSInputForgetGate - w_i[j].wInputForgetGate * beta2;
-                          w_i[j].wInputOutputGate += alpha * gradientOutputGate * neuFeatures[j].cellOutput - w_i[j].wInputOutputGate * beta2;
+                          w_i[j].wInputOutputGate += alpha * gradientOutputGate * neuFeatures[j] - w_i[j].wInputOutputGate * beta2;
                       }
                       else
                       {
                           w_i[j].wInputCell += alpha * cellStateError * w_i[j].dSInputCell;
                           w_i[j].wInputInputGate += alpha * cellStateError * w_i[j].dSInputInputGate;
                           w_i[j].wInputForgetGate += alpha * cellStateError * w_i[j].dSInputForgetGate;
-                          w_i[j].wInputOutputGate += alpha * gradientOutputGate * neuFeatures[j].cellOutput;
+                          w_i[j].wInputOutputGate += alpha * gradientOutputGate * neuFeatures[j];
                       }
 
                   }
@@ -683,9 +699,9 @@ public override void computeNet(State state, double[] doutput)
                     for (int i = 0; i < fea_size; i++)
                     {
                         LSTMWeight w = mat_feature2hidden[j][i];
-                        cell_j.netForget += neuFeatures[i].cellOutput * w.wInputForgetGate;
-                        cell_j.netCellState += neuFeatures[i].cellOutput * w.wInputCell;
-                        cell_j.netOut += neuFeatures[i].cellOutput * w.wInputOutputGate;
+                        cell_j.netForget += neuFeatures[i] * w.wInputForgetGate;
+                        cell_j.netCellState += neuFeatures[i] * w.wInputCell;
+                        cell_j.netOut += neuFeatures[i] * w.wInputOutputGate;
                     }
                 }
 
@@ -731,16 +747,9 @@ public override void computeNet(State state, double[] doutput)
         public override void netFlush()   //cleans all activations and error vectors
         {
             int a;
-            for (a = 0; a < L0; a++)
-            {
-                neuInput[a].cellOutput = 0;
-                neuInput[a].er = 0;
-            }
-
             for (a = 0; a < fea_size; a++)
             {
-                neuFeatures[a].cellOutput = 0;
-                neuFeatures[a].er = 0;
+                neuFeatures[a] = 0;
             }
 
             for (int i = 0; i < L1; i++)

RNNSharp/RNN.cs

@@ -54,29 +54,19 @@ abstract public class RNN
         public int L1;
         public int L2;
         protected int fea_size;
-
         protected float randrng;
-
         protected int final_stop;
 
-        protected neuron[] neuInput;		//neurons in input layer
-        protected neuron[] neuFeatures;		//features in input layer
-        protected neuron[] neuHidden;		//neurons in hidden layer
-        public neuron[] neuOutput;		//neurons in output layer
 
+        protected double[] neuFeatures;		//features in input layer
+        public neuron[] neuOutput;		//neurons in output layer
+        public Matrix mat_hidden2output;
 
         protected const int MAX_RNN_HIST = 512;
 
         protected Matrix m_RawOutput;
         protected int counterTokenForLM;
 
-        protected Matrix mat_input2hidden = new Matrix();
-        public Matrix mat_hidden2output = new Matrix();
-        protected Matrix mat_feature2hidden = new Matrix();
-
-
-        protected double md_beta = 1.0;
-
         protected DataSet m_TrainingSet;
         protected DataSet m_ValidationSet;
 
@@ -163,7 +153,7 @@ public void setInputLayer(State state, int curState, int numStates, int[] predic
             var dense = state.GetDenseData();
             for (int i = 0; i < dense.GetDimension(); i++)
             {
-                neuFeatures[i].cellOutput = dense[i];
+                neuFeatures[i] = dense[i];
             }
         }
 
@@ -198,9 +188,7 @@ public RNN()
 
             fea_size = 0;
 
-            neuInput = null;
             neuFeatures = null;
-            neuHidden = null;
             neuOutput = null;
         }
 
@@ -492,45 +480,9 @@ public void ForwardBackward(int numStates, Matrix m_RawOutput)
             }
         }
 
-        public void computeHiddenActivity()
-        {
-            for (int a = 0; a < L1; a++)
-            {
-                if (neuHidden[a].cellOutput > 50) neuHidden[a].cellOutput = 50;  //for numerical stability
-                if (neuHidden[a].cellOutput < -50) neuHidden[a].cellOutput = -50;  //for numerical stability
-                neuHidden[a].cellOutput = 1.0 / (1.0 + Math.Exp(-neuHidden[a].cellOutput));
-            }
-        }
-
-        public virtual void initWeights()
-        {
-            int b, a;
-            for (b = 0; b < L1; b++)
-            {
-                for (a = 0; a < L0 - L1; a++)
-                {
-                    mat_input2hidden[b][a] = random(-randrng, randrng) + random(-randrng, randrng) + random(-randrng, randrng);
-                }
-            }
 
 
-            for (b = 0; b < L1; b++)
-            {
-                for (a = 0; a < fea_size; a++)
-                {
-                    mat_feature2hidden[b][a] = random(-randrng, randrng) + random(-randrng, randrng) + random(-randrng, randrng);
-
-                }
-            }
-
-            for (b = 0; b < mat_hidden2output.GetHeight(); b++)
-            {
-                for (a = 0; a < L1; a++)
-                {
-                    mat_hidden2output[b][a] = random(-randrng, randrng) + random(-randrng, randrng) + random(-randrng, randrng);
-                }
-            }
-        }
+        public abstract void initWeights();
 
         public double random(double min, double max)
         {
@@ -616,60 +568,7 @@ public int randNext()
             return rand.Next();
         }
 
-        public virtual void initMem()
-        {
-            CreateCells();
-
-            mat_input2hidden = new Matrix(L1, L0 - L1);
-            mat_feature2hidden = new Matrix(L1, fea_size);
-            mat_hidden2output = new Matrix(L2, L1);
-
-            Console.WriteLine("[TRACE] Initializing weights, random value is {0}", random(-1.0, 1.0));// yy debug
-            initWeights();
-
-            for (int i = 0; i < MAX_RNN_HIST; i++)
-            {
-                m_Diff[i] = new double[L2];
-            }
-
-            m_tagBigramTransition = new Matrix(L2, L2);
-            m_DeltaBigramLM = new Matrix(L2, L2);
-
-        }
-
-        protected void CreateCells()
-        {
-
-            neuInput = new neuron[L0];
-            neuFeatures = new neuron[fea_size];
-            neuHidden = new neuron[L1];
-            neuOutput = new neuron[L2];
-
-            for (int a = 0; a < L0; a++)
-            {
-                neuInput[a].cellOutput = 0;
-                neuInput[a].er = 0;
-            }
-
-            for (int a = 0; a < fea_size; a++)
-            {
-                neuFeatures[a].cellOutput = 0;
-                neuFeatures[a].er = 0;
-            }
-
-            for (int a = 0; a < L1; a++)
-            {
-                neuHidden[a].cellOutput = 0;
-                neuHidden[a].er = 0;
-            }
-
-            for (int a = 0; a < L2; a++)
-            {
-                neuOutput[a].cellOutput = 0;
-                neuOutput[a].er = 0;
-            }
-        }
-
+        public abstract void initMem();
 
         public abstract void saveNetBin(string filename);
         public abstract void loadNetBin(string filename);
@@ -686,6 +585,18 @@ public static MODELTYPE CheckModelFileType(string filename)
             return type;
         }
 
+        public void matrixXvectorADD(neuron[] dest, double[] srcvec, Matrix srcmatrix, int from, int to, int from2, int to2)
+        {
+            //ac mod
+            Parallel.For(0, (to - from), parallelOption, i =>
+            {
+                for (int j = 0; j < to2 - from2; j++)
+                {
+                    dest[i + from].cellOutput += srcvec[j + from2] * srcmatrix[i][j];
+                }
+            });
+        }
+
         public void matrixXvectorADD(neuron[] dest, neuron[] srcvec, Matrix srcmatrix, int from, int to, int from2, int to2, int type)
         {
             if (type == 0)
@@ -723,34 +634,7 @@ public void matrixXvectorADD(neuron[] dest, neuron[] srcvec, Matrix srcmatrix, i
             }
         }
 
-        public virtual void netFlush()   //cleans all activations and error vectors
-        {
-            int a;
-            for (a = 0; a < L0 - L1; a++)
-            {
-                neuInput[a].cellOutput = 0;
-                neuInput[a].er = 0;
-            }
-
-            for (a = L0 - L1; a < L0; a++)
-            {   
-                //last hidden layer is initialized to vector of 0.1 values to prevent unstability
-                neuInput[a].cellOutput = 0.1;
-                neuInput[a].er = 0;
-            }
-
-            for (a = 0; a < L1; a++)
-            {
-                neuHidden[a].cellOutput = 0;
-                neuHidden[a].er = 0;
-            }
-
-            for (a = 0; a < L2; a++)
-            {
-                neuOutput[a].cellOutput = 0;
-                neuOutput[a].er = 0;
-            }
-        }
+        public abstract void netFlush();
 
         public int[] DecodeNN(Sequence seq)
         {