#1. Bugs fixed

#2. Code refactoring
#3. Performance improvement

Author: zhongkaifu

README.md

@@ -61,7 +61,7 @@ major      | JJ   | S
 financial  | JJ   | S
 centers    | NNS  | S
 .          | PUN  | S
-           |      |    
+---empty line---
 !          | PUN  | S
 p          | FW   | S
 '          | PUN  | S
@@ -162,21 +162,21 @@ PRETRAIN_TYPE = Embedding
 WORDEMBEDDING_FILENAME = Data\WordEmbedding\wordvec_chs.bin  
 \#The context range of word embedding. In below example, the context is current token, previous token and next token  
 \#If more than one token are combined, this feature would use a plenty of memory.  
-WORDEMBEDDING_CONTEXT: -1,0,1  
+WORDEMBEDDING_CONTEXT = -1,0,1  
 \#The column index applied word embedding feature  
 WORDEMBEDDING_COLUMN = 0  
 
 \#The following setting is for pretrained model in 'Autoencoder' type.  
 \#The feature configuration file for pretrained model.  
-AUTOENCODER_CONFIG: D:\RNNSharpDemoPackage\config_autoencoder.txt  
+AUTOENCODER_CONFIG = D:\RNNSharpDemoPackage\config_autoencoder.txt  
 
 \#The following setting is the configuration file for source sequence encoder which is only for sequence-to-sequence task that MODEL_TYPE equals to SEQ2SEQ.  
 \#In this example, since MODEL_TYPE is SEQLABEL, so we comment it out.  
-\#SEQ2SEQ_AUTOENCODER_CONFIG: D:\RNNSharpDemoPackage\config_seq2seq_autoencoder.txt  
+\#SEQ2SEQ_AUTOENCODER_CONFIG = D:\RNNSharpDemoPackage\config_seq2seq_autoencoder.txt  
 
 \#The context range of run time feature. In below example, RNNSharp will use the output of previous token as run time feature for current token  
 \#Note that, bi-directional model does not support run time feature, so we comment it out.  
-\#RTFEATURE_CONTEXT: -1
+\#RTFEATURE_CONTEXT = -1
 
 ## Training file format
 
@@ -199,7 +199,7 @@ major      | JJ   | S
 financial  | JJ   | S
 centers    | NNS  | S
 .          | PUN  | S
-           |      |    
+---empty line---
 !          | PUN  | S
 p          | FW   | S
 '          | PUN  | S
@@ -233,7 +233,7 @@ is        |
 your      | 
 name      | 
 ?         | 
-          | 
+---empty line---
 I         | 
 am        | 
 Zhongkai  | 

RNNSharp/Config.cs

@@ -200,6 +200,10 @@ public void LoadFeatureConfigFromFile(string configFilePath)
             {
                 NetworkType = NETWORKTYPE.BiDirectionalAverage;
             }
+            else if (networkType.Equals(NETWORKTYPE.ForwardSeq2SeqLabeling.ToString(), StringComparison.InvariantCultureIgnoreCase))
+            {
+                NetworkType = NETWORKTYPE.ForwardSeq2SeqLabeling;
+            }
             else
             {
                 throw new ArgumentException($"Invalidated network type: {networkType}");
@@ -212,7 +216,7 @@ public void LoadFeatureConfigFromFile(string configFilePath)
             SetTFeatures();
 
             //Load auto-encoder model for sequence-to-sequence. This model is used to encode source sequence
-            if (NetworkType == NETWORKTYPE.ForwardSeq2Seq)
+            if (NetworkType == NETWORKTYPE.ForwardSeq2Seq || NetworkType == NETWORKTYPE.ForwardSeq2SeqLabeling)
             {
                 var seqAutoEncoderConfigFilePath = GetFilePath(currentDirectory,
                     config.GetValueRequired(SEQ2SEQ_AUTOENCODER_CONFIG));
@@ -360,6 +364,10 @@ var type in
                     OutputLayerConfig = new SimpleLayerConfig();
                     Logger.WriteLine("Initialize configuration for simple layer.");
                     break;
+
+                default:
+                    Logger.WriteLine($"Invalidated output layer type {sLayerType}");
+                    throw new ArgumentException($"Invalidated output layer type {sLayerType}");
             }
         }
 

RNNSharp/Enums.cs

@@ -27,6 +27,7 @@ public enum NETWORKTYPE
     {
         Forward = 0,
         ForwardSeq2Seq,
+        ForwardSeq2SeqLabeling,
         BiDirectional,
         BiDirectionalAverage
     }

RNNSharp/Layers/DropoutLayer.cs

@@ -1,6 +1,7 @@
 using AdvUtils;
 using System;
 using System.IO;
+using System.Numerics;
 
 namespace RNNSharp
 {
@@ -42,12 +43,35 @@ public override Neuron CopyNeuronTo(Neuron neuron)
             return dropoutNeuron;
         }
 
+        public override void PreUpdateWeights(Neuron neuron, float[] errs)
+        {
+            DropoutNeuron dropoutNeuron = neuron as DropoutNeuron;
+            dropoutNeuron.Cells.CopyTo(Cells, 0);
+            for (int i = 0; i < LayerSize; i++)
+            {
+                if (dropoutNeuron.mask[i])
+                {
+                    Errs[i] = 0;
+                }
+                else
+                {
+                    Errs[i] = errs[i];
+                }
+            }
+        }
+
         public override void InitializeWeights(int sparseFeatureSize, int denseFeatureSize)
         {
+            SparseFeatureSize = sparseFeatureSize;
+            DenseFeatureSize = denseFeatureSize;
+            if (DenseFeatureSize % Vector<float>.Count != 0)
+            {
+                DenseFeatureSize += (Vector<float>.Count - (DenseFeatureSize % Vector<float>.Count));
+            }
+
             if (denseFeatureSize > 0)
             {
                 Logger.WriteLine("Initializing dense feature matrix. layer size = {0}, feature size = {1}", LayerSize, denseFeatureSize);
-                DenseFeatureSize = denseFeatureSize;
                 DenseWeights = new Matrix<float>(LayerSize, denseFeatureSize);
                 for (var i = 0; i < DenseWeights.Height; i++)
                 {
@@ -61,7 +85,6 @@ public override void InitializeWeights(int sparseFeatureSize, int denseFeatureSi
             if (sparseFeatureSize > 0)
             {
                 Logger.WriteLine("Initializing sparse feature matrix. layer size = {0}, feature size = {1}", LayerSize, sparseFeatureSize);
-                SparseFeatureSize = sparseFeatureSize;
                 SparseWeights = new Matrix<float>(LayerSize, SparseFeatureSize);
                 for (var i = 0; i < SparseWeights.Height; i++)
                 {
@@ -111,19 +134,6 @@ public override void BackwardPass()
         {
         }
 
-        public override void ComputeLayerErr(SimpleLayer nextLayer, float[] destErrLayer, float[] srcErrLayer, Neuron neuron)
-        {
-            DropoutNeuron dropoutNeuron = neuron as DropoutNeuron;
-            base.ComputeLayerErr(nextLayer, destErrLayer, srcErrLayer, dropoutNeuron);
-            for (var i = 0; i < LayerSize; i++)
-            {
-                if (dropoutNeuron.mask[i])
-                {
-                    destErrLayer[i] = 0;
-                }
-            }
-        }
-
         public override void ComputeLayerErr(SimpleLayer nextLayer)
         {
             base.ComputeLayerErr(nextLayer);