Merge pull request #1 from IITM-DONLAB/devel

Added DNN

Author: abilng

config/DNN/README

@@ -0,0 +1,31 @@
+{
+    "validation": {
+	"base_path" : "data/NPFILE",
+	"filename" : "val.dat",
+	"partition" : 200,
+	"random" : true,
+	"random_seed" : 123,
+	"keep_flatten" : true,
+	"reader_type" : "NP"
+    },
+
+    "training" : {
+	"base_path" : "data/NPFILE",
+	"filename" : "train.dat",
+	"partition" : 200,
+	"random" : true,
+	"random_seed" : 123,
+	"keep_flatten" : true,
+	"reader_type" : "NP"
+    },
+
+    "testing" : {
+	"base_path" : "data/NPFILE",
+	"filename" : "train.dat",
+	"partition" : 200,
+	"random" : true,
+	"random_seed" : 123,
+	"keep_flatten" : true,
+	"reader_type" : "NP"
+    }
+}

config/DNN/data_spec.json

@@ -0,0 +1,29 @@
+
+{
+
+  "comment" : "layers :: RBM layer configuration (No: of Nodes)",
+  "hidden_layers": [ 2350,1024,1024,1024,1024,1901 ],
+
+  "comment" : "activation :: sigmoid or tanh",
+  "activation" : "sigmoid",
+
+  "comment" : "pretrained_layers:number of layers to be pre-trained",
+  "pretrained_layers" : 5,
+
+  "comment" : "random_seed::",
+  "random_seed" : 89677,
+
+  "comment" :"regularization for hidden layer parameter",
+  "max_col_norm":null,
+  "l1_reg":null,
+  "l2_reg":null,
+
+  "do_maxout":false,
+  "pool_size":1,
+  "do_pnorm": false,
+  "pnorm_order": 1,
+
+  "do_dropout": false,
+  "dropout_factor": [0.0],
+  "input_dropout_factor": 0.0
+}

config/DNN/dnn_spec.json

@@ -0,0 +1,55 @@
+{	
+    "comment" : "nnetType :: (Mandatory) specify Type of Network (CNN,RBM) ",
+    "nnetType" : "DNN",
+
+    "comment" : "train_data :: (Mandatory) specify the working directory containing data configuration and output ",
+    "wdir" : "wdir",
+
+    "comment" : "valid_data (Mandatory) specify the path of the validation data relative current directory",
+    "data_spec" : "data_spec.json",
+    
+    "comment" : "rbm_nnet_spec:: (Mandatory) specify the path of RBM network configuration specification relative to current directory",
+    "nnet_spec" : "dnn_spec.json",
+
+    "comment" : "output_file :: (Mandatory) specify the path of RBM network output file relative to working directory",
+    "output_file" : "dnn_out.model",
+    
+    "comment" : "batch_size :: specify the mini batch size while training, default 128",
+    "batch_size" : 128,
+
+    "comment": "",
+    "n_ins":2352,
+
+    "comment":"",
+    "n_outs":200,
+
+    "comment" : "input_file :: (Mandatory) specify the path of PreTrained network input file relative to working directory",
+    "input_file" : "rbm_in.model",
+
+    "comment" : "finetune_method::  Two methods are supported  C: Constant learning rate and E : Exponential decay",
+    "finetune_method":"C",
+
+    "comment" : "finetune_rate :: learning rate configuration",
+    "finetune_rate" : {
+	"learning_rate" : 0.08,
+	"epoch_num" : 10,
+
+	"start_rate" : 0.08,
+	"scale_by" : 0.5,
+	"min_derror_decay_start" : 0.05,
+	"min_derror_stop" : 0.05,
+	"min_epoch_decay_start" : 15,
+	"init_error" :100
+    },
+
+    "comment" : "finetune_momentum ::  Specify the momentum factor while finetuning",
+    "finetune_momentum": 0.5,
+
+    "processes":{
+	"pretraining":false,
+	"finetuning":true,
+	"testing":false,
+	"export_data":false
+    }
+
+}

config/DNN/model_conf.json

@@ -64,6 +64,12 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, activation=T.tanh,
 		super(DropoutHiddenLayer, self).__init__(rng=rng, input=input, n_in=n_in, n_out=n_out, W=W, b=b,
 				activation=activation, do_maxout = do_maxout, pool_size = pool_size)
 		self.theano_rng = RandomStreams(rng.randint(2 ** 30))
-		dropout_prob = self.theano_rng.binomial(n=1, p=1-dropout_factor, size=self.output.shape, dtype=theano.config.floatX)	
+		dropout_prob = self.theano_rng.binomial(n=1, p=1-dropout_factor, size=self.output.shape,
+			dtype=theano.config.floatX)	
 		self.dropout_output = dropout_prob * self.output
 
+def _dropout_from_layer(theano_rng, hid_out, p):
+    """ p is the factor for dropping a unit """
+    # p=1-p because 1's indicate keep and p is prob of dropping
+    return theano_rng.binomial(n=1, p=1-p, size=hid_out.shape,dtype=theano.config.floatX) * hid_out
+

layers/mlp.py

@@ -20,6 +20,7 @@
 import time
 
 import numpy
+from collections import OrderedDict
 
 import theano
 import theano.tensor as T
@@ -115,20 +116,47 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
                 W = self.params[i * 2]
                 self.finetune_cost += self.l2_reg * T.sqr(W).sum()
 
+        self.output = self.logLayer.prediction();
+        self.features = self.sigmoid_layers[-2].output;
+
     def build_finetune_functions(self, train_shared_xy, valid_shared_xy, batch_size):
+        """
+        Generates a function `train` that implements one step of
+        finetuning and a function `validate` that computes the error on 
+        a batch from the validation set 
+
+        :type train_shared_xy: pairs of theano.tensor.TensorType
+        :param train_shared_xy: It is a list that contain all the train dataset, 
+            pair is formed of two Theano variables, one for the datapoints,
+            the other for the labels
+
+        :type valid_shared_xy: pairs of theano.tensor.TensorType
+        :param valid_shared_xy: It is a list that contain all the valid dataset, 
+            pair is formed of two Theano variables, one for the datapoints,
+            the other for the labels
+
+        :type batch_size: int
+        :param batch_size: size of a minibatch
+
+        :returns (theano.function,theano.function)
+        * A function for training takes minibatch_index,learning_rate,momentum 
+        which updates weights,and return error rate
+        * A function for validation takes minibatch_indexand return error rate
+        
+        """
 
         (train_set_x, train_set_y) = train_shared_xy
         (valid_set_x, valid_set_y) = valid_shared_xy
 
         index = T.lscalar('index')  # index to a [mini]batch
-        learning_rate = T.fscalar('learning_rate')
-        momentum = T.fscalar('momentum')
+        learning_rate = T.scalar('learning_rate',dtype=theano.config.floatX)
+        momentum = T.scalar('momentum',dtype=theano.config.floatX)
 
         # compute the gradients with respect to the model parameters
         gparams = T.grad(self.finetune_cost, self.params)
 
         # compute list of fine-tuning updates
-        updates = {}
+        updates = OrderedDict()
         for dparam, gparam in zip(self.delta_params, gparams):
             updates[dparam] = momentum * dparam - gparam*learning_rate
         for dparam, param in zip(self.delta_params, self.params):

models/dnn.py

@@ -20,6 +20,7 @@
 import time
 
 import numpy
+from collections import OrderedDict
 
 import theano
 import theano.tensor as T
@@ -30,6 +31,7 @@
 
 from models import nnet
 
+
 class DNN_Dropout(nnet):
 
     def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
@@ -127,6 +129,9 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
         self.finetune_cost = self.dropout_logLayer.negative_log_likelihood(self.y)
         self.errors = self.logLayer.errors(self.y)
 
+        self.output = self.logLayer.prediction();
+        self.features = self.sigmoid_layers[-2].output;
+
         if self.l1_reg is not None:
             for i in xrange(self.n_layers):
                 W = self.params[i * 2]
@@ -138,19 +143,43 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
                 self.finetune_cost += self.l2_reg * T.sqr(W).sum()
 
     def build_finetune_functions(self, train_shared_xy, valid_shared_xy, batch_size):
+        """
+        Generates a function `train` that implements one step of
+        finetuning and a function `validate` that computes the error on 
+        a batch from the validation set 
+
+        :type train_shared_xy: pairs of theano.tensor.TensorType
+        :param train_shared_xy: It is a list that contain all the train dataset, 
+            pair is formed of two Theano variables, one for the datapoints,
+            the other for the labels
+
+        :type valid_shared_xy: pairs of theano.tensor.TensorType
+        :param valid_shared_xy: It is a list that contain all the valid dataset, 
+            pair is formed of two Theano variables, one for the datapoints,
+            the other for the labels
+
+        :type batch_size: int
+        :param batch_size: size of a minibatch
+
+        :returns (theano.function,theano.function)
+        * A function for training takes minibatch_index,learning_rate,momentum 
+        which updates weights,and return error rate
+        * A function for validation takes minibatch_indexand return error rate
+        
+        """
 
         (train_set_x, train_set_y) = train_shared_xy
         (valid_set_x, valid_set_y) = valid_shared_xy
 
         index = T.lscalar('index')  # index to a [mini]batch
-        learning_rate = T.fscalar('learning_rate')
-        momentum = T.fscalar('momentum')
+        learning_rate = T.scalar('learning_rate',dtype=theano.config.floatX)
+        momentum = T.scalar('momentum',dtype=theano.config.floatX)
 
         # compute the gradients with respect to the model parameters
         gparams = T.grad(self.finetune_cost, self.params)
 
         # compute list of fine-tuning updates
-        updates = {}
+        updates = OrderedDict()
         for dparam, gparam in zip(self.delta_params, gparams):
             updates[dparam] = momentum * dparam - gparam*learning_rate
         for dparam, param in zip(self.delta_params, self.params):

models/dropout_nnet.py

@@ -54,6 +54,8 @@ def runNet(modelConfig):
 		from run.run_DBN import runRBM as runModel
 	elif nnetType == 'SDA':
 		from run.run_SDA import runSdA as runModel
+	elif nnetType == 'DNN':
+		from run.run_DNN import runDNN as runModel
 	else :
 		logger.error('Unknown nnet Type')
 		return 1