Add: Activation in each layer cnn/cnn3d

Author: abilng

src/layers/mlp.py

@@ -33,26 +33,26 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, activation=T.tanh,
 		self.delta_W = theano.shared(value = numpy.zeros((n_in,n_out),dtype=theano.config.floatX), name='delta_W')
 		self.delta_b = theano.shared(value = numpy.zeros_like(self.b.get_value(borrow=True), dtype=theano.config.floatX), name='delta_b')
 
-		lin_output = T.dot(input, self.W) + self.b
+		self.lin_output = T.dot(input, self.W) + self.b
 
 		if adv_activation_method == 'maxout':	# pooling of output of neuron based on poolsize
 			self.last_start = n_out - pool_size
-			self.tmp_output = lin_output[:,0:self.last_start+1:pool_size]
+			self.tmp_output = self.lin_output[:,0:self.last_start+1:pool_size]
 			for i in range(1, pool_size):
-				cur = lin_output[:,i:self.last_start+i+1:pool_size]
+				cur = self.lin_output[:,i:self.last_start+i+1:pool_size]
 				self.tmp_output = T.maximum(cur, self.tmp_output)
 			self.output = activation(self.tmp_output)
 		elif adv_activation_method == 'pnorm': # pooling of output of neuron based on poolsize and normalizing the output
 			self.last_start = n_out - pool_size
-			self.tmp_output = abs(lin_output[:,0:self.last_start+1:pool_size]) ** pnorm_order
+			self.tmp_output = abs(self.lin_output[:,0:self.last_start+1:pool_size]) ** pnorm_order
 			for i in range(1, pool_size):
-				cur = abs(lin_output[:,i:self.last_start+i+1:pool_size]) ** pnorm_order
+				cur = abs(self.lin_output[:,i:self.last_start+i+1:pool_size]) ** pnorm_order
 				self.tmp_output = self.tmp_output + cur
 			self.tmp_output = self.tmp_output ** (1.0 / pnorm_order)
 			self.output = activation(self.tmp_output)
 		else:
-			self.output = (lin_output if activation is None
-						   else activation(lin_output))
+			self.output = (self.lin_output if activation is None
+						   else activation(self.lin_output))
 
 		# parameters of the model
 		self.params = [self.W, self.b]

src/models/cnn.py

@@ -10,6 +10,7 @@
 from collections import OrderedDict
 from io_modules.file_reader import read_dataset
 from utils.plotter import plot
+from utils.utils import parse_activation
 
 import logging
 logger = logging.getLogger(__name__)
@@ -157,8 +158,7 @@ def plot_layer_output(self,plot_spec,plot_path,max_images=10):
 class CNN(CNNBase):
 	""" Instantiation of Convolution neural network ... """
 	def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs, hidden_layer_configs, 
-			use_fast=False,conv_activation = T.nnet.sigmoid,hidden_activation = T.nnet.sigmoid,
-			l1_reg=None,l2_reg=None,max_col_norm=None):
+			use_fast=False,hidden_activation = T.nnet.sigmoid,l1_reg=None,l2_reg=None,max_col_norm=None):
 
 		super(CNN, self).__init__(conv_layer_configs, hidden_layer_configs,l1_reg,l2_reg,max_col_norm)
 		if not theano_rng:
@@ -171,7 +171,7 @@ def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs,
 				input = self.layers[-1].output #output of previous layer
 
 			config = conv_layer_configs[i]
-	
+			conv_activation= parse_activation(config['activation'])
 			conv_layer = ConvLayer(numpy_rng=numpy_rng, input=input,input_shape=config['input_shape'],
 				filter_shape=config['filter_shape'],poolsize=config['poolsize'],
 				activation = conv_activation, use_fast = use_fast)
@@ -255,8 +255,8 @@ def save_mlp2dict(self,withfinal=True,max_layer_num=-1):
 class DropoutCNN(CNNBase):
 	""" Instantiation of Convolution neural network ... """
 	def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs, hidden_layer_configs, 
-			use_fast=False,conv_activation = T.nnet.sigmoid,hidden_activation = T.nnet.sigmoid,
-			l1_reg=None,l2_reg=None,max_col_norm=None,input_dropout_factor=0.0):
+			use_fast=False,hidden_activation = T.nnet.sigmoid,l1_reg=None,l2_reg=None,
+			max_col_norm=None,input_dropout_factor=0.0):
 
 		super(DropoutCNN, self).__init__(conv_layer_configs,hidden_layer_configs,l1_reg,l2_reg,max_col_norm)
 		self.input_dropout_factor = input_dropout_factor;
@@ -277,7 +277,7 @@ def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs,
 				dropout_conv_input = self.dropout_layers[-1].dropout_output;
 
 			config = conv_layer_configs[i]
-			
+			activation = conv_activation
 			dropout_conv_layer = DropoutConvLayer(numpy_rng=numpy_rng, input=dropout_conv_input,
 				input_shape=config['input_shape'],filter_shape=config['filter_shape'],poolsize=config['poolsize'],
 				activation = conv_activation, use_fast = use_fast,dropout_factor=conv_layer_configs[i]['dropout_factor'])

src/models/cnn3d.py

@@ -10,6 +10,7 @@
 from collections import OrderedDict
 from io_modules.file_reader import read_dataset
 from utils.plotter import plot
+from utils.utils import parse_activation
 
 import logging
 logger = logging.getLogger(__name__)
@@ -158,8 +159,8 @@ def plot_layer_output(self,plot_spec,plot_path,max_images=10):
 
 class CNN3D(CNN3DBase):
 	""" Instantiation of Convolution neural network ... """
-	def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs, hidden_layer_configs, 
-			conv_activation = T.nnet.sigmoid,hidden_activation = T.nnet.sigmoid,l1_reg=None,l2_reg=None,max_col_norm=None):
+	def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs, hidden_layer_configs,
+		hidden_activation = T.nnet.sigmoid,l1_reg=None,l2_reg=None,max_col_norm=None):
 
 		super(CNN3D, self).__init__(conv_layer_configs, hidden_layer_configs,l1_reg,l2_reg,max_col_norm)
 		if not theano_rng:
@@ -172,7 +173,7 @@ def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs,
 				input = self.layers[-1].output #output of previous layer
 
 			config = conv_layer_configs[i]
-	
+			conv_activation = parse_activation(config['activation']);
 			conv_layer = ConvLayer(numpy_rng=numpy_rng, input=input,input_shape=config['input_shape'],
 				filter_shape=config['filter_shape'],poolsize=config['poolsize'],
 				activation = conv_activation)

src/run/run_CNN.py

@@ -25,9 +25,8 @@
 
 from utils.load_conf import load_model,load_conv_spec,load_data_spec
 from io_modules.file_reader import read_dataset
-from utils.utils import parse_activation
 from io_modules import setLogger
-
+from utils.utils import parse_activation
 from run import fineTunning,testing,exportFeatures,createDir
 
 import logging
@@ -53,8 +52,7 @@ def runCNN(arg):
 	theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
 
 	logger.info('> ... building the model')
-	conv_activation = parse_activation(conv_config['activation']);
-	hidden_activation = parse_activation(mlp_config['activation']);
+	activationFn = parse_activation(mlp_config['activation']);
 
 	createDir(model_config['wdir']);
 	#create working dir
@@ -64,14 +62,14 @@ def runCNN(arg):
 		logger.info('>Initializing dropout cnn model')
 		cnn = DropoutCNN(numpy_rng,theano_rng,conv_layer_configs = conv_layer_config, batch_size = batch_size,
 				n_outs=model_config['n_outs'],hidden_layer_configs=mlp_config, 
-				conv_activation = conv_activation,hidden_activation = hidden_activation,
+				hidden_activation = activationFn,
 				use_fast = conv_config['use_fast'],l1_reg = mlp_config['l1_reg'],
 				l2_reg = mlp_config['l1_reg'],max_col_norm = mlp_config['max_col_norm'],
 				input_dropout_factor=conv_config['input_dropout_factor'])
 	else:
 		cnn = CNN(numpy_rng,theano_rng,conv_layer_configs = conv_layer_config, batch_size = batch_size,
 				n_outs=model_config['n_outs'],hidden_layer_configs=mlp_config, 
-				conv_activation = conv_activation,hidden_activation = hidden_activation,
+				hidden_activation = activationFn,
 				use_fast = conv_config['use_fast'],l1_reg = mlp_config['l1_reg'],
 				l2_reg = mlp_config['l1_reg'],max_col_norm = mlp_config['max_col_norm'])
 

src/run/run_CNN3d.py

@@ -15,7 +15,7 @@
 # See the Apache 2 License for the specific language governing permissions and
 # limitations under the License.
 
-import cPickle, gzip, os, time,sys
+import sys
 from models.cnn3d import CNN3D
 import numpy
 
@@ -53,7 +53,6 @@ def runCNN3D(arg):
 	theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
 
 	logger.info('> ... building the model')
-	conv_activation = parse_activation(conv_config['activation']);
 	hidden_activation = parse_activation(mlp_config['activation']);
 
 	createDir(model_config['wdir']);
@@ -63,8 +62,7 @@ def runCNN3D(arg):
 	batch_size = model_config['batch_size'];
 
 	cnn = CNN3D(numpy_rng,theano_rng,conv_layer_configs = conv_layer_config, batch_size = batch_size,
-			n_outs=model_config['n_outs'],hidden_layer_configs=mlp_config, 
-			conv_activation = conv_activation,hidden_activation = hidden_activation,
+			n_outs=model_config['n_outs'],hidden_layer_configs=mlp_config,hidden_activation = hidden_activation,
 			l1_reg = mlp_config['l1_reg'],l2_reg = mlp_config['l1_reg'],max_col_norm = mlp_config['max_col_norm'])
 
 

src/utils/load_conf.py

@@ -277,6 +277,9 @@ def load_conv_spec(input_file,batch_size,input_shape):
 	else:
 		cnn_data['input_dropout_factor']=0.0;
 
+	if not cnn_data.has_key('activation'):
+		cnn_data['activation'] =  "sigmoid";
+
 	#layer_configs
 	layer_configs=cnn_data.pop('layers');
 	conv_configs = cnn_data;
@@ -310,13 +313,18 @@ def load_conv_spec(input_file,batch_size,input_shape):
 			input_shape.append((inp-wdim+1)/pool);
 		layer_configs[layer_index]['output_shape'].extend(input_shape)
 
+		if not layer_configs[layer_index].has_key('activation'):
+			layer_configs[layer_index]['activation'] =  cnn_data['activation'];
+
 		if (do_dropout and (not layer_configs[layer_index].has_key('dropout_factor') and 
 				not type(layer_configs[layer_index]['dropout_factor']) is float)):
 			logger.critical("dropout_factor of cnn layer  %d is not present (or not a float) in mlp_configuration"%layer_index)
 			exit(1);
 		elif not do_dropout:
 			layer_configs[layer_index]['dropout_factor']=0.0;
 
+
+
 		prev_map_number = current_map_number