Fix:Config+Added Main

abilng · abilng · commit 3857a92f8f73 · 2014-08-27T12:49:16.000+05:30
diff --git a/config/CNN/model_conf.json b/config/CNN/model_conf.json
@@ -6,13 +6,13 @@
 	"wdir" : "wdir",
 
 	"comment" : "valid_data (Mandatory) specify the path of the validation data relative to the working directory",
-	"data_spec" : "config/CNN/data_spec.json",
+	"data_spec" : "data_spec.json",
 	
 	"comment" : "hidden_nnet_spec :: (Mandatory) specify the path of hidden network configuration specification relative to working directory",
-	"hidden_nnet_spec" : "config/CNN/mlp_spec.json",
+	"hidden_nnet_spec" : "mlp_spec.json",
 	
 	"comment" : "conv_nnet_spec :: (Mandatory) specify the path of convolution network configuration specification relative to working directory",
-	"conv_nnet_spec" : "config/CNN/conv_spec.json",
+	"conv_nnet_spec" : "conv_spec.json",
 
 	"comment" : "hidden_output_file :: (Mandatory) specify the path of convolution network output file relative to working directory",
 	"hidden_output_file" : "hidden_out.model",
diff --git a/config/CNN1/model_conf.json b/config/CNN1/model_conf.json
@@ -6,13 +6,13 @@
 	"wdir" : "wdir",
 
 	"comment" : "valid_data (Mandatory) specify the path of the validation data relative to the working directory",
-	"data_spec" : "config/CNN1/data_spec.json",
+	"data_spec" : "data_spec.json",
 	
 	"comment" : "hidden_nnet_spec :: (Mandatory) specify the path of hidden network configuration specification relative to working directory",
-	"hidden_nnet_spec" : "config/CNN1/mlp_spec.json",
+	"hidden_nnet_spec" : "mlp_spec.json",
 	
 	"comment" : "conv_nnet_spec :: (Mandatory) specify the path of convolution network configuration specification relative to working directory",
-	"conv_nnet_spec" : "config/CNN1/conv_spec.json",
+	"conv_nnet_spec" : "conv_spec.json",
 
 	"comment" : "hidden_output_file :: (Mandatory) specify the path of convolution network output file relative to working directory",
 	"hidden_output_file" : "hidden_out.model",
diff --git a/config/DBN/model_conf.json b/config/DBN/model_conf.json
@@ -6,10 +6,10 @@
 	"wdir" : "wdir",
 
 	"comment" : "valid_data (Mandatory) specify the path of the validation data relative to the working directory",
-	"data_spec" : "config/DBN/data_spec.json",
+	"data_spec" : "data_spec.json",
 	
 	"comment" : "rbm_nnet_spec:: (Mandatory) specify the path of RBM network configuration specification relative to working directory",
-	"rbm_nnet_spec" : "config/DBN/rbm_spec.json",
+	"rbm_nnet_spec" : "rbm_spec.json",
 
 	"comment" : "output_file :: (Mandatory) specify the path of RBM network output file relative to working directory",
 	"output_file" : "rbm_out.model",
diff --git a/config/SDA/model_conf.json b/config/SDA/model_conf.json
@@ -6,10 +6,10 @@
 	"wdir" : "wdir",
 
 	"comment" : "valid_data (Mandatory) specify the path of the validation data relative to the working directory",
-	"data_spec" : "config/SDA/data_spec.json",
+	"data_spec" : "data_spec.json",
 	
 	"comment" : "sda_nnet_spec:: (Mandatory) specify the path of SDA network configuration specification relative to working directory",
-	"sda_nnet_spec" : "config/SDA/sda_spec.json",
+	"sda_nnet_spec" : "sda_spec.json",
 
 	"comment" : "output_file :: (Mandatory) specify the path of SDA network output file relative to working directory",
 	"output_file" : "sda_out.model",
diff --git a/io_modules/__init__.py b/io_modules/__init__.py
@@ -1,5 +1,5 @@
 
-def setLogger(level="INFO",stderr=True,name=None,logFile='python-dnn.log'):
+def setLogger(level="INFO",stderr=True,logFile='python-dnn.log',name=None):
 	import logging,sys
 
 	#get Logger
diff --git a/models/__init__.py b/models/__init__.py
@@ -11,6 +11,7 @@ def __init__(self):
 		self.params = [];
 		self.delta_params = [];
 		self.n_layers = 0;
+		self.type = None;
 
 		# allocate symbolic variables for the data
 		self.x = T.matrix('x')  # the data is presented as rasterized images
@@ -22,6 +23,9 @@ def __init__(self):
 		self.errors = None
 		self.finetune_cost = None
 
+	def getType(self):
+		return self.type
+
 	def pretraining_functions(self, train_x, batch_size):
 		"""
 		Should be implemeted by derived class
diff --git a/run.py b/run.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python2.7
+# Copyright 2014	G.K SUDHARSHAN <sudharpun90@gmail.com>	IIT Madras
+# Copyright 2014	Abil N George<mail@abilng.in>	IIT Madras
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+from utils.load_conf import load_model
+import sys
+
+import logging
+logger = logging.getLogger(__name__)
+from io_modules import setLogger
+
+
+def setLoggerLevel(modelConfig):
+	# Set the level which determines what you see
+	try:
+		level = modelConfig['logger_level']
+	except KeyError:
+		level = "INFO"
+
+	if level == "INFO":
+		logger.setLevel(logging.INFO)
+	elif level == "DEBUG":
+		logger.setLevel(logging.DEBUG)
+	elif level == "ERROR":
+		logger.setLevel(logging.ERROR)
+	else:
+		logger.setLevel(logging.WARNING)
+
+
+def runNet(modelConfig):
+	nnetType = modelConfig ['nnetType']
+	logger.info("Loading Other Configuration for %s",nnetType);
+	if nnetType == 'CNN':
+		from run.run_CNN import runCNN as runModel
+	elif nnetType == 'RBM':
+		from run.run_DBN import runRBM as runModel
+	elif nnetType == 'SDA':
+		from run.run_SDA import runSdA as runModel
+	else :
+		logger.error('Unknown nnet Type')
+		return 1
+	runModel(modelConfig)
+
+
+if __name__ == '__main__':
+	setLogger();
+	modelConfig = load_model(sys.argv[1])
+	setLoggerLevel(modelConfig)
+	runNet(modelConfig)
diff --git a/run/__init__.py b/run/__init__.py
diff --git a/run/run_CNN.py b/run/run_CNN.py
@@ -36,8 +36,13 @@
 logger = logging.getLogger(__name__)
 
 
-def runCNN(configFile):
-	model_configs = load_model(configFile,'CNN')
+def runCNN(arg):
+	
+	if type(arg) is dict:
+		model_configs = arg
+	else :
+		model_configs = load_model(arg,'CNN')
+	
 	conv_configs,conv_layer_configs = load_conv_spec(model_configs['conv_nnet_spec'],model_configs['batch_size'],
 				model_configs['input_shape'])
 
diff --git a/run/run_CNNFeat.py b/run/run_CNNFeat.py
@@ -32,8 +32,14 @@
 import logging
 logger = logging.getLogger(__name__)
 
-def runCNNFeat(configFile):
-	model_configs = load_model(configFile,'CNN')
+def runCNNFeat(arg):
+
+	if type(arg) is dict:
+		model_configs = arg
+	else :
+		model_configs = load_model(arg,'CNN')
+	
+	
 	batch_size = model_configs['batch_size'];
 	conv_configs,conv_layer_configs = load_conv_spec(model_configs['conv_nnet_spec'],model_configs['batch_size'],
 				model_configs['input_shape'])
diff --git a/run/run_DBN.py b/run/run_DBN.py
@@ -89,9 +89,13 @@ def preTraining(nnetModel,train_sets,train_xy,train_x,train_y,model_config):
     end_time = time.clock()
     logger.info('The PreTraing ran for %.2fm' % ((end_time - start_time) / 60.))
 
-def runRBM(configFile):
+def runRBM(arg):
+
+    if type(arg) is dict:
+        model_config = arg
+    else :
+        model_config = load_model(arg,'RBM')
 
-    model_config = load_model(configFile)
     rbm_config = load_rbm_spec(model_config['rbm_nnet_spec'])
     data_spec =  load_data_spec(model_config['data_spec']);
 
diff --git a/run/run_SDA.py b/run/run_SDA.py
@@ -58,9 +58,13 @@ def preTraining(layers,epochs,pretrainfns,train_sets,train_xy,corruptions,lr,bat
     return err
 
 
-def runSdA(configFile):
+def runSdA(arg):
 
-    model_config = load_model(configFile)
+    if type(arg) is dict:
+        model_config = arg
+    else :
+        model_config = load_model(arg,'SDA')
+        
     sda_config = load_sda_spec(model_config['sda_nnet_spec'])
     data_spec =  load_data_spec(model_config['data_spec']);
 
diff --git a/utils/load_conf.py b/utils/load_conf.py
@@ -1,4 +1,5 @@
 from json import load as jsonLoad
+from utils import makeAbsolute,pathSep
 
 import logging
 logger = logging.getLogger(__name__)
@@ -32,6 +33,16 @@ def load_model(input_file,nnetType=None):
 		logger.critical(" the mandatory arguments are missing in model properties file..")
 		exit(1)
 
+	specPaths=['data_spec','conv_nnet_spec','hidden_nnet_spec',
+				'rbm_nnet_spec','sda_nnet_spec']
+
+	data = correctPath(data,specPaths,input_file);
+
+	outputFiles=['output_file','conv_output_file',
+				'hidden_output_file','export_path']
+
+	data = correctPath(data,outputFiles,data['wdir']+pathSep);
+
 	#init Default Values or update from Json.
 	if nnetType == 'CNN':
 		data = initModelCNN(data)
@@ -46,7 +57,11 @@ def load_model(input_file,nnetType=None):
 	#__debugPrintData__(data,'model');
 	return data;
 
-
+def correctPath(data,keys,basePath):
+	for key in keys:
+		if data.has_key(key):
+			data[key] = makeAbsolute(data[key],basePath)
+	return data
 
 def checkConfig(data,nnetType):
 	if not data.has_key('data_spec'): 
@@ -209,7 +224,7 @@ def initModelRBM(data):
 	learning_rate = 0.08
 	batch_size=128
 	epochs=10
-        keep_layer_num=0	
+	keep_layer_num=0
 
 	# momentum; more complicated than dnn 
 	initial_momentum = 0.5	 # initial momentum 
@@ -302,4 +317,4 @@ def __debugPrintData__(data,name=None):
 #	#config_list = load_model(sys.argv[1])
 #	config_list = load_conv_spec(sys.argv[1],256,[1,29,29,29])
 #	print config_list;
-	
+
diff --git a/utils/utils.py b/utils/utils.py
@@ -1,5 +1,8 @@
 import theano.tensor as T
-from learn_rates import LearningRateConstant, LearningRateExpDecay
+
+from os.path import isabs,dirname,abspath,join
+from os.path import sep as pathSep
+
 
 def string_2_bool(string):
     if string == 'true':
@@ -31,72 +34,7 @@ def dimshuffle(a,shuffle):
 			shuffle[index],shuffle[swapwith]=shuffle[swapwith],shuffle[index]
 	return a
 
-'''
-#older code
-def parse_lrate(lrate_string):
-    elements = lrate_string.split(":")
-    # 'D:0.08:0.5:0.05,0.05:15'
-    if elements[0] == 'D':  # ExpDecay
-        if (len(elements) != 5):
-            return None
-        values = elements[3].split(',')
-        lrate = LearningRateExpDecay(start_rate=float(elements[1]),
-                                 scale_by = float(elements[2]),
-                                 min_derror_decay_start = float(values[0]),
-                                 min_derror_stop = float(values[1]),
-                                 init_error = 100,
-                                 min_epoch_decay_start=int(elements[4]))
-        return lrate
-
-    # 'C:0.08:15'
-    if elements[0] == 'C':  # Constant
-        if (len(elements) != 3):
-            return None
-        lrate = LearningRateConstant(learning_rate=float(elements[1]),
-                                 epoch_num = int(elements[2]))
-        return lrate
-
-def parse_conv_spec(conv_spec, batch_size):
-    # "1x29x29:100,5x5,p2x2:200,4x4,p2x2,f"
-    conv_spec = conv_spec.replace('X', 'x')
-    structure = conv_spec.split(':')
-    conv_layer_configs = []
-    for i in range(1, len(structure)):
-        config = {}
-        elements = structure[i].split(',')
-        if i == 1:
-            input_dims = structure[i - 1].split('x')
-            prev_map_number = int(input_dims[0])
-            prev_feat_dim_x = int(input_dims[1])
-            prev_feat_dim_y = int(input_dims[2])
-        else:
-            prev_map_number = conv_layer_configs[-1]['output_shape'][1]
-            prev_feat_dim_x = conv_layer_configs[-1]['output_shape'][2]
-            prev_feat_dim_y = conv_layer_configs[-1]['output_shape'][3]
 
-        current_map_number = int(elements[0])
-        filter_xy = elements[1].split('x')
-        filter_size_x = int(filter_xy[0])
-        filter_size_y = int(filter_xy[1])
-        pool_xy = elements[2].replace('p','').replace('P','').split('x')
-        pool_size_x = int(pool_xy[0])
-        pool_size_y = int(pool_xy[1])
-        output_dim_x = (prev_feat_dim_x - filter_size_x + 1) / pool_size_x
-        output_dim_y = (prev_feat_dim_y - filter_size_y + 1) / pool_size_y
-
-        config['input_shape'] = (batch_size, prev_map_number, prev_feat_dim_x, prev_feat_dim_y)
-        config['filter_shape'] = (current_map_number, prev_map_number, filter_size_x, filter_size_y)
-        config['poolsize'] = (pool_size_x, pool_size_y)
-        config['output_shape'] = (batch_size, current_map_number, output_dim_x, output_dim_y)
-        if len(elements) == 4 and elements[3] == 'f':
-            config['flatten'] = True
-        else:
-            config['flatten'] = False
-
-        conv_layer_configs.append(config)
-    return conv_layer_configs
-'''
- 
 def parse_activation(act_str):
     if act_str == 'sigmoid':
         return T.nnet.sigmoid
@@ -116,3 +54,13 @@ def parse_two_integers(argument_str):
     return int(int_strs[0]), int(int_strs[1])
 
 
+
+def makeAbsolute(path,base='/'):
+    if isabs(path) :
+        return path
+    else :
+        if not base[-1] == pathSep : 
+            base = dirname(base);
+        base = abspath(base)
+        path = join(base,path);
+        return path
