Fix:Minor:FILE

Author: abilng

config/DBN/data_spec.json

@@ -1,19 +1,37 @@
 {
 	"validation": {
-		"path" : "data/val.dat",
+		"base_path" : "data/NPFILE",
+		"filename" : "val.dat",
 		"partition" : 200,
 		"random" : true,
 		"random_seed" : 123,
-		"keep_flatten": true,
-		"batch_size": 256
+		"dim_shuffle" : [0,3,2,1],
+		"batch_size": 256,
+		"keep_flatten" : true,
+		"reader_type" : "NP"
 	},
 
 	"training" : {
-		"path" : "data/train.dat",
+		"base_path" : "data/NPFILE",
+		"filename" : "train.dat",
 		"partition" : 200,
 		"random" : true,
 		"random_seed" : 123,
-		"keep_flatten": true,
-		"batch_size": 256
+		"dim_shuffle" : [0,3,2,1],
+		"batch_size": 256,
+		"keep_flatten" : true,
+		"reader_type" : "NP"
+	},
+
+	"testing" : {
+		"base_path" : "data/NPFILE",
+		"filename" : "train.dat",
+		"partition" : 200,
+		"random" : true,
+		"random_seed" : 123,
+		"dim_shuffle" : [0,3,2,1],
+		"batch_size": 256,
+		"keep_flatten" : true,
+		"reader_type" : "NP"
 	}
 }

config/DBN/mlp_spec.json

@@ -1,19 +1,37 @@
 {
 	"validation": {
-		"path" : "data/val.dat",
+		"base_path" : "data/NPFILE",
+		"filename" : "val.dat",
 		"partition" : 200,
 		"random" : true,
 		"random_seed" : 123,
-		"keep_flatten": true,
-		"batch_size": 256
+		"dim_shuffle" : [0,3,2,1],
+		"batch_size": 256,
+		"keep_flatten" : true,
+		"reader_type" : "NP"
 	},
 
 	"training" : {
-		"path" : "data/train.dat",
+		"base_path" : "data/NPFILE",
+		"filename" : "train.dat",
 		"partition" : 200,
 		"random" : true,
 		"random_seed" : 123,
-		"keep_flatten": true,
-		"batch_size": 256
+		"dim_shuffle" : [0,3,2,1],
+		"batch_size": 256,
+		"keep_flatten" : true,
+		"reader_type" : "NP"
+	},
+
+	"testing" : {
+		"base_path" : "data/NPFILE",
+		"filename" : "train.dat",
+		"partition" : 200,
+		"random" : true,
+		"random_seed" : 123,
+		"dim_shuffle" : [0,3,2,1],
+		"batch_size": 256,
+		"keep_flatten" : true,
+		"reader_type" : "NP"
 	}
 }

config/SDA/data_spec.json

@@ -23,10 +23,12 @@ def setLogger(level="INFO",stderr=True,name=None,logFile='python-dnn.log'):
 	# Set the level which determines what you see
 	if level == "INFO":
 		logger.setLevel(logging.INFO)
-	elif level=="DEBUG":
+	elif level == "DEBUG":
 		logger.setLevel(logging.DEBUG)
+	elif level == "ERROR":
+		logger.setLevel(logging.ERROR)		
 	else:
-		logger.setLevel(logging.ERROR)
+		logger.setLevel(logging.WARNING)
 
 def create_folder_structure_if_not_exists(filepath):
 	import logging,os,errno

io_modules/__init__.py

@@ -165,6 +165,7 @@ def read_next_partition_data(self,already_read=0,pad_zeros=False):
 			self.partition_num = self.partition_num + 1
 			logger.debug('TD Filereader : %d frames read from %s' % (self.cur_frame_num,self.filepath))
 			if not self.options['keep_flatten'] :	#reshape the vector if needed
+				logger.debug('TD Filereader : Reshape input...')
 				shape = [self.cur_frame_num];
 				shape.extend(self.options['input_shape']);
 				self.feat = self.feat.reshape(shape);

io_modules/file_io.py

@@ -26,9 +26,10 @@
 #module imports
 from utils.load_conf import load_model,load_rbm_spec,load_data_spec
 from models.dbn import DBN
-from io_modules.file_io import read_dataset
+from io_modules.file_reader import read_dataset
 from io_modules import setLogger
 from utils.learn_rates import LearningRate
+from io_modules.model_io import _nnet2file, _file2nnet
 
 from models import fineTunning,testing
 
@@ -73,7 +74,8 @@ def preTraining(nnetModel,train_sets,train_xy,train_x,train_y,model_config):
             while not train_sets.is_finish():
                 train_sets.make_partition_shared(train_xy)
                 #train_sets.load_next_partition(train_xy)
-                for batch_index in xrange(train_sets.cur_frame_num / batch_size):  # loop over mini-batches
+                for batch_index in xrange(train_sets.cur_frame_num / batch_size):  
+                    # loop over mini-batches
                     #logger.info("Training For epoch %d and batch %d",epoch,batch_index)
                     [reconstruction_cost, free_energy_cost] = pretraining_fns[i](index=batch_index,
                                                                              lr=pretrain_lr,
@@ -82,7 +84,8 @@ def preTraining(nnetModel,train_sets,train_xy,train_x,train_y,model_config):
                     fe_c.append(free_energy_cost)
                 train_sets.read_next_partition_data()
             train_sets.initialize_read()
-            logger.info('Training layer %i, epoch %d, r_cost %f, fe_cost %f' % (i, epoch, numpy.mean(r_c), numpy.mean(fe_c)))
+            logger.info('Training layer %i, epoch %d, r_cost %f, fe_cost %f',
+                i, epoch, numpy.mean(r_c), numpy.mean(fe_c))
     end_time = time.clock()
     logger.info('The PreTraing ran for %.2fm' % ((end_time - start_time) / 60.))
 
@@ -105,19 +108,22 @@ def runRBM(configFile):
 
     train_sets, train_xy, train_x, train_y = read_dataset(data_spec['training'])
 
-    keep_layer_num=model_config['keep_layer_num']
-    batch_size=model_config['batch_size']
+    keep_layer_num = model_config['keep_layer_num']
+    batch_size = model_config['batch_size']
+    wdir = model_config['wdir']
 
     if keep_layer_num > 0:
-        #current_nnet = wdir + 'nnet.ptr.current'
+        current_nnet = wdir + '/nnet.ptr.current'
         logger.info('Initializing model from ' + str(current_nnet) + '....')
         # load model
-        #_file2nnet(dbn.sigmoid_layers, set_layer_num = keep_layer_num, filename = current_nnet, withfinal=False)
+        _file2nnet(dbn.sigmoid_layers, set_layer_num = keep_layer_num, 
+            filename = current_nnet, withfinal=False)
 
     preTraining(dbn,train_sets,train_xy,train_x,train_y,model_config)
 
     # save the pretrained nnet to file
-    #_nnet2file(dbn.sigmoid_layers, filename=output_file, withfinal=True)
+    logger.info('Saving model to ' + str(model_config['output_file']) + '....')
+    _nnet2file(dbn.sigmoid_layers, filename=model_config['output_file'], withfinal=True)
 
 
     ########################
@@ -146,6 +152,8 @@ def runRBM(configFile):
     fineTunning(dbn,train_sets,train_xy,train_x,train_y,
         valid_sets,valid_xy,valid_x,valid_y,lrate,momentum,batch_size)
 
+    logger.info('Saving model to ' + str(model_config['output_file']) + '.final ....')
+    _nnet2file(dbn.sigmoid_layers, filename=model_config['output_file']+'.final', withfinal=True)
 
     try:
         test_sets, test_xy, test_x, test_y = read_dataset(data_spec['testing']) 

io_modules/file_reader.py

@@ -23,11 +23,11 @@
 
 #module imports
 from utils.load_conf import load_model,load_sda_spec,load_data_spec
-from io_modules.file_io import read_dataset
+from io_modules.file_reader import read_dataset
 from io_modules import setLogger
 from utils.learn_rates import LearningRate
 from models.sda import SDA
-
+from io_modules.model_io import _nnet2file, _file2nnet
 from models import fineTunning,testing
 
 
@@ -104,6 +104,10 @@ def runSdA(configFile):
 
     logger.info('The PreTraing ran for %.2fm' % ((end_time - start_time) / 60.))
 
+    # save the pretrained nnet to file
+    logger.info('Saving model to ' + str(model_config['output_file']) + '....')
+    _nnet2file(dbn.sigmoid_layers, filename=model_config['output_file'], withfinal=True)
+
 
     ########################
     # FINETUNING THE MODEL #
@@ -130,6 +134,11 @@ def runSdA(configFile):
     fineTunning(sda,train_sets,train_xy,train_x,train_y,
         valid_sets,valid_xy,valid_x,valid_y,lrate,momentum,batch_size);
 
+        # save the pretrained nnet to file
+    logger.info('Saving model to ' + str(model_config['output_file']) + '.final ....')
+    _nnet2file(dbn.sigmoid_layers, filename=model_config['output_file']+'.final', withfinal=True)
+
+
     try:
         test_sets, test_xy, test_x, test_y = read_dataset(data_spec['testing'])        
     except KeyError: