Added:checkpointing of model

Author: abilng

sample_config/MNIST/CNN/model_conf.json

@@ -8,6 +8,7 @@
 	"input_shape" : [1,28,28],
 	"batch_size" : 256,
 	"n_outs" : 10,
+	"save_feq":4,
 	"finetune_params" : {
 		"method" : "E",
 		"momentum" : 0.5,

src/pythonDnn/run/__init__.py

@@ -1,4 +1,5 @@
 import time,numpy,os
+import threading
 import logging
 logger = logging.getLogger(__name__)
 
@@ -44,7 +45,7 @@ def testing(nnetModel,data_spec,saveLabel=True,outFile='test.out'):
 			saveLabels(nnetModel,outFile,data_spec['testing'])
 
 
-def _fineTunning(nnetModel,train_sets,valid_sets,lrate,momentum):
+def _fineTunning(nnetModel,train_sets,valid_sets,lrate,momentum,saveFeq=0,outFile=''):
 
 	train_xy = train_sets.shared_xy
 	train_x = train_sets.shared_x
@@ -85,6 +86,17 @@ def valid_score():
 				logger.debug('Training batch %d error %f',batch_index, numpy.mean(train_error))
 			train_sets.read_next_partition_data()
 		logger.info('Fine Tunning:epoch %d, training error %f',lrate.epoch, numpy.mean(train_error));
+
+		#savemodel
+		savethread=None
+
+		if(saveFeq!=0 and lrate.epoch%saveFeq==0):
+			logger.info('Saving partial model to:%s',outFile)
+			savethread=threading.Thread(target=nnetModel.save,kwargs={'filename':outFile});
+			savethread.start()
+			#nnetModel.save(filename=outFile);
+
+
 		train_sets.initialize_read()
 
 		valid_error = valid_score()
@@ -93,6 +105,9 @@ def valid_score():
 		logger.info('Fine Tunning:epoch %d, validation error %f',lrate.epoch, valid_error);
 		lrate.get_next_rate(current_error = 100 * valid_error)
 
+		if savethread!=None:
+			savethread.join();
+
 	end_time = time.clock()
 
 	logger.info('Best validation error %f',best_validation_loss)
@@ -111,6 +126,8 @@ def fineTunning(nnetModel,model_config,data_spec):
 		logger.info("No validation/training set:Skiping Fine tunning");
 	else:
 		try:
+			outFile=model_config['output_file']
+			saveFeq=model_config['save_feq']
 			finetune_config = model_config['finetune_params']
 			momentum = finetune_config['momentum']
 			lrate = LearningRate.get_instance(finetune_config);
@@ -119,8 +136,7 @@ def fineTunning(nnetModel,model_config,data_spec):
 			logger.critical("Fine tunning Paramters Missing")
 			exit(2)
 
-
-		_fineTunning(nnetModel,train_sets,valid_sets,lrate,momentum)
+		_fineTunning(nnetModel,train_sets,valid_sets,lrate,momentum,saveFeq,outFile)
 
 
 def exportFeatures(nnetModel,model_config,data_spec):

src/pythonDnn/utils/load_conf.py

@@ -39,6 +39,9 @@ def load_model(input_file,nnetType=None):
 	if not data.has_key('random_seed') or not type(data['random_seed']) is int:
 		data['random_seed'] = None
 
+	if not data.has_key('save_feq') or not type(data['save_feq']) is int:
+		data['save_feq'] = 0
+
 	if data.has_key('n_ins') or data.has_key('input_shape'):
 		pass
 	else: