Major update: ready for optimization tests

Author: ArbinTimilsina

README.md

@@ -11,8 +11,8 @@ cd DeepLearningWithProtoDUNE
 
 ### Download the dataset
 ```
-# 320x320 inputs
-wget -O input_files.zip https://www.dropbox.com/sh/3529klxprjgswlv/AAAuD-HN3oOzpY1-rlkaokt8a?dl=1
+# 320x320 inputs; min 2000 beam hits 
+wget -O input_files.zip https://www.dropbox.com/sh/a00fbuye3i1c0sj/AACeI2l-iEpIoeDbDBtogjJKa?dl=1
 unzip input_files.zip -d input_files
 rm -rf input_files.zip
 ```

analyze_model.py

@@ -6,8 +6,8 @@
 from keras import backend as K
 from keras.models import load_model
 from tools.plotting_tools import plot_feature_label_prediction
+from tools.loss_metrics_tools import intersection_over_union, mean_iou, focal_loss
 from tools.data_tools import DataSequence, get_data_generator, preprocess_feature, preprocess_label
-from tools.loss_metrics_tools import intersection_over_union, mean_iou, weighted_categorical_crossentropy, focal_loss
 
 def argument_parser():
     ap = argparse.ArgumentParser()
@@ -79,8 +79,7 @@ def main():
 
     # Get the model
     model_path = os.path.join("saved_models", "model_and_weights.hdf5")
-    model = load_model(model_path, custom_objects={"loss": weighted_categorical_crossentropy(WEIGHTS),
-                                                   "loss": focal_loss(),
+    model = load_model(model_path, custom_objects={"loss": focal_loss(),
                                                    "mean_iou": mean_iou})
 
     # Make comparision plots

calculate_weights.py

@@ -39,7 +39,7 @@ def main():
         for index, weight in class_weights.items():
             weights[index].append(weight)
 
-    ranges = [(0.5,1.5), (100, 225), (5, 35)]
+    ranges = [(0.5,1.5), (100, 500), (5, 35)]
     plot_path = os.path.join("plots", "extra", "weights_median.pdf")
     plot_weights_median(weights, ranges, CLASS_NAMES, plot_path)
     print("\nDone! Plot with median weights for each class is saved at {}!\n".format(plot_path))

configurations/fnal_tev_configuration.ini

@@ -5,23 +5,23 @@ IMAGE_DEPTH = 1
 
 CLASS_NAMES = Background Beam Not-Beam
 
-FEATURE_FILE_TRAINING = /data/arbint/input_files/training
-LABEL_FILE_TRAINING = /data/arbint/input_files/training
+FEATURE_FILE_TRAINING = /data/arbint/input_files/training/feature.csv
+LABEL_FILE_TRAINING = /data/arbint/input_files/training/label.csv
 
-FEATURE_FILE_VALIDATION = /data/arbint/input_files/validation
-LABEL_FILE_VALIDATION = /data/arbint/input_files/validation
+FEATURE_FILE_VALIDATION = /data/arbint/input_files/validation/feature.csv
+LABEL_FILE_VALIDATION = /data/arbint/input_files/validation/label.csv
 
-FEATURE_FILE_TESTING = /data/arbint/input_files/testing
-LABEL_FILE_TESTING = /data/arbint/input_files/testing
+FEATURE_FILE_TESTING = /data/arbint/input_files/testing/feature.csv
+LABEL_FILE_TESTING = /data/arbint/input_files/testing/label.csv
 
 WEIGHTS = 1.0 1.0 1.0
 BATCH_SIZE = 2
 
 [TRAINING]
-NUM_TRAINING = 1835
-NUM_VALIDATION = 171
-NUM_TESTING = 38
-NUM_EPOCHS = 50
+NUM_TRAINING = 2637
+NUM_VALIDATION = 246
+NUM_TESTING = 62
+NUM_EPOCHS = 150
 
 [DEVELOPMENT]
 NUM_TRAINING = 40

configurations/hpc1_configuration.ini

@@ -5,22 +5,22 @@ IMAGE_DEPTH = 1
 
 CLASS_NAMES = Background Beam Not-Beam
 
-FEATURE_FILE_TRAINING = /scratch/arbint/input_files/training
-LABEL_FILE_TRAINING = /scratch/arbint/input_files/training
+FEATURE_FILE_TRAINING = /scratch/arbint/input_files/training/feature.csv
+LABEL_FILE_TRAINING = /scratch/arbint/input_files/training/label.csv
 
-FEATURE_FILE_VALIDATION = /scratch/arbint/input_files/validation
-LABEL_FILE_VALIDATION = /scratch/arbint/input_files/validation
+FEATURE_FILE_VALIDATION = /scratch/arbint/input_files/validation/feature.csv
+LABEL_FILE_VALIDATION = /scratch/arbint/input_files/validation/label.csv
 
-FEATURE_FILE_TESTING = /scratch/arbint/input_files/testing
-LABEL_FILE_TESTING = /scratch/arbint/input_files/testing
+FEATURE_FILE_TESTING = /scratch/arbint/input_files/testing/feature.csv
+LABEL_FILE_TESTING = /scratch/arbint/input_files/testing/label.csv
 
 WEIGHTS = 1.0 1.0 1.0
 BATCH_SIZE = 1
 
 [TRAINING]
-NUM_TRAINING = 1835
-NUM_VALIDATION = 171
-NUM_TESTING = 38
+NUM_TRAINING = 2637
+NUM_VALIDATION = 246
+NUM_TESTING = 62
 NUM_EPOCHS = 150
 
 [DEVELOPMENT]

configurations/local_configuration.ini

@@ -5,26 +5,26 @@ IMAGE_DEPTH = 1
 
 CLASS_NAMES = Background Beam Not-Beam
 
-FEATURE_FILE_TRAINING = input_files/training
-LABEL_FILE_TRAINING = input_files/training
+FEATURE_FILE_TRAINING = input_files/training/feature.csv
+LABEL_FILE_TRAINING = input_files/training/label.csv
 
-FEATURE_FILE_VALIDATION = input_files/validation
-LABEL_FILE_VALIDATION = input_files/validation
+FEATURE_FILE_VALIDATION = input_files/validation/feature.csv
+LABEL_FILE_VALIDATION = input_files/validation/label.csv
 
-FEATURE_FILE_TESTING = input_files/testing
-LABEL_FILE_TESTING = input_files/testing
+FEATURE_FILE_TESTING = input_files/testing/feature.csv
+LABEL_FILE_TESTING = input_files/testing/label.csv
 
 WEIGHTS = 1.0 1.0 1.0
 BATCH_SIZE = 1
 
 [TRAINING]
-NUM_TRAINING = 1835
-NUM_VALIDATION = 171
-NUM_TESTING = 38
+NUM_TRAINING = 2637
+NUM_VALIDATION = 246
+NUM_TESTING = 62
 NUM_EPOCHS = 150
 
 [DEVELOPMENT]
-NUM_TRAINING = 20
-NUM_VALIDATION = 10
+NUM_TRAINING = 40
+NUM_VALIDATION = 20
 NUM_TESTING = 10
-NUM_EPOCHS = 2
+NUM_EPOCHS = 3

configurations/master_configuration.ini

@@ -1 +1 @@
-fnal_tev_configuration.ini
+local_configuration.ini

from_dune_tpc/DeepLearning/Tools/Constants.h~

@@ -0,0 +1,22 @@
+#ifndef CONSTANTS_H
+#define CONSTANTS_H
+
+namespace ProtoDuneDL
+{
+    enum Labels
+    {
+	Background = 0,
+	Beam = 1,
+	NotBeam = 2,
+    };
+
+    const unsigned int WiresPerPlane[3] = {1148, 1148, 480}; // Plane: U, V, W
+    const unsigned int WiresOffsetPerTPC[3] = {400, 400, 480};
+    const unsigned int GlobalWiresPerPlane[3] = {1948, 1948, 1440}; // 2 * WiresOffsetPerTPC + WiresPerPlane
+    const unsigned int TDCsPerPlane[3] = {6000, 6000, 6000};
+
+    const unsigned int MaxPlanes = 3;
+    const unsigned int MaxWires = 448;
+    const unsigned int MaxTDCs = 448;
+}
+#endif