Merge pull request #34 from SchwarzNeuroconLab/dev_bsoidupdate

updated likelihood handling for BSOID classification
DLC pose estimation can now be filtered by likelihood in advanced settings. Values below the threshold will be set to NaN and need to be handled by calculate skeleton

Author: JensBlack

DeepLabStream.py

@@ -35,6 +35,7 @@
     CROP,
     CROP_X,
     CROP_Y,
+    USE_DLSTREAM_POSTURE_DETECTION,
 )
 from utils.plotter import plot_bodyparts, plot_metadata_frame
 from utils.poser import (
@@ -336,11 +337,14 @@ def get_pose_mp(input_q, output_q):
                         scmap, locref, pose = get_pose(
                             frame, config, sess, inputs, outputs
                         )
-                        peaks = find_local_peaks_new(
-                            scmap, locref, ANIMALS_NUMBER, config
-                        )
+                        if USE_DLSTREAM_POSTURE_DETECTION:
+                            """ This is a legacy function that was used in earlier versions"""
+                            peaks = find_local_peaks_new(
+                                scmap, locref, ANIMALS_NUMBER, config
+                            )
                         # Use the line below to use raw DLC output rather then DLStream optimization
-                        #peaks = pose
+                        else:
+                            peaks = pose
                     if MODEL_ORIGIN == "MADLC":
                         peaks = get_ma_pose(frame, config, sess, inputs, outputs)
                     analysis_time = time.time() - start_time

convert_classifier.py

@@ -1,29 +1,52 @@
 import pickle
+import joblib
 import os
 from pure_sklearn.map import convert_estimator
 
 
-def load_classifier(path_to_sav):
+def load_classifier_SIMBA(path_to_sav):
     """Load saved classifier"""
     file = open(path_to_sav, "rb")
     classifier = pickle.load(file)
     file.close()
     return classifier
 
+def load_classifier_BSOID(path_to_sav):
+    """Load saved classifier"""
+    file = open(path_to_sav, "rb")
+    clf = joblib.load(file)
+    file.close()
+    return clf
 
-def convert_classifier(path):
+
+def convert_classifier(path, origin: str):
     # convert to pure python estimator
-    print("Loading classifier...")
-    clf = load_classifier(path)
     dir_path = os.path.dirname(path)
     filename = os.path.basename(path)
     filename, _ = filename.split(".")
-    clf_pure_predict = convert_estimator(clf)
-    with open(dir_path + "/" + filename + "_pure.sav", "wb") as f:
-        pickle.dump(clf_pure_predict, f)
+
+    print("Loading classifier...")
+    if origin.lower() == 'simba':
+        clf = load_classifier_SIMBA(path)
+        clf_pure_predict = convert_estimator(clf)
+        with open(dir_path + "/" + filename + "_pure.sav", "wb") as f:
+            pickle.dump(clf_pure_predict, f)
+
+    elif origin.lower() == 'bsoid':
+        clf_pack = load_classifier_BSOID(path)
+        # bsoid exported classfier has format [a, b, c, clf, d, e]
+        clf_pure_predict = convert_estimator(clf_pack[3])
+        clf_pack[3] =clf_pure_predict
+        with open(dir_path + "/" + filename + "_pure.sav", "wb") as f:
+            joblib.dump(clf_pack, f)
+    else:
+        raise ValueError(f'{origin} is not a valid classifier origin.')
+
     print(f"Converted Classifier {filename}")
 
 
 if __name__ == "__main__":
+
+    """Converted BSOID Classifiers are not integrated yet, although you can already convert them here"""
     path_to_classifier = "PATH_TO_CLASSIFIER"
-    convert_classifier(path_to_classifier)
+    convert_classifier(path_to_classifier, origin= 'SIMBA')

experiments/custom/classifier.py

@@ -10,7 +10,7 @@
 import time
 import pickle
 
-from utils.configloader import PATH_TO_CLASSIFIER, TIME_WINDOW
+from utils.configloader import PATH_TO_CLASSIFIER, TIME_WINDOW, FRAMERATE
 from experiments.custom.featureextraction import (
     SimbaFeatureExtractor,
     SimbaFeatureExtractorStandard14bp,
@@ -117,6 +117,7 @@ def classify(self, features):
         Adapted from BSOID; https://github.com/YttriLab/B-SOID
         """
         labels_fslow = []
+        # TODO: adapt to pure version of BSOID Classifier
         for i in range(0, len(features)):
             labels = self._classifier.predict(features[i].T)
             labels_fslow.append(labels)
@@ -183,21 +184,26 @@ def simba_feat_classifier_pool_run(input_q: mp.Queue, output_q: mp.Queue):
 
 
 def bsoid_feat_classifier_pool_run(input_q: mp.Queue, output_q: mp.Queue):
-    feature_extractor = BsoidFeatureExtractor(TIME_WINDOW)
+    feature_extractor = BsoidFeatureExtractor()
     classifier = BsoidClassifier()  # initialize classifier
     while True:
         skel_time_window = None
         feature_id = 0
         if input_q.full():
             skel_time_window, feature_id = input_q.get()
         if skel_time_window is not None:
-            start_time = time.time()
+            start_time_feat = time.time()
             features = feature_extractor.extract_features(skel_time_window)
+            end_time_feat = time.time()
+            start_time_clf = time.time()
             last_prob = classifier.classify(features)
             output_q.put((last_prob, feature_id))
             end_time = time.time()
-            # print("Classification time: {:.2f} msec".format((end_time-start_time)*1000))
-            # print("Feature ID: "+ feature_id)
+            print("Feature Extraction time: {:.2f} msec".format((end_time_feat - start_time_feat) * 1000))
+            print("Classification time: {:.2f} msec".format((end_time-start_time_clf)*1000))
+            print("Total time: {:.2f} msec".format((end_time-start_time_feat)*1000))
+            print("Current motif: ", *last_prob)
+            print("Feature ID: "+ str(feature_id))
         else:
             pass
 
@@ -275,6 +281,8 @@ def pass_time_window(self, skel_time_window: tuple, debug: bool = False):
         :param debug bool: reporting of process + feature id to identify discrepancies in processing sequence
         """
         for process in self._process_pool:
+            #if the process is not already busy, feed it some new input and break the loop
+            #this should only be valid the first time the process is fed.
             if not process["running"]:
                 if process["input"].empty():
                     process["input"].put(skel_time_window)
@@ -287,6 +295,8 @@ def pass_time_window(self, skel_time_window: tuple, debug: bool = False):
                         )
                     break
 
+            #if the process is busy but finished (has output), feed it some new input.
+            #this should be the normal case
             elif process["input"].empty() and process["output"].full():
                 process["input"].put(skel_time_window)
                 if debug:
@@ -306,7 +316,12 @@ def get_result(self, debug: bool = False):
         """
         result = (None, 0)
         for process in self._process_pool:
+            #check if process is finished
             if process["output"].full():
+                #take result and break the loop. This way two simultaneously finished processes are emptied in sequence
+                #rather then overwriting the results of each other
+                #the disadvantage is that the result won't be the latest classification but in the next in sequential order (to the last).
+                #the advantage is that we won't miss any results this way and have "consistent" latency, which is the intended behavior.
                 result = process["output"].get()
                 if debug:
                     print("Output", process["process"].name, "ID: " + str(result[1]))
@@ -381,13 +396,15 @@ def simba_classifier_run(input_q: mp.Queue, output_q: mp.Queue):
 
 
 def bsoid_classifier_run(input_q: mp.Queue, output_q: mp.Queue):
+    #takes features from input and feeds them into classifier. Outputs classification
     classifier = BsoidClassifier()  # initialize classifier
     while True:
         features = None
         if input_q.full():
             features = input_q.get()
         if features is not None:
             start_time = time.time()
+            #last prob is a missleading name that comes from a binary classifier. B-SOID's output is a cluster id rather then the probability.
             last_prob = classifier.classify(features)
             output_q.put((last_prob))
             end_time = time.time()