Updating NLL and reference to cheat sheet

Author: Carole Sudre

MetricsReloaded/metrics/calibration_measures.py

@@ -460,18 +460,22 @@ def negative_log_likelihood(self):
 
         George Cybenko, Dianne P O’Leary, and Jorma Rissanen. 1998. The Mathematics of Information Coding, Extraction
         and Distribution. Vol. 107. Springer Science & Business Media.
+        Cheat Sheet p 116 - Figure SN 3.71
 
         .. math::
 
-            NLL = -\sum_{i=1}{N} log(p_{i,k} | y_i=k)
+            NLL = -\dfrac{1}{N}\sum_{i=1}^{N}\sum_{k=1}^{C} y_{ik} \dot log(p_{i,k})
+
+        where :math: `y_{ik}` the outcome is 1 if the class of :math: `y_{i}` is k and :math: `p_{ik}` is the predicted 
+        probability for sample :math: `x_i` and class k
 
         :return: NLL
 
         """
         log_pred = np.log(self.pred)
         numb_samples = self.pred.shape[0]
         ll = np.sum(log_pred[range(numb_samples), self.ref])
-        nll = -1 * ll
+        nll = -1/numb_samples * ll
         return nll
 
     def to_dict_meas(self, fmt="{:.4f}"):

test/test_metrics/test_calibration_metrics.py

@@ -35,6 +35,9 @@ def test_expected_calibration_error():
     assert_allclose(value_test3, expected_ece3, atol=0.01)
 
 def test_maximum_calibration_error():
+    """
+    Using figure 2.24 p67 of pitfalls as reference
+    """
     ppm1 = CalibrationMeasures(pred_224, ref_224, dict_args={"bins_mce": 2})
     ppm2 = CalibrationMeasures(pred_224, ref_224, dict_args={'bins_mce':5})
     ppm3 = CalibrationMeasures(pred_224, ref_224)
@@ -88,7 +91,7 @@ def test_negative_log_likelihood():
     pred_nll = [[0.1, 0.8, 0.05, 0.1], [0.6, 0.1, 0, 0.7], [0.3, 0.1, 0.95, 0.2]]
     ref_nll = np.asarray(ref_nll)
     pred_nll = np.asarray(pred_nll).T
-    expected_nll = -1 * (np.log(0.8) + np.log(0.6) + np.log(0.7) + np.log(0.95))
+    expected_nll = -1/4 * (np.log(0.8) + np.log(0.6) + np.log(0.7) + np.log(0.95))
     cm = CalibrationMeasures(pred_nll, ref_nll)
     value_test = cm.negative_log_likelihood()
     assert_allclose(value_test, expected_nll)

test/test_utility/test_assignment_localization.py

@@ -11,22 +11,46 @@
 from sklearn.metrics import matthews_corrcoef as mcc
 
 #Data for figure 6a testing of assignment and average precision
-ref6c1 = np.asarray([3,2,7,5])
-ref6c2 = np.asarray([7,9,8,11])
-ref6c3 = np.asarray([1,16,3,18])
-ref6c4 = np.asarray([14,14,16,18])
+ref6a1 = np.asarray([3,2,7,5])
+ref6a2 = np.asarray([7,9,8,11])
+ref6a3 = np.asarray([1,16,3,18])
+ref6a4 = np.asarray([14,14,16,18])
 
-pred6c1 = np.asarray([2,3,6,6])
-pred6c2 = np.asarray([2,15,4,17])
-pred6c3 = np.asarray([13,13,15,17])
-pred6c4 = np.asarray([16,7,19,10])
-pred6c5 = np.asarray([12,2,15,4])
+pred6a1 = np.asarray([2,3,6,6])
+pred6a2 = np.asarray([2,15,4,17])
+pred6a3 = np.asarray([13,13,15,17])
+pred6a4 = np.asarray([16,7,19,10])
+pred6a5 = np.asarray([12,2,15,4])
 
-pred_proba_6c = [[0.05, 0.95],[0.30,0.70],[0.20,0.80],[0.20,0.80],[0.10,0.90]]
+pred_proba_6a = [[0.05, 0.95],[0.30,0.70],[0.20,0.80],[0.20,0.80],[0.10,0.90]]
 
-pred_boxes_6c = [pred6c1, pred6c2, pred6c3, pred6c4, pred6c5]
-ref_boxes_6c = [ref6c1, ref6c2, ref6c3, ref6c4]
+pred_boxes_6a = [pred6a1, pred6a2, pred6a3, pred6a4, pred6a5]
+ref_boxes_6a = [ref6a1, ref6a2, ref6a3, ref6a4]
 
+#Data from Panoptic Quality - 3.51 p96
+#Figure 3.51 p96
+pq_pred1 = np.zeros([18, 18])
+pq_pred1[ 3:7,1:3] = 1
+pq_pred1[3:6,3:7]=1
+pq_pred2 = np.zeros([18, 18])
+pq_pred2[13:16,4:6] = 1
+pq_pred3 = np.zeros([18, 18])
+pq_pred3[7:12,13:17] = 1
+pq_pred4 = np.zeros([18, 18])
+pq_pred4[13:15,13:17] = 1
+pq_pred4[15,15] = 1
+
+pq_ref1 = np.zeros([18, 18])
+pq_ref1[2:7, 1:3] = 1
+pq_ref1[2:5,3:6] = 1
+pq_ref2 = np.zeros([18, 18])
+pq_ref2[6:12,12:17] = 1
+pq_ref3 = np.zeros([18, 18])
+pq_ref3[14:15:,7:10] = 1
+pq_ref3[13:16,8:9] = 1
+
+ref_351 = [pq_ref1, pq_ref2, pq_ref3]
+pred_351 = [pq_pred1, pq_pred2, pq_pred3,pq_pred4]
 
 
 ## Data for figure 59 and testing of localisation
@@ -41,7 +65,7 @@
 f59_pred2[4:8, 5:9] = 1
 
 def test_assignment_6c():
-    asm1 = AssignmentMapping(pred_loc=pred_boxes_6c, ref_loc=ref_boxes_6c, pred_prob=pred_proba_6c, thresh=0.1,localization='box_iou')
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
     df_matching, df_fn, df_fp, list_valid = asm1.initial_mapping()
     print(asm1.matrix, df_matching, df_fp, df_fn, list_valid)
     numb_fn = df_fn.shape[0]
@@ -52,6 +76,7 @@ def test_assignment_6c():
     assert expected_fp == numb_fp
     assert_array_almost_equal(np.asarray(list_valid),np.asarray([0,1,2]))
 
+
 def test_check_localization():
     ref_box = [[2,2,4,4]]
     ref_com = [[3,3]]
@@ -209,6 +234,84 @@ def test_pairwise_pointcomdist():
     expected_matrix = np.asarray([[0, 9.22],[8.49, 1]])
     assert_allclose(am.matrix, expected_matrix,atol=0.01)
 
+def test_pairwise_boxiou_6a():
+    """
+    Using figure 6a p14 pitfalls as illustration for boxiou pairwise example
+    """
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
+    df_matching, df_fn, df_fp, list_valid = asm1.initial_mapping()
+    expected_matrix = np.asarray([[0.42857143, 0.  ,       0.  ,       0.        ], 
+                                  [0. ,        0.  ,       0.28571429, 0.        ],
+                                  [0. ,        0.  ,       0.     ,    0.36363636],
+                                  [0.  ,       0.  ,       0.     ,    0.        ],
+                                  [0. ,        0.  ,       0.    ,     0.        ]])
+    assert_array_almost_equal(expected_matrix, asm1.matrix)
+
+def test_com_from_refbox_6a():
+    """
+    Using figure 6a as illustration
+    """
+
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
+    asm1.com_fromrefbox()
+    test_com = asm1.ref_loc_mod
+    expected_ref_com = [[5,3.5],[7.5,10],[2,17],[15,16]]
+    assert_array_almost_equal(np.asarray(expected_ref_com), test_com)
+
+def test_com_from_predbox_6a():
+    """
+    Using figure 6a as illustration
+    """
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
+    asm1.com_frompredbox()
+    test_com = asm1.pred_loc_mod
+    print(test_com)
+    expected_pred_com = [[4,4.5],[3,16],[14,15],[17.5,8.5],[13.5,3]]
+    assert_array_almost_equal(np.asarray(expected_pred_com), test_com,decimal=2)
+
+def test_comfrompredmask_351():
+    """
+    Using figure 3.51 (p96 Panoptic quality) as illustration
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    expected_predcom = [[4.2,3.3],[14,4.5],[9,14.5],[13.66,14.55]]
+    asm.com_frompredmask()
+    test_com = asm.pred_loc_mod
+    assert_array_almost_equal(np.asarray(expected_predcom),test_com,decimal=2)
+
+def test_comfromrefmask_351():
+    """
+    Using figure 3.51 (p96 Panoptic quality) as illustrative example
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    expected_refcom = [[3.53,2.68],[8.5,14],[14,8]]
+    asm.com_fromrefmask()
+    test_com = asm.ref_loc_mod
+    assert_array_almost_equal(np.asarray(expected_refcom),test_com,decimal=2)
+
+def test_box_fromrefmask():
+    """
+    Using fig 3.51 p96 as illustrative example
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    asm.box_fromrefmask()
+    test_box = asm.ref_loc_mod
+    expected_box = [[2,1,6,5],[6,12,11,16],[13,7,15,9]]
+    assert_array_almost_equal(np.asarray(expected_box),test_box)
+
+def test_box_frompredmask():
+    """
+    Using fig 3.51 p96 as illustrative example
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    asm.box_frompredmask()
+    test_box = asm.pred_loc_mod
+    expected_box = [[3,1,6,6],[13,4,15,5],[7,13,11,16],[13,13,15,16]]
+    assert_array_almost_equal(np.asarray(expected_box),test_box)
+
+
+
+
 
 def test_localization():
     ref = [f59_ref1, f59_ref2]