Fix issue that not the full match is used to estimate the non-prefix regex matches

Author: thomaskrause

core/src/annostorage/inmemory.rs

@@ -778,7 +778,7 @@ where
             } else {
                 // For regular expressions without a prefix the worst case would be `.*[X].*` where `[X]` are the most common characters.
                 // Sample values from the histogram to get a better estimation of how many percent of the actual values could match.
-                if let Ok(pattern) = regex::Regex::new(pattern) {
+                if let Ok(pattern) = regex::Regex::new(&full_match_pattern) {
                     let mut rng = thread_rng();
                     let qualified_keys: Vec<_> = match ns {
                         Some(ns) => vec![AnnoKey {

core/src/annostorage/inmemory/tests.rs

@@ -171,10 +171,14 @@ fn get_node_id_from_name() {
     assert_eq!(false, a.has_node_name("somenode").unwrap());
 }
 
-#[test]
-fn regex_search() {
+/// Inserts the following strings as node names:
+/// - _ABC
+/// - AAA
+/// - AAB
+/// - AAC
+/// - B
+fn insert_test_strings(a: &mut AnnoStorageImpl<NodeID>) {
     let key = NODE_NAME_KEY.as_ref().clone();
-    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new();
     a.insert(
         0,
         Annotation {
@@ -216,6 +220,12 @@ fn regex_search() {
         },
     )
     .unwrap();
+}
+
+#[test]
+fn regex_search() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new();
+    insert_test_strings(&mut a);
 
     let result: Result<Vec<_>> = a
         .regex_anno_search(Some(ANNIS_NS), NODE_NAME, "A.*", false)
@@ -261,3 +271,20 @@ fn regex_search() {
 
     assert_eq!(0, result.len());
 }
+
+#[test]
+fn regex_estimation_without_prefix() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new();
+    insert_test_strings(&mut a);
+    a.calculate_statistics().unwrap();
+
+    // Test with namespace
+    // Since there are very less than 250 items the histogram based statistics should be exact.
+    assert_eq!(
+        3,
+        a.guess_max_count_regex(Some(ANNIS_NS), NODE_NAME, ".A.")
+            .unwrap()
+    );
+    // Test without namespace
+    assert_eq!(3, a.guess_max_count_regex(None, NODE_NAME, ".A.").unwrap());
+}

core/src/annostorage/ondisk.rs

@@ -887,7 +887,8 @@ where
 
         if sum_histogram_buckets > 0 {
             let selectivity: f64 = (count_matches as f64) / (sum_histogram_buckets as f64);
-            Ok((selectivity * (universe_size as f64)).round() as usize)
+            let estimation = (selectivity * (universe_size as f64)).round() as usize;
+            Ok(estimation)
         } else {
             Ok(0)
         }
@@ -921,7 +922,7 @@ where
             } else {
                 // For regular expressions without a prefix the worst case would be `.*[X].*` where `[X]` are the most common characters.
                 // Sample values from the histogram to get a better estimation of how many percent of the actual values could match.
-                if let Ok(pattern) = regex::Regex::new(pattern) {
+                if let Ok(pattern) = regex::Regex::new(&full_match_pattern) {
                     let mut rng = thread_rng();
                     let qualified_keys = match ns {
                         Some(ns) => vec![AnnoKey {
@@ -939,6 +940,7 @@ where
                         if let Some(histo) = self.histogram_bounds.get(&anno_key) {
                             if !histo.is_empty() {
                                 let sampled_values = histo.iter().choose_multiple(&mut rng, 20);
+
                                 let matches = sampled_values
                                     .iter()
                                     .filter(|v| pattern.is_match(v))

core/src/annostorage/ondisk/tests.rs

@@ -180,10 +180,14 @@ fn get_node_id_from_name() {
     assert_eq!(false, a.has_node_name("somenode").unwrap());
 }
 
-#[test]
-fn regex_search() {
+/// Inserts the following strings as node names:
+/// - _ABC
+/// - AAA
+/// - AAB
+/// - AAC
+/// - B
+fn insert_test_strings(a: &mut AnnoStorageImpl<NodeID>) {
     let key = NODE_NAME_KEY.as_ref().clone();
-    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new(None).unwrap();
     a.insert(
         0,
         Annotation {
@@ -225,6 +229,12 @@ fn regex_search() {
         },
     )
     .unwrap();
+}
+
+#[test]
+fn regex_search() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new(None).unwrap();
+    insert_test_strings(&mut a);
 
     // Test with namespace
     let result: Result<Vec<_>> = a
@@ -270,7 +280,7 @@ fn regex_search() {
     let result = result.unwrap();
     assert_eq!(0, result.len());
 
-    // lso test without namepsace
+    // also test without namepsace
     let result: Result<Vec<_>> = a
         .regex_anno_search(None, NODE_NAME, "A.*", false)
         .map_ok(|m| m.node)
@@ -297,3 +307,20 @@ fn regex_search() {
     result.sort();
     assert_eq!(vec![0, 4], result);
 }
+
+#[test]
+fn regex_estimation_without_prefix() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new(None).unwrap();
+    insert_test_strings(&mut a);
+    a.calculate_statistics().unwrap();
+
+    // Test with namespace
+    // Since there are very less than 250 items the histogram based statistics should be exact.
+    assert_eq!(
+        3,
+        a.guess_max_count_regex(Some(ANNIS_NS), NODE_NAME, ".A.")
+            .unwrap()
+    );
+    // Test without namespace
+    assert_eq!(3, a.guess_max_count_regex(None, NODE_NAME, ".A.").unwrap());
+}