Merge pull request #325 from korpling/better-regex-estimation

Better estimation for queries with regular expressions without prefix

Author: thomaskrause

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Avoid loading the node annotation storage when listing the components for a
 corpus in the `CorpusStorage`. Before this change, querying for components via
 the webservice could block the corpus cache.
+- Better estimation for queries with regular expressions without prefix.
+
 
 ## [3.8.0] - 2025-05-14
 

core/src/annostorage/inmemory.rs

@@ -7,6 +7,8 @@ use crate::util::{self};
 use crate::{annostorage::symboltable::SymbolTable, errors::GraphAnnisCoreError};
 use core::ops::Bound::*;
 use itertools::Itertools;
+use rand::seq::IteratorRandom;
+use rand::thread_rng;
 use rustc_hash::FxHashSet;
 use smartstring::alias::String;
 use smartstring::{LazyCompact, SmartString};
@@ -775,9 +777,50 @@ where
                 }
             } else {
                 // For regular expressions without a prefix the worst case would be `.*[X].*` where `[X]` are the most common characters.
-                // Assume that a generic percentage (here 5%) of all nodes match the regex.
-                // TODO: find better ways of estimating this constant
-                guessed_count = (0.05 * (total as f64)) as usize;
+                // Sample values from the histogram to get a better estimation of how many percent of the actual values could match.
+                if let Ok(pattern) = regex::Regex::new(&full_match_pattern) {
+                    let mut rng = thread_rng();
+                    let qualified_keys: Vec<_> = match ns {
+                        Some(ns) => vec![AnnoKey {
+                            name: name.into(),
+                            ns: ns.into(),
+                        }],
+                        None => self.get_qnames(name)?,
+                    }
+                    .into_iter()
+                    .filter_map(|key| self.anno_keys.get_symbol(&key).map(|symbol| (symbol, key)))
+                    .collect();
+                    for (anno_key_symbol, anno_key) in qualified_keys {
+                        let anno_size = self
+                            .anno_key_sizes
+                            .get(&anno_key)
+                            .copied()
+                            .unwrap_or_default();
+
+                        if let Some(histo) = self.histogram_bounds.get(&anno_key_symbol) {
+                            if !histo.is_empty() {
+                                let sampled_values = histo.iter().choose_multiple(&mut rng, 20);
+                                let matches = sampled_values
+                                    .iter()
+                                    .filter(|v| pattern.is_match(v))
+                                    .count();
+                                if sampled_values.len() == matches {
+                                    // Assume all values match
+                                    guessed_count += anno_size;
+                                } else if matches == 0 {
+                                    // No match found, but use the bucket size as pessimistic guess
+                                    guessed_count +=
+                                        (anno_size as f64 / sampled_values.len() as f64) as usize;
+                                } else {
+                                    // Use the percent of matched values to guess the overall number
+                                    let match_ratio =
+                                        (matches as f64) / (sampled_values.len() as f64);
+                                    guessed_count += ((anno_size as f64) * match_ratio) as usize;
+                                }
+                            }
+                        }
+                    }
+                }
             }
 
             Ok(guessed_count.min(total))

core/src/annostorage/inmemory/tests.rs

@@ -171,10 +171,14 @@ fn get_node_id_from_name() {
     assert_eq!(false, a.has_node_name("somenode").unwrap());
 }
 
-#[test]
-fn regex_search() {
+/// Inserts the following strings as node names:
+/// - _ABC
+/// - AAA
+/// - AAB
+/// - AAC
+/// - B
+fn insert_test_strings(a: &mut AnnoStorageImpl<NodeID>) {
     let key = NODE_NAME_KEY.as_ref().clone();
-    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new();
     a.insert(
         0,
         Annotation {
@@ -216,6 +220,12 @@ fn regex_search() {
         },
     )
     .unwrap();
+}
+
+#[test]
+fn regex_search() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new();
+    insert_test_strings(&mut a);
 
     let result: Result<Vec<_>> = a
         .regex_anno_search(Some(ANNIS_NS), NODE_NAME, "A.*", false)
@@ -261,3 +271,20 @@ fn regex_search() {
 
     assert_eq!(0, result.len());
 }
+
+#[test]
+fn regex_estimation_without_prefix() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new();
+    insert_test_strings(&mut a);
+    a.calculate_statistics().unwrap();
+
+    // Test with namespace
+    // Since there are very less than 250 items the histogram based statistics should be exact.
+    assert_eq!(
+        3,
+        a.guess_max_count_regex(Some(ANNIS_NS), NODE_NAME, ".A.")
+            .unwrap()
+    );
+    // Test without namespace
+    assert_eq!(3, a.guess_max_count_regex(None, NODE_NAME, ".A.").unwrap());
+}

core/src/annostorage/ondisk.rs

@@ -10,6 +10,7 @@ use crate::{try_as_boxed_iter, util};
 use core::ops::Bound::*;
 use itertools::Itertools;
 use rand::seq::IteratorRandom;
+use rand::thread_rng;
 use regex_syntax::hir::literal::Seq;
 use regex_syntax::Parser;
 use serde_bytes::ByteBuf;
@@ -886,7 +887,8 @@ where
 
         if sum_histogram_buckets > 0 {
             let selectivity: f64 = (count_matches as f64) / (sum_histogram_buckets as f64);
-            Ok((selectivity * (universe_size as f64)).round() as usize)
+            let estimation = (selectivity * (universe_size as f64)).round() as usize;
+            Ok(estimation)
         } else {
             Ok(0)
         }
@@ -919,9 +921,47 @@ where
                 }
             } else {
                 // For regular expressions without a prefix the worst case would be `.*[X].*` where `[X]` are the most common characters.
-                // Assume that a generic percentage (here 5%) of all nodes match the regex.
-                // TODO: find better ways of estimating this constant
-                guessed_count = (0.05 * (total as f64)) as usize;
+                // Sample values from the histogram to get a better estimation of how many percent of the actual values could match.
+                if let Ok(pattern) = regex::Regex::new(&full_match_pattern) {
+                    let mut rng = thread_rng();
+                    let qualified_keys = match ns {
+                        Some(ns) => vec![AnnoKey {
+                            name: name.into(),
+                            ns: ns.into(),
+                        }],
+                        None => self.get_qnames(name)?,
+                    };
+                    for anno_key in qualified_keys {
+                        let anno_size = self
+                            .anno_key_sizes
+                            .get(&anno_key)
+                            .copied()
+                            .unwrap_or_default();
+                        if let Some(histo) = self.histogram_bounds.get(&anno_key) {
+                            if !histo.is_empty() {
+                                let sampled_values = histo.iter().choose_multiple(&mut rng, 20);
+
+                                let matches = sampled_values
+                                    .iter()
+                                    .filter(|v| pattern.is_match(v))
+                                    .count();
+                                if sampled_values.len() == matches {
+                                    // Assume all values match
+                                    guessed_count += anno_size;
+                                } else if matches == 0 {
+                                    // No match found, but use the bucket size as pessimistic guess
+                                    guessed_count +=
+                                        (anno_size as f64 / sampled_values.len() as f64) as usize;
+                                } else {
+                                    // Use the percent of matched values to guess the overall number
+                                    let match_ratio =
+                                        (matches as f64) / (sampled_values.len() as f64);
+                                    guessed_count += ((anno_size as f64) * match_ratio) as usize;
+                                }
+                            }
+                        }
+                    }
+                }
             }
 
             Ok(guessed_count.min(total))

core/src/annostorage/ondisk/tests.rs

@@ -180,10 +180,14 @@ fn get_node_id_from_name() {
     assert_eq!(false, a.has_node_name("somenode").unwrap());
 }
 
-#[test]
-fn regex_search() {
+/// Inserts the following strings as node names:
+/// - _ABC
+/// - AAA
+/// - AAB
+/// - AAC
+/// - B
+fn insert_test_strings(a: &mut AnnoStorageImpl<NodeID>) {
     let key = NODE_NAME_KEY.as_ref().clone();
-    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new(None).unwrap();
     a.insert(
         0,
         Annotation {
@@ -225,6 +229,12 @@ fn regex_search() {
         },
     )
     .unwrap();
+}
+
+#[test]
+fn regex_search() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new(None).unwrap();
+    insert_test_strings(&mut a);
 
     // Test with namespace
     let result: Result<Vec<_>> = a
@@ -270,7 +280,7 @@ fn regex_search() {
     let result = result.unwrap();
     assert_eq!(0, result.len());
 
-    // lso test without namepsace
+    // also test without namepsace
     let result: Result<Vec<_>> = a
         .regex_anno_search(None, NODE_NAME, "A.*", false)
         .map_ok(|m| m.node)
@@ -297,3 +307,20 @@ fn regex_search() {
     result.sort();
     assert_eq!(vec![0, 4], result);
 }
+
+#[test]
+fn regex_estimation_without_prefix() {
+    let mut a: AnnoStorageImpl<NodeID> = AnnoStorageImpl::new(None).unwrap();
+    insert_test_strings(&mut a);
+    a.calculate_statistics().unwrap();
+
+    // Test with namespace
+    // Since there are very less than 250 items the histogram based statistics should be exact.
+    assert_eq!(
+        3,
+        a.guess_max_count_regex(Some(ANNIS_NS), NODE_NAME, ".A.")
+            .unwrap()
+    );
+    // Test without namespace
+    assert_eq!(3, a.guess_max_count_regex(None, NODE_NAME, ".A.").unwrap());
+}