Merge pull request #322 from korpling/fix-partof-estimation

Improve selectivity estimation for @-Queries

Author: thomaskrause

.gitattributes

@@ -1 +1,3 @@
-*.graphml text eol=lf
+*.graphml text eol=lf
+third-party-licenses.html linguist-generated
+/graphannis/tests/data/ linguist-generated=true

.github/workflows/verify.yml

@@ -1,9 +1,6 @@
 name: Verify
 
 on:
-  push:
-    branches:
-      - main
   pull_request:
 
 jobs:
@@ -66,4 +63,3 @@ jobs:
             ```
             ${{env.COVERAGE_INFO}}
             ```
-      

CHANGELOG.md

@@ -16,13 +16,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - New optional `file` option for the `[logging]` section in the webservice
 configuration. Can be used to additionally output all log messages to the given
 file.
+- Add number of root nodes to graph storage statistics. This changes the way
+most of the graph storages store their statistics. You can use old imported data
+files, but to make use of the new information you queries, you have to
+**reimport** your corpora.
 - `Graph:ensure_loaded_parallel` returns the actually loaded components that did
 exist.
 
 ### Fixed
 
 - Less frequent corpus cache status updates in log. Before, every corpus access
 could trigger an entry into the log which is not desired under heavy load.
+- Improve query execution planning by assuming all annotations can be matched in
+regular expressions without a prefix.
 
 ## [3.7.1] - 2025-04-14
 

cli/src/bin/annis.rs

@@ -321,7 +321,7 @@ impl AnnisRunner {
                 format = ExportFormat::GraphMLZip;
             } else if file_ext.to_string_lossy() == ".graphml" && self.current_corpus.len() != 1 {
                 bail!(
-                    r##"You need to select a *single* corpus first with the \"corpus\" command when exporting to a GraphML file. 
+                    r##"You need to select a *single* corpus first with the \"corpus\" command when exporting to a GraphML file.
                 To export multiple corpora, select a directory as output or a ZIP file (ending with .zip)"##
                 );
             }
@@ -442,7 +442,7 @@ impl AnnisRunner {
                 "unsorted" => ResultOrder::NotSorted,
                 _ => {
                     return Err(anyhow!(
-                        "Non-existing order with name {}. 
+                        "Non-existing order with name {}.
                         Must be one of \"normal\", \"inverted\", \"random\", \"unsorted\"",
                         args
                     ));

cli/tests/cli.rs

@@ -26,7 +26,7 @@ fn show_corpus_info() -> Result<(), Box<dyn std::error::Error>> {
 
     cmd.arg("../graphannis/tests/data/")
         .arg("-c")
-        .arg("corpus sample-disk-based-3.3")
+        .arg("corpus sample-disk-based-3.8")
         .arg("-c")
         .arg("preload")
         .arg("-c")

cli/tests/snapshots/cli__list_corpora_fully_loaded.snap

@@ -20,6 +20,7 @@ exit_code: 0
 sample-disk-based-1.5 (not loaded)
 sample-disk-based-3.2 (not loaded)
 sample-disk-based-3.3 (fully loaded)
+sample-disk-based-3.8 (not loaded)
 sample-memory-based-1.5 (not loaded)
 sample-memory-based-3.2 (not loaded)
 sample-memory-based-3.3 (not loaded)

cli/tests/snapshots/cli__list_corpora_not_loaded.snap

@@ -13,10 +13,10 @@ exit_code: 0
 sample-disk-based-1.5 (not loaded)
 sample-disk-based-3.2 (not loaded)
 sample-disk-based-3.3 (not loaded)
+sample-disk-based-3.8 (not loaded)
 sample-memory-based-1.5 (not loaded)
 sample-memory-based-3.2 (not loaded)
 sample-memory-based-3.3 (not loaded)
 graphANNIS says good-bye!
 
 ----- stderr -----
-

cli/tests/snapshots/cli__list_corpora_partially_loaded.snap

@@ -21,6 +21,7 @@ result: 44 matches in 4 documents
 sample-disk-based-1.5 (not loaded)
 sample-disk-based-3.2 (not loaded)
 sample-disk-based-3.3 (partially loaded)
+sample-disk-based-3.8 (not loaded)
 sample-memory-based-1.5 (not loaded)
 sample-memory-based-3.2 (not loaded)
 sample-memory-based-3.3 (not loaded)

cli/tests/snapshots/cli__show_corpus_info.snap

@@ -5,7 +5,7 @@ info:
   args:
     - "../graphannis/tests/data/"
     - "-c"
-    - corpus sample-disk-based-3.3
+    - corpus sample-disk-based-3.8
     - "-c"
     - preload
     - "-c"
@@ -14,59 +14,59 @@ info:
 success: true
 exit_code: 0
 ----- stdout -----
-12:00:00[INFO] Loaded corpus sample-disk-based-3.3
-12:00:00[INFO] Corpus cache after preloading sample-disk-based-3.3: 100MB / 300MB - loaded corpora [sample-disk-based-3.3]
+12:00:00[INFO] Loaded corpus sample-disk-based-3.8
+12:00:00[INFO] Corpus cache after preloading sample-disk-based-3.8: 100MB / 300MB - loaded corpora [sample-disk-based-3.8]
 12:00:00[INFO] Preloaded corpus in 10 ms
 Status: "fully loaded"
 Token search shortcut possible: true
 ------------
 Component Coverage//: 0 annnotations
-Stats: nodes=92, avg_fan_out=2.17, max_fan_out=11, max_depth=1
+Stats: nodes=92, root nodes=48, avg_fan_out=2.17, max_fan_out=11, fan_out_99%=11, inv_fan_out_99%=9, max_depth=1
 Implementation: DiskAdjacencyListV1
 Status: "fully loaded"
 ------------
 Component Coverage/annis/: 0 annnotations
-Stats: nodes=0, avg_fan_out=0.00, max_fan_out=0, max_depth=1, tree
+Stats: nodes=0, root nodes=0, avg_fan_out=0.00, max_fan_out=0, fan_out_99%=0, inv_fan_out_99%=0, max_depth=1, tree
 Implementation: DiskAdjacencyListV1
 Status: "fully loaded"
 ------------
 Component Coverage/default_ns/: 0 annnotations
-Stats: nodes=56, avg_fan_out=0.93, max_fan_out=10, max_depth=1
+Stats: nodes=56, root nodes=12, avg_fan_out=0.93, max_fan_out=10, fan_out_99%=10, inv_fan_out_99%=2, max_depth=1
 Implementation: DiskAdjacencyListV1
 Status: "fully loaded"
 ------------
 Component Coverage/annis/inherited-coverage: 0 annnotations
-Stats: nodes=0, avg_fan_out=0.00, max_fan_out=0, max_depth=1, tree
+Stats: nodes=0, root nodes=0, avg_fan_out=0.00, max_fan_out=0, fan_out_99%=0, inv_fan_out_99%=0, max_depth=1, tree
 Implementation: DiskAdjacencyListV1
 Status: "fully loaded"
 ------------
 Component Dominance/syntax/: 0 annnotations
-Stats: nodes=92, avg_fan_out=0.96, max_fan_out=3, max_depth=9, tree
+Stats: nodes=92, root nodes=4, avg_fan_out=0.96, max_fan_out=3, fan_out_99%=3, inv_fan_out_99%=1, max_depth=9, tree
 Implementation: PrePostOrderO16L8V1
 Status: "fully loaded"
 ------------
 Component Pointing/default_ns/anaphoric: 0 annnotations
-Stats: nodes=8, avg_fan_out=0.50, max_fan_out=1, max_depth=1, tree
+Stats: nodes=8, root nodes=4, avg_fan_out=0.50, max_fan_out=1, fan_out_99%=1, inv_fan_out_99%=1, max_depth=1, tree
 Implementation: DiskPathV1_D15
 Status: "fully loaded"
 ------------
 Component Ordering/annis/: 0 annnotations
-Stats: nodes=44, avg_fan_out=0.91, max_fan_out=1, max_depth=10, tree
+Stats: nodes=44, root nodes=4, avg_fan_out=0.91, max_fan_out=1, fan_out_99%=1, inv_fan_out_99%=1, max_depth=10, tree
 Implementation: DiskPathV1_D15
 Status: "fully loaded"
 ------------
 Component LeftToken/annis/: 0 annnotations
-Stats: nodes=92, avg_fan_out=0.65, max_fan_out=1, max_depth=1
+Stats: nodes=92, root nodes=60, avg_fan_out=0.65, max_fan_out=1, fan_out_99%=1, inv_fan_out_99%=3, max_depth=1
 Implementation: DiskPathV1_D15
 Status: "fully loaded"
 ------------
 Component RightToken/annis/: 0 annnotations
-Stats: nodes=84, avg_fan_out=0.71, max_fan_out=1, max_depth=1
+Stats: nodes=84, root nodes=60, avg_fan_out=0.71, max_fan_out=1, fan_out_99%=1, inv_fan_out_99%=8, max_depth=1
 Implementation: DiskPathV1_D15
 Status: "fully loaded"
 ------------
 Component PartOf/annis/: 0 annnotations
-Stats: nodes=115, avg_fan_out=0.99, max_fan_out=1, max_depth=4
+Stats: nodes=115, root nodes=104, avg_fan_out=0.99, max_fan_out=1, fan_out_99%=1, inv_fan_out_99%=26, max_depth=4
 Implementation: DiskPathV1_D15
 Status: "fully loaded"
 ------------

core/src/annostorage/inmemory.rs

@@ -720,7 +720,6 @@ where
                 if let Some(anno_key) = self.anno_keys.get_symbol(&anno_key) {
                     if let Some(histo) = self.histogram_bounds.get(&anno_key) {
                         // find the range in which the value is contained
-
                         // we need to make sure the histogram is not empty -> should have at least two bounds
                         if histo.len() >= 2 {
                             sum_histogram_buckets += histo.len() - 1;
@@ -752,6 +751,10 @@ where
     fn guess_max_count_regex(&self, ns: Option<&str>, name: &str, pattern: &str) -> Result<usize> {
         let full_match_pattern = util::regex_full_match(pattern);
 
+        // Get the total number of annotations with the namespace/name. We
+        // can't get larger than this number
+        let total = self.number_of_annotations_by_name(ns, name)?;
+
         // Try to parse the regular expression
         let parsed = regex_syntax::Parser::new().parse(&full_match_pattern);
         if let Ok(parsed) = parsed {
@@ -770,11 +773,13 @@ where
                         guessed_count += self.guess_max_count(ns, name, lower_val, &upper_val)?;
                     }
                 }
+            } else {
+                // For regular expressions without a prefix the worst case would be `.*[X].*` where `[X]` are the most common characters.
+                // Assume that a generic percentage (here 5%) of all nodes match the regex.
+                // TODO: find better ways of estimating this constant
+                guessed_count = (0.05 * (total as f64)) as usize;
             }
 
-            // Get the total number of annotations with the namespace/name. We
-            // can't get larger than this number
-            let total = self.number_of_annotations_by_name(ns, name)?;
             Ok(guessed_count.min(total))
         } else {
             Ok(0)