Add support for coverage edges between spans and segmentation nodes

Author: thomaskrause

CHANGELOG.md

@@ -5,6 +5,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added support for coverage edges between span nodes an segmentation nodes when
+  calculating the AQL model index.
+
+### Fixed
+
+- Do not use recursion to calculate the indirect coverage edges in the model
+  index, since this could fail for deeply nested structures. 
+
 ## [3.3.3] - 2024-07-12
 
 ### Fixed

cli/src/bin/annis.rs

@@ -175,7 +175,7 @@ impl AnnisRunner {
             let readline = rl.readline(&prompt);
             match readline {
                 Ok(line) => {
-                    rl.add_history_entry(&line.clone());
+                    rl.add_history_entry(line.clone());
                     if !self.exec(&line) {
                         break;
                     }

graphannis/src/annis/db/aql/model.rs

@@ -112,60 +112,66 @@ fn calculate_inherited_coverage_edges(
     graph: &mut AnnotationGraph,
     n: NodeID,
     all_cov_components: &[AnnotationComponent],
-    all_dom_gs: &[Arc<dyn GraphStorage>],
+    all_dom_components: &[AnnotationComponent],
 ) -> std::result::Result<FxHashSet<NodeID>, ComponentTypeError> {
-    let mut directly_covered_token = FxHashSet::default();
-
-    for c in all_cov_components.iter() {
-        if let Some(gs) = graph.get_graphstorage_as_ref(c) {
-            let out: Result<Vec<u64>, graphannis_core::errors::GraphAnnisCoreError> =
-                gs.get_outgoing_edges(n).collect();
-            directly_covered_token.extend(out?);
-        }
-    }
-
-    if directly_covered_token.is_empty() {
-        let has_token_anno = graph
-            .get_node_annos()
-            .get_value_for_item(&n, &TOKEN_KEY)?
-            .is_some();
-        if has_token_anno {
-            // Even if technically a token does not cover itself, if we need to abort the recursion
-            // with the basic case
-            directly_covered_token.insert(n);
+    // Iterate over all  all nodes that are somehow covered (by coverage or
+    // dominance edges) starting from the given node.
+    let all_text_coverage_components: Vec<AnnotationComponent> =
+        [all_cov_components, all_dom_components].concat();
+
+    let all_text_coverage_gs: Vec<_> = all_text_coverage_components
+        .iter()
+        .filter_map(|c| graph.get_graphstorage_as_ref(c))
+        .map(|gs| gs.as_edgecontainer())
+        .collect();
+    let combined_gs = UnionEdgeContainer::new(all_text_coverage_gs);
+
+    // Remember the non-token and token nodes, so we can connect all non-token
+    // nodes to the covered token.
+    let mut indirectly_covered_nodes = FxHashSet::default();
+    indirectly_covered_nodes.insert(n);
+    let mut covered_token = FxHashSet::default();
+
+    let tok_helper = TokenHelper::new(graph)?;
+
+    for step in CycleSafeDFS::new(&combined_gs, n, 1, usize::MAX) {
+        let step = step?;
+        if tok_helper.is_token(step.node)? {
+            covered_token.insert(step.node);
+        } else {
+            indirectly_covered_nodes.insert(step.node);
         }
     }
 
-    let mut indirectly_covered_token = FxHashSet::default();
-    // recursivly get the covered token from all children connected by a dominance relation
-    for dom_gs in all_dom_gs {
-        for out in dom_gs.get_outgoing_edges(n) {
-            let out = out?;
-            indirectly_covered_token.extend(calculate_inherited_coverage_edges(
-                graph,
-                out,
-                all_cov_components,
-                all_dom_gs,
-            )?);
-        }
-    }
+    let coverage_gs = tok_helper.get_gs_coverage().clone();
 
     if let Ok(gs_cov) = graph.get_or_create_writable(&AnnotationComponent::new(
         AnnotationComponentType::Coverage,
         ANNIS_NS.into(),
         "inherited-coverage".into(),
     )) {
-        // Ignore all already directly covered token when creating the inherited coverage edges
-        for t in indirectly_covered_token.difference(&directly_covered_token) {
-            gs_cov.add_edge(Edge {
-                source: n,
-                target: *t,
-            })?;
+        // Connect all non-token nodes to the covered token nodes if no such direct coverage already exists
+        for source in indirectly_covered_nodes {
+            for target in &covered_token {
+                let mut needs_edge = true;
+                for gs in coverage_gs.iter() {
+                    if gs.is_connected(source, *target, 1, std::ops::Bound::Included(1))? {
+                        needs_edge = false;
+                        break;
+                    }
+                }
+
+                if needs_edge {
+                    gs_cov.add_edge(Edge {
+                        source,
+                        target: *target,
+                    })?;
+                }
+            }
         }
     }
 
-    directly_covered_token.extend(indirectly_covered_token);
-    Ok(directly_covered_token)
+    Ok(covered_token)
 }
 
 pub struct AQLUpdateGraphIndex {
@@ -276,17 +282,18 @@ impl AQLUpdateGraphIndex {
 
         let all_cov_components =
             graph.get_all_components(Some(AnnotationComponentType::Coverage), None);
-        let all_dom_gs: Vec<Arc<dyn GraphStorage>> = graph
-            .get_all_components(Some(AnnotationComponentType::Dominance), Some(""))
-            .into_iter()
-            .filter_map(|c| graph.get_graphstorage(&c))
-            .collect();
+        let all_dom_components =
+            graph.get_all_components(Some(AnnotationComponentType::Dominance), Some(""));
 
         // go over each node and calculate the left-most and right-most token
         for invalid in self.invalid_nodes.iter()? {
             let (n, _) = invalid?;
-            let covered_token =
-                calculate_inherited_coverage_edges(graph, n, &all_cov_components, &all_dom_gs)?;
+            let covered_token = calculate_inherited_coverage_edges(
+                graph,
+                n,
+                &all_cov_components,
+                &all_dom_components,
+            )?;
             self.calculate_token_alignment(
                 graph,
                 n,

graphannis/src/annis/db/corpusstorage.rs

@@ -1078,7 +1078,7 @@ impl CorpusStorage {
         for node in file_nodes? {
             // Get the linked file for this node
             if let Some(original_path) = node_annos.get_value_for_item(&node, &linked_file_key)? {
-                let original_path = old_base_path.join(&PathBuf::from(original_path.as_ref()));
+                let original_path = old_base_path.join(PathBuf::from(original_path.as_ref()));
                 if original_path.is_file() {
                     if let Some(node_name) = node_annos.get_value_for_item(&node, &NODE_NAME_KEY)? {
                         // Create a new file name based on the node name and copy the file
@@ -1583,6 +1583,7 @@ impl CorpusStorage {
 
     /// Count the number of results for a `query`.
     /// - `query` - The search query definition.
+    ///
     /// Returns the count as number.
     pub fn count<S: AsRef<str>>(&self, query: SearchQuery<S>) -> Result<u64> {
         let timeout = TimeoutCheck::new(query.timeout);

graphannis/src/annis/util/mod.rs

@@ -73,6 +73,7 @@ impl From<SearchDefRaw> for SearchDef {
 /// Returns a vector over all query definitions defined in a CSV file.
 /// - `file` - The CSV file path.
 /// - `panic_on_invalid` - If true, an invalid query definition will trigger a panic, otherwise it will be ignored.
+///
 /// Can be used if this query is called in a test case to fail the test.
 pub fn get_queries_from_csv(file: &Path, panic_on_invalid: bool) -> Vec<SearchDef> {
     if let Ok(mut reader) = csv::Reader::from_path(file) {

graphannis/src/lib.rs

@@ -22,9 +22,6 @@ extern crate lazy_static;
 #[macro_use]
 extern crate lalrpop_util;
 
-#[cfg(feature = "c-api")]
-extern crate simplelog;
-
 mod annis;
 
 pub use crate::annis::db::corpusstorage::CorpusStorage;