Merge pull request #324 from korpling/get-components-without-loading-db

Avoid loading the node annotation storage when listing the components

Author: thomaskrause

CHANGELOG.md

@@ -5,6 +5,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Avoid loading the node annotation storage when listing the components for a
+corpus in the `CorpusStorage`. Before this change, querying for components via
+the webservice could block the corpus cache.
+
 ## [3.8.0] - 2025-05-14
 
 ### Changed

core/src/graph/mod.rs

@@ -14,7 +14,6 @@ use crate::{
 use clru::CLruCache;
 use rayon::prelude::*;
 use smartstring::alias::String as SmartString;
-use std::io::prelude::*;
 use std::ops::Bound::Included;
 use std::path::{Path, PathBuf};
 use std::string::ToString;
@@ -23,6 +22,7 @@ use std::{
     sync::{Arc, Mutex},
 };
 use std::{collections::BTreeMap, num::NonZeroUsize};
+use std::{collections::BTreeSet, io::prelude::*};
 use update::{GraphUpdate, UpdateEvent};
 
 pub const ANNIS_NS: &str = "annis";
@@ -113,6 +113,66 @@ fn component_path<CT: ComponentType>(
     }
 }
 
+/// List all the components that belong to corpus in the given directory.
+pub fn find_components_from_disk<CT: ComponentType, P: AsRef<Path>>(
+    location: P,
+) -> Result<BTreeSet<Component<CT>>> {
+    let mut result = BTreeSet::new();
+    // for all component types
+    for c in CT::all_component_types().into_iter() {
+        let cpath = PathBuf::from(location.as_ref())
+            .join("gs")
+            .join(c.to_string());
+
+        if cpath.is_dir() {
+            // get all the namespaces/layers
+            for layer in cpath.read_dir()? {
+                let layer = layer?;
+                if layer.path().is_dir() {
+                    // try to load the component with the empty name
+                    let layer_file_name = layer.file_name();
+                    let layer_name_from_file = layer_file_name.to_string_lossy();
+                    let layer_name: SmartString = if layer_name_from_file == DEFAULT_EMPTY_LAYER {
+                        SmartString::default()
+                    } else {
+                        layer_name_from_file.into()
+                    };
+                    let empty_name_component =
+                        Component::new(c.clone(), layer_name.clone(), SmartString::default());
+                    {
+                        let cfg_file = PathBuf::from(location.as_ref())
+                            .join(component_to_relative_path(&empty_name_component))
+                            .join("impl.cfg");
+
+                        if cfg_file.is_file() {
+                            result.insert(empty_name_component.clone());
+                            debug!("Registered component {}", empty_name_component);
+                        }
+                    }
+                    // also load all named components
+                    for name in layer.path().read_dir()? {
+                        let name = name?;
+                        let named_component = Component::new(
+                            c.clone(),
+                            layer_name.clone(),
+                            name.file_name().to_string_lossy().into(),
+                        );
+                        let cfg_file = PathBuf::from(location.as_ref())
+                            .join(component_to_relative_path(&named_component))
+                            .join("impl.cfg");
+
+                        if cfg_file.is_file() {
+                            result.insert(named_component.clone());
+                            debug!("Registered component {}", named_component);
+                        }
+                    }
+                }
+            }
+        }
+    } // end for all components
+    Ok(result)
+}
+
 impl<CT: ComponentType> Graph<CT> {
     /// Create a new and empty instance without any location on the disk.
     pub fn new(disk_based: bool) -> Result<Self> {
@@ -233,7 +293,10 @@ impl<CT: ComponentType> Graph<CT> {
 
         let logfile_exists = log_path.exists() && log_path.is_file();
 
-        self.find_components_from_disk(&dir2load)?;
+        self.components = find_components_from_disk(&dir2load)?
+            .into_iter()
+            .map(|c| (c, None))
+            .collect();
 
         // If backup is active or a write log exists, always  a pre-load to get the complete corpus.
         if logfile_exists | load_from_backup {
@@ -287,63 +350,6 @@ impl<CT: ComponentType> Graph<CT> {
         Ok(())
     }
 
-    fn find_components_from_disk(&mut self, location: &Path) -> Result<()> {
-        self.components.clear();
-
-        // for all component types
-        for c in CT::all_component_types().into_iter() {
-            let cpath = PathBuf::from(location).join("gs").join(c.to_string());
-
-            if cpath.is_dir() {
-                // get all the namespaces/layers
-                for layer in cpath.read_dir()? {
-                    let layer = layer?;
-                    if layer.path().is_dir() {
-                        // try to load the component with the empty name
-                        let layer_file_name = layer.file_name();
-                        let layer_name_from_file = layer_file_name.to_string_lossy();
-                        let layer_name: SmartString = if layer_name_from_file == DEFAULT_EMPTY_LAYER
-                        {
-                            SmartString::default()
-                        } else {
-                            layer_name_from_file.into()
-                        };
-                        let empty_name_component =
-                            Component::new(c.clone(), layer_name.clone(), SmartString::default());
-                        {
-                            let cfg_file = PathBuf::from(location)
-                                .join(component_to_relative_path(&empty_name_component))
-                                .join("impl.cfg");
-
-                            if cfg_file.is_file() {
-                                self.components.insert(empty_name_component.clone(), None);
-                                debug!("Registered component {}", empty_name_component);
-                            }
-                        }
-                        // also load all named components
-                        for name in layer.path().read_dir()? {
-                            let name = name?;
-                            let named_component = Component::new(
-                                c.clone(),
-                                layer_name.clone(),
-                                name.file_name().to_string_lossy().into(),
-                            );
-                            let cfg_file = PathBuf::from(location)
-                                .join(component_to_relative_path(&named_component))
-                                .join("impl.cfg");
-
-                            if cfg_file.is_file() {
-                                self.components.insert(named_component.clone(), None);
-                                debug!("Registered component {}", named_component);
-                            }
-                        }
-                    }
-                }
-            }
-        } // end for all components
-        Ok(())
-    }
-
     fn internal_save(&self, location: &Path) -> Result<()> {
         let location = PathBuf::from(location);
 
@@ -376,7 +382,7 @@ impl<CT: ComponentType> Graph<CT> {
 
     fn get_cached_node_id_from_name(
         &self,
-        node_name: Cow<String>,
+        node_name: Cow<str>,
         cache: &mut CLruCache<String, Option<NodeID>>,
     ) -> Result<Option<NodeID>> {
         if let Some(id) = cache.get(node_name.as_ref()) {

graphannis/src/annis/db/aql/model.rs

@@ -167,6 +167,10 @@ pub struct AQLUpdateGraphIndex {
 }
 
 impl AQLUpdateGraphIndex {
+    #[allow(
+        clippy::owned_cow,
+        reason = "We don't want to copy the node_name if not necessary and also have to look up &String"
+    )]
     fn get_cached_node_id_from_name(
         &mut self,
         node_name: Cow<String>,

graphannis/src/annis/db/corpusstorage.rs

@@ -2387,11 +2387,41 @@ impl CorpusStorage {
         ctype: Option<AnnotationComponentType>,
         name: Option<&str>,
     ) -> Result<Vec<Component<AnnotationComponentType>>> {
-        if let Ok(db_entry) = self.get_loaded_entry(corpus_name, false, false) {
-            let lock = db_entry.read()?;
-            if let Ok(db) = get_read_or_error(&lock) {
-                return Ok(db.get_all_components(ctype, name));
-            }
+        if let Ok(db_entry) = self.get_entry(corpus_name) {
+            // Lock the entry so no-one else can write to it
+            let _lock = db_entry.read()?;
+            // Get the components by listing the directories on the disk We
+            // could use `Graph:get_all_components() but this needs to load the
+            // annotation storage which can be costly.
+            let db_path = self.corpus_directory_on_disk(corpus_name);
+            let backup = db_path.join("backup");
+
+            let dir2load = if backup.exists() && backup.is_dir() {
+                backup.clone()
+            } else {
+                db_path.join("current")
+            };
+            let components = graphannis_core::graph::find_components_from_disk::<
+                AnnotationComponentType,
+                _,
+            >(&dir2load)?
+            .into_iter()
+            .filter(|c| {
+                if let Some(ctype) = ctype.as_ref() {
+                    &c.get_type() == ctype
+                } else {
+                    true
+                }
+            })
+            .filter(|c| {
+                if let Some(name) = name {
+                    c.name.as_str() == name
+                } else {
+                    true
+                }
+            })
+            .collect();
+            return Ok(components);
         }
         Ok(vec![])
     }

webservice/src/main.rs

@@ -33,7 +33,7 @@ use simplelog::{
 };
 use std::fs::OpenOptions;
 use std::{
-    io::{Error, ErrorKind, Result},
+    io::{Error, Result},
     path::PathBuf,
 };
 
@@ -270,12 +270,8 @@ async fn get_api_html_docs(_req: HttpRequest) -> HttpResponse {
 #[actix_web::main]
 async fn main() -> Result<()> {
     // Initialize application and its state
-    let (cs, settings, db_pool) = init_app_state().map_err(|e| {
-        Error::new(
-            ErrorKind::Other,
-            format!("Could not initialize graphANNIS service: {:?}", e),
-        )
-    })?;
+    let (cs, settings, db_pool) = init_app_state()
+        .map_err(|e| Error::other(format!("Could not initialize graphANNIS service: {:?}", e)))?;
 
     let bind_address = format!("{}:{}", &settings.bind.host, &settings.bind.port);
     let cs = web::Data::new(cs);