perf: optimize directory walking to reduce syscalls and allocations

Author: richiemcilroy

src/glob.rs

@@ -318,10 +318,19 @@ impl Glob {
             (None, Some(d)) => Some(d + 1),   // Pattern depth + 1 (for root directory)
             (None, None) => None,             // Unlimited (has **)
         };
+
+        // Optimization: Only enable accurate symlink detection when needed.
+        // The `mark` option requires knowing whether an entry is a symlink to avoid
+        // adding a trailing slash. When following symlinks, walkdir reports the TARGET
+        // type, so we need an extra syscall to detect the symlink. Skip this overhead
+        // when not needed.
+        let need_accurate_symlink_detection = mark && follow;
+
         let walk_options = WalkOptions::new()
             .follow_symlinks(follow)
             .max_depth(walker_max_depth)
-            .dot(true);
+            .dot(true)
+            .need_accurate_symlink_detection(need_accurate_symlink_detection);
 
         // Pre-compute: check if any pattern requires directory matching (ends with /)
         let any_pattern_requires_dir = patterns.iter().any(|p| p.requires_dir());
@@ -448,6 +457,9 @@ impl Glob {
         let walker = Walker::new(walk_root.clone(), adjusted_walk_options)
             .with_dir_prune_filter(prune_filter);
 
+        // Optimization: Check if we have any ignore patterns to avoid unnecessary work
+        let has_ignore_filter = self.ignore_filter.is_some();
+
         for entry in walker.walk() {
             let path = entry.path();
 
@@ -479,13 +491,6 @@ impl Glob {
                 rel_str_from_walk_root.into_owned()
             };
 
-            // For operations that need the actual relative path from cwd
-            let rel_path = if prefix_to_strip.is_some() {
-                std::path::PathBuf::from(&normalized)
-            } else {
-                rel_path_from_walk_root.to_path_buf()
-            };
-
             // Check if this path is inside an ignored directory.
             // Optimization: Use byte-level comparison instead of char iteration.
             if !ignored_dirs.is_empty() {
@@ -502,22 +507,30 @@ impl Glob {
             }
 
             // Check ignore patterns
-            if let Some(ref ignore_filter) = self.ignore_filter {
+            // Optimization: Only create rel_path and abs_path when we have ignore patterns
+            if has_ignore_filter {
+                // For operations that need the actual relative path from cwd
+                let rel_path = if prefix_to_strip.is_some() {
+                    std::path::PathBuf::from(&normalized)
+                } else {
+                    rel_path_from_walk_root.to_path_buf()
+                };
                 let abs_path = abs_cwd.join(&rel_path);
+                let ignore_filter = self.ignore_filter.as_ref().unwrap();
 
                 // Check if this specific path should be ignored
                 if ignore_filter.should_ignore(&normalized, &abs_path) {
                     // If children are also ignored, mark this directory
                     if entry.is_dir() && ignore_filter.children_ignored(&normalized, &abs_path) {
-                        ignored_dirs.insert(normalized.to_string());
+                        ignored_dirs.insert(normalized.clone());
                     }
                     continue;
                 }
 
                 // Also check if this is a directory whose children should be ignored
                 // (for optimization - skip traversing)
                 if entry.is_dir() && ignore_filter.children_ignored(&normalized, &abs_path) {
-                    ignored_dirs.insert(normalized.to_string());
+                    ignored_dirs.insert(normalized.clone());
                 }
             }
 
@@ -633,7 +646,9 @@ impl Glob {
 
                 let result = if self.absolute {
                     // Return absolute path
-                    let abs_path = abs_cwd.join(&rel_path);
+                    // Optimization: Construct the absolute path from normalized string
+                    // instead of creating a PathBuf first
+                    let abs_path = abs_cwd.join(&normalized);
                     let mut path = self.format_path(&abs_path);
                     if self.mark && should_mark_as_dir {
                         path = self.ensure_trailing_slash(&path);

src/walker.rs

@@ -12,6 +12,10 @@ pub struct WalkOptions {
     pub max_depth: Option<usize>,
     /// Include dotfiles (files starting with .)
     pub dot: bool,
+    /// Whether to accurately detect symlinks even when following them.
+    /// This is needed for the `mark` option to correctly NOT add trailing slashes to symlinks.
+    /// When false, avoids an extra stat call per file (faster).
+    pub need_accurate_symlink_detection: bool,
 }
 
 /// A filter function that can prune directories during walking.
@@ -37,6 +41,11 @@ impl WalkOptions {
         self.dot = include_dot;
         self
     }
+
+    pub fn need_accurate_symlink_detection(mut self, need: bool) -> Self {
+        self.need_accurate_symlink_detection = need;
+        self
+    }
 }
 
 /// A single entry returned from the walker
@@ -50,6 +59,25 @@ pub struct WalkEntry {
 }
 
 impl WalkEntry {
+    /// Create a WalkEntry from a walkdir DirEntry without checking symlink metadata.
+    /// This is faster but may not correctly detect symlinks when following links.
+    /// Use this when `mark` option is false and you don't need accurate symlink detection.
+    #[inline]
+    pub fn from_dir_entry_fast(entry: &DirEntry) -> Self {
+        let file_type = entry.file_type();
+        Self {
+            path: entry.path().to_path_buf(),
+            depth: entry.depth(),
+            is_dir: file_type.is_dir(),
+            is_file: file_type.is_file(),
+            is_symlink: file_type.is_symlink(),
+        }
+    }
+
+    /// Create a WalkEntry from a walkdir DirEntry with full symlink detection.
+    /// This is slower because it makes an extra syscall for symlink_metadata,
+    /// but correctly detects symlinks even when following them.
+    /// Use this when `mark` option is true.
     pub fn from_dir_entry(entry: &DirEntry) -> Self {
         let file_type = entry.file_type();
         // When following symlinks, walkdir reports the TARGET type, not the symlink type.
@@ -97,6 +125,12 @@ impl WalkEntry {
     pub fn file_name(&self) -> Option<&std::ffi::OsStr> {
         self.path.file_name()
     }
+
+    /// Get the file name as a string slice if possible, for fast comparisons.
+    #[inline]
+    pub fn file_name_str(&self) -> Option<&str> {
+        self.path.file_name().and_then(|s| s.to_str())
+    }
 }
 
 /// Directory walker that can traverse filesystem trees
@@ -152,6 +186,15 @@ impl Walker {
 
         let dot = self.options.dot;
         let root = self.root.clone();
+        let need_accurate_symlink = self.options.need_accurate_symlink_detection;
+
+        // Choose the appropriate entry creation function based on whether we need
+        // accurate symlink detection. This avoids an extra syscall per file when not needed.
+        let create_entry = if need_accurate_symlink {
+            WalkEntry::from_dir_entry
+        } else {
+            WalkEntry::from_dir_entry_fast
+        };
 
         // If we have a pruning filter, we need to use it in filter_entry
         if let Some(ref prune_filter) = self.dir_prune_filter {
@@ -161,6 +204,7 @@ impl Walker {
                 .into_iter()
                 .filter_entry(|e| {
                     // Filter dot files if dot option is false
+                    // Optimization: Use bytes comparison for dot check
                     if !dot {
                         if let Some(name) = e.file_name().to_str() {
                             if e.depth() > 0 && name.starts_with('.') {
@@ -183,7 +227,7 @@ impl Walker {
                     true
                 })
                 .filter_map(|result| match result {
-                    Ok(entry) => Some(WalkEntry::from_dir_entry(&entry)),
+                    Ok(entry) => Some(create_entry(&entry)),
                     Err(err) => {
                         if let Some(path) = err.path() {
                             if let Ok(meta) = path.symlink_metadata() {
@@ -221,9 +265,9 @@ impl Walker {
                         }
                         true
                     })
-                    .filter_map(|result| {
+                    .filter_map(move |result| {
                         match result {
-                            Ok(entry) => Some(WalkEntry::from_dir_entry(&entry)),
+                            Ok(entry) => Some(create_entry(&entry)),
                             Err(err) => {
                                 // For broken symlinks (or other IO errors), try to extract the path
                                 // and return it as an entry. This handles the case where follow_links
@@ -638,8 +682,14 @@ mod tests {
         // Create a symlink to the directory
         symlink(base.join("real-dir"), base.join("symlink-to-dir")).unwrap();
 
-        // Walk WITH following symlinks - check what is_symlink reports
-        let walker = Walker::new(base.to_path_buf(), WalkOptions::new().follow_symlinks(true));
+        // Walk WITH following symlinks AND accurate symlink detection enabled
+        // (This is needed to correctly detect symlinks when following them)
+        let walker = Walker::new(
+            base.to_path_buf(),
+            WalkOptions::new()
+                .follow_symlinks(true)
+                .need_accurate_symlink_detection(true),
+        );
         let entries: Vec<_> = walker.walk_sync();
 
         // Find the symlink entry