Refactor glob expansion with ignore and globset

Moved input glob expansion logic from main.rs to a new path_glob.rs module, now using the ignore and globset crates for faster, .gitignore-aware, parallel file matching. Updated dependencies in Cargo.toml to include rayon, ignore, and globset.

Author: WendellXY

langcodec-cli/Cargo.toml

@@ -22,6 +22,9 @@ serde_json = "1.0"
 serde_yaml = "0.9"
 unic-langid = "0.9.6"
 glob = "0.3"
+rayon = "1.10"
+ignore = "0.4"
+globset = "0.4"
 
 [dev-dependencies]
 tempfile = "3.8"

langcodec-cli/src/main.rs

@@ -1,6 +1,7 @@
 mod debug;
 mod formats;
 mod merge;
+mod path_glob;
 mod transformers;
 mod validation;
 mod view;
@@ -15,7 +16,6 @@ use clap::{CommandFactory, Parser, Subcommand};
 use clap_complete::{Shell, generate};
 
 use langcodec::{Codec, convert_auto, formats::FormatType};
-use std::collections::HashSet;
 use std::fs::File;
 use std::io::BufWriter;
 
@@ -202,7 +202,7 @@ fn main() {
         } => {
             // Expand any glob patterns in inputs (e.g., *.strings, **/*.xml)
             println!("Expanding glob patterns in inputs: {:?}", inputs);
-            let expanded_inputs = match expand_input_globs(&inputs) {
+            let expanded_inputs = match path_glob::expand_input_globs(&inputs) {
                 Ok(list) => list,
                 Err(e) => {
                     eprintln!("❌ Failed to expand input patterns: {}", e);
@@ -736,51 +736,4 @@ fn read_resources_from_any_input(
     ))
 }
 
-/// Expand possible glob patterns in a list of input strings into concrete file paths
-fn expand_input_globs(inputs: &Vec<String>) -> Result<Vec<String>, String> {
-    use rayon::prelude::*;
-
-    // Expand each pattern concurrently, collecting per-pattern results first
-    let expanded: Vec<String> = inputs
-        .par_iter()
-        .map(|pattern| {
-            // Try treating the input as a glob pattern. If it fails to parse as a glob,
-            // just treat it as a literal path.
-            match glob::glob(pattern) {
-                Ok(paths) => {
-                    let mut out: Vec<String> = Vec::new();
-                    let mut matched = false;
-                    for entry in paths {
-                        match entry {
-                            Ok(path) => {
-                                if path.is_file() {
-                                    out.push(path.to_string_lossy().to_string());
-                                    matched = true;
-                                }
-                            }
-                            Err(e) => {
-                                return Err(format!("Glob error for '{}': {}", pattern, e));
-                            }
-                        }
-                    }
-                    if matched { Ok(out) } else { Ok(vec![pattern.clone()]) }
-                }
-                Err(_) => Ok(vec![pattern.clone()]),
-            }
-        })
-        .collect::<Result<Vec<Vec<String>>, String>>()?
-        .into_iter()
-        .flatten()
-        .collect();
-
-    // Deduplicate while preserving the first-seen order
-    let mut seen: HashSet<String> = HashSet::new();
-    let mut results: Vec<String> = Vec::with_capacity(expanded.len());
-    for s in expanded {
-        if seen.insert(s.clone()) {
-            results.push(s);
-        }
-    }
-
-    Ok(results)
-}
+// Path glob expansion is implemented in path_glob.rs

langcodec-cli/src/path_glob.rs

@@ -0,0 +1,115 @@
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use globset::{GlobBuilder, GlobSetBuilder};
+use ignore::WalkBuilder;
+use rayon::prelude::*;
+
+/// Expand possible glob patterns in a list of input strings into concrete file paths.
+/// Uses ignore + globset for fast, parallel, .gitignore-aware traversal.
+pub fn expand_input_globs(inputs: &Vec<String>) -> Result<Vec<String>, String> {
+    fn has_glob_meta(s: &str) -> bool {
+        s.bytes().any(|b| matches!(b, b'*' | b'?' | b'[' | b'{'))
+    }
+
+    // Extract a static directory prefix before the first glob meta-character
+    fn static_prefix_dir(pattern: &str) -> PathBuf {
+        let bytes = pattern.as_bytes();
+        let mut idx = 0usize;
+        while idx < bytes.len() {
+            match bytes[idx] {
+                b'*' | b'?' | b'[' | b'{' => break,
+                _ => idx += 1,
+            }
+        }
+        let prefix = &pattern[..idx];
+        let p = Path::new(prefix);
+        if p.is_dir() {
+            p.to_path_buf()
+        } else {
+            p.parent()
+                .map(|pp| pp.to_path_buf())
+                .unwrap_or_else(|| PathBuf::from("."))
+        }
+    }
+
+    // Build one GlobSet for all patterns (literal_separator to avoid '/' matching)
+    let mut builder = GlobSetBuilder::new();
+    for pat in inputs {
+        let glob = GlobBuilder::new(pat)
+            .literal_separator(true)
+            .build()
+            .map_err(|e| format!("Invalid glob pattern '{}': {}", pat, e))?;
+        builder.add(glob);
+    }
+    let set = builder
+        .build()
+        .map_err(|e| format!("Failed to build glob set: {}", e))?;
+
+    // Collect unique roots to minimize directory walks
+    let mut roots: Vec<PathBuf> = Vec::new();
+    for pat in inputs {
+        let root = if has_glob_meta(pat) {
+            static_prefix_dir(pat)
+        } else {
+            Path::new(pat)
+                .parent()
+                .map(|p| p.to_path_buf())
+                .unwrap_or_else(|| PathBuf::from("."))
+        };
+        if !roots.iter().any(|r| r == &root) {
+            roots.push(root);
+        }
+    }
+
+    // Walk roots in parallel and match files against the GlobSet
+    let collected: Vec<String> = roots
+        .par_iter()
+        .map(|root| {
+            let mut out: Vec<String> = Vec::new();
+            let walker = WalkBuilder::new(root)
+                .git_ignore(true)
+                .git_global(true)
+                .git_exclude(true)
+                .hidden(false)
+                .ignore(true)
+                .parents(true)
+                .build();
+
+            for dent in walker {
+                let dent = match dent {
+                    Ok(d) => d,
+                    Err(_e) => continue,
+                };
+                let ftype = match dent.file_type() {
+                    Some(t) => t,
+                    None => continue,
+                };
+                if !ftype.is_file() {
+                    continue;
+                }
+                let s = dent.path().to_string_lossy();
+                if set.is_match(s.as_ref()) {
+                    out.push(s.to_string());
+                }
+            }
+            out
+        })
+        .flatten()
+        .collect();
+
+    // If nothing matched, preserve original inputs to surface errors later
+    if collected.is_empty() {
+        return Ok(inputs.clone());
+    }
+
+    // Deduplicate while preserving order
+    let mut seen: HashSet<String> = HashSet::new();
+    let mut results: Vec<String> = Vec::with_capacity(collected.len());
+    for s in collected {
+        if seen.insert(s.clone()) {
+            results.push(s);
+        }
+    }
+    Ok(results)
+}