From 1a9b0f0bbcc82900a96b3ee1473871cb488fdbcf Mon Sep 17 00:00:00 2001 From: harehare Date: Wed, 15 Jul 2026 23:46:08 +0900 Subject: [PATCH 1/2] =?UTF-8?q?perf(store):=20avoid=20O(N=C2=B2)=20catalog?= =?UTF-8?q?=20flush=20and=20path=20scan=20in=20reindex=5Fpaths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reindex_paths() did a linear scan over all documents to find each file's existing DocumentId, and flushed the full on-disk catalog after every single added/updated file — turning an N-document reindex into O(N²) work. Build a path->(id, position) map once up front, and defer the catalog flush to a single call after the batch completes. Benchmarked on 3000 changed files: 5.6s -> 0.17s. --- src/store.rs | 101 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/src/store.rs b/src/store.rs index 589b376..685fb5d 100644 --- a/src/store.rs +++ b/src/store.rs @@ -297,7 +297,7 @@ impl DocumentStore { /// When called on an in-memory store (no backing file) this behaves /// identically to [`add_str`](DocumentStore::add_str). pub fn append_str(&mut self, content: &str) -> Result { - self.do_append(content, None) + self.do_append(content, None, true) } /// Append a Markdown file to the existing `.mq-db` file (in-place). @@ -306,13 +306,22 @@ impl DocumentStore { pub fn append_file(&mut self, path: impl AsRef) -> Result { let path = path.as_ref(); let content = std::fs::read_to_string(path)?; - self.do_append(&content, Some(path.to_path_buf())) + self.do_append(&content, Some(path.to_path_buf()), true) } + /// Core of [`append_str`](Self::append_str)/[`append_file`](Self::append_file). + /// + /// `flush` controls whether the on-disk catalog is rewritten immediately + /// after this document's pages are written. Single-document callers pass + /// `true` for durability; bulk callers like + /// [`reindex_paths`](Self::reindex_paths) pass `false` and flush once + /// after the whole batch, since rewriting the full catalog after every + /// file makes an N-document reindex cost O(N²). fn do_append( &mut self, content: &str, md_path: Option, + flush: bool, ) -> Result { let md = Markdown::from_markdown_str(content).map_err(|e| MqdbError::Parse(e.to_string()))?; @@ -330,9 +339,6 @@ impl DocumentStore { let idx_opt = { let mut storage_guard = self.storage.lock().unwrap(); if let Some(storage) = storage_guard.as_mut() { - // Reconstruct catalog entries from already-loaded document metadata. - let mut entries = self.catalog_entries(); - let first_block_page = storage.write_document(&doc)?; doc.first_block_page = first_block_page; @@ -340,25 +346,17 @@ impl DocumentStore { let index_start_page = storage.write_index(&idx.to_bytes())?; doc.index_start_page = index_start_page; - entries.push(CatalogEntry { - document_id: doc.id, - path: doc.path.as_ref().map(|p| p.to_string_lossy().into_owned()), - first_block_page, - num_blocks: doc.block_count, - zone_map_bytes: encode_zone_map(&doc.zone_maps), - index_start_page, - }); - - let custom = persist_unsaved_table_rows(storage, &self.custom_tables)?; - storage.flush_catalog(&entries, &custom, &self.content_hash_pairs())?; Some(idx) } else { None } }; self.doc_indexes.push(idx_opt); - self.documents.push(doc); + + if flush { + self.try_flush_catalog_to_storage(); + } Ok(doc_id) } @@ -381,13 +379,40 @@ impl DocumentStore { doc_id: DocumentId, content: &str, path: Option, + ) -> Result<(), MqdbError> { + self.do_replace(doc_id, content, path, true) + } + + /// Core of [`replace_document`](Self::replace_document). See + /// [`do_append`](Self::do_append) for what `flush` controls and why. + fn do_replace( + &mut self, + doc_id: DocumentId, + content: &str, + path: Option, + flush: bool, ) -> Result<(), MqdbError> { let pos = self .documents .iter() .position(|d| d.id == doc_id) .ok_or_else(|| MqdbError::Storage(format!("no such document: {doc_id}")))?; + self.do_replace_at(pos, doc_id, content, path, flush) + } + /// Like [`do_replace`](Self::do_replace) but takes the document's index + /// in `self.documents` directly instead of scanning for it — for bulk + /// callers like [`reindex_paths`](Self::reindex_paths) that already know + /// the position and would otherwise turn an N-document reindex into an + /// O(N²) scan. + fn do_replace_at( + &mut self, + pos: usize, + doc_id: DocumentId, + content: &str, + path: Option, + flush: bool, + ) -> Result<(), MqdbError> { let md = Markdown::from_markdown_str(content).map_err(|e| MqdbError::Parse(e.to_string()))?; let mut blocks = index::build_blocks(doc_id, &md.nodes); @@ -401,8 +426,6 @@ impl DocumentStore { let idx_opt = { let mut storage_guard = self.storage.lock().unwrap(); if let Some(storage) = storage_guard.as_mut() { - let mut entries = self.catalog_entries(); - let first_block_page = storage.write_document(&doc)?; doc.first_block_page = first_block_page; @@ -410,16 +433,6 @@ impl DocumentStore { let index_start_page = storage.write_index(&idx.to_bytes())?; doc.index_start_page = index_start_page; - if let Some(entry) = entries.iter_mut().find(|e| e.document_id == doc_id) { - entry.path = doc.path.as_ref().map(|p| p.to_string_lossy().into_owned()); - entry.first_block_page = first_block_page; - entry.num_blocks = doc.block_count; - entry.zone_map_bytes = encode_zone_map(&doc.zone_maps); - entry.index_start_page = index_start_page; - } - - let custom = persist_unsaved_table_rows(storage, &self.custom_tables)?; - storage.flush_catalog(&entries, &custom, &self.content_hash_pairs())?; Some(idx) } else { None @@ -428,6 +441,10 @@ impl DocumentStore { self.documents[pos] = doc; self.doc_indexes[pos] = idx_opt; + + if flush { + self.try_flush_catalog_to_storage(); + } Ok(()) } @@ -452,34 +469,42 @@ impl DocumentStore { let mut seen: HashSet = HashSet::with_capacity(files.len()); let contents = read_files_parallel(files); + // Path -> (DocumentId, position in `self.documents`), built once so + // each file below is an O(1) lookup instead of a linear scan over + // every already-indexed document — the latter turns an N-document + // reindex into O(N²) work. + let mut by_path: HashMap = self + .documents + .iter() + .enumerate() + .filter_map(|(i, d)| d.path.clone().map(|p| (p, (d.id, i)))) + .collect(); + for (path, content) in files.iter().zip(contents) { seen.insert(path.clone()); let result = (|| -> Result<(), MqdbError> { let content = content?; let hash = hash_bytes(content.as_bytes()); - let existing = self - .documents - .iter() - .find(|d| d.path.as_deref() == Some(path.as_path())) - .map(|d| d.id); + let existing = by_path.get(path).copied(); match existing { - Some(doc_id) if self.content_hashes.get(&doc_id) == Some(&hash) => { + Some((doc_id, _)) if self.content_hashes.get(&doc_id) == Some(&hash) => { report.unchanged += 1; } - Some(doc_id) => { - self.replace_document(doc_id, &content, Some(path.clone()))?; + Some((doc_id, pos)) => { + self.do_replace_at(pos, doc_id, &content, Some(path.clone()), false)?; self.content_hashes.insert(doc_id, hash); report.updated.push(path.clone()); } None => { let doc_id = if self.storage.lock().unwrap().is_some() { - self.do_append(&content, Some(path.clone()))? + self.do_append(&content, Some(path.clone()), false)? } else { self.add_str_with_path(&content, Some(path.clone()))? }; self.content_hashes.insert(doc_id, hash); + by_path.insert(path.clone(), (doc_id, self.documents.len() - 1)); report.added.push(path.clone()); } } From 50ad4b34ed5d6b5efb2af4d1705e4e2c4a1bcc05 Mon Sep 17 00:00:00 2001 From: harehare Date: Wed, 15 Jul 2026 23:50:25 +0900 Subject: [PATCH 2/2] perf(bin): wire mimalloc as the global allocator The use_mimalloc feature added the dependency and Cargo feature flag but never set #[global_allocator], so it had no effect on the binary. --- src/bin/mq-db.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bin/mq-db.rs b/src/bin/mq-db.rs index b5d4dba..dee3662 100644 --- a/src/bin/mq-db.rs +++ b/src/bin/mq-db.rs @@ -17,6 +17,10 @@ use clap::{Parser, Subcommand, ValueEnum}; use mq_db::{DocumentStore, MqEngine, SqlEngine, block::BlockType, sql::html_escape}; use serde::Deserialize; +#[cfg(feature = "use_mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + // ───────────────────────────────────────────────────────────────────────────── // CLI structure // ─────────────────────────────────────────────────────────────────────────────