✨ feat: add full-text search

README.md

@@ -35,10 +35,11 @@ flowchart TD
 
 - **Flat block storage** — every Markdown element becomes a typed `Block` with row-polymorphic properties
 - **O(1) hierarchy queries** — interval index (`pre`/`post`) makes ancestor/descendant checks a single integer comparison
-- **Three-layer secondary indexes** — `BitmapIndex` (block type), `BTreeIndex` (pre/post), `HashIndex` (content/lang/depth) for fast SQL predicate pushdown
+- **Four-layer secondary indexes** — `BitmapIndex` (block type), `BTreeIndex` (pre/post), `HashIndex` (content/lang/depth), `TermIndex` (tokenized content, full-text) for fast SQL predicate pushdown
 - **Zone Maps** — per-document statistics skip irrelevant files before scanning any blocks
 - **Dual query engines** — SQL via a custom `sqlparser`-based evaluator, and `mq` via `mq-lang`
 - **`WITH` (CTE) support** — non-recursive common table expressions, usable in `FROM`, `JOIN`, and subqueries
+- **Full-text search** — `match()`/`score()` SQL functions backed by a persisted per-document inverted index
 - **Incremental re-indexing** — re-running `index` skips unchanged files (content-hash based), replaces changed ones in place (same `DocumentId`), and can `--prune` deleted ones
 - **SQL `INSERT`/`UPDATE`/`DELETE` with write-back** — add, edit, or remove `blocks` and push the change back to the source Markdown file, opt-in via `--write-back`
 - **DDL support** — `CREATE TABLE`, `INSERT INTO`, `DROP TABLE` for in-memory custom tables
@@ -179,6 +180,17 @@ mq-db sql "
 `documents`, or a custom table shadows it for the duration of the `WITH`
 clause's scope.
 
+**Full-text search with `match()`/`score()`** — index-accelerated term matching and simple TF-based ranking:
+
+```bash
+mq-db sql "
+  SELECT content, score(content, 'error handling') AS relevance
+  FROM blocks
+  WHERE match(content, 'error handling')
+  ORDER BY relevance DESC
+" --db store.mq-db
+```
+
 ### INSERT / UPDATE / DELETE with write-back
 
 `INSERT`/`UPDATE`/`DELETE` on `blocks` write the change back to the
@@ -468,6 +480,8 @@ mq-db-specific:
 | `under(pre, post, anc_pre, anc_post)` | O(1) interval ancestor check               |
 | `mq(program, content)`                | Run an mq program against Markdown content |
 | `json_extract(json, path)`            | Extract a value from a JSON string         |
+| `match(content, query)`               | Full-text search — true iff every tokenized term in `query` appears in `content`; index-accelerated when `content` is a bare column reference and `query` is a string literal |
+| `score(content, query)`               | Simple term-frequency relevance score for `query` against `content` (no IDF — see [Storage format](#storage-format) note) |
 
 String:
 
@@ -580,6 +594,12 @@ SELECT d.path, h.n
 FROM heading_counts h
 JOIN documents d ON d.id = h.document_id
 ORDER BY h.n DESC;
+
+-- Full-text search, ranked by relevance
+SELECT content, score(content, 'error handling') AS relevance
+FROM blocks
+WHERE match(content, 'error handling')
+ORDER BY relevance DESC;
 ```
 
 ## Architecture
@@ -699,6 +719,13 @@ graph TD
 
 Writes are atomic: data goes to `<path>.tmp` then renamed to `<path>` on success.
 
+> [!IMPORTANT]
+> File format version 5: `DocumentIndex` gained a `TermIndex` (full-text
+> search postings) appended after the pre-existing index sections. There is
+> no migration path — a store written by an older `mq-db` (version 4 or
+> earlier) is rejected with a clear "unsupported file version" error; run
+> `mq-db index` again to recreate it.
+
 ## License
 
 [MIT](LICENSE)

src/bin/mq-db.rs

@@ -21,9 +21,7 @@ use serde::Deserialize;
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-// ─────────────────────────────────────────────────────────────────────────────
 // CLI structure
-// ─────────────────────────────────────────────────────────────────────────────
 
 #[derive(Parser)]
 #[command(
@@ -211,9 +209,7 @@ impl std::fmt::Display for ReplMode {
     }
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Helpers
-// ─────────────────────────────────────────────────────────────────────────────
 
 fn load_store(db: &Path) -> anyhow::Result<DocumentStore> {
     if !db.exists() {
@@ -282,16 +278,14 @@ fn block_type_icon(bt: &BlockType) -> &'static str {
     }
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Main
-// ─────────────────────────────────────────────────────────────────────────────
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let cli = Cli::parse();
 
     match cli.command {
-        // ── index ────────────────────────────────────────────────────────────
+        // index
         Commands::Index {
             paths,
             output,
@@ -354,7 +348,7 @@ async fn main() -> anyhow::Result<()> {
             );
         }
 
-        // ── list ─────────────────────────────────────────────────────────────
+        // list
         Commands::List { db, format } => {
             // Catalog-only: skip deserialising all block data for a listing.
             let store = load_catalog_store(&db)?;
@@ -460,7 +454,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
 
-        // ── mq ───────────────────────────────────────────────────────────────
+        // mq
         Commands::Mq { code, db, format } => {
             let store = load_store(&db)?;
             let results =
@@ -518,7 +512,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
 
-        // ── sql ──────────────────────────────────────────────────────────────
+        // sql
         Commands::Sql {
             query,
             db,
@@ -558,7 +552,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
 
-        // ── repl ─────────────────────────────────────────────────────────────
+        // repl
         Commands::Repl {
             db,
             mode,
@@ -568,7 +562,7 @@ async fn main() -> anyhow::Result<()> {
             run_repl(store, mode, write_back)?;
         }
 
-        // ── lint ─────────────────────────────────────────────────────────────
+        // lint
         Commands::Lint { db, depth } => {
             let store = load_store(&db)?;
             let q = store.query();
@@ -605,7 +599,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
 
-        // ── stats ─────────────────────────────────────────────────────────────
+        // stats
         Commands::Stats { db } => {
             let store = load_store(&db)?;
             let stats = store.stats();
@@ -649,7 +643,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
 
-        // ── show ─────────────────────────────────────────────────────────────
+        // show
         Commands::Show { doc_id, db } => {
             let store = load_store(&db)?;
             let doc = store
@@ -738,7 +732,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
 
-        // ── tui ──────────────────────────────────────────────────────────────
+        // tui
         Commands::Tui { db } => {
             let store = if db.exists() {
                 DocumentStore::load(&db).map_err(|e| anyhow::anyhow!("{}", e))?
@@ -752,7 +746,7 @@ async fn main() -> anyhow::Result<()> {
             mq_db::tui::run(store).map_err(|e| anyhow::anyhow!("{}", e))?;
         }
 
-        // ── serve ─────────────────────────────────────────────────────────────
+        // serve
         Commands::Serve { db, host, port } => {
             let store = Arc::new(load_store(&db)?);
             let addr = format!("{}:{}", host, port);
@@ -776,9 +770,7 @@ async fn main() -> anyhow::Result<()> {
     Ok(())
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // HTTP server handlers
-// ─────────────────────────────────────────────────────────────────────────────
 
 type SharedStore = Arc<DocumentStore>;
 
@@ -955,9 +947,7 @@ fn md_block_to_html(s: &str) -> String {
     format!("<p>{}</p>", html_escape(trimmed))
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // REPL
-// ─────────────────────────────────────────────────────────────────────────────
 
 /// Crude prefix check used to gate write-back before even attempting a
 /// parse. Only `INSERT INTO blocks` requires `--write-back`; custom tables

src/block.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 
 /// Unique identifier for a block within a document.
 pub type BlockId = u32;
@@ -175,7 +175,7 @@ impl From<bool> for PropertyValue {
 /// Open-record property bag – acts like extra virtual columns in a
 /// row-polymorphic schema. Keyed by string name, typed by [`PropertyValue`].
 #[derive(Debug, Clone, Default, PartialEq)]
-pub struct Properties(HashMap<String, PropertyValue>);
+pub struct Properties(FxHashMap<String, PropertyValue>);
 
 impl Properties {
     pub fn new() -> Self {

src/index.rs

@@ -2,9 +2,7 @@ use mq_markdown::Node;
 
 use crate::block::{Block, BlockId, BlockType, DocumentId, Properties, PropertyValue, Span};
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Helper: extract span from a node's position
-// ─────────────────────────────────────────────────────────────────────────────
 
 fn node_to_span(node: &Node) -> Option<Span> {
     node.position().map(|p| Span {
@@ -15,9 +13,7 @@ fn node_to_span(node: &Node) -> Option<Span> {
     })
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Helper: if a node is a Heading, return its depth
-// ─────────────────────────────────────────────────────────────────────────────
 
 fn heading_depth(node: &Node) -> Option<u8> {
     if let Node::Heading(h) = node {
@@ -27,9 +23,7 @@ fn heading_depth(node: &Node) -> Option<u8> {
     }
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Helper: convert a serde_yaml Value to a PropertyValue
-// ─────────────────────────────────────────────────────────────────────────────
 
 fn yaml_value_to_property(v: serde_yaml::Value) -> PropertyValue {
     match v {
@@ -53,10 +47,8 @@ fn yaml_value_to_property(v: serde_yaml::Value) -> PropertyValue {
     }
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Core conversion: Node → (BlockType, content, Properties)
 // Returns None for nodes that should be skipped (Fragment, Empty).
-// ─────────────────────────────────────────────────────────────────────────────
 
 fn node_to_parts(node: &Node) -> Option<(BlockType, String, Properties)> {
     let mut props = Properties::new();
@@ -190,9 +182,7 @@ fn node_to_parts(node: &Node) -> Option<(BlockType, String, Properties)> {
     }
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Public API: build_blocks
-// ─────────────────────────────────────────────────────────────────────────────
 
 /// Converts a flat mq-markdown node list into a [`Vec<Block>`] with
 /// **Nested Set / Pre-Post Order** interval indexing.
@@ -220,7 +210,7 @@ pub fn build_blocks(doc_id: DocumentId, nodes: &[Node]) -> Vec<Block> {
         return Vec::new();
     }
 
-    // ── Phase 1: build section tree ──────────────────────────────────────────
+    // Phase 1: build section tree
     //
     // `children[slot]` holds the tree-slot indices of slot's direct children.
     // Slot 0 is the virtual document root; nodes start at slot 1.
@@ -257,7 +247,7 @@ pub fn build_blocks(doc_id: DocumentId, nodes: &[Node]) -> Vec<Block> {
         }
     }
 
-    // ── Phase 2: iterative DFS to assign pre/post numbers ────────────────────
+    // Phase 2: iterative DFS to assign pre/post numbers
 
     let num_slots = children.len();
     let mut pre = vec![0u32; num_slots];
@@ -287,7 +277,7 @@ pub fn build_blocks(doc_id: DocumentId, nodes: &[Node]) -> Vec<Block> {
         }
     }
 
-    // ── Phase 3: construct Block objects ─────────────────────────────────────
+    // Phase 3: construct Block objects
 
     let mut blocks: Vec<Block> = Vec::with_capacity(n);
     let mut next_id: BlockId = 0;
@@ -312,9 +302,7 @@ pub fn build_blocks(doc_id: DocumentId, nodes: &[Node]) -> Vec<Block> {
     blocks
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
 // Tests
-// ─────────────────────────────────────────────────────────────────────────────
 
 #[cfg(test)]
 mod tests {