feat: add full-text search, CTE support, and INSERT write-back

README.md

@@ -35,11 +35,13 @@ flowchart TD
 
 - **Flat block storage** — every Markdown element becomes a typed `Block` with row-polymorphic properties
 - **O(1) hierarchy queries** — interval index (`pre`/`post`) makes ancestor/descendant checks a single integer comparison
-- **Three-layer secondary indexes** — `BitmapIndex` (block type), `BTreeIndex` (pre/post), `HashIndex` (content/lang/depth) for fast SQL predicate pushdown
+- **Four-layer secondary indexes** — `BitmapIndex` (block type), `BTreeIndex` (pre/post), `HashIndex` (content/lang/depth), `TermIndex` (tokenized content, full-text) for fast SQL predicate pushdown
 - **Zone Maps** — per-document statistics skip irrelevant files before scanning any blocks
 - **Dual query engines** — SQL via a custom `sqlparser`-based evaluator, and `mq` via `mq-lang`
+- **`WITH` (CTE) support** — non-recursive common table expressions, usable in `FROM`, `JOIN`, and subqueries
+- **Full-text search** — `match()`/`score()` SQL functions backed by a persisted per-document inverted index
 - **Incremental re-indexing** — re-running `index` skips unchanged files (content-hash based), replaces changed ones in place (same `DocumentId`), and can `--prune` deleted ones
-- **SQL `UPDATE`/`DELETE` with write-back** — edit `blocks` and push the change back to the source Markdown file, opt-in via `--write-back`
+- **SQL `INSERT`/`UPDATE`/`DELETE` with write-back** — add, edit, or remove `blocks` and push the change back to the source Markdown file, opt-in via `--write-back`
 - **DDL support** — `CREATE TABLE`, `INSERT INTO`, `DROP TABLE` for in-memory custom tables
 - **Comprehensive SQL function library** — string, numeric, null-handling, `CASE`, and aggregate functions comparable to a general-purpose RDBMS
 - **`mq()` scalar function** — run an mq program against Markdown content inline in SQL
@@ -165,23 +167,57 @@ mq-db sql "
 mq-db sql "SELECT mq('.h1 | to_text', content) AS title FROM blocks WHERE block_type = 'code'" --db store.mq-db
 ```
 
-### UPDATE / DELETE with write-back
+**Full-text search with `match()`/`score()`** — index-accelerated term matching and simple TF-based ranking:
 
-`UPDATE`/`DELETE` on `blocks` write the change back to the block's *source
-Markdown file* (re-parsed in place, same `DocumentId`) — pass `--write-back`
-to allow it; without the flag the statement is rejected:
+```bash
+mq-db sql "
+  SELECT content, score(content, 'error handling') AS relevance
+  FROM blocks
+  WHERE match(content, 'error handling')
+  ORDER BY relevance DESC
+" --db store.mq-db
+```
+
+**CTE (`WITH`)** — name an intermediate result and reuse it in the main query, a join, or a subquery:
+
+```bash
+mq-db sql "
+  WITH headings AS (SELECT content, pre, post FROM blocks WHERE block_type = 'heading')
+  SELECT content FROM headings WHERE pre < 10 ORDER BY pre
+" --db store.mq-db
+```
+
+`WITH RECURSIVE` is not supported. A CTE name identical to `blocks`,
+`documents`, or a custom table shadows it for the duration of the `WITH`
+clause's scope.
+
+### INSERT / UPDATE / DELETE with write-back
+
+`INSERT`/`UPDATE`/`DELETE` on `blocks` write the change back to the
+document's *source Markdown file* (re-parsed in place, same `DocumentId`)
+— pass `--write-back` to allow it; without the flag the statement is
+rejected:
 
 ```bash
 mq-db sql "UPDATE blocks SET content = 'New Title' WHERE block_type = 'heading' AND content = 'Old Title'" \
   --db store.mq-db --write-back
 
 mq-db sql "DELETE FROM blocks WHERE content = 'Outdated paragraph'" \
   --db store.mq-db --write-back
+
+# after_pre anchors the new block right after an existing block's `pre`;
+# omit it to append at the end of the document.
+mq-db sql "INSERT INTO blocks (document_id, block_type, content, depth, after_pre) VALUES (0, 'heading', 'New Section', 2, 4)" \
+  --db store.mq-db --write-back
+
+mq-db sql "INSERT INTO blocks (document_id, block_type, content) VALUES (0, 'paragraph', 'Appended at the end')" \
+  --db store.mq-db --write-back
 ```
 
 Limitations in this version:
 
-- `UPDATE ... SET content` only supports `heading`/`paragraph` blocks (not tables, code, lists, ...)
+- `UPDATE ... SET content` and `INSERT INTO blocks` only support `heading`/`paragraph` blocks (not tables, code, lists, ...)
+- `INSERT INTO blocks` requires an explicit column list drawn from `document_id`, `block_type`, `content`, `depth` (required for `heading`, 1-6), `after_pre` (optional) — `INSERT ... SELECT` is not supported, only `VALUES`
 - Only documents indexed **with spans** (the default; not `--no-spans`) and from a real file (not added via the library's `add_str`) are eligible
 - Not available over `serve`'s HTTP endpoint or from `mq-mcp` — CLI (`sql`/`repl` with `--write-back`) and the library (`DocumentStore::execute_sql_mut`) only
 
@@ -444,6 +480,8 @@ mq-db-specific:
 | `under(pre, post, anc_pre, anc_post)` | O(1) interval ancestor check               |
 | `mq(program, content)`                | Run an mq program against Markdown content |
 | `json_extract(json, path)`            | Extract a value from a JSON string         |
+| `match(content, query)`               | Full-text search — true iff every tokenized term in `query` appears in `content`; index-accelerated when `content` is a bare column reference and `query` is a string literal |
+| `score(content, query)`               | Simple term-frequency relevance score for `query` against `content` (no IDF — see [Storage format](#storage-format) note) |
 
 String:
 
@@ -544,6 +582,24 @@ SELECT
 FROM blocks
 WHERE block_type = 'heading'
 GROUP BY CASE WHEN depth <= 1 THEN 'top-level' ELSE 'nested' END;
+
+-- Full-text search, ranked by relevance
+SELECT content, score(content, 'error handling') AS relevance
+FROM blocks
+WHERE match(content, 'error handling')
+ORDER BY relevance DESC;
+
+-- CTE: rank documents by how many headings they contain
+WITH heading_counts AS (
+  SELECT document_id, count(*) AS n
+  FROM blocks
+  WHERE block_type = 'heading'
+  GROUP BY document_id
+)
+SELECT d.path, h.n
+FROM heading_counts h
+JOIN documents d ON d.id = h.document_id
+ORDER BY h.n DESC;
 ```
 
 ## Architecture
@@ -663,6 +719,13 @@ graph TD
 
 Writes are atomic: data goes to `<path>.tmp` then renamed to `<path>` on success.
 
+> [!IMPORTANT]
+> File format version 5: `DocumentIndex` gained a `TermIndex` (full-text
+> search postings) appended after the pre-existing index sections. There is
+> no migration path — a store written by an older `mq-db` (version 4 or
+> earlier) is rejected with a clear "unsupported file version" error; run
+> `mq-db index` again to recreate it.
+
 ## License
 
 [MIT](LICENSE)

src/bin/mq-db.rs

@@ -17,9 +17,9 @@ use clap::{Parser, Subcommand, ValueEnum};
 use mq_db::{DocumentStore, MqEngine, SqlEngine, block::BlockType, sql::html_escape};
 use serde::Deserialize;
 
-// ─────────────────────────────────────────────────────────────────────────────
-// CLI structure
-// ─────────────────────────────────────────────────────────────────────────────
+#[cfg(feature = "use_mimalloc")]
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 #[derive(Parser)]
 #[command(
@@ -957,10 +957,30 @@ fn md_block_to_html(s: &str) -> String {
 
 /// Crude prefix check used to gate write-back before even attempting a
 /// parse — mirrors the DESC/SHOW TABLES prefix check in `SqlEngine::execute`.
+///
+/// `INSERT INTO <custom table>` is deliberately *not* flagged here — it
+/// never writes back to a Markdown file and works without `--write-back`.
+/// Only `INSERT INTO blocks` does; real enforcement of that distinction
+/// lives in `DocumentStore::execute_sql_mut` (dispatch by parsed table
+/// name), this is just a friendlier pre-parse error message.
 fn is_write_statement(sql: &str) -> bool {
     let trimmed = sql.trim().trim_end_matches(';');
     let upper = trimmed.to_ascii_uppercase();
-    upper.starts_with("UPDATE ") || upper.starts_with("DELETE ")
+    upper.starts_with("UPDATE ") || upper.starts_with("DELETE ") || is_insert_into_blocks(&upper)
+}
+
+fn is_insert_into_blocks(upper: &str) -> bool {
+    let Some(after_into) = upper.strip_prefix("INSERT INTO") else {
+        return false;
+    };
+    let table = after_into
+        .trim_start()
+        .trim_start_matches(['"', '`'])
+        .trim_start();
+    table == "BLOCKS"
+        || ["BLOCKS ", "BLOCKS(", "BLOCKS\"", "BLOCKS`"]
+            .iter()
+            .any(|p| table.starts_with(p))
 }
 
 fn run_repl(