Added support for path and snippet in the memory_search virtual table

Author: marcobambini

API.md

@@ -342,16 +342,16 @@ SELECT * FROM memory_search WHERE query = 'search text';
 |--------|------|-------------|
 | `query` | TEXT (HIDDEN) | Search query (required in WHERE clause) |
 | `hash` | INTEGER | Content hash identifier |
-| `path` | TEXT | Source path or generated UUID |
+| `path` | TEXT | Source file path or generated UUID for text content |
 | `context` | TEXT | Context label (NULL if not set) |
-| `snippet` | TEXT | Text snippet from matching chunk |
-| `score` | REAL | Combined similarity score (0.0 - 1.0) |
+| `snippet` | TEXT | Text snippet from the matching chunk |
+| `ranking` | REAL | Combined similarity score (0.0 - 1.0) |
 
 **Notes:**
 - Requires sqlite-vector extension loaded first
 - Performs hybrid search combining vector similarity and FTS5
 - Results are ranked by combined score
-- Limited by `max_items` setting (default: 20)
+- Limited by `max_results` setting (default: 20)
 - Filtered by `min_score` setting (default: 0.7)
 - Updates `last_accessed` timestamp if `update_access` is enabled
 
@@ -360,11 +360,11 @@ SELECT * FROM memory_search WHERE query = 'search text';
 -- Basic search
 SELECT * FROM memory_search WHERE query = 'database indexing strategies';
 
--- Search with score filter
-SELECT path, snippet, score
+-- Search with ranking filter
+SELECT path, snippet, ranking
 FROM memory_search
 WHERE query = 'how to optimize queries'
-AND score > 0.8;
+AND ranking > 0.8;
 
 -- Search within a specific context
 SELECT * FROM memory_search
@@ -389,7 +389,7 @@ AND context = 'meetings';
 | `skip_html` | INTEGER | 1 | Strip HTML tags when parsing |
 | `extensions` | TEXT | "md,mdx" | Comma-separated file extensions to process |
 | `engine_warmup` | INTEGER | 0 | Warm up engine on model load (compiles GPU shaders) |
-| `max_items` | INTEGER | 20 | Maximum search results |
+| `max_results` | INTEGER | 20 | Maximum search results |
 | `fts_enabled` | INTEGER | 1 | Enable FTS5 in hybrid search |
 | `vector_weight` | REAL | 0.5 | Weight for vector similarity in scoring |
 | `text_weight` | REAL | 0.5 | Weight for FTS in scoring |
@@ -449,7 +449,7 @@ SELECT memory_add_text('SQLite is a C library that provides a lightweight disk-b
 SELECT memory_add_directory('/docs/sqlite', 'sqlite-docs');
 
 -- Search
-SELECT path, snippet, score
+SELECT path, snippet, ranking
 FROM memory_search
 WHERE query = 'how does SQLite store data on disk';
 

README.md

@@ -92,17 +92,17 @@ systems, and AI applications.', 'concepts');
 SELECT memory_add_directory('/path/to/docs', 'project-docs');
 
 -- Search your memory semantically
-SELECT path, snippet, score
+SELECT path, snippet, ranking
 FROM memory_search
 WHERE query = 'how do databases store information efficiently';
 
 -- Results ranked by semantic similarity + keyword matching
--- ┌──────────────┬─────────────────────────────────────┬───────┐
--- │     path     │               snippet               │ score │
--- ├──────────────┼─────────────────────────────────────┼───────┤
--- │ (uuid)       │ SQLite is a C-language library...   │ 0.89  │
--- │ (uuid)       │ Vector databases store data as...   │ 0.82  │
--- └──────────────┴─────────────────────────────────────┴───────┘
+-- ┌──────────────┬─────────────────────────────────────┬─────────┐
+-- │     path     │               snippet               │ ranking │
+-- ├──────────────┼─────────────────────────────────────┼─────────┤
+-- │ (uuid)       │ SQLite is a C-language library...   │ 0.89    │
+-- │ (uuid)       │ Vector databases store data as...   │ 0.82    │
+-- └──────────────┴─────────────────────────────────────┴─────────┘
 ```
 
 ### Example: Building an AI Agent with Memory
@@ -127,9 +127,9 @@ def remember(content, context="conversation"):
 # Retrieve relevant memories
 def recall(query, min_score=0.7):
     cursor = conn.execute("""
-        SELECT snippet, score FROM memory_search
-        WHERE query = ? AND score > ?
-        ORDER BY score DESC
+        SELECT snippet, ranking FROM memory_search
+        WHERE query = ? AND ranking > ?
+        ORDER BY ranking DESC
     """, (query, min_score))
     return cursor.fetchall()
 
@@ -161,7 +161,7 @@ SELECT memory_set_option('max_tokens', 512);      -- Tokens per chunk
 SELECT memory_set_option('overlay_tokens', 100);  -- Overlap between chunks
 
 -- Search behavior
-SELECT memory_set_option('max_items', 30);        -- Max results
+SELECT memory_set_option('max_results', 30);      -- Max search results
 SELECT memory_set_option('min_score', 0.75);      -- Score threshold
 SELECT memory_set_option('vector_weight', 0.6);   -- Vector vs FTS balance
 SELECT memory_set_option('text_weight', 0.4);

src/dbmem-search.c

@@ -30,6 +30,8 @@ SQLITE_EXTENSION_INIT3
 #define SEARCH_COLUMN_HASH                      3
 #define SEARCH_COLUMN_SEQ                       4
 #define SEARCH_COLUMN_RANKING                   5
+#define SEARCH_COLUMN_PATH                      6
+#define SEARCH_COLUMN_SNIPPET                   7
 
 typedef struct {
     sqlite3_vtab    base;                   // Base class - must be first
@@ -39,7 +41,7 @@ typedef struct {
 
 typedef struct {
     sqlite3_vtab_cursor base;               // Base class - must be first
-    int             max_items;
+    int             max_results;
     bool            perform_fts;
 
     int             index;
@@ -117,7 +119,7 @@ int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perfo
     if (!buffer) return SQLITE_NOMEM;
 
     // adjust all internal pointers
-    c->max_items = entries;
+    c->max_results = entries;
     c->perform_fts = perform_fts;
     c->buffer = buffer;
 
@@ -162,7 +164,7 @@ int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perfo
     return SQLITE_OK;
 }
 
-int vMemorySearchCursorMerge(vMemorySearchCursor *c, double vectorWeight, double textWeight, double min_score, int max_items) {
+int vMemorySearchCursorMerge(vMemorySearchCursor *c, double vectorWeight, double textWeight, double min_score, int max_results) {
     c->merge.count = 0;
 
     // add semantic (vector) results
@@ -244,7 +246,7 @@ int vMemorySearchCursorMerge(vMemorySearchCursor *c, double vectorWeight, double
 
     // copy top results to final cursor arrays, filtering by min_score
     c->count = 0;
-    for (int i = 0; i < c->merge.count && c->count < max_items; i++) {
+    for (int i = 0; i < c->merge.count && c->count < max_results; i++) {
         if (c->merge.textScore[i] < min_score) break;  // sorted descending, so stop at first below threshold
         c->hash[c->count] = c->merge.hash[i];
         c->seq[c->count] = c->merge.seq[i];
@@ -430,7 +432,7 @@ static int vMemorySearchConnect (sqlite3 *db, void *pAux, int argc, const char *
     }
 
     // https://www.sqlite.org/vtab.html#table_valued_functions
-    int rc = sqlite3_declare_vtab(db, "CREATE TABLE x(query hidden, max_entries hidden, context hidden, hash, seq, ranking);");
+    int rc = sqlite3_declare_vtab(db, "CREATE TABLE x(query hidden, max_entries hidden, context hidden, hash, seq, ranking, path, snippet);");
     if (rc != SQLITE_OK) return rc;
 
     vMemorySearchTable *vtab = (vMemorySearchTable *)dbmem_zeroalloc(sizeof(vMemorySearchTable));
@@ -506,20 +508,37 @@ static int vMemorySearchCursorEof (sqlite3_vtab_cursor *cur){
 }
 
 static int vMemorySearchCursorColumn (sqlite3_vtab_cursor *cur, sqlite3_context *context, int iCol) {
+    static const char *path_sql = "SELECT path FROM dbmem_content WHERE hash = ?1;";
+    static const char *snippet_sql = "SELECT substr(c.value, v.offset + 1, v.length) FROM dbmem_vault v JOIN dbmem_content c ON v.hash = c.hash WHERE v.hash = ?1 AND v.seq = ?2;";
+    
     vMemorySearchCursor *c = (vMemorySearchCursor *)cur;
+    sqlite3 *db = ((vMemorySearchTable *)cur->pVtab)->db;
 
     switch (iCol) {
         case SEARCH_COLUMN_HASH:
-            sqlite3_result_int64(context, (sqlite3_int64)c->hash[c->index]);
+            sqlite3_result_int64(context, c->hash[c->index]);
             break;
 
         case SEARCH_COLUMN_SEQ:
-            sqlite3_result_int64(context, (sqlite3_int64)c->seq[c->index]);
+            sqlite3_result_int64(context, c->seq[c->index]);
             break;
 
         case SEARCH_COLUMN_RANKING:
             sqlite3_result_double(context, c->rank[c->index]);
             break;
+            
+        case SEARCH_COLUMN_PATH:
+        case SEARCH_COLUMN_SNIPPET:{
+            const char *sql = (iCol == SEARCH_COLUMN_PATH) ? path_sql : snippet_sql;
+            sqlite3_stmt *vm = NULL;
+            if (sqlite3_prepare_v2(db, sql, -1, &vm, NULL) == SQLITE_OK) {
+                sqlite3_bind_int64(vm, 1, c->hash[c->index]);
+                if (iCol == SEARCH_COLUMN_SNIPPET) sqlite3_bind_int64(vm, 2, c->seq[c->index]);
+                if (sqlite3_step(vm) == SQLITE_ROW) sqlite3_result_value(context, sqlite3_column_value(vm, 0));
+            }
+            if (vm) sqlite3_finalize(vm);
+        }
+            break;
     }
 
     return SQLITE_OK;
@@ -540,7 +559,7 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
     sqlite3 *db = searchTab->db;
 
     // check and retrieve arguments
-    int  max_items = dbmem_context_max_items(ctx);
+    int  max_results = dbmem_context_max_results(ctx);
     bool perform_fts = dbmem_context_perform_fts(ctx);
     const char *query = NULL;
     const char *context = NULL;
@@ -559,14 +578,14 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
     if (argc > 1) {
         // only the next two arguments are handled
         for (int i=1; i<argc && i<=2; ++i) {
-            if (sqlite3_value_type(argv[i]) == SQLITE_INTEGER) max_items = sqlite3_value_int(argv[i]);
+            if (sqlite3_value_type(argv[i]) == SQLITE_INTEGER) max_results = sqlite3_value_int(argv[i]);
             else if (sqlite3_value_type(argv[i]) == SQLITE_TEXT) context = (const char *)sqlite3_value_text(argv[i]);
             // ignore any other type
         }
     }
 
     // allocate internal cursor buffer
-    int rc = vMemorySearchCursorAllocate(c, max_items, perform_fts);
+    int rc = vMemorySearchCursorAllocate(c, max_results, perform_fts);
     if (rc != SQLITE_OK) return SQLITE_NOMEM;
 
     // perform semantic search
@@ -608,7 +627,7 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
     }
 
     // perform search
-    rc = dbmem_semantic_search(db, c, result.embedding, (int)(result.n_embd * sizeof(float)), context, max_items);
+    rc = dbmem_semantic_search(db, c, result.embedding, (int)(result.n_embd * sizeof(float)), context, max_results);
     if (rc != 0) {
         sqlvTab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
         return SQLITE_ERROR;
@@ -617,7 +636,7 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
     // perform fts search
     if (perform_fts) {
         // in case of FTS error ignore its contribution
-        rc = dbmem_fts_search(db, c, query, context, max_items);
+        rc = dbmem_fts_search(db, c, query, context, max_results);
         if (rc != SQLITE_OK) perform_fts = false;
     }
 
@@ -632,7 +651,7 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
     double vectorWeight = dbmem_context_vector_weight(ctx);
     double textWeight = dbmem_context_text_weight(ctx);
     double min_score = dbmem_context_min_score(ctx);
-    vMemorySearchCursorMerge(c, vectorWeight, textWeight, min_score, max_items);
+    vMemorySearchCursorMerge(c, vectorWeight, textWeight, min_score, max_results);
 
     // update last_accessed timestamps for returned results
     if (dbmem_context_update_access(ctx) && c->count > 0) {

src/sqlite-memory.c

@@ -50,7 +50,7 @@ SQLITE_EXTENSION_INIT1
 #define DBMEM_SETTINGS_KEY_SKIP_HTML            "skip_html"
 #define DBMEM_SETTINGS_KEY_EXTENSIONS           "extensions"
 #define DBMEM_SETTINGS_KEY_ENGINE_WARMUP        "engine_warmup"
-#define DBMEM_SETTINGS_KEY_MAX_ITEMS            "max_items"
+#define DBMEM_SETTINGS_KEY_MAX_RESULTS          "max_results"
 #define DBMEM_SETTINGS_KEY_FTS_ENABLED          "fts_enabled"
 #define DBMEM_SETTINGS_KEY_VECTOR_WEIGHT        "vector_weight"
 #define DBMEM_SETTINGS_KEY_TEXT_WEIGHT          "text_weight"
@@ -62,7 +62,7 @@ SQLITE_EXTENSION_INIT1
 #define DEFAULT_MAX_TOKENS                      400
 #define DEFAULT_OVERLAY_TOKENS                  80
 #define DEFAULT_MAX_SNIPPET_CHARS               700
-#define DEFAULT_MAX_ITEMS                       20
+#define DEFAULT_MAX_RESULTS                     20
 #define DEFAULT_VECTOR_WEIGHT                   0.5
 #define DEFAULT_TEXT_WEIGHT                     0.5
 #define DEFAULT_MIN_SCORE                       0.7
@@ -99,7 +99,7 @@ struct dbmem_context {
     bool        dimension_saved;                // Embedding dimension needs to be automatically serialized
 
     // Settings
-    int         max_items;                      // Maximum number of items to be returned from a search
+    int         max_results;                    // Maximum number of results to be returned from a search
     double      vector_weight;                  // Weight of the vector results during the merge of the result
     double      text_weight;                    // Weight of the FTS results during the merge of the result
     double      min_score;                      // Minimum score threshold to filter irrelevant results
@@ -212,9 +212,9 @@ static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_val
         return 0;
     }
 
-    if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_ITEMS) == 0) {
+    if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_RESULTS) == 0) {
         int n = sqlite3_value_int(value);
-        if (n > 0) ctx->max_items = n;
+        if (n > 0) ctx->max_results = n;
         return 0;
     }
 
@@ -470,7 +470,7 @@ static void *dbmem_context_create (sqlite3 *db) {
     ctx->engine_warmup = false;
 
     ctx->perform_fts = fts5_is_available;
-    ctx->max_items = DEFAULT_MAX_ITEMS;
+    ctx->max_results = DEFAULT_MAX_RESULTS;
     ctx->vector_weight = DEFAULT_VECTOR_WEIGHT;
     ctx->text_weight = DEFAULT_TEXT_WEIGHT;
     ctx->min_score = DEFAULT_MIN_SCORE;
@@ -567,8 +567,8 @@ bool dbmem_context_perform_fts (dbmem_context *ctx) {
     return ctx->perform_fts;
 }
 
-int dbmem_context_max_items (dbmem_context *ctx) {
-    return ctx->max_items;
+int dbmem_context_max_results (dbmem_context *ctx) {
+    return ctx->max_results;
 }
 
 double dbmem_context_vector_weight (dbmem_context *ctx) {

src/sqlite-memory.h

@@ -26,7 +26,7 @@
 extern "C" {
 #endif
 
-#define SQLITE_DBMEMORY_VERSION "0.5.1"
+#define SQLITE_DBMEMORY_VERSION "0.5.2"
 
 // public API
 SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
@@ -38,7 +38,7 @@ void  *dbmem_context_engine (dbmem_context *ctx, bool *is_local);
 bool   dbmem_context_load_vector (dbmem_context *ctx);
 bool   dbmem_context_load_sync (dbmem_context *ctx);
 bool   dbmem_context_perform_fts (dbmem_context *ctx);
-int    dbmem_context_max_items (dbmem_context *ctx);
+int    dbmem_context_max_results (dbmem_context *ctx);
 double dbmem_context_vector_weight (dbmem_context *ctx);
 double dbmem_context_text_weight (dbmem_context *ctx);
 double dbmem_context_min_score (dbmem_context *ctx);