feat(chunk): remove overlapping from snippet result

Author: Daniele Briggi

src/sqlite_rag/chunker.py

@@ -161,10 +161,16 @@ def _apply_overlap(self, chunks: List[Chunk]) -> List[Chunk]:
 
             if overlap_text:
                 combined_content = overlap_text + " " + current_content
+                # Core content starts after overlap and separator
+                core_start_pos = len(overlap_text) + 1
             else:
                 combined_content = current_content
+                # No overlap, core starts at beginning
+                core_start_pos = 0
 
-            overlapped_chunks.append(Chunk(content=combined_content))
+            overlapped_chunks.append(
+                Chunk(content=combined_content, core_start_pos=core_start_pos)
+            )
 
         return overlapped_chunks
 

src/sqlite_rag/database.py

@@ -78,6 +78,7 @@ def _create_schema(conn: sqlite3.Connection, settings: Settings):
                 document_id TEXT,
                 content TEXT,
                 embedding BLOB,
+                core_start_pos INTEGER DEFAULT 0,
                 FOREIGN KEY (document_id) REFERENCES documents (id) ON DELETE CASCADE
             );
         """

src/sqlite_rag/engine.py

@@ -170,6 +170,7 @@ def search(self, query: str, limit: int = 10) -> list[DocumentResult]:
                 documents.content as document_content,
                 documents.metadata,
                 chunks.content AS snippet,
+                chunks.core_start_pos,
                 vec_rank,
                 fts_rank,
                 combined_rank,
@@ -200,7 +201,8 @@ def search(self, query: str, limit: int = 10) -> list[DocumentResult]:
                     content=row["document_content"],
                     metadata=json.loads(row["metadata"]) if row["metadata"] else {},
                 ),
-                snippet=row["snippet"],
+                # remove overlapping text from the snippet
+                snippet=row["snippet"][row["core_start_pos"] :],
                 vec_rank=row["vec_rank"],
                 fts_rank=row["fts_rank"],
                 combined_rank=row["combined_rank"],

src/sqlite_rag/formatters.py

@@ -180,7 +180,7 @@ def _clean_and_wrap_snippet(self, snippet: str, width: int = 75) -> List[str]:
 
 
 class TableDebugFormatter(SearchResultFormatter):
-    """Legacy debug formatter for backwards compatibility."""
+    """Table view debug formatter."""
 
     def format_results(self, results: List[DocumentResult], query: str) -> None:
         if not results:
@@ -222,34 +222,6 @@ def format_results(self, results: List[DocumentResult], query: str) -> None:
             )
 
 
-class LegacyCompactFormatter(SearchResultFormatter):
-    """Legacy compact formatter for backwards compatibility."""
-
-    def format_results(self, results: List[DocumentResult], query: str) -> None:
-        if not results:
-            typer.echo("No documents found matching the query.")
-            return
-
-        typer.echo(f"Found {len(results)} documents:")
-
-        # Clean simple table for normal view
-        typer.echo(f"{'#':<3} {'Preview':<60} {'URI':<40}")
-        typer.echo("─" * 105)
-
-        for idx, doc in enumerate(results, 1):
-            # Clean snippet display
-            snippet = doc.snippet.replace("\n", " ").replace("\r", "")
-            if len(snippet) > 57:
-                snippet = snippet[:54] + "..."
-
-            # Clean URI display
-            uri = doc.document.uri or "N/A"
-            if len(uri) > 37:
-                uri = "..." + uri[-34:]
-
-            typer.echo(f"{idx:<3} {snippet:<60} {uri:<40}")
-
-
 def get_formatter(
     debug: bool = False, table_view: bool = False
 ) -> SearchResultFormatter:

src/sqlite_rag/models/chunk.py

@@ -7,3 +7,4 @@ class Chunk:
     document_id: int | None = None
     content: str = ""
     embedding: str | bytes = b""
+    core_start_pos: int = 0

src/sqlite_rag/repository.py

@@ -30,8 +30,8 @@ def add_document(self, document: Document) -> str:
         for chunk in document.chunks:
             # TODO: use the right vector_convert function based on the vector type
             cursor.execute(
-                "INSERT INTO chunks (document_id, content, embedding) VALUES (?, ?, vector_as_f16(?))",
-                (document_id, chunk.content, chunk.embedding),
+                "INSERT INTO chunks (document_id, content, embedding, core_start_pos) VALUES (?, ?, ?, ?)",
+                (document_id, chunk.content, chunk.embedding, chunk.core_start_pos),
             )
             cursor.execute(
                 "INSERT INTO chunks_fts (rowid, content) VALUES (last_insert_rowid(), ?)",