perf(sync): return final markdown from batch indexer

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/indexing/__init__.py

@@ -8,6 +8,7 @@
     IndexFileMetadata,
     IndexFileWriter,
     IndexFrontmatterUpdate,
+    IndexFrontmatterWriteResult,
     IndexingBatchResult,
     IndexInputFile,
     IndexProgress,
@@ -20,6 +21,7 @@
     "IndexFileMetadata",
     "IndexFileWriter",
     "IndexFrontmatterUpdate",
+    "IndexFrontmatterWriteResult",
     "IndexingBatchResult",
     "IndexInputFile",
     "IndexProgress",

src/basic_memory/indexing/batch_indexer.py

@@ -13,7 +13,7 @@
 from sqlalchemy.exc import IntegrityError
 
 from basic_memory.config import BasicMemoryConfig
-from basic_memory.file_utils import compute_checksum, has_frontmatter
+from basic_memory.file_utils import compute_checksum, has_frontmatter, remove_frontmatter
 from basic_memory.markdown.schemas import EntityMarkdown
 from basic_memory.indexing.models import (
     IndexedEntity,
@@ -47,6 +47,7 @@ class _PreparedEntity:
     checksum: str
     content_type: str | None
     search_content: str | None
+    markdown_content: str | None = None
 
 
 class BatchIndexer:
@@ -253,6 +254,7 @@ async def _normalize_markdown_file(
         reserved_permalinks: set[str],
     ) -> _PreparedMarkdownFile:
         final_checksum = prepared.final_checksum
+        final_content = prepared.content
         final_permalink = await self._resolve_batch_permalink(prepared, reserved_permalinks)
 
         # Trigger: markdown file has no frontmatter and sync enforcement is enabled.
@@ -264,9 +266,11 @@ async def _normalize_markdown_file(
                 "type": prepared.markdown.frontmatter.type,
                 "permalink": final_permalink,
             }
-            final_checksum = await self.file_writer.write_frontmatter(
+            write_result = await self.file_writer.write_frontmatter(
                 IndexFrontmatterUpdate(path=prepared.file.path, metadata=frontmatter_updates)
             )
+            final_checksum = write_result.checksum
+            final_content = write_result.content
             prepared.markdown.frontmatter.metadata.update(frontmatter_updates)
 
         # Trigger: existing markdown frontmatter may lack the canonical permalink.
@@ -278,16 +282,18 @@ async def _normalize_markdown_file(
             and final_permalink != prepared.markdown.frontmatter.permalink
         ):
             prepared.markdown.frontmatter.metadata["permalink"] = final_permalink
-            final_checksum = await self.file_writer.write_frontmatter(
+            write_result = await self.file_writer.write_frontmatter(
                 IndexFrontmatterUpdate(
                     path=prepared.file.path,
                     metadata={"permalink": final_permalink},
                 )
             )
+            final_checksum = write_result.checksum
+            final_content = write_result.content
 
         return _PreparedMarkdownFile(
             file=prepared.file,
-            content=prepared.content,
+            content=final_content,
             final_checksum=final_checksum,
             markdown=prepared.markdown,
             file_contains_frontmatter=prepared.file_contains_frontmatter,
@@ -351,7 +357,8 @@ async def _upsert_markdown_file(self, prepared: _PreparedMarkdownFile) -> _Prepa
             entity_id=updated.id,
             checksum=prepared.final_checksum,
             content_type=prepared.file.content_type,
-            search_content=prepared.markdown.content,
+            search_content=remove_frontmatter(prepared.content),
+            markdown_content=prepared.content,
         )
 
     async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity:
@@ -412,6 +419,7 @@ async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity:
             checksum=checksum,
             content_type=file.content_type,
             search_content=None,
+            markdown_content=None,
         )
 
     # --- Relations ---
@@ -487,6 +495,7 @@ async def _refresh_search_index(
             permalink=entity.permalink,
             checksum=prepared.checksum,
             content_type=prepared.content_type,
+            markdown_content=prepared.markdown_content,
         )
 
     # --- Helpers ---

src/basic_memory/indexing/models.py

@@ -55,6 +55,14 @@ class IndexFrontmatterUpdate:
     metadata: dict[str, Any]
 
 
+@dataclass(slots=True)
+class IndexFrontmatterWriteResult:
+    """Typed result for a frontmatter write performed during indexing."""
+
+    checksum: str
+    content: str
+
+
 @dataclass(slots=True)
 class IndexedEntity:
     """Stable output describing one file that finished indexing successfully."""
@@ -64,6 +72,7 @@ class IndexedEntity:
     permalink: str | None
     checksum: str
     content_type: str | None = None
+    markdown_content: str | None = None
 
 
 @dataclass(slots=True)
@@ -80,4 +89,6 @@ class IndexingBatchResult:
 class IndexFileWriter(Protocol):
     """Narrow protocol for frontmatter writes during indexing."""
 
-    async def write_frontmatter(self, update: IndexFrontmatterUpdate) -> str: ...
+    async def write_frontmatter(
+        self, update: IndexFrontmatterUpdate
+    ) -> IndexFrontmatterWriteResult: ...

src/basic_memory/services/file_service.py

@@ -3,6 +3,7 @@
 import asyncio
 import hashlib
 import mimetypes
+from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
@@ -25,6 +26,14 @@
 from loguru import logger
 
 
+@dataclass(slots=True)
+class FrontmatterUpdateResult:
+    """Final content emitted by a frontmatter rewrite without a follow-up reread."""
+
+    checksum: str
+    content: str
+
+
 class FileService:
     """Service for handling file operations with concurrency control.
 
@@ -401,12 +410,14 @@ async def move_file(self, source: FilePath, destination: FilePath) -> None:
             )
             raise FileOperationError(f"Failed to move file {source} -> {destination}: {e}")
 
-    async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
-        """Update frontmatter fields in a file while preserving all content.
+    async def update_frontmatter_with_result(
+        self, path: FilePath, updates: Dict[str, Any]
+    ) -> FrontmatterUpdateResult:
+        """Update frontmatter and return the exact final written markdown content.
 
         Only modifies the frontmatter section, leaving all content untouched.
         Creates frontmatter section if none exists.
-        Returns checksum of updated file.
+        Returns both checksum and final content so callers do not need a reread.
 
         Uses aiofiles for true async I/O (non-blocking).
 
@@ -415,7 +426,7 @@ async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> s
             updates: Dict of frontmatter fields to update
 
         Returns:
-            Checksum of updated file
+            Typed result containing checksum and final content
 
         Raises:
             FileOperationError: If file operations fail
@@ -467,7 +478,10 @@ async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> s
                 if formatted_content is not None:
                     content_for_checksum = formatted_content  # pragma: no cover
 
-            return await file_utils.compute_checksum(content_for_checksum)
+            return FrontmatterUpdateResult(
+                checksum=await file_utils.compute_checksum(content_for_checksum),
+                content=content_for_checksum,
+            )
 
         except Exception as e:  # pragma: no cover
             # Only log real errors (not YAML parsing, which is handled above)
@@ -479,6 +493,11 @@ async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> s
                 )
             raise FileOperationError(f"Failed to update frontmatter: {e}")
 
+    async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
+        """Update frontmatter fields in a file while preserving all content."""
+        result = await self.update_frontmatter_with_result(path, updates)
+        return result.checksum
+
     async def compute_checksum(self, path: FilePath) -> str:
         """Compute checksum for a file using true async I/O.
 

src/basic_memory/sync/sync_service.py

@@ -21,7 +21,12 @@
 from basic_memory.file_utils import has_frontmatter
 from basic_memory.indexing import BatchIndexer, IndexFileMetadata, IndexInputFile, IndexProgress
 from basic_memory.indexing.batching import build_index_batches
-from basic_memory.indexing.models import IndexedEntity, IndexFileWriter, IndexFrontmatterUpdate
+from basic_memory.indexing.models import (
+    IndexedEntity,
+    IndexFileWriter,
+    IndexFrontmatterUpdate,
+    IndexFrontmatterWriteResult,
+)
 from basic_memory.ignore_utils import load_bmignore_patterns, should_ignore_path
 from basic_memory.markdown import EntityParser, MarkdownProcessor
 from basic_memory.models import Entity, Project
@@ -127,8 +132,13 @@ class _FileServiceIndexWriter(IndexFileWriter):
     def __init__(self, file_service: FileService) -> None:
         self.file_service = file_service
 
-    async def write_frontmatter(self, update: IndexFrontmatterUpdate) -> str:
-        return await self.file_service.update_frontmatter(update.path, update.metadata)
+    async def write_frontmatter(
+        self, update: IndexFrontmatterUpdate
+    ) -> IndexFrontmatterWriteResult:
+        result = await self.file_service.update_frontmatter_with_result(
+            update.path, update.metadata
+        )
+        return IndexFrontmatterWriteResult(checksum=result.checksum, content=result.content)
 
 
 class SyncService:

tests/indexing/test_batch_indexer.py

@@ -9,7 +9,12 @@
 import pytest
 from sqlalchemy import text
 
-from basic_memory.indexing import BatchIndexer, IndexFrontmatterUpdate, IndexInputFile
+from basic_memory.indexing import (
+    BatchIndexer,
+    IndexFrontmatterUpdate,
+    IndexFrontmatterWriteResult,
+    IndexInputFile,
+)
 
 
 class _TestFileWriter:
@@ -18,8 +23,13 @@ class _TestFileWriter:
     def __init__(self, file_service) -> None:
         self.file_service = file_service
 
-    async def write_frontmatter(self, update: IndexFrontmatterUpdate) -> str:
-        return await self.file_service.update_frontmatter(update.path, update.metadata)
+    async def write_frontmatter(
+        self, update: IndexFrontmatterUpdate
+    ) -> IndexFrontmatterWriteResult:
+        result = await self.file_service.update_frontmatter_with_result(
+            update.path, update.metadata
+        )
+        return IndexFrontmatterWriteResult(checksum=result.checksum, content=result.content)
 
 
 async def _create_file(path: Path, content: str | bytes) -> None:
@@ -214,6 +224,51 @@ async def spy_upsert(*args, **kwargs):
     assert result.errors == []
 
 
+@pytest.mark.asyncio
+async def test_batch_indexer_returns_original_markdown_content_when_no_frontmatter_rewrite(
+    app_config,
+    entity_service,
+    entity_repository,
+    relation_repository,
+    search_service,
+    file_service,
+    project_config,
+):
+    app_config.disable_permalinks = True
+
+    path = "notes/original.md"
+    original_content = dedent(
+        """
+        ---
+        title: Original
+        type: note
+        ---
+        # Original
+        """
+    ).strip()
+    await _create_file(project_config.home / path, original_content)
+
+    files = {path: await _load_input(file_service, path)}
+    batch_indexer = _make_batch_indexer(
+        app_config,
+        entity_service,
+        entity_repository,
+        relation_repository,
+        search_service,
+        file_service,
+    )
+
+    result = await batch_indexer.index_files(
+        files,
+        max_concurrent=1,
+        parse_max_concurrent=1,
+    )
+
+    assert result.errors == []
+    assert len(result.indexed) == 1
+    assert result.indexed[0].markdown_content == original_content
+
+
 @pytest.mark.asyncio
 async def test_batch_indexer_indexes_non_markdown_files(
     app_config,
@@ -249,6 +304,7 @@ async def test_batch_indexer_indexes_non_markdown_files(
     )
 
     assert {indexed.path for indexed in result.indexed} == {pdf_path, image_path}
+    assert all(indexed.markdown_content is None for indexed in result.indexed)
 
     pdf_entity = await entity_repository.get_by_file_path(pdf_path)
     image_entity = await entity_repository.get_by_file_path(image_path)
@@ -377,6 +433,11 @@ async def test_batch_indexer_assigns_unique_permalinks_for_batch_local_conflicts
         path_one: await _load_input(file_service, path_one),
         path_two: await _load_input(file_service, path_two),
     }
+    original_contents = {
+        path: file.content.decode("utf-8")
+        for path, file in files.items()
+        if file.content is not None
+    }
     batch_indexer = _make_batch_indexer(
         app_config,
         entity_service,
@@ -393,6 +454,17 @@ async def test_batch_indexer_assigns_unique_permalinks_for_batch_local_conflicts
     )
 
     assert result.errors == []
+    indexed_by_path = {indexed.path: indexed for indexed in result.indexed}
+    assert indexed_by_path[path_one].markdown_content is not None
+    assert indexed_by_path[path_two].markdown_content is not None
+    assert indexed_by_path[path_one].markdown_content != original_contents[path_one]
+    assert indexed_by_path[path_two].markdown_content != original_contents[path_two]
+    assert indexed_by_path[path_one].markdown_content == await file_service.read_file_content(
+        path_one
+    )
+    assert indexed_by_path[path_two].markdown_content == await file_service.read_file_content(
+        path_two
+    )
 
     entities = await entity_repository.find_all()
     assert len(entities) == 2