fix: address review feedback on stale embedding stats

1. Remove unused valid_entity_ids parameter from _purge_stale_search_rows
2. Add entity_exists filter to total_embeddings and orphaned_chunks
   queries for consistency (previously only applied to 3 of 5 queries)
3. Strengthen test assertion to exact match instead of <=

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/services/project_service.py

@@ -1000,6 +1000,10 @@ async def get_embedding_status(self, project_id: int) -> EmbeddingStatus:
         # Filter by entity existence to exclude stale rows from deleted entities
         # that remain in derived search tables (search_index, search_vector_chunks)
         entity_exists = "AND entity_id IN (SELECT id FROM entity WHERE project_id = :project_id)"
+        # Same filter for aliased chunks table (used in JOIN queries below)
+        chunk_entity_exists = (
+            "AND c.entity_id IN (SELECT id FROM entity WHERE project_id = :project_id)"
+        )
 
         si_result = await self.repository.execute_query(
             text(
@@ -1034,13 +1038,13 @@ async def get_embedding_status(self, project_id: int) -> EmbeddingStatus:
                 embeddings_sql = text(
                     "SELECT COUNT(*) FROM search_vector_chunks c "
                     "JOIN search_vector_embeddings e ON e.chunk_id = c.id "
-                    "WHERE c.project_id = :project_id"
+                    f"WHERE c.project_id = :project_id {chunk_entity_exists}"
                 )
             else:
                 embeddings_sql = text(
                     "SELECT COUNT(*) FROM search_vector_chunks c "
                     "JOIN search_vector_embeddings e ON e.rowid = c.id "
-                    "WHERE c.project_id = :project_id"
+                    f"WHERE c.project_id = :project_id {chunk_entity_exists}"
                 )
 
             embeddings_result = await self.repository.execute_query(
@@ -1053,13 +1057,13 @@ async def get_embedding_status(self, project_id: int) -> EmbeddingStatus:
                 orphan_sql = text(
                     "SELECT COUNT(*) FROM search_vector_chunks c "
                     "LEFT JOIN search_vector_embeddings e ON e.chunk_id = c.id "
-                    "WHERE c.project_id = :project_id AND e.chunk_id IS NULL"
+                    f"WHERE c.project_id = :project_id AND e.chunk_id IS NULL {chunk_entity_exists}"
                 )
             else:
                 orphan_sql = text(
                     "SELECT COUNT(*) FROM search_vector_chunks c "
                     "LEFT JOIN search_vector_embeddings e ON e.rowid = c.id "
-                    "WHERE c.project_id = :project_id AND e.rowid IS NULL"
+                    f"WHERE c.project_id = :project_id AND e.rowid IS NULL {chunk_entity_exists}"
                 )
 
             orphan_result = await self.repository.execute_query(

src/basic_memory/services/search_service.py

@@ -406,7 +406,7 @@ async def reindex_vectors(self, progress_callback=None) -> dict:
 
         # Clean up stale rows in search_index and search_vector_chunks
         # that reference entity_ids no longer in the entity table
-        await self._purge_stale_search_rows(set(entity_ids))
+        await self._purge_stale_search_rows()
 
         batch_result = await self.repository.sync_entity_vectors_batch(
             entity_ids,
@@ -424,7 +424,7 @@ async def reindex_vectors(self, progress_callback=None) -> dict:
 
         return stats
 
-    async def _purge_stale_search_rows(self, valid_entity_ids: set[int]) -> None:
+    async def _purge_stale_search_rows(self) -> None:
         """Remove rows from search_index and search_vector_chunks for deleted entities.
 
         Trigger: entities are deleted but their derived search rows remain

tests/services/test_project_service_embedding_status.py

@@ -282,14 +282,20 @@ async def test_embedding_status_excludes_stale_entity_ids(
     ):
         status = await project_service.get_embedding_status(test_project.id)
 
-    # The stale entity_id should NOT be counted in total_indexed_entities
-    real_entity_result = await project_service.repository.execute_query(
-        text("SELECT COUNT(*) FROM entity WHERE project_id = :pid"),
+    # The stale entity_id should NOT be counted in total_indexed_entities.
+    # Count real entities that have search_index rows (the stale one should be excluded).
+    real_indexed_result = await project_service.repository.execute_query(
+        text(
+            "SELECT COUNT(DISTINCT si.entity_id) FROM search_index si "
+            "JOIN entity e ON e.id = si.entity_id "
+            "WHERE si.project_id = :pid"
+        ),
         {"pid": test_project.id},
     )
-    real_entity_count = real_entity_result.scalar() or 0
+    real_indexed_count = real_indexed_result.scalar() or 0
 
-    assert status.total_indexed_entities <= real_entity_count
+    # Exact match — stale entity_id must not inflate the count
+    assert status.total_indexed_entities == real_indexed_count
 
 
 @pytest.mark.asyncio