fix: resolve FTS5 search syntax errors with special characters

Enhances search term preparation to handle special characters gracefully while preserving functionality:

- Improves FTS5 query preparation with targeted special character handling
- Preserves boolean operators (AND, OR, NOT) without modification
- Quotes problematic characters that cause syntax errors
- Maintains wildcard patterns for legitimate use cases
- Adds comprehensive error handling with graceful fallback

Includes extensive test coverage:
- 10 new test cases for various search scenarios
- Programming terms (C++, function(), email@domain.com) now searchable
- Malformed syntax handled without crashes
- Boolean and wildcard functionality preserved

Fixes search crashes when users enter queries containing special characters.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: phernandez

src/basic_memory/repository/search_repository.py

@@ -128,25 +128,32 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
             is_prefix: Whether to add prefix search capability (* suffix)
 
         For FTS5:
-        - Special characters and phrases need to be quoted
-        - Terms with spaces or special chars need quotes
         - Boolean operators (AND, OR, NOT) are preserved for complex queries
+        - Terms with FTS5 special characters are quoted to prevent syntax errors
+        - Simple terms get prefix wildcards for better matching
         """
-        if "*" in term:
-            return term
-
         # Check for explicit boolean operators - if present, return the term as is
         boolean_operators = [" AND ", " OR ", " NOT "]
         if any(op in f" {term} " for op in boolean_operators):
             return term
 
-        # List of FTS5 special characters that need escaping/quoting
-        special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
-
-        # Check if term contains any special characters
-        needs_quotes = any(c in term for c in special_chars)
+        # Check if term is already a proper wildcard pattern (alphanumeric + *)
+        # e.g., "hello*", "test*world" - these should be left alone
+        if "*" in term and all(c.isalnum() or c in "*_-" for c in term):
+            return term
 
-        if needs_quotes:
+        # Characters that can cause FTS5 syntax errors when used as operators
+        # We're more conservative here - only quote when we detect problematic patterns
+        problematic_chars = ['"', "'", "(", ")", "[", "]", "+", "!", "@", "#", "$", "%", "^", "&", "=", "|", "\\", "~", "`"]
+        
+        # Characters that indicate we should quote (spaces, dots, colons, etc.)
+        needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/"]
+        
+        # Check if term needs quoting
+        has_problematic = any(c in term for c in problematic_chars)
+        has_spaces_or_special = any(c in term for c in needs_quoting_chars)
+        
+        if has_problematic or has_spaces_or_special:
             # Escape any existing quotes by doubling them
             escaped_term = term.replace('"', '""')
             # Quote the entire term to handle special characters safely
@@ -273,9 +280,20 @@ async def search(
         """
 
         logger.trace(f"Search {sql} params: {params}")
-        async with db.scoped_session(self.session_maker) as session:
-            result = await session.execute(text(sql), params)
-            rows = result.fetchall()
+        try:
+            async with db.scoped_session(self.session_maker) as session:
+                result = await session.execute(text(sql), params)
+                rows = result.fetchall()
+        except Exception as e:
+            # Handle FTS5 syntax errors and provide user-friendly feedback
+            if "fts5: syntax error" in str(e).lower():
+                logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
+                # Return empty results rather than crashing
+                return []
+            else:
+                # Re-raise other database errors
+                logger.error(f"Database error during search: {e}")
+                raise
 
         results = [
             SearchIndexRow(

src/basic_memory/services/project_service.py

@@ -159,7 +159,9 @@ async def _ensure_single_default_project(self) -> None:
         multiple projects might have is_default=True or no project is marked as default.
         """
         if not self.repository:
-            raise ValueError("Repository is required for _ensure_single_default_project") # pragma: no cover
+            raise ValueError(
+                "Repository is required for _ensure_single_default_project"
+            )  # pragma: no cover
 
         # Get all projects with is_default=True
         db_projects = await self.repository.find_all()
@@ -597,4 +599,4 @@ def get_system_status(self) -> SystemStatus:
             database_size=db_size_readable,
             watch_status=watch_status,
             timestamp=datetime.now(),
-        )
+        )

tests/repository/test_search_repository.py

@@ -301,3 +301,87 @@ def test_directory_property():
         project_id=1,
     )
     assert row3.directory == ""
+
+
+class TestSearchTermPreparation:
+    """Test cases for FTS5 search term preparation."""
+
+    def test_simple_terms_get_prefix_wildcard(self, search_repository):
+        """Simple alphanumeric terms should get prefix matching."""
+        assert search_repository._prepare_search_term("hello") == "hello*"
+        assert search_repository._prepare_search_term("project") == "project*"
+        assert search_repository._prepare_search_term("test123") == "test123*"
+
+    def test_terms_with_existing_wildcard_unchanged(self, search_repository):
+        """Terms that already contain * should remain unchanged."""
+        assert search_repository._prepare_search_term("hello*") == "hello*"
+        assert search_repository._prepare_search_term("test*world") == "test*world"
+
+    def test_boolean_operators_preserved(self, search_repository):
+        """Boolean operators should be preserved without modification."""
+        assert search_repository._prepare_search_term("hello AND world") == "hello AND world"
+        assert search_repository._prepare_search_term("cat OR dog") == "cat OR dog" 
+        assert search_repository._prepare_search_term("project NOT meeting") == "project NOT meeting"
+        assert search_repository._prepare_search_term("(hello AND world) OR test") == "(hello AND world) OR test"
+
+    def test_programming_terms_should_work(self, search_repository):
+        """Programming-related terms with special chars should be searchable."""
+        # These should be quoted to handle special characters safely
+        assert search_repository._prepare_search_term("C++") == '"C++"*'
+        assert search_repository._prepare_search_term("function()") == '"function()"*'
+        assert search_repository._prepare_search_term("email@domain.com") == '"email@domain.com"*'
+        assert search_repository._prepare_search_term("array[index]") == '"array[index]"*'
+        assert search_repository._prepare_search_term("config.json") == '"config.json"*'
+
+    def test_malformed_fts5_syntax_quoted(self, search_repository):
+        """Malformed FTS5 syntax should be quoted to prevent errors."""
+        # Multiple operators without proper syntax
+        assert search_repository._prepare_search_term("+++invalid+++") == '"+++invalid+++"*'
+        assert search_repository._prepare_search_term("!!!error!!!") == '"!!!error!!!"*'
+        assert search_repository._prepare_search_term("@#$%^&*()") == '"@#$%^&*()"*'
+
+    def test_quoted_strings_handled_properly(self, search_repository):
+        """Strings with quotes should have quotes escaped."""
+        assert search_repository._prepare_search_term('say "hello"') == '"say ""hello"""*'
+        assert search_repository._prepare_search_term("it's working") == '"it\'s working"*'
+
+    def test_file_paths_no_prefix_wildcard(self, search_repository):
+        """File paths should not get prefix wildcards."""
+        assert search_repository._prepare_search_term("config.json", is_prefix=False) == '"config.json"'
+        assert search_repository._prepare_search_term("docs/readme.md", is_prefix=False) == '"docs/readme.md"'
+
+    def test_spaces_handled_correctly(self, search_repository):
+        """Terms with spaces should be quoted."""
+        assert search_repository._prepare_search_term("hello world") == '"hello world"*'
+        assert search_repository._prepare_search_term("project planning") == '"project planning"*'
+
+    @pytest.mark.asyncio
+    async def test_search_with_special_characters_returns_results(self, search_repository):
+        """Integration test: search with special characters should work gracefully."""
+        # This test ensures the search doesn't crash with FTS5 syntax errors
+        
+        # These should all return empty results gracefully, not crash
+        results1 = await search_repository.search(search_text="C++")
+        assert isinstance(results1, list)  # Should not crash
+        
+        results2 = await search_repository.search(search_text="function()")
+        assert isinstance(results2, list)  # Should not crash
+        
+        results3 = await search_repository.search(search_text="+++malformed+++")
+        assert isinstance(results3, list)  # Should not crash, return empty results
+        
+        results4 = await search_repository.search(search_text="email@domain.com")
+        assert isinstance(results4, list)  # Should not crash
+
+    @pytest.mark.asyncio
+    async def test_boolean_search_still_works(self, search_repository):
+        """Boolean search operations should continue to work."""
+        # These should not crash and should respect boolean logic
+        results1 = await search_repository.search(search_text="hello AND world")
+        assert isinstance(results1, list)
+        
+        results2 = await search_repository.search(search_text="cat OR dog")
+        assert isinstance(results2, list)
+        
+        results3 = await search_repository.search(search_text="project NOT meeting")
+        assert isinstance(results3, list)