fix: handle Boolean search syntax with hyphenated terms (#180)

Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
Co-authored-by: Paul Hernandez <phernandez@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>

Author: 3 people

src/basic_memory/repository/search_repository.py

@@ -1,6 +1,7 @@
 """Repository for search operations."""
 
 import json
+import re
 import time
 from dataclasses import dataclass
 from datetime import datetime
@@ -120,23 +121,113 @@ async def init_search_index(self):
             logger.error(f"Error initializing search index: {e}")
             raise e
 
-    def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
-        """Prepare a search term for FTS5 query.
-
+    def _prepare_boolean_query(self, query: str) -> str:
+        """Prepare a Boolean query by quoting individual terms while preserving operators.
+        
         Args:
-            term: The search term to prepare
+            query: A Boolean query like "tier1-test AND unicode" or "(hello OR world) NOT test"
+            
+        Returns:
+            A properly formatted Boolean query with quoted terms that need quoting
+        """
+        # Define Boolean operators and their boundaries
+        boolean_pattern = r'(\bAND\b|\bOR\b|\bNOT\b)'
+        
+        # Split the query by Boolean operators, keeping the operators
+        parts = re.split(boolean_pattern, query)
+        
+        processed_parts = []
+        for part in parts:
+            part = part.strip()
+            if not part:
+                continue
+                
+            # If it's a Boolean operator, keep it as is
+            if part in ['AND', 'OR', 'NOT']:
+                processed_parts.append(part)
+            else:
+                # Handle parentheses specially - they should be preserved for grouping
+                if '(' in part or ')' in part:
+                    # Parse parenthetical expressions carefully
+                    processed_part = self._prepare_parenthetical_term(part)
+                    processed_parts.append(processed_part)
+                else:
+                    # This is a search term - for Boolean queries, don't add prefix wildcards
+                    prepared_term = self._prepare_single_term(part, is_prefix=False)
+                    processed_parts.append(prepared_term)
+        
+        return " ".join(processed_parts)
+    
+    def _prepare_parenthetical_term(self, term: str) -> str:
+        """Prepare a term that contains parentheses, preserving the parentheses for grouping.
+        
+        Args:
+            term: A term that may contain parentheses like "(hello" or "world)" or "(hello OR world)"
+            
+        Returns:
+            A properly formatted term with parentheses preserved
+        """
+        # Handle terms that start/end with parentheses but may contain quotable content
+        result = ""
+        i = 0
+        while i < len(term):
+            if term[i] in '()':
+                # Preserve parentheses as-is
+                result += term[i]
+                i += 1
+            else:
+                # Find the next parenthesis or end of string
+                start = i
+                while i < len(term) and term[i] not in '()':
+                    i += 1
+                
+                # Extract the content between parentheses
+                content = term[start:i].strip()
+                if content:
+                    # Only quote if it actually needs quoting (has hyphens, special chars, etc)
+                    # but don't quote if it's just simple words
+                    if self._needs_quoting(content):
+                        escaped_content = content.replace('"', '""')
+                        result += f'"{escaped_content}"'
+                    else:
+                        result += content
+        
+        return result
+    
+    def _needs_quoting(self, term: str) -> bool:
+        """Check if a term needs to be quoted for FTS5 safety.
+        
+        Args:
+            term: The term to check
+            
+        Returns:
+            True if the term should be quoted
+        """
+        if not term or not term.strip():
+            return False
+            
+        # Characters that indicate we should quote (excluding parentheses which are valid syntax)
+        needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/", "-", "'", '"', 
+                              "[", "]", "{", "}", "+", "!", "@", "#", "$", "%", "^", "&", 
+                              "=", "|", "\\", "~", "`"]
+        
+        return any(c in term for c in needs_quoting_chars)
+    
+    def _prepare_single_term(self, term: str, is_prefix: bool = True) -> str:
+        """Prepare a single search term (no Boolean operators).
+        
+        Args:
+            term: A single search term
             is_prefix: Whether to add prefix search capability (* suffix)
-
-        For FTS5:
-        - Boolean operators (AND, OR, NOT) are preserved for complex queries
-        - Terms with FTS5 special characters are quoted to prevent syntax errors
-        - Simple terms get prefix wildcards for better matching
+            
+        Returns:
+            A properly formatted single term
         """
-        # Check for explicit boolean operators - if present, return the term as is
-        boolean_operators = [" AND ", " OR ", " NOT "]
-        if any(op in f" {term} " for op in boolean_operators):
+        if not term or not term.strip():
             return term
-
+            
+        term = term.strip()
+        
         # Check if term is already a proper wildcard pattern (alphanumeric + *)
         # e.g., "hello*", "test*world" - these should be left alone
         if "*" in term and all(c.isalnum() or c in "*_-" for c in term):
@@ -218,6 +309,26 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
 
         return term
 
+    def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
+        """Prepare a search term for FTS5 query.
+
+        Args:
+            term: The search term to prepare
+            is_prefix: Whether to add prefix search capability (* suffix)
+
+        For FTS5:
+        - Boolean operators (AND, OR, NOT) are preserved for complex queries
+        - Terms with FTS5 special characters are quoted to prevent syntax errors
+        - Simple terms get prefix wildcards for better matching
+        """
+        # Check for explicit boolean operators - if present, process as Boolean query
+        boolean_operators = [" AND ", " OR ", " NOT "]
+        if any(op in f" {term} " for op in boolean_operators):
+            return self._prepare_boolean_query(term)
+
+        # For non-Boolean queries, use the single term preparation logic
+        return self._prepare_single_term(term, is_prefix)
+
     async def search(
         self,
         search_text: Optional[str] = None,
@@ -242,19 +353,10 @@ async def search(
                 # For wildcard searches, don't add any text conditions - return all results
                 pass
             else:
-                # Check for explicit boolean operators - only detect them in proper boolean contexts
-                has_boolean = any(op in f" {search_text} " for op in [" AND ", " OR ", " NOT "])
-
-                if has_boolean:
-                    # If boolean operators are present, use the raw query
-                    # No need to prepare it, FTS5 will understand the operators
-                    params["text"] = search_text
-                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
-                else:
-                    # Standard search with term preparation
-                    processed_text = self._prepare_search_term(search_text.strip())
-                    params["text"] = processed_text
-                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                # Use _prepare_search_term to handle both Boolean and non-Boolean queries
+                processed_text = self._prepare_search_term(search_text.strip())
+                params["text"] = processed_text
+                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
 
         # Handle title match search
         if title:

tests/repository/test_search_repository.py

@@ -329,6 +329,21 @@ def test_boolean_operators_preserved(self, search_repository):
             == "(hello AND world) OR test"
         )
 
+    def test_hyphenated_terms_with_boolean_operators(self, search_repository):
+        """Hyphenated terms with Boolean operators should be properly quoted."""
+        # Test the specific case from the GitHub issue
+        result = search_repository._prepare_search_term("tier1-test AND unicode")
+        assert result == '"tier1-test" AND unicode'
+        
+        # Test other hyphenated Boolean combinations
+        assert search_repository._prepare_search_term("multi-word OR single") == '"multi-word" OR single'
+        assert search_repository._prepare_search_term("well-formed NOT badly-formed") == '"well-formed" NOT "badly-formed"'
+        assert search_repository._prepare_search_term("test-case AND (hello OR world)") == '"test-case" AND (hello OR world)'
+        
+        # Test mixed special characters with Boolean operators
+        assert search_repository._prepare_search_term("config.json AND test-file") == '"config.json" AND "test-file"'
+        assert search_repository._prepare_search_term("C++ OR python-script") == '"C++" OR "python-script"'
+
     def test_programming_terms_should_work(self, search_repository):
         """Programming-related terms with special chars should be searchable."""
         # These should be quoted to handle special characters safely