Merge pull request #148 from Dooders/dev

Dev

Author: csmangum

converter/config.py

@@ -35,7 +35,7 @@ class ConverterConfig:
     memory_type_mapper: Optional[MemoryTypeMapper] = None
 
     # Tiering strategy
-    tiering_strategy_type: str = "step_based"  # One of: "step_based", "importance_aware"
+    tiering_strategy_type: str = "simple"  # One of: "simple", "step_based", "importance_aware"
     tiering_strategy: Optional[TieringStrategy] = None
 
     # Import settings
@@ -81,7 +81,7 @@ def _validate_batch_size(self):
 
     def _validate_tiering_strategy(self):
         """Validate tiering strategy settings."""
-        valid_types = ["step_based", "importance_aware"]
+        valid_types = ["simple", "step_based", "importance_aware"]
         if self.tiering_strategy_type not in valid_types:
             raise ValueError(
                 f"Invalid tiering_strategy_type: {self.tiering_strategy_type}. "
@@ -100,7 +100,7 @@ def _validate_tiering_strategy(self):
         'ActionModel': 'action',
         'SocialInteractionModel': 'interaction'
     },
-    'tiering_strategy_type': 'step_based',
+    'tiering_strategy_type': 'simple',
     'import_mode': 'full',
     'selective_agents': None
 } 

converter/converter.py

@@ -7,7 +7,7 @@
 
 from sqlalchemy.exc import SQLAlchemyError
 
-from memory.config import MemoryConfig
+from memory.config import MemoryConfig, RedisSTMConfig
 from memory.core import AgentMemorySystem
 
 from .agent_import import AgentImporter, AgentMetadata
@@ -102,7 +102,15 @@ def from_agent_farm(db_path: str, config: Optional[Dict] = None) -> AgentMemoryS
         logger.info(f"Successfully imported {len(all_memories)} total memories")
 
         # Create memory system configuration
-        memory_config = MemoryConfig(use_mock_redis=True, logging_level="INFO")
+        memory_config = MemoryConfig(
+            use_mock_redis=True,
+            logging_level="INFO",
+            stm_config=RedisSTMConfig(
+                memory_limit=10000,  # Increase STM memory limit
+                ttl=86400,  # 24 hours
+                namespace="agent-stm",
+            ),
+        )
 
         # Create and configure memory system
         memory_system = AgentMemorySystem.get_instance(memory_config)

converter/memory_import.py

@@ -265,14 +265,14 @@ def _import_memory(
         )
         tier = self.tiering_strategy.determine_tier(tiering_context)
 
-        # Get the correct ID field based on model type
+        # Get the correct ID field based on model type and ensure uniqueness
         memory_id = None
         if model_name == 'ActionModel':
-            memory_id = getattr(memory, 'action_id', None)
+            memory_id = f"action_{getattr(memory, 'action_id', None)}_step_{getattr(memory, 'step_number', 0)}"
         elif model_name == 'SocialInteractionModel':
-            memory_id = getattr(memory, 'interaction_id', None)
+            memory_id = f"interaction_{getattr(memory, 'interaction_id', None)}_step_{getattr(memory, 'step_number', 0)}"
         elif model_name == 'AgentStateModel':
-            memory_id = getattr(memory, 'id', None)
+            memory_id = f"state_{getattr(memory, 'id', None)}_step_{getattr(memory, 'step_number', 0)}"
 
         if memory_id is None:
             logger.warning(f"Could not find ID for {model_name} memory")

converter/tiering.py

@@ -31,6 +31,15 @@ def determine_tier(self, context: TieringContext) -> str:
         """
         pass
 
+class SimpleTieringStrategy(TieringStrategy):
+    """
+    Simple tiering strategy that puts all memories in STM.
+    """
+    
+    def determine_tier(self, context: TieringContext) -> str:
+        """Always return STM tier."""
+        return "stm"
+
 class StepBasedTieringStrategy(TieringStrategy):
     """
     Step-based time decay tiering strategy.
@@ -86,12 +95,12 @@ def determine_tier(self, context: TieringContext) -> str:
 
         return base_tier
 
-def create_tiering_strategy(strategy_type: str = "step_based") -> TieringStrategy:
+def create_tiering_strategy(strategy_type: str = "simple") -> TieringStrategy:
     """
     Factory function to create tiering strategies.
     
     Args:
-        strategy_type: Type of strategy to create ("step_based" or "importance_aware")
+        strategy_type: Type of strategy to create ("simple", "step_based", or "importance_aware")
         
     Returns:
         Configured TieringStrategy instance
@@ -100,6 +109,7 @@ def create_tiering_strategy(strategy_type: str = "step_based") -> TieringStrateg
         ValueError: If strategy_type is invalid
     """
     strategies = {
+        "simple": SimpleTieringStrategy,
         "step_based": StepBasedTieringStrategy,
         "importance_aware": ImportanceAwareTieringStrategy
     }

debug_converter.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+"""
+Script to run the AgentFarm to Memory System converter.
+"""
+
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from converter.config import DEFAULT_CONFIG
+from converter.converter import from_agent_farm
+
+def main():
+    db_path = "data/simulation.db"
+    output = "validation/memory_samples/agent_farm_memories.json"
+    validate = False
+    error_handling = "skip"
+    use_mock_redis = True
+    batch_size = 100
+    tiering_strategy = "simple"
+    log_file = None
+
+    # Create logs directory if it doesn't exist
+    log_dir = Path("logs")
+    log_dir.mkdir(exist_ok=True)
+
+    # Generate default log filename if not provided
+    if not log_file:
+        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        log_file = log_dir / f"converter_{timestamp}.log"
+
+    # Configure logging
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=[
+            logging.FileHandler(log_file),
+            logging.StreamHandler(sys.stdout),
+        ],
+        force=True,
+    )
+    logger = logging.getLogger(__name__)
+
+    # Set all SQLAlchemy loggers to WARNING level to suppress most logs
+    logging.getLogger("sqlalchemy").setLevel(logging.WARNING)
+    logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
+    logging.getLogger("sqlalchemy.orm").setLevel(logging.WARNING)
+    logging.getLogger("sqlalchemy.pool").setLevel(logging.WARNING)
+
+    # Set all other loggers to DEBUG level
+    for name in logging.root.manager.loggerDict:
+        if not name.startswith("sqlalchemy"):
+            logging.getLogger(name).setLevel(logging.DEBUG)
+
+    logger.info(f"Logging to file: {log_file}")
+
+    # Prepare configuration
+    config = DEFAULT_CONFIG.copy()
+    config.update(
+        {
+            "validate": validate,
+            "error_handling": error_handling,
+            "use_mock_redis": True,
+            "batch_size": batch_size,
+            "tiering_strategy_type": tiering_strategy,
+        }
+    )
+
+    # Run converter
+    logger.info(f"Converting database: {db_path}")
+    memory_system = from_agent_farm(db_path, config)
+    
+    return memory_system
+
+if __name__ == "__main__":
+    memory_system = main()
+    agent = memory_system.agents['nWpvyFJReoFD5Fnq7AEggt']
+    stats = agent.get_memory_statistics('nWpvyFJReoFD5Fnq7AEggt')
+    print(f"STM count: {stats['tiers']['stm']['count']}")
+    print(f"IM count: {stats['tiers']['im']['count']}")
+    print(f"LTM count: {stats['tiers']['ltm']['count']}")

memory/agent_memory.py

@@ -3,6 +3,7 @@
 import logging
 import math
 import time
+import uuid
 from typing import Any, Dict, List, Optional, Union
 
 from memory.config import MemoryConfig
@@ -220,7 +221,7 @@ def _create_memory_entry(
         Returns:
             Formatted memory entry
         """
-        # Generate unique memory ID
+        # Generate memory ID using agent_id, step number and timestamp
         timestamp = int(time.time())
         memory_id = f"{self.agent_id}-{step_number}-{timestamp}"
 

memory/core.py

@@ -360,24 +360,27 @@ def add_memory(self, memory_data: Dict[str, Any]) -> str:
         step_number = memory_data.get("step_number", 0)
         priority = memory_data.get("metadata", {}).get("importance_score", 1.0)
         memory_type = memory_data.get("type", "generic")
+        tier = memory_data.get("metadata", {}).get(
+            "tier", "stm"
+        )  # Get tier from metadata
 
         # Choose appropriate method based on memory type
         if memory_type == "state":
             memory_agent.store_state(
-                memory_data.get("content", {}), step_number, priority
+                memory_data.get("content", {}), step_number, priority, tier
             )
         elif memory_type == "interaction":
             memory_agent.store_interaction(
-                memory_data.get("content", {}), step_number, priority
+                memory_data.get("content", {}), step_number, priority, tier
             )
         elif memory_type == "action":
             memory_agent.store_action(
-                memory_data.get("content", {}), step_number, priority
+                memory_data.get("content", {}), step_number, priority, tier
             )
         else:
             # For generic types, use store_state as a fallback
             memory_agent.store_state(
-                memory_data.get("content", {}), step_number, priority
+                memory_data.get("content", {}), step_number, priority, tier
             )
 
         return memory_id

memory/search/strategies/similarity.py

@@ -99,8 +99,11 @@ def search(
         tiers_to_search = ["stm", "im", "ltm"] if tier is None else [tier]
         logger.debug("Searching memory tiers: %s", tiers_to_search)
 
-        # Get the memory agent
-        memory_agent = self.memory_system.get_memory_agent(agent_id)
+        if isinstance(self.memory_system, AgentMemorySystem):
+            # Get the memory agent
+            memory_agent = self.memory_system.get_memory_agent(agent_id)
+        else:
+            memory_agent = self.memory_system
 
         for current_tier in tiers_to_search:
             # Skip if tier is not supported
@@ -221,6 +224,10 @@ def search(
                             )
                             continue
 
+                    # Ensure memory has required fields
+                    if "id" not in memory:
+                        memory["id"] = memory_id
+                    
                     # Attach similarity score and tier information
                     if "metadata" not in memory:
                         memory["metadata"] = {}

tests/converter/test_config.py

@@ -4,7 +4,7 @@
 
 import pytest
 from converter.config import ConverterConfig
-from converter.tiering import StepBasedTieringStrategy, ImportanceAwareTieringStrategy
+from converter.tiering import SimpleTieringStrategy, StepBasedTieringStrategy, ImportanceAwareTieringStrategy
 from converter.mapping import MemoryTypeMapper
 
 def test_default_config():
@@ -17,8 +17,8 @@ def test_default_config():
     assert config.show_progress is True
     assert config.import_mode == "full"
     assert config.selective_agents is None
-    assert config.tiering_strategy_type == "step_based"
-    assert isinstance(config.tiering_strategy, StepBasedTieringStrategy)
+    assert config.tiering_strategy_type == "simple"
+    assert isinstance(config.tiering_strategy, SimpleTieringStrategy)
     assert isinstance(config.memory_type_mapper, MemoryTypeMapper)
     assert config.memory_type_mapping == {
         'AgentStateModel': 'state',

tests/test_memory_agent.py

@@ -28,7 +28,7 @@
 import pytest
 
 from memory.agent_memory import MemoryAgent
-from memory.config import AutoencoderConfig, MemoryConfig
+from memory.config import MemoryConfig
 
 
 @pytest.fixture
@@ -98,24 +98,25 @@ def memory_agent(
     agent_id = "test-agent"
     config = MemoryConfig()
     config.autoencoder_config.use_neural_embeddings = True
-    config.text_model_name = "test-model"
+    config.text_model_name = (
+        "sentence-transformers/all-MiniLM-L6-v2"  # Use a real model
+    )
     config.ltm_config.db_path = "test_memory.db"  # Set a valid db path
 
-    # Mock store classes before instantiating the agent
-    with mock.patch("memory.agent_memory.RedisSTMStore") as mock_stm_class, mock.patch(
+    with mock.patch("memory.agent_memory.RedisSTMStore") as mock_stm, mock.patch(
         "memory.agent_memory.RedisIMStore"
-    ) as mock_im_class, mock.patch(
+    ) as mock_im, mock.patch(
         "memory.agent_memory.SQLiteLTMStore"
-    ) as mock_ltm_class, mock.patch(
+    ) as mock_ltm, mock.patch(
         "memory.agent_memory.CompressionEngine"
-    ), mock.patch(
-        "memory.embeddings.text_embeddings.TextEmbeddingEngine"
-    ):
+    ) as mock_ce, mock.patch(
+        "memory.agent_memory.TextEmbeddingEngine"
+    ) as mock_ae:
 
         # Configure the mock classes to return our mock instances
-        mock_stm_class.return_value = mock_stm_store
-        mock_im_class.return_value = mock_im_store
-        mock_ltm_class.return_value = mock_ltm_store
+        mock_stm.return_value = mock_stm_store
+        mock_im.return_value = mock_im_store
+        mock_ltm.return_value = mock_ltm_store
 
         agent = MemoryAgent(agent_id, config)
 
@@ -147,7 +148,7 @@ def test_init(self):
         ) as mock_ltm, mock.patch(
             "memory.agent_memory.CompressionEngine"
         ) as mock_ce, mock.patch(
-            "memory.embeddings.text_embeddings.TextEmbeddingEngine"
+            "memory.agent_memory.TextEmbeddingEngine"
         ) as mock_ae:
 
             agent = MemoryAgent(agent_id, config)
@@ -174,7 +175,7 @@ def test_init_without_neural_embeddings(self):
         ), mock.patch("memory.agent_memory.SQLiteLTMStore"), mock.patch(
             "memory.agent_memory.CompressionEngine"
         ), mock.patch(
-            "memory.embeddings.text_embeddings.TextEmbeddingEngine"
+            "memory.agent_memory.TextEmbeddingEngine"
         ) as mock_te:
 
             agent = MemoryAgent(agent_id, config)