refactor(cache): optimize SqliteDataCache with lazy deserialization and per-section tables

Replace single cache_entries table with per-section tables (libdoc,
variables, resource, namespace) for better query performance.

Introduce CacheEntry with lazy meta/data deserialization — data blobs
are only unpickled on cache hit, avoiding expensive deserialization on
meta mismatch.

Move version checking from per-entry meta_version field to DB-level
app_version parameter. On version mismatch all tables are dropped and
recreated automatically.

Simplify cache API from 3 methods (cache_data_exists, read_cache_data,
save_cache_data) to 2 (read_entry, save_entry), reducing DB
round-trips from 3 to 1 for reads and 2 to 1 for writes.

Remove filepath_base property (adler32 hash-based keys) from all
metadata classes — cache keys now use source paths or library names
directly.

Author: d-biehl

packages/robot/src/robotcode/robot/diagnostics/data_cache.py

@@ -1,13 +1,11 @@
 import pickle
 import sqlite3
-from abc import ABC, abstractmethod
 from enum import Enum
 from pathlib import Path
-from typing import Any, Tuple, Type, TypeVar, Union, cast
+from typing import Any, Generic, Optional, Tuple, Type, TypeVar, Union, cast
 
-from robotcode.core.utils.dataclasses import as_json, from_json
-
-_T = TypeVar("_T")
+_M = TypeVar("_M")
+_D = TypeVar("_D")
 
 
 class CacheSection(Enum):
@@ -17,90 +15,64 @@ class CacheSection(Enum):
     NAMESPACE = "namespace"
 
 
-class DataCache(ABC):
-    @abstractmethod
-    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool: ...
-
-    @abstractmethod
-    def read_cache_data(
-        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
-    ) -> _T: ...
-
-    @abstractmethod
-    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None: ...
-
-    def close(self) -> None:
-        pass
-
-
-class FileCacheDataBase(DataCache, ABC):
-    def __init__(self, cache_dir: Path) -> None:
-        self.cache_dir = cache_dir
-
-        if not Path.exists(self.cache_dir):
-            Path.mkdir(self.cache_dir, parents=True)
-            Path(self.cache_dir / ".gitignore").write_text(
-                "# Created by robotcode\n*\n",
-                "utf-8",
-            )
-
-
-class JsonDataCache(FileCacheDataBase):
-    def build_cache_data_filename(self, section: CacheSection, entry_name: str) -> Path:
-        return self.cache_dir / section.value / (entry_name + ".json")
-
-    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool:
-        cache_file = self.build_cache_data_filename(section, entry_name)
-        return cache_file.exists()
-
-    def read_cache_data(
-        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
-    ) -> _T:
-        cache_file = self.build_cache_data_filename(section, entry_name)
-        return from_json(cache_file.read_text("utf-8"), types)
-
-    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None:
-        cached_file = self.build_cache_data_filename(section, entry_name)
-
-        cached_file.parent.mkdir(parents=True, exist_ok=True)
-        cached_file.write_text(as_json(data), "utf-8")
-
-
-class PickleDataCache(FileCacheDataBase):
-    def build_cache_data_filename(self, section: CacheSection, entry_name: str) -> Path:
-        return self.cache_dir / section.value / (entry_name + ".pkl")
-
-    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool:
-        cache_file = self.build_cache_data_filename(section, entry_name)
-        return cache_file.exists()
-
-    def read_cache_data(
-        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
-    ) -> _T:
-        cache_file = self.build_cache_data_filename(section, entry_name)
-
-        with cache_file.open("rb") as f:
-            result = pickle.load(f)
-
-            if isinstance(result, types):
-                return cast(_T, result)
-
-            raise TypeError(f"Expected {types} but got {type(result)}")
-
-    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None:
-        cached_file = self.build_cache_data_filename(section, entry_name)
-
-        cached_file.parent.mkdir(parents=True, exist_ok=True)
-        with cached_file.open("wb") as f:
-            pickle.dump(data, f)
-
-
-class SqliteDataCache(DataCache):
-    """Cache backend using a single SQLite database with zlib-compressed pickle blobs."""
-
-    _SCHEMA_VERSION = 1
-
-    def __init__(self, cache_dir: Path) -> None:
+class CacheEntry(Generic[_M, _D]):
+    """Lazy-deserializing cache entry.
+
+    Meta and data blobs are deserialized on first property access, not when read from DB.
+    """
+
+    def __init__(
+        self,
+        meta_blob: Optional[bytes],
+        data_blob: bytes,
+        meta_type: Union[Type[_M], Tuple[Type[_M], ...]],
+        data_type: Union[Type[_D], Tuple[Type[_D], ...]],
+    ) -> None:
+        self._meta_blob = meta_blob
+        self._data_blob = data_blob
+        self._meta_type = meta_type
+        self._data_type = data_type
+        self._meta_cache: Optional[_M] = None
+        self._data_cache: Optional[_D] = None
+        self._meta_loaded = False
+        self._data_loaded = False
+
+    @property
+    def meta(self) -> Optional[_M]:
+        if not self._meta_loaded:
+            if self._meta_blob is not None:
+                result = pickle.loads(self._meta_blob)
+                if not isinstance(result, self._meta_type):
+                    raise TypeError(f"Expected {self._meta_type} but got {type(result)}")
+                self._meta_cache = cast(_M, result)
+            self._meta_loaded = True
+        return self._meta_cache
+
+    @property
+    def data(self) -> _D:
+        if not self._data_loaded:
+            result = pickle.loads(self._data_blob)
+            if not isinstance(result, self._data_type):
+                raise TypeError(f"Expected {self._data_type} but got {type(result)}")
+            self._data_cache = cast(_D, result)
+            self._data_loaded = True
+
+        assert self._data_cache is not None
+        return self._data_cache
+
+
+_TABLE_NAMES = [s.value for s in CacheSection]
+
+
+class SqliteDataCache:
+    """Cache backend using a single SQLite database with per-section tables.
+
+    Each CacheSection gets its own table with entry_name as PK, plus meta and data
+    BLOB columns. An app_version is stored in a metadata table; on version mismatch
+    all tables are dropped and recreated.
+    """
+
+    def __init__(self, cache_dir: Path, app_version: str = "") -> None:
         self.cache_dir = cache_dir
 
         if not cache_dir.exists():
@@ -116,46 +88,55 @@ def __init__(self, cache_dir: Path) -> None:
         self._conn.execute("PRAGMA synchronous=NORMAL")
         self._conn.execute("PRAGMA cache_size=-8000")
         self._conn.execute("PRAGMA mmap_size=67108864")
-        self._conn.execute(
-            "CREATE TABLE IF NOT EXISTS cache_entries ("
-            "  section TEXT NOT NULL,"
-            "  entry_name TEXT NOT NULL,"
-            "  data BLOB NOT NULL,"
-            "  PRIMARY KEY (section, entry_name)"
-            ")"
-        )
-        self._conn.commit()
 
-    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool:
-        row = self._conn.execute(
-            "SELECT 1 FROM cache_entries WHERE section = ? AND entry_name = ?",
-            (section.value, entry_name),
-        ).fetchone()
-        return row is not None
+        self._ensure_schema(app_version)
 
-    def read_cache_data(
-        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
-    ) -> _T:
-        row = self._conn.execute(
-            "SELECT data FROM cache_entries WHERE section = ? AND entry_name = ?",
-            (section.value, entry_name),
-        ).fetchone()
+    def _ensure_schema(self, app_version: str) -> None:
+        self._conn.execute("CREATE TABLE IF NOT EXISTS _meta (  key TEXT PRIMARY KEY,  value TEXT NOT NULL)")
 
-        if row is None:
-            raise FileNotFoundError(f"No cache entry for {section.value}/{entry_name}")
+        row = self._conn.execute("SELECT value FROM _meta WHERE key = 'app_version'").fetchone()
+        stored_version = row[0] if row else None
 
-        result = pickle.loads(row[0])
+        if stored_version != app_version:
+            for table in _TABLE_NAMES:
+                self._conn.execute(f"DROP TABLE IF EXISTS {table}")
+            self._conn.execute("INSERT OR REPLACE INTO _meta (key, value) VALUES ('app_version', ?)", (app_version,))
 
-        if isinstance(result, types):
-            return cast(_T, result)
+        for table in _TABLE_NAMES:
+            self._conn.execute(
+                f"CREATE TABLE IF NOT EXISTS {table} (  entry_name TEXT PRIMARY KEY,  meta BLOB,  data BLOB NOT NULL)"
+            )
+        self._conn.commit()
 
-        raise TypeError(f"Expected {types} but got {type(result)}")
+    def read_entry(
+        self,
+        section: CacheSection,
+        entry_name: str,
+        meta_type: Union[Type[_M], Tuple[Type[_M], ...]],
+        data_type: Union[Type[_D], Tuple[Type[_D], ...]],
+    ) -> Optional[CacheEntry[_M, _D]]:
+        row = self._conn.execute(
+            f"SELECT meta, data FROM {section.value} WHERE entry_name = ?",
+            (entry_name,),
+        ).fetchone()
 
-    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None:
-        blob = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
+        if row is None:
+            return None
+
+        return CacheEntry(row[0], row[1], meta_type, data_type)
+
+    def save_entry(
+        self,
+        section: CacheSection,
+        entry_name: str,
+        meta: Any,
+        data: Any,
+    ) -> None:
+        meta_blob = pickle.dumps(meta, protocol=pickle.HIGHEST_PROTOCOL) if meta is not None else None
+        data_blob = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
         self._conn.execute(
-            "INSERT OR REPLACE INTO cache_entries (section, entry_name, data) VALUES (?, ?, ?)",
-            (section.value, entry_name, blob),
+            f"INSERT OR REPLACE INTO {section.value} (entry_name, meta, data) VALUES (?, ?, ?)",
+            (entry_name, meta_blob, data_blob),
         )
         self._conn.commit()
 

packages/robot/src/robotcode/robot/diagnostics/document_cache_helper.py

@@ -482,37 +482,27 @@ def _try_load_cached_namespace(
         if not Path(source).exists():
             return None
 
-        # Compute filepath_base from source path
-        temp_filepath_base = NamespaceMetaData(
-            meta_version="",
-            source=source,
-            source_mtime_ns=0,
-            config_fingerprint="",
-        ).filepath_base
-
-        meta_file = temp_filepath_base + ".meta"
-        if not data_cache.cache_data_exists(CacheSection.NAMESPACE, meta_file):
-            return None
-
         try:
-            saved_meta = data_cache.read_cache_data(CacheSection.NAMESPACE, meta_file, NamespaceMetaData)
+            entry = data_cache.read_entry(CacheSection.NAMESPACE, source, NamespaceMetaData, NamespaceData)
         except (SystemExit, KeyboardInterrupt):
             raise
         except BaseException as e:
             ex = e
             self._logger.debug(
-                lambda: f"Failed to read namespace meta for {source}: {ex}",
+                lambda: f"Failed to read namespace cache for {source}: {ex}",
                 context_name="import",
             )
             return None
 
-        if not imports_manager.validate_namespace_meta(saved_meta):
+        if entry is None or entry.meta is None:
             return None
 
-        # Meta is valid — load the full NamespaceData
-        data_file = temp_filepath_base + ".data"
+        if not imports_manager.validate_namespace_meta(entry.meta):
+            return None
+
+        # Meta is valid — load the full NamespaceData (lazy deserialization)
         try:
-            namespace_data = data_cache.read_cache_data(CacheSection.NAMESPACE, data_file, NamespaceData)
+            namespace_data = entry.data
         except (SystemExit, KeyboardInterrupt):
             raise
         except BaseException as e:
@@ -564,11 +554,7 @@ def _save_namespace_to_cache(
             data = namespace.to_data()
 
             data_cache = imports_manager.data_cache
-            data_file = meta.filepath_base + ".data"
-            meta_file = meta.filepath_base + ".meta"
-
-            data_cache.save_cache_data(CacheSection.NAMESPACE, data_file, data)
-            data_cache.save_cache_data(CacheSection.NAMESPACE, meta_file, meta)
+            data_cache.save_entry(CacheSection.NAMESPACE, source, meta, data)
         except (SystemExit, KeyboardInterrupt):
             raise
         except BaseException as e: