Merge pull request #295 from fgcz/rel-20250919_1

Release 1.13.34

Author: leoschwarz

.github/workflows/pr_release_preview.yml

@@ -40,7 +40,7 @@ jobs:
           echo -e "$table" >> $GITHUB_OUTPUT
           echo "TABLE_EOF" >> $GITHUB_OUTPUT
       - name: Create or update PR comment
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const packages = JSON.parse('${{ steps.check-versions.outputs.packages-to-release }}');

.github/workflows/publish_release.yml

@@ -58,7 +58,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.11'
       - name: Install hatch

.github/workflows/run_unit_tests.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.11
       - name: Install nox
@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.11
       - name: Install nox
@@ -49,7 +49,7 @@ jobs:
           head-ref: ${{ github.sha }}
       - name: Comment on PR with "TODO" changes
         if: steps.todo-diff.outputs.has-changes == 'true'
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const summary = `${{ steps.todo-diff.outputs.summary }}`;

bfabric/docs/changelog.md

@@ -10,6 +10,18 @@ Versioning currently follows `X.Y.Z` where
 
 ## \[Unreleased\]
 
+## \[1.13.34\] - 2025-09-19
+
+### Added
+
+- `bfabric.experimental.cache` which implements re-entrant lookup caching for entities (when retrieved by ID).
+- Fields `filename`, `storage_relative_path`, `storage_absolute_path` have been added to `Resource`. This should be used
+    in the future, to ensure path handling is performed consistently
+
+### Removed
+
+- `bfabric.experimental.entity_lookup_cache` has been removed in favor of the new (experimental) API.
+
 ## \[1.13.33\] - 2025-08-26
 
 ### Added

bfabric/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "bfabric"
 description = "Python client for the B-Fabric API"
 readme = "../README.md"
-version = "1.13.33"
+version = "1.13.34"
 license = { text = "GPL-3.0" }
 authors = [
     { name = "Christian Panse", email = "cp@fgcz.ethz.ch" },

bfabric/src/bfabric/entities/core/entity.py

@@ -6,7 +6,7 @@
 from loguru import logger
 
 from bfabric.experimental import MultiQuery
-from bfabric.experimental.entity_lookup_cache import EntityLookupCache
+from bfabric.experimental.cache.context import get_cache_stack
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -47,41 +47,41 @@ def _client(self) -> Bfabric | None:
     @classmethod
     def find(cls, id: int, client: Bfabric) -> Self | None:
         """Finds an entity by its ID, if it does not exist `None` is returned."""
-        cache = EntityLookupCache.instance()
-        if cache and cache.contains(entity_type=cls, entity_id=id):
-            return cache.get(entity_type=cls, entity_id=id)
-        else:
-            result = client.read(cls.ENDPOINT, obj={"id": int(id)})
-            entity = cls(result[0], client=client) if len(result) == 1 else None
-            if cache:
-                cache.put(entity_type=cls, entity_id=id, entity=entity)
-            return entity
+        cache_stack = get_cache_stack()
+        cache_entry = cache_stack.item_get(entity_type=cls, entity_id=id)
+        if cache_entry:
+            return cache_entry
+
+        result = client.read(cls.ENDPOINT, obj={"id": int(id)})
+        entity = cls(result[0], client=client) if len(result) == 1 else None
+        cache_stack.item_put(entity_type=cls, entity_id=id, entity=entity)
+        return entity
 
     @classmethod
     def find_all(cls, ids: list[int], client: Bfabric) -> dict[int, Self]:
         """Returns a dictionary of entities with the given IDs. The order will generally match the input, however,
         if some entities are not found they will be omitted and a warning will be logged.
         """
-        cache = EntityLookupCache.instance()
+        cache_stack = get_cache_stack()
         ids_requested = cls.__check_ids_list(ids)
 
         # retrieve entities from cache and from B-Fabric as needed
-        results_cached = cache.get_all(entity_type=cls, entity_ids=ids) if cache else {}
+        results_cached = cache_stack.item_get_all(entity_type=cls, entity_ids=ids)
         results_fresh = cls.__retrieve_entities(
             client=client, ids_requested=ids_requested, ids_cached=results_cached.keys()
         )
 
-        if cache:
-            for entity_id, entity in results_fresh.items():
-                cache.put(entity_type=cls, entity_id=entity_id, entity=entity)
-
+        cache_stack.item_put_all(entity_type=cls, entities=results_fresh)
         return cls.__ensure_results_order(ids_requested, results_cached, results_fresh)
 
     @classmethod
     def find_by(cls, obj: dict[str, Any], client: Bfabric, max_results: int | None = 100) -> dict[int, Self]:
         """Returns a dictionary of entities that match the given query."""
         result = client.read(cls.ENDPOINT, obj=obj, max_results=max_results)
-        return {x["id"]: cls(x, client=client) for x in result}
+        cache_stack = get_cache_stack()
+        entities = {x["id"]: cls(x, client=client) for x in result}
+        cache_stack.item_put_all(entity_type=cls, entities=entities)
+        return entities
 
     def dump_yaml(self, path: Path) -> None:
         """Writes the entity's data dictionary to a YAML file."""

bfabric/src/bfabric/entities/resource.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from pathlib import Path
 from typing import Any, TYPE_CHECKING
 
 from bfabric.entities.core.entity import Entity
@@ -21,3 +22,20 @@ def __init__(self, data_dict: dict[str, Any], client: Bfabric | None = None) ->
     storage: HasOne[Storage] = HasOne("Storage", bfabric_field="storage")
     workunit: HasOne[Workunit] = HasOne("Workunit", bfabric_field="workunit")
     sample: HasOne[Sample] = HasOne("Sample", bfabric_field="sample", optional=True)
+
+    @property
+    def storage_relative_path(self) -> Path:
+        """Returns the relative path of the resource in the storage as a Path object."""
+        return Path(self["relativepath"].lstrip("/"))
+
+    @property
+    def storage_absolute_path(self) -> Path:
+        """Returns the absolute path of the resource in the storage as a Path object."""
+        return Path(self.storage.base_path) / self.storage_relative_path
+
+    @property
+    def filename(self) -> str:
+        """Returns the filename of the actual path, i.e. not necessarily the resource name but rather the name
+        as the file is stored.
+        """
+        return self["relativepath"].split("/")[-1]

bfabric/src/bfabric/experimental/cache/__init__.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from bfabric.entities.core.entity import Entity
+    from bfabric.experimental.cache._entity_memory_cache import EntityMemoryCache
+
+
+class CacheStack:
+    """Implements a stack of entity caches.
+
+    Retrieval is performed by looking first at the most recently added cache.
+    Items are put into all caches in the stack (they may each have different sizes and inclusion rules) so that nesting
+    contexts has no effect on cache hits.
+    """
+
+    def __init__(self) -> None:
+        self._stack: list[EntityMemoryCache] = []
+
+    def cache_push(self, cache: EntityMemoryCache) -> None:
+        self._stack.append(cache)
+
+    def cache_pop(self) -> None:
+        self._stack.pop()
+
+    def item_contains(self, entity_type: type[Entity], entity_id: int) -> bool:
+        return any(cache.contains(entity_type, entity_id) for cache in reversed(self._stack))
+
+    def item_get(self, entity_type: type[Entity], entity_id: int) -> Entity | None:
+        for cache in reversed(self._stack):
+            entity = cache.get(entity_type, entity_id)
+            if entity is not None:
+                return entity
+        return None
+
+    def item_get_all(self, entity_type: type[Entity], entity_ids: list[int]) -> dict[int, Entity]:
+        results = {}
+        pending = set(entity_ids)
+        for cache in reversed(self._stack):
+            if not pending:
+                break
+            matches = cache.get_all(entity_type, list(pending))
+            results.update(matches)
+            pending.difference_update(matches)
+        return results
+
+    def item_put(self, entity_type: type[Entity], entity_id: int, entity: Entity | None) -> None:
+        for cache in reversed(self._stack):
+            cache.put(entity_type, entity_id, entity)
+
+    def item_put_all(self, entity_type: type[Entity], entities: dict[int, Entity | None]) -> None:
+        for entity_id, entity in entities.items():
+            self.item_put(entity_type, entity_id, entity)

bfabric/src/bfabric/experimental/cache/_cache_stack.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from typing import TypeVar, TYPE_CHECKING
+
+from loguru import logger
+
+from bfabric.experimental.cache._fifo_cache import FifoCache
+
+if TYPE_CHECKING:
+    from bfabric.entities.core.entity import Entity
+
+    E = TypeVar("E", bound="Entity")
+
+
+class EntityMemoryCache:
+    """Implements a configurable cache for different entity types in memory.
+
+    Only entity types specified in the config will be cached, and at most the specified number of entities will be
+    cached for each type.
+    """
+
+    def __init__(self, config: dict[type[Entity], int]) -> None:
+        self._config = config
+        self._caches = {entity_type: FifoCache(max_size=max_size) for entity_type, max_size in config.items()}
+
+    def contains(self, entity_type: type[Entity], entity_id: int) -> bool:
+        """Returns whether the cache contains an entity with the given type and ID."""
+        if entity_type not in self._caches:
+            return False
+        else:
+            return entity_id in self._caches[entity_type]
+
+    def get(self, entity_type: type[E], entity_id: int) -> E | None:
+        """Returns the entity with the given type and ID, if it exists in the cache."""
+        if entity_type not in self._caches:
+            return None
+
+        if self._caches[entity_type].get(entity_id):
+            logger.debug(f"Cache hit for entity {entity_type} with ID {entity_id}")
+            return self._caches[entity_type].get(entity_id)
+        else:
+            logger.debug(f"Cache miss for entity {entity_type} with ID {entity_id}")
+            return None
+
+    def get_all(self, entity_type: type[Entity], entity_ids: list[int]) -> dict[int, Entity]:
+        """Returns a dictionary of entities with the given type and IDs,
+        containing only the entities that exist in the cache.
+        """
+        results = {entity_id: self.get(entity_type, entity_id) for entity_id in entity_ids}
+        return {entity_id: result for entity_id, result in results.items() if result is not None}
+
+    def put(self, entity_type: type[Entity], entity_id: int, entity: Entity | None) -> None:
+        """Puts an entity with the given type and ID into the cache."""
+        if entity_type not in self._caches:
+            return
+
+        logger.debug(f"Caching entity {entity_type} with ID {entity_id}")
+        self._caches[entity_type].put(entity_id, entity)