Fix eviction flow and removeCb calls

Without this fix removeCb called even in case when Item is moved between
tiers.

Author: Sergei Vinogradov

cachelib/allocator/CacheAllocator-inl.h

@@ -1463,10 +1463,17 @@ CacheAllocator<CacheTrait>::findEviction(TierId tid, PoolId pid, ClassId cid) {
     // for chained items, the ownership of the parent can change. We try to
     // evict what we think as parent and see if the eviction of parent
     // recycles the child we intend to.
-    auto toReleaseHandle =
-        itr->isChainedItem()
-            ? advanceIteratorAndTryEvictChainedItem(tid, pid, itr)
-            : advanceIteratorAndTryEvictRegularItem(tid, pid, mmContainer, itr);
+    
+    ItemHandle toReleaseHandle = tryEvictToNextMemoryTier(tid, pid, itr);
+    bool movedToNextTier = false;
+    if(toReleaseHandle) {
+      movedToNextTier = true;
+    } else {
+      toReleaseHandle =
+          itr->isChainedItem()
+              ? advanceIteratorAndTryEvictChainedItem(tid, pid, itr)
+              : advanceIteratorAndTryEvictRegularItem(tid, pid, mmContainer, itr);
+    }
 
     if (toReleaseHandle) {
       if (toReleaseHandle->hasChainedItem()) {
@@ -1497,7 +1504,7 @@ CacheAllocator<CacheTrait>::findEviction(TierId tid, PoolId pid, ClassId cid) {
       // recycle the candidate.
       if (ReleaseRes::kRecycled ==
           releaseBackToAllocator(itemToRelease, RemoveContext::kEviction,
-                                 /* isNascent */ false, candidate)) {
+                                 /* isNascent */ movedToNextTier, candidate)) {
         return candidate;
       }
     }
@@ -1564,6 +1571,7 @@ template <typename ItemPtr>
 typename CacheAllocator<CacheTrait>::ItemHandle
 CacheAllocator<CacheTrait>::tryEvictToNextMemoryTier(
     TierId tid, PoolId pid, ItemPtr& item) {
+  if(item->isChainedItem()) return {}; // TODO: We do not support ChainedItem yet
   if(item->isExpired()) return acquire(item);
 
   TierId nextTier = tid; // TODO - calculate this based on some admission policy
@@ -1597,9 +1605,6 @@ template <typename CacheTrait>
 typename CacheAllocator<CacheTrait>::ItemHandle
 CacheAllocator<CacheTrait>::advanceIteratorAndTryEvictRegularItem(
     TierId tid, PoolId pid, MMContainer& mmContainer, EvictionIterator& itr) {
-  auto evictHandle = tryEvictToNextMemoryTier(tid, pid, itr);
-  if(evictHandle) return evictHandle;
-
   Item& item = *itr;
 
   const bool evictToNvmCache = shouldWriteToNvmCache(item);
@@ -1618,7 +1623,7 @@ CacheAllocator<CacheTrait>::advanceIteratorAndTryEvictRegularItem(
   // if we remove the item from both access containers and mm containers
   // below, we will need a handle to ensure proper cleanup in case we end up
   // not evicting this item
-  evictHandle = accessContainer_->removeIf(item, &itemEvictionPredicate);
+  auto evictHandle = accessContainer_->removeIf(item, &itemEvictionPredicate);
 
   if (!evictHandle) {
     ++itr;