Merge pull request #589 from evoskuil/master

Integration of hashmap sieve.

Author: evoskuil

include/bitcoin/database/impl/primitives/hashhead.ipp

@@ -115,7 +115,7 @@ TEMPLATE
 inline Link CLASS::index(const Key& key) const NOEXCEPT
 {
     using namespace system;
-    const auto index = possible_narrow_cast<link>(keys::hash<Key>(key));
+    const auto index = possible_narrow_cast<link>(keys::hash(key));
     return bit_and<link>(mask_, index);
 }
 
@@ -129,37 +129,28 @@ TEMPLATE
 inline Link CLASS::top(const Key& key) const NOEXCEPT
 {
     const auto value = get_cell(index(key));
-    if (is_collision(value, key))
+    if (screened(value, key))
         return to_link(value);
-    
+
+    // Conflict (body) search is bypassed by filter when key is not screened.
+    // If terminal here it is assured that table does not contain the key.
     return {};
 }
 
 TEMPLATE
 inline bool CLASS::push(const Link& current, bytes& next,
     const Key& key) NOEXCEPT
 {
-    return push(current, next, index(key));
-}
-
-TEMPLATE
-inline bool CLASS::push(const Link& current, bytes& next,
-    const Link& index) NOEXCEPT
-{
-    return set_cell(next, current, index) != terminal;
+    bool unused{};
+    return push(unused, current, next, key);
 }
 
 TEMPLATE
 inline bool CLASS::push(bool& collision, const Link& current, bytes& next,
     const Key& key) NOEXCEPT
 {
-    const auto previous = set_cell(next, current, index(key));
-    if (previous == terminal)
-        return false;
-
-    // Caller searches Link{ next } for duplicate in case of filter collision.
-    collision = is_collision(previous, key);
-    return true;
+    // next holds previous top and can searched for dups if collision is true.
+    return set_cell(collision, next, current, key);
 }
 
 // protected
@@ -181,31 +172,31 @@ inline CLASS::cell CLASS::get_cell(const Link& index) const NOEXCEPT
         ////const std::atomic_ref<cell> top(unsafe_byte_cast<cell>(raw));
         const auto& top = *pointer_cast<std::atomic<cell>>(raw);
 
-        // Acquire is necessary to synchronize with push release.
+        // Acquire is necessary to synchronize with set_cell release.
         // Relaxed would miss next updates, so acquire is optimal.
         return top.load(std::memory_order_acquire);
     }
     else
     {
-        cell top{};
-        const auto& head = cell_array(raw);
+        const auto& top = cell_array(raw);
+        cell head{};
 
         mutex_.lock_shared();
-        cell_array(top) = head;
+        cell_array(head) = top;
         mutex_.unlock_shared();
 
-        return top;
+        return head;
     }
 }
 
 TEMPLATE
-inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
-    const Link& index) NOEXCEPT
+inline bool CLASS::set_cell(bool& collision, bytes& next, const Link& current,
+    const Key& key) NOEXCEPT
 {
     using namespace system;
-    const auto raw = file_.get_raw(link_to_position(index));
+    const auto raw = file_.get_raw(link_to_position(index(key)));
     if (is_null(raw))
-        return terminal;
+        return false;
 
     if constexpr (aligned)
     {
@@ -214,28 +205,30 @@ inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
         ////const std::atomic_ref<cell> head(unsafe_byte_cast<cell>(raw));
         auto& top = *pointer_cast<std::atomic<cell>>(raw);
         auto head = top.load(std::memory_order_acquire);
+        cell update{};
         do
         {
             // Compiler could order this after top.store, which would expose key
             // to search before next entry is linked. Thread fence imposes order.
             // A release fence ensures that all prior writes (like next) are
             // completed before any subsequent atomic store.
             next = link_array(head);
+            update = to_cell(collision, head, current, key);
             std::atomic_thread_fence(std::memory_order_release);
         }
-        while (!top.compare_exchange_weak(head, to_cell(head, current),
+        while (!top.compare_exchange_weak(head, update,
             std::memory_order_release, std::memory_order_acquire));
     }
     else
     {
-        cell top{};
-        auto& head = cell_array(raw);
+        auto& top = cell_array(raw);
+        cell head{};
 
         mutex_.lock();
-        cell_array(top) = head;
-        next = link_array(top);
-        auto bytes = to_cell(top, current);
-        head = cell_array(bytes);
+        cell_array(head) = top;
+        next = link_array(head);
+        auto update = to_cell(collision, head, current, key);
+        top = cell_array(update);
         mutex_.unlock();
     }
 
@@ -247,31 +240,67 @@ inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
 // filters
 
 TEMPLATE
-constexpr CLASS::link CLASS::to_link(cell value) NOEXCEPT
+INLINE constexpr bool CLASS::screened(cell value, const Key& key) NOEXCEPT
 {
-    if (value == terminal)
-        return {};
+    if constexpr (sieve_t::disabled)
+    {
+        return true;
+    }
+    else
+    {
+        return sieve_t::is_screened(to_filter(value), fingerprint(key));
+    }
+}
 
+TEMPLATE
+INLINE constexpr CLASS::filter CLASS::fingerprint(const Key& key) NOEXCEPT
+{
     using namespace system;
-    constexpr auto mask = unmask_right<cell>(link_bits);
-    return possible_narrow_cast<link>(bit_and(value, mask));
+    return possible_narrow_cast<filter>(keys::thumb(key));
 }
 
 TEMPLATE
-constexpr CLASS::cell CLASS::to_cell(cell, link current) NOEXCEPT
+INLINE constexpr CLASS::filter CLASS::to_filter(cell value) NOEXCEPT
 {
-    // TODO:
-    return current;
+    using namespace system;
+    return possible_narrow_cast<filter>(shift_right(value, link_bits));
 }
 
 TEMPLATE
-constexpr bool CLASS::is_collision(cell value, const Key&) NOEXCEPT
+INLINE constexpr CLASS::link CLASS::to_link(cell value) NOEXCEPT
 {
-    if (value == terminal)
-        return false;
+    if constexpr (sieve_t::disabled)
+    {
+        return system::possible_narrow_cast<link>(value);
+    }
+    else
+    {
+        using namespace system;
+        if (value == terminal)
+            return {};
 
-    // TODO:
-    return true;
+        constexpr auto mask = unmask_right<cell>(link_bits);
+        return possible_narrow_cast<link>(bit_and(value, mask));
+    }
+}
+
+TEMPLATE
+INLINE constexpr CLASS::cell CLASS::to_cell(bool& collision, cell previous,
+    link current, const Key& key) NOEXCEPT
+{
+    if constexpr (sieve_t::disabled)
+    {
+        collision = true;
+        return current;
+    }
+    else
+    {
+        using namespace system;
+        const auto sieve = to_filter(previous);
+        const auto next = sieve_t::screen(sieve, fingerprint(key));
+        collision = (next == sieve || sieve_t::is_saturated(next));
+        return bit_or<cell>(shift_left<cell>(next, link_bits), current);
+    }
 }
 
 } // namespace database

include/bitcoin/database/impl/primitives/hashmap.ipp

@@ -373,11 +373,16 @@ ELEMENT_CONSTRAINT
 inline bool CLASS::put(bool& duplicate, const memory_ptr& ptr,
     const Link& link, const Key& key, const Element& element) NOEXCEPT
 {
-    Link previous_head{};
-    if (!write(previous_head, ptr, link, key, element))
+    Link previous{};
+    if (!write(previous, ptr, link, key, element))
         return false;
 
-    duplicate = !first(ptr, previous_head, key).is_terminal();
+    if (previous.is_terminal())
+        ncounter_.fetch_add(one, std::memory_order_relaxed);
+    else
+        pcounter_.fetch_add(one, std::memory_order_relaxed);
+
+    duplicate = !first(ptr, previous, key).is_terminal();
     return true;
 }
 
@@ -411,7 +416,7 @@ bool CLASS::commit(const memory_ptr& ptr, const Link& link,
 
     // Commit element to search index (terminal is a valid bucket index).
     auto& next = unsafe_array_cast<uint8_t, Link::size>(offset);
-    return head_.push(link, next, head_.index(key));
+    return head_.push(link, next, key);
 }
 
 // protected
@@ -509,7 +514,7 @@ bool CLASS::write(const memory_ptr& ptr, const Link& link, const Key& key,
     // Commit element to body and search (terminal is a valid bucket index).
     if constexpr (!is_slab) { BC_DEBUG_ONLY(sink.set_limit(RowSize * element.count());) }
     auto& next = unsafe_array_cast<uint8_t, Link::size>(offset);
-    return element.to_data(sink) && head_.push(link, next, head_.index(key));
+    return element.to_data(sink) && head_.push(link, next, key);
 }
 
 TEMPLATE
@@ -548,11 +553,15 @@ bool CLASS::write(Link& previous, const memory_ptr& ptr, const Link& link,
 
     // Commit element to search (terminal is a valid bucket index).
     bool collision{};
-    if (!head_.push(collision, link, next, head_.index(key)))
+    if (!head_.push(collision, link, next, key))
         return false;
 
     // If filter collision set previous stack head for conflict resolution.
-    previous = collision ? next : Link::terminal;
+    if (collision)
+        previous = next;
+    else
+        previous = Link::terminal;
+
     return true;
 }