Integration of hashmap sieve.

Author: evoskuil

include/bitcoin/database/impl/primitives/hashhead.ipp

@@ -129,36 +129,33 @@ TEMPLATE
 inline Link CLASS::top(const Key& key) const NOEXCEPT
 {
     const auto value = get_cell(index(key));
-    if (is_collision(value, key))
+    if (screened(value, key))
         return to_link(value);
-    
+
+    // Conflict (body) search is bypassed by filter when key is not screened.
+    // If terminal here it is assured that table does not contain the key.
     return {};
 }
 
 TEMPLATE
 inline bool CLASS::push(const Link& current, bytes& next,
     const Key& key) NOEXCEPT
 {
-    return push(current, next, index(key));
-}
-
-TEMPLATE
-inline bool CLASS::push(const Link& current, bytes& next,
-    const Link& index) NOEXCEPT
-{
-    return set_cell(next, current, index) != terminal;
+    bool unused{};
+    return push(unused, current, next, key);
 }
 
 TEMPLATE
 inline bool CLASS::push(bool& collision, const Link& current, bytes& next,
     const Key& key) NOEXCEPT
 {
-    const auto previous = set_cell(next, current, index(key));
+    const auto previous = set_cell(next, current, key);
     if (previous == terminal)
         return false;
 
-    // Caller searches Link{ next } for duplicate in case of filter collision.
-    collision = is_collision(previous, key);
+    // Conflict (body) search is bypassed by filter when key is not screened.
+    // If collision false it is assured that table does not contain the key.
+    collision = screened(previous, key);
     return true;
 }
 
@@ -200,10 +197,10 @@ inline CLASS::cell CLASS::get_cell(const Link& index) const NOEXCEPT
 
 TEMPLATE
 inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
-    const Link& index) NOEXCEPT
+    const Key& key) NOEXCEPT
 {
     using namespace system;
-    const auto raw = file_.get_raw(link_to_position(index));
+    const auto raw = file_.get_raw(link_to_position(index(key)));
     if (is_null(raw))
         return terminal;
 
@@ -223,7 +220,8 @@ inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
             next = link_array(head);
             std::atomic_thread_fence(std::memory_order_release);
         }
-        while (!top.compare_exchange_weak(head, to_cell(head, current),
+        while (!top.compare_exchange_weak(head,
+            to_cell(head, current, key),
             std::memory_order_release, std::memory_order_acquire));
     }
     else
@@ -234,7 +232,7 @@ inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
         mutex_.lock();
         cell_array(top) = head;
         next = link_array(top);
-        auto bytes = to_cell(top, current);
+        auto bytes = to_cell(top, current, key);
         head = cell_array(bytes);
         mutex_.unlock();
     }
@@ -247,31 +245,67 @@ inline CLASS::cell CLASS::set_cell(bytes& next, const Link& current,
 // filters
 
 TEMPLATE
-constexpr CLASS::link CLASS::to_link(cell value) NOEXCEPT
+INLINE constexpr bool CLASS::screened(cell value, const Key& key) NOEXCEPT
 {
-    if (value == terminal)
-        return {};
+    if constexpr (sieve::disabled)
+    {
+        return true;
+    }
+    else
+    {
+        return sieve{ to_filter(value) }.screened(fingerprint(key));
+    }
+}
 
+TEMPLATE
+INLINE constexpr CLASS::filter CLASS::fingerprint(const Key& key) NOEXCEPT
+{
     using namespace system;
-    constexpr auto mask = unmask_right<cell>(link_bits);
-    return possible_narrow_cast<link>(bit_and(value, mask));
+    return possible_narrow_cast<filter>(keys::thumb<Key>(key));
 }
 
 TEMPLATE
-constexpr CLASS::cell CLASS::to_cell(cell, link current) NOEXCEPT
+INLINE constexpr CLASS::filter CLASS::to_filter(cell value) NOEXCEPT
 {
-    // TODO:
-    return current;
+    using namespace system;
+    return possible_narrow_cast<filter>(shift_right(value, link_bits));
 }
 
 TEMPLATE
-constexpr bool CLASS::is_collision(cell value, const Key&) NOEXCEPT
+INLINE constexpr CLASS::link CLASS::to_link(cell value) NOEXCEPT
 {
-    if (value == terminal)
-        return false;
+    if constexpr (sieve::disabled)
+    {
+        return system::possible_narrow_cast<link>(value);
+    }
+    else
+    {
+        using namespace system;
+        if (value == terminal)
+            return {};
 
-    // TODO:
-    return true;
+        constexpr auto mask = unmask_right<cell>(link_bits);
+        return possible_narrow_cast<link>(bit_and(value, mask));
+    }
+}
+
+TEMPLATE
+INLINE constexpr CLASS::cell CLASS::to_cell(cell previous, link current,
+    const Key& key) NOEXCEPT
+{
+    if constexpr (sieve::disabled)
+    {
+        return current;
+    }
+    else
+    {
+        using namespace system;
+        static_assert(sizeof(filter) <= sizeof(cell));
+        sieve filter{ to_filter(previous) };
+        filter.screen(fingerprint(key));
+        const auto shifted = shift_left<cell>(filter.value(), link_bits);
+        return bit_or<cell>(shifted, current);
+    }
 }
 
 } // namespace database

include/bitcoin/database/impl/primitives/hashmap.ipp

@@ -411,7 +411,7 @@ bool CLASS::commit(const memory_ptr& ptr, const Link& link,
 
     // Commit element to search index (terminal is a valid bucket index).
     auto& next = unsafe_array_cast<uint8_t, Link::size>(offset);
-    return head_.push(link, next, head_.index(key));
+    return head_.push(link, next, key);
 }
 
 // protected
@@ -509,7 +509,7 @@ bool CLASS::write(const memory_ptr& ptr, const Link& link, const Key& key,
     // Commit element to body and search (terminal is a valid bucket index).
     if constexpr (!is_slab) { BC_DEBUG_ONLY(sink.set_limit(RowSize * element.count());) }
     auto& next = unsafe_array_cast<uint8_t, Link::size>(offset);
-    return element.to_data(sink) && head_.push(link, next, head_.index(key));
+    return element.to_data(sink) && head_.push(link, next, key);
 }
 
 TEMPLATE
@@ -548,7 +548,7 @@ bool CLASS::write(Link& previous, const memory_ptr& ptr, const Link& link,
 
     // Commit element to search (terminal is a valid bucket index).
     bool collision{};
-    if (!head_.push(collision, link, next, head_.index(key)))
+    if (!head_.push(collision, link, next, key))
         return false;
 
     // If filter collision set previous stack head for conflict resolution.

include/bitcoin/database/impl/primitives/sieve.ipp

@@ -22,8 +22,8 @@
 #include <bitcoin/system.hpp>
 #include <bitcoin/database/define.hpp>
 
-// [--------------------sieve-----------][--------------link--------------]
-// [[selector][--------screens---------]][--------------link--------------]
+// [------------------sieve-----------][--------------link--------------]
+// [[select][--------screens---------]][--------------link--------------]
 
 // [[111][1111111111111111111111111111]] empty/default
 // [[000][1111111111111111111111111111]] 1 screen
@@ -49,6 +49,10 @@
 namespace libbitcoin {
 namespace database {
 
+// Suppress bogus warnings to use constexpr when function is consteval.
+BC_PUSH_WARNING(USE_CONSTEXPR_FOR_FUNCTION)
+BC_PUSH_WARNING(NO_ARRAY_INDEXING)
+
 TEMPLATE
 constexpr CLASS::sieve() NOEXCEPT
   : sieve(empty)
@@ -87,87 +91,95 @@ constexpr CLASS::type CLASS::masks(size_t row, size_t column) const NOEXCEPT
 TEMPLATE
 constexpr bool CLASS::screened(type fingerprint) const NOEXCEPT
 {
-    using namespace system;
-    if constexpr (is_zero(limit))
+    if constexpr (disabled)
+    {
         return true;
-
-    const auto row = shift_right(sieve_, screen_bits);
-    if (row == limit)
+    }
+    else
     {
-        if (sieve_ == empty)
-            return false;
+        using namespace system;
+        const auto row = shift_right(sieve_, screen_bits);
+        if (row == limit)
+        {
+            if (sieve_ == empty)
+                return false;
 
-        if (sieve_ == saturated)
-            return true;
-    }
+            if (sieve_ == saturated)
+                return true;
+        }
 
-    // Compare masked fingerprint to masked sieve, for all masks of the screen.
-    for (type segment{}; segment <= row; ++segment)
-    {
-        const auto mask = masks(row, segment);
-        if (bit_and(fingerprint, mask) == bit_and(sieve_, mask))
-            return true;
-    }
+        // Compare masked fingerprint to sieve, for all masks of screen.
+        for (type segment{}; segment <= row; ++segment)
+        {
+            const auto mask = masks(row, segment);
+            if (bit_and(fingerprint, mask) == bit_and(sieve_, mask))
+                return true;
+        }
 
-    // Not empty, not saturated, not aligned with active screen (full if max).
-    return false;
+        // Not empty or saturated, not aligned with screen (full if max).
+        return false;
+    }
 }
 
 TEMPLATE
 constexpr bool CLASS::screen(type fingerprint) NOEXCEPT
 {
-    using namespace system;
-    if constexpr (is_zero(limit))
-        return false;
-
-    auto row = shift_right(sieve_, screen_bits);
-    if (row == limit)
+    if constexpr (disabled)
     {
-        if (sieve_ == empty)
-        {
-            // Reset empty sentinel (not screened) for first screen.
-            zeroize(row);
-        }
-        else
-        {
-            // Sieve was full, now saturated (all screened).
-            sieve_ = saturated;
-            return false;
-        }
+        return false;
     }
     else
     {
-        if (screened(fingerprint))
+        using namespace system;
+        auto row = shift_right(sieve_, screen_bits);
+        if (row == limit)
         {
-            // Screened, bucket may contain element.
-            return false;
+            if (sieve_ == empty)
+            {
+                // Reset empty sentinel (not screened) for first screen.
+                zeroize(row);
+            }
+            else
+            {
+                // Sieve was full, now saturated (all screened).
+                sieve_ = saturated;
+                return false;
+            }
         }
         else
         {
-            // Not screened, empty, or full - add screen.
-            ++row;
+            if (screened(fingerprint))
+            {
+                // Screened, bucket may contain element.
+                return false;
+            }
+            else
+            {
+                // Not screened, empty, or full - add screen.
+                ++row;
+            }
         }
-    }
 
-    // Both indexes are screen selector, as each new screen adds one mask.
-    const auto mask = masks(row, row);
+        // Both indexes are screen selector, as each new screen adds one mask.
+        const auto mask = masks(row, row);
 
-    // Merge incremented selector, current sieve, and new fingerprint.
-    sieve_ = bit_or
-    (
-        shift_left(row, screen_bits),
-        bit_and
+        // Merge incremented selector, current sieve, and new fingerprint.
+        sieve_ = bit_or
         (
-            selector_mask,
-            bit_or
+            shift_left(row, screen_bits),
+            bit_and
             (
-                bit_and(fingerprint, mask),
-                bit_and(sieve_, bit_not(mask))
+                selector_mask,
+                bit_or
+                (
+                    bit_and(fingerprint, mask),
+                    bit_and(sieve_, bit_not(mask))
+                )
             )
-        )
-    );
+        );
 
-    return true;
+        return true;
+    }
 }
 
 // protected/static
@@ -178,7 +190,7 @@ CONSTEVAL CLASS::offsets_t CLASS::generate_offsets() NOEXCEPT
 
     // Generate compression offsets at compile, generally 16 or 32 elements.
     offsets_t offsets{};
-    for (size_t index{}; index < screens; ++index)
+    for (type index{}; index < screens; ++index)
         offsets[index] = to_half(ceilinged_multiply(index, add1(index)));
 
     return offsets;
@@ -208,7 +220,11 @@ CONSTEVAL CLASS::masks_t CLASS::generate_masks() NOEXCEPT
         return div + to_int(column < mod);
     };
 
+    // Start with all screen bits set.
     masks(0, 0) = first_mask;
+
+    // Progressively divide previous row masks into subsequent row.
+    // Each row adds one mask, with previous row masks sacrificing bits for it.
     for (type row = 1; row < screens; ++row)
     {
         for (type col = 0; col < row; ++col)
@@ -233,9 +249,17 @@ CONSTEVAL CLASS::masks_t CLASS::generate_masks() NOEXCEPT
         }
     }
 
+    // Unset last row high order screen bit to avoid sentinel conflict.
+    // This may result in one empty mask, and therefore a false positive.
+    for (type col = 0; col < screens; ++col)
+        set_right_into(masks(sub1(screens), col), sentinel, false);
+
     return out;
 }
 
+BC_POP_WARNING()
+BC_POP_WARNING()
+
 } // namespace database
 } // namespace libbitcoin