Merge pull request #575 from evoskuil/master

Implement lock-free hashmap and arraymap heads.

Author: evoskuil

include/bitcoin/database/impl/primitives/arrayhead.ipp

@@ -55,17 +55,7 @@ bool CLASS::enabled() const NOEXCEPT
 TEMPLATE
 inline Link CLASS::index(size_t key) const NOEXCEPT
 {
-    // buckets a table lock via file.size().
-    ////if (key >= buckets()) return {};
-    ////BC_ASSERT_MSG(key < buckets(), "index overflow");
-
-    // Put index does not validate, allowing for head expansion.
-    return putter_index(key);
-}
-
-TEMPLATE
-inline Link CLASS::putter_index(size_t key) const NOEXCEPT
-{
+    // Does not validate, allowing for head expansion.
     // Key is the logical bucket index (no-hash).
     return body::cast_link(key);
 }
@@ -109,27 +99,29 @@ TEMPLATE
 bool CLASS::get_body_count(Link& count) const NOEXCEPT
 {
     const auto ptr = file_.get();
-    if (!ptr || Link::size > size())
+    if (!ptr || size_ > size())
         return false;
 
-    count = array_cast<Link::size>(ptr->data());
+    count = to_array<Link::size>(ptr->data());
     return true;
 }
 
 TEMPLATE
 bool CLASS::set_body_count(const Link& count) NOEXCEPT
 {
     const auto ptr = file_.get();
-    if (!ptr || Link::size > size())
+    if (!ptr || size_ > size())
         return false;
 
-    array_cast<Link::size>(ptr->data()) = count;
+    // If head is padded then last bytes are fill (0xff).
+    to_array<Link::size>(ptr->data()) = count;
     return true;
 }
 
 TEMPLATE
 Link CLASS::at(size_t key) const NOEXCEPT
 {
+    using namespace system;
     const auto link = index(key);
     if (link.is_terminal())
         return {};
@@ -138,30 +130,55 @@ Link CLASS::at(size_t key) const NOEXCEPT
     if (is_null(ptr))
         return {};
 
-    const auto& head = array_cast<Link::size>(ptr->data());
-
-    mutex_.lock_shared();
-    const auto top = head;
-    mutex_.unlock_shared();
-    return top;
+    if constexpr (Align)
+    {
+        // Reads full padded word.
+        const auto raw = ptr->data();
+        // xcode clang++16 does not support C++20 std::atomic_ref.
+        ////const std::atomic_ref<integer> head(unsafe_byte_cast<integer>(raw));
+        const auto& head = *pointer_cast<std::atomic<integer>>(raw);
+        return head.load(std::memory_order_acquire);
+    }
+    else
+    {
+        const auto& head = to_array<size_>(ptr->data());
+        mutex_.lock_shared();
+        const auto top = head;
+        mutex_.unlock_shared();
+        return top;
+    }
 }
 
 TEMPLATE
-bool CLASS::push(const bytes& current, const Link& index) NOEXCEPT
+bool CLASS::push(const Link& link, const Link& index) NOEXCEPT
 {
-    constexpr auto fill = system::bit_all<uint8_t>;
+    using namespace system;
+    constexpr auto fill = bit_all<uint8_t>;
 
     // Allocate as necessary and fill allocations.
-    const auto ptr = file_.set(link_to_position(index), Link::size, fill);
-
+    const auto ptr = file_.set(link_to_position(index), size_, fill);
     if (is_null(ptr))
         return false;
 
-    auto& head = array_cast<Link::size>(ptr->data());
+    if constexpr (Align)
+    {
+        // Writes full padded word (0x00 fill).
+        const auto raw = ptr->data();
+        // xcode clang++16 does not support C++20 std::atomic_ref.
+        ////const std::atomic_ref<integer> head(unsafe_byte_cast<integer>(raw));
+        auto& head = *pointer_cast<std::atomic<integer>>(raw);
+        head.store(link, std::memory_order_acq_rel);
+    }
+    else
+    {
+        bytes current = link;
+        auto& head = to_array<size_>(ptr->data());
+
+        mutex_.lock();
+        head = std::move(current);
+        mutex_.unlock();
+    }
 
-    mutex_.lock();
-    head = current;
-    mutex_.unlock();
     return true;
 }
 

include/bitcoin/database/impl/primitives/arraymap.ipp

@@ -171,7 +171,7 @@ bool CLASS::put(size_t key, const Element& element) NOEXCEPT
     finalizer sink{ stream };
 
     if constexpr (!is_slab) { BC_DEBUG_ONLY(sink.set_limit(Size * element.count());) }
-    return element.to_data(sink) && head_.push(link, head_.putter_index(key));
+    return element.to_data(sink) && head_.push(link, head_.index(key));
 }
 
 // protected

include/bitcoin/database/impl/primitives/hashhead.ipp

@@ -87,7 +87,7 @@ bool CLASS::get_body_count(Link& count) const NOEXCEPT
     if (!ptr)
         return false;
 
-    count = array_cast<Link::size>(ptr->data());
+    count = to_array<Link::size>(ptr->data());
     return true;
 }
 
@@ -98,7 +98,8 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT
     if (!ptr)
         return false;
 
-    array_cast<Link::size>(ptr->data()) = count;
+    // If head is padded then last bytes are fill (0xff).
+    to_array<Link::size>(ptr->data()) = count;
     return true;
 }
 
@@ -123,39 +124,62 @@ inline Link CLASS::top(const Key& key) const NOEXCEPT
 TEMPLATE
 inline Link CLASS::top(const Link& index) const NOEXCEPT
 {
+    using namespace system;
     const auto raw = file_.get_raw(link_to_position(index));
     if (is_null(raw))
         return {};
 
-    const auto& head = array_cast<Link::size>(raw);
-
-    mutex_.lock_shared();
-    const auto top = head;
-    mutex_.unlock_shared();
-    return top;
+    if constexpr (Align)
+    {
+        // Reads full padded word.
+        // xcode clang++16 does not support C++20 std::atomic_ref.
+        ////const std::atomic_ref<integer> head(unsafe_byte_cast<integer>(raw));
+        const auto& head =* pointer_cast<std::atomic<integer>>(raw);
+        return head.load(std::memory_order_acquire);
+    }
+    else
+    {
+        const auto& head = to_array<size_>(raw);
+        mutex_.lock_shared();
+        const auto top = head;
+        mutex_.unlock_shared();
+        return top;
+    }
 }
 
 TEMPLATE
-inline bool CLASS::push(const bytes& current, bytes& next,
+inline bool CLASS::push(const Link& current, bytes& next,
     const Key& key) NOEXCEPT
 {
     return push(current, next, index(key));
 }
 
 TEMPLATE
-inline bool CLASS::push(const bytes& current, bytes& next,
+inline bool CLASS::push(const Link& current, bytes& next,
     const Link& index) NOEXCEPT
 {
+    using namespace system;
     const auto raw = file_.get_raw(link_to_position(index));
     if (is_null(raw))
         return false;
 
-    auto& head = array_cast<Link::size>(raw);
+    if constexpr (Align)
+    {
+        // Writes full padded word (0x00 fill).
+        // xcode clang++16 does not support C++20 std::atomic_ref.
+        ////const std::atomic_ref<integer> head(unsafe_byte_cast<integer>(raw));
+        auto& head = *pointer_cast<std::atomic<integer>>(raw);
+        next = Link(head.exchange(current, std::memory_order_acq_rel));
+    }
+    else
+    {
+        auto& head = to_array<size_>(raw);
+        mutex_.lock();
+        next = head;
+        head = current;
+        mutex_.unlock();
+    }
 
-    mutex_.lock();
-    next = head;
-    head = current;
-    mutex_.unlock();
     return true;
 }
 

include/bitcoin/database/primitives/arrayhead.hpp

@@ -19,6 +19,7 @@
 #ifndef LIBBITCOIN_DATABASE_PRIMITIVES_ARRAYHEAD_HPP
 #define LIBBITCOIN_DATABASE_PRIMITIVES_ARRAYHEAD_HPP
 
+#include <atomic>
 #include <shared_mutex>
 #include <bitcoin/system.hpp>
 #include <bitcoin/database/define.hpp>
@@ -29,7 +30,7 @@ namespace database {
 
 /// Dynamically expanding array map header.
 /// Less efficient than a fixed-size header.
-template <typename Link>
+template <typename Link, bool Align>
 class arrayhead
 {
 public:
@@ -63,42 +64,45 @@ class arrayhead
     /// Convert natural key to head bucket index (unvalidated).
     inline Link index(size_t key) const NOEXCEPT;
 
-    /// Convert natural key to head bucket index (unvalidated).
-    inline Link putter_index(size_t key) const NOEXCEPT;
-
     /// Unsafe if verify false.
     Link at(size_t key) const NOEXCEPT;
 
-    /// Assign value to bucket index.
-    bool push(const bytes& current, const Link& index) NOEXCEPT;
+    /// Assign link value to bucket index.
+    bool push(const Link& link, const Link& index) NOEXCEPT;
 
 private:
+    using integer = Link::integer;
+    static_assert(std::atomic<integer>::is_always_lock_free);
+    static constexpr auto size_ = Align ? sizeof(integer) : Link::size;
+
+    // Body does not use padded link size.
     using body = manager<Link, system::data_array<zero>, Link::size>;
 
     template <size_t Bytes>
-    static inline auto& array_cast(memory::iterator buffer) NOEXCEPT
+    static inline auto& to_array(memory::iterator it) NOEXCEPT
     {
-        return system::unsafe_array_cast<uint8_t, Bytes>(buffer);
+        return system::unsafe_array_cast<uint8_t, Bytes>(it);
     }
 
     static constexpr Link position_to_link(size_t position) NOEXCEPT
     {
         using namespace system;
-        static_assert(is_nonzero(Link::size));
-        const auto link = floored_subtract(position / Link::size, one);
-        return possible_narrow_cast<typename Link::integer>(link);
+        static_assert(is_nonzero(size_));
+        const auto link = floored_subtract(position / size_, one);
+        return possible_narrow_cast<integer>(link);
     }
 
     // Byte offset of bucket index within head file.
     // [body_size][[bucket[0]...bucket[buckets-1]]]
     static constexpr size_t link_to_position(const Link& index) NOEXCEPT
     {
         using namespace system;
-        BC_ASSERT(!is_multiply_overflow<size_t>(index, Link::size));
-        BC_ASSERT(!is_add_overflow(Link::size, index * Link::size));
-        return possible_narrow_cast<size_t>(Link::size + index * Link::size);
+        BC_ASSERT(!is_multiply_overflow<size_t>(index, size_));
+        BC_ASSERT(!is_add_overflow(size_, index * size_));
+        return possible_narrow_cast<size_t>(size_ + index * size_);
     }
 
+    // These are thread safe.
     storage& file_;
     const Link initial_buckets_;
     mutable std::shared_mutex mutex_{};
@@ -107,8 +111,8 @@ class arrayhead
 } // namespace database
 } // namespace libbitcoin
 
-#define TEMPLATE template <typename Link>
-#define CLASS arrayhead<Link>
+#define TEMPLATE template <typename Link, bool Align>
+#define CLASS arrayhead<Link, Align>
 
 #include <bitcoin/database/impl/primitives/arrayhead.ipp>
 

include/bitcoin/database/primitives/arraymap.hpp

@@ -35,7 +35,7 @@ namespace database {
 /// Readers and writers are always prepositioned at data, and are limited to
 /// the extent the record/slab size is known (limit can always be removed).
 /// Streams are always initialized from first element byte up to file limit.
-template <typename Link, size_t Size>
+template <typename Link, size_t Size, bool Align>
 class arraymap
 {
 public:
@@ -113,7 +113,7 @@ class arraymap
 private:
     static constexpr auto is_slab = (Size == max_size_t);
 
-    using head = database::arrayhead<Link>;
+    using head = database::arrayhead<Link, Align>;
     using body = database::manager<Link, system::data_array<0>, Size>;
 
     // Thread safe (index/top/push).
@@ -125,13 +125,13 @@ class arraymap
 };
 
 template <typename Element>
-using array_map = arraymap<linkage<Element::pk>, Element::size>;
+using array_map = arraymap<linkage<Element::pk>, Element::size, Element::align>;
 
 } // namespace database
 } // namespace libbitcoin
 
-#define TEMPLATE template <typename Link, size_t Size>
-#define CLASS arraymap<Link, Size>
+#define TEMPLATE template <typename Link, size_t Size, bool Align>
+#define CLASS arraymap<Link, Size, Align>
 
 #include <bitcoin/database/impl/primitives/arraymap.ipp>