Merge pull request #725 from evoskuil/master

Fix merkle branch/proof queries.

Author: evoskuil

include/bitcoin/database/impl/query/merkle.ipp

@@ -77,10 +77,14 @@ CLASS::hash_option CLASS::get_confirmed_interval(size_t height) const NOEXCEPT
 TEMPLATE
 void CLASS::merge_merkle(hashes& to, hashes&& from, size_t first) NOEXCEPT
 {
+    // from is either even or has one additional element of reserved space.
+    if (!is_one(from.size()) && is_odd(from.size()))
+        from.push_back(from.back());
+
     using namespace system;
     for (const auto& row: block::merkle_branch(first, from.size()))
     {
-        BC_ASSERT(row.sibling * add1(row.width) <= from.size());
+        BC_ASSERT(add1(row.sibling) * row.width <= from.size());
         const auto it = std::next(from.begin(), row.sibling * row.width);
         const auto mover = std::make_move_iterator(it);
         to.push_back(merkle_root({ mover, std::next(mover, row.width) }));
@@ -108,15 +112,52 @@ code CLASS::get_merkle_proof(hashes& proof, hashes roots, size_t target,
     return error::success;
 }
 
+// static/private
+TEMPLATE
+hash_digest CLASS::merkle_subroot(hashes&& tree, size_t span) NOEXCEPT
+{
+    using namespace system;
+
+    // Tree cannot be empty or exceed span (span is power of two).
+    if (tree.empty() || tree.size() > span)
+        return {};
+
+    // zero depth implies single tree element, which is the root.
+    const auto depth = ceilinged_log2(span);
+    if (is_zero(depth))
+        return tree.front();
+
+    // Merkle root treats a single hash as top/complete, but for a non-zero
+    // depth subtree, any odd leaf requires duplication including a single.
+    if (is_odd(tree.size()))
+        tree.push_back(tree.back());
+
+    // Log2 of the evened breadth gives the elevation by merkle_root.
+    const auto partial = ceilinged_log2(tree.size());
+
+    // Partial cannot exceed depth, since tree.size() <= span (a power of 2).
+    if (is_subtract_overflow(depth, partial))
+        return {};
+
+    // Elevate hashes to partial level.
+    auto hash = merkle_root(std::move(tree));
+
+    // Elevate hashes from partial to depth level.
+    for (size_t level{}; level < (depth - partial); ++level)
+        hash = sha256::double_hash(hash, hash);
+
+    return hash;
+}
+
 // protected
 TEMPLATE
-code CLASS::get_merkle_tree(hashes& tree, size_t waypoint) const NOEXCEPT
+code CLASS::get_merkle_subroots(hashes& roots, size_t waypoint) const NOEXCEPT
 {
     const auto span = interval_span();
     BC_ASSERT(!is_zero(span));
 
     const auto range = add1(waypoint);
-    tree.reserve(system::ceilinged_divide(range, span));
+    roots.reserve(system::ceilinged_divide(range, span));
     for (size_t first{}; first < range; first += span)
     {
         const auto last = std::min(sub1(first + span), waypoint);
@@ -126,13 +167,13 @@ code CLASS::get_merkle_tree(hashes& tree, size_t waypoint) const NOEXCEPT
         {
             auto interval = get_confirmed_interval(last);
             if (!interval.has_value()) return error::merkle_interval;
-            tree.push_back(std::move(interval.value()));
+            roots.push_back(std::move(interval.value()));
         }
         else
         {
-            auto confirmed = get_confirmed_hashes(first, size);
-            if (confirmed.empty()) return error::merkle_hashes;
-            tree.push_back(system::merkle_root(std::move(confirmed)));
+            auto partial = get_confirmed_hashes(first, size);
+            if (partial.empty()) return error::merkle_hashes;
+            roots.push_back(merkle_subroot(std::move(partial), span));
         }
     }
 
@@ -150,7 +191,7 @@ code CLASS::get_merkle_root_and_proof(hash_digest& root, hashes& proof,
         return error::not_found;
 
     hashes tree{};
-    if (const auto ec = get_merkle_tree(tree, waypoint))
+    if (const auto ec = get_merkle_subroots(tree, waypoint))
         return ec;
 
     proof.clear();
@@ -165,7 +206,7 @@ TEMPLATE
 hash_digest CLASS::get_merkle_root(size_t height) const NOEXCEPT
 {
     hashes tree{};
-    if (const auto ec = get_merkle_tree(tree, height))
+    if (const auto ec = get_merkle_subroots(tree, height))
         return {};
 
     return system::merkle_root(std::move(tree));

include/bitcoin/database/query.hpp

@@ -767,6 +767,8 @@ class query
     /// merkle
     /// -----------------------------------------------------------------------
 
+    // merkle related utilities
+    static hash_digest merkle_subroot(hashes&& tree, size_t span) NOEXCEPT;
     static void merge_merkle(hashes& branch, hashes&& hashes,
         size_t first) NOEXCEPT;
 
@@ -776,7 +778,7 @@ class query
 
     hash_option get_confirmed_interval(size_t height) const NOEXCEPT;
     hash_option create_interval(header_link link, size_t height) const NOEXCEPT;
-    code get_merkle_tree(hashes& roots, size_t waypoint) const NOEXCEPT;
+    code get_merkle_subroots(hashes& roots, size_t waypoint) const NOEXCEPT;
     code get_merkle_proof(hashes& proof, hashes roots, size_t target,
         size_t waypoint) const NOEXCEPT;
 

test/mocks/blocks.hpp

@@ -23,37 +23,25 @@
 
 namespace test {
 
+constexpr auto block0_hash = system::base16_hash("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f");
+constexpr auto block1_hash = system::base16_hash("00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048");
+constexpr auto block2_hash = system::base16_hash("000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd");
+constexpr auto block3_hash = system::base16_hash("0000000082b5015589a3fdf2d4baff403e6f0be035a5d9742c1cae6295464449");
+constexpr auto block4_hash = system::base16_hash("000000004ebadb55ee9096c9a2f8880e09da59c0d68b1c228da88e48844a1485");
+constexpr auto block5_hash = system::base16_hash("000000009b7262315dbf071787ad3656097b892abffd1f95a1a022f896f533fc");
+constexpr auto block6_hash = system::base16_hash("000000003031a0e73735690c5a1ff2a4be82553b2a12b776fbd3a215dc8f778d");
+constexpr auto block7_hash = system::base16_hash("0000000071966c2b1d065fd446b1e485b2c9d9594acd2007ccbd5441cfc89444");
+constexpr auto block8_hash = system::base16_hash("00000000408c48f847aa786c2268fc3e6ec2af68e8468a34a28c61b7f1de0dc6");
+
 // blockchain.info/rawblock/[block-hash]?format=hex
-constexpr auto block1_hash = system::base16_hash(
-    "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048");
-constexpr auto block1_data = system::base16_array(
-    "010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d61900"
-    "00000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e8"
-    "57233e0e61bc6649ffff001d01e3629901010000000100000000000000000000"
-    "00000000000000000000000000000000000000000000ffffffff0704ffff001d"
-    "0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec1"
-    "1600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781"
-    "e62294721166bf621e73a82cbf2342c858eeac00000000");
-constexpr auto block2_hash = system::base16_hash(
-    "000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd");
-constexpr auto block2_data = system::base16_array(
-    "010000004860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a83"
-    "00000000d5fdcc541e25de1c7a5addedf24858b8bb665c9f36ef744ee42c3160"
-    "22c90f9bb0bc6649ffff001d08d2bd6101010000000100000000000000000000"
-    "00000000000000000000000000000000000000000000ffffffff0704ffff001d"
-    "010bffffffff0100f2052a010000004341047211a824f55b505228e4c3d5194c"
-    "1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167"
-    "c13e236446b417ab79a0fcae412ae3316b77ac00000000");
-constexpr auto block3_hash = system::base16_hash(
-    "0000000082b5015589a3fdf2d4baff403e6f0be035a5d9742c1cae6295464449");
-constexpr auto block3_data = system::base16_array(
-    "01000000bddd99ccfda39da1b108ce1a5d70038d0a967bacb68b6b63065f626a"
-    "0000000044f672226090d85db9a9f2fbfe5f0f9609b387af7be5b7fbb7a1767c"
-    "831c9e995dbe6649ffff001d05e0ed6d01010000000100000000000000000000"
-    "00000000000000000000000000000000000000000000ffffffff0704ffff001d"
-    "010effffffff0100f2052a0100000043410494b9d3e76c5b1629ecf97fff95d7"
-    "a4bbdac87cc26099ada28066c6ff1eb9191223cd897194a08d0c2726c5747f1d"
-    "b49e8cf90e75dc3e3550ae9b30086f3cd5aaac00000000");
+constexpr auto block1_data = system::base16_array("010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e362990101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000");
+constexpr auto block2_data = system::base16_array("010000004860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a8300000000d5fdcc541e25de1c7a5addedf24858b8bb665c9f36ef744ee42c316022c90f9bb0bc6649ffff001d08d2bd610101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d010bffffffff0100f2052a010000004341047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac00000000");
+constexpr auto block3_data = system::base16_array("01000000bddd99ccfda39da1b108ce1a5d70038d0a967bacb68b6b63065f626a0000000044f672226090d85db9a9f2fbfe5f0f9609b387af7be5b7fbb7a1767c831c9e995dbe6649ffff001d05e0ed6d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d010effffffff0100f2052a0100000043410494b9d3e76c5b1629ecf97fff95d7a4bbdac87cc26099ada28066c6ff1eb9191223cd897194a08d0c2726c5747f1db49e8cf90e75dc3e3550ae9b30086f3cd5aaac00000000");
+constexpr auto block4_data = system::base16_array("010000004944469562ae1c2c74d9a535e00b6f3e40ffbad4f2fda3895501b582000000007a06ea98cd40ba2e3288262b28638cec5337c1456aaf5eedc8e9e5a20f062bdf8cc16649ffff001d2bfee0a90101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d011affffffff0100f2052a01000000434104184f32b212815c6e522e66686324030ff7e5bf08efb21f8b00614fb7690e19131dd31304c54f37baa40db231c918106bb9fd43373e37ae31a0befc6ecaefb867ac00000000");
+constexpr auto block5_data = system::base16_array("0100000085144a84488ea88d221c8bd6c059da090e88f8a2c99690ee55dbba4e00000000e11c48fecdd9e72510ca84f023370c9a38bf91ac5cae88019bee94d24528526344c36649ffff001d1d03e4770101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0120ffffffff0100f2052a0100000043410456579536d150fbce94ee62b47db2ca43af0a730a0467ba55c79e2a7ec9ce4ad297e35cdbb8e42a4643a60eef7c9abee2f5822f86b1da242d9c2301c431facfd8ac00000000");
+constexpr auto block6_data = system::base16_array("01000000fc33f596f822a0a1951ffdbf2a897b095636ad871707bf5d3162729b00000000379dfb96a5ea8c81700ea4ac6b97ae9a9312b2d4301a29580e924ee6761a2520adc46649ffff001d189c4c970101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0123ffffffff0100f2052a0100000043410408ce279174b34c077c7b2043e3f3d45a588b85ef4ca466740f848ead7fb498f0a795c982552fdfa41616a7c0333a269d62108588e260fd5a48ac8e4dbf49e2bcac00000000");
+constexpr auto block7_data = system::base16_array("010000008d778fdc15a2d3fb76b7122a3b5582bea4f21f5a0c693537e7a03130000000003f674005103b42f984169c7d008370967e91920a6a5d64fd51282f75bc73a68af1c66649ffff001d39a59c860101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d012bffffffff0100f2052a01000000434104a59e64c774923d003fae7491b2a7f75d6b7aa3f35606a8ff1cf06cd3317d16a41aa16928b1df1f631f31f28c7da35d4edad3603adb2338c4d4dd268f31530555ac00000000");
+constexpr auto block8_data = system::base16_array("010000004494c8cf4154bdcc0720cd4a59d9c9b285e4b146d45f061d2b6c967100000000e3855ed886605b6d4a99d5fa2ef2e9b0b164e63df3c4136bebf2d0dac0f1f7a667c86649ffff001d1c4b56660101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d012cffffffff0100f2052a01000000434104cc8d85f5e7933cb18f13b97d165e1189c1fb3e9c98b0dd5446b2a1989883ff9e740a8a75da99cc59a21016caf7a7afd3e4e9e7952983e18d1ff70529d62e0ba1ac00000000");
 
 constexpr hash_digest two_hash = system::from_uintx(uint256_t(two));
 constexpr database::context context
@@ -72,6 +60,12 @@ const auto genesis = system::settings{ selection::mainnet }.genesis_block;
 const block block1{ test::block1_data, true };
 const block block2{ test::block2_data, true };
 const block block3{ test::block3_data, true };
+const block block4{ test::block4_data, true };
+const block block5{ test::block5_data, true };
+const block block6{ test::block6_data, true };
+const block block7{ test::block7_data, true };
+const block block8{ test::block8_data, true };
+
 const block block1a
 {
     header