Skip to content

Commit f138ee2

Browse files
authored
Merge pull request #740 from evoskuil/master
Optimizing block/header/tx wire serialization queries with tests.
2 parents 88a85df + bca6641 commit f138ee2

20 files changed

Lines changed: 1476 additions & 838 deletions

File tree

Makefile.am

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ test_libbitcoin_database_test_SOURCES = \
101101
test/query/optional.cpp \
102102
test/query/translate.cpp \
103103
test/query/validate.cpp \
104+
test/query/wire.cpp \
104105
test/tables/archives/header.cpp \
105106
test/tables/archives/input.cpp \
106107
test/tables/archives/output.cpp \
@@ -192,7 +193,8 @@ include_bitcoin_database_impl_query_HEADERS = \
192193
include/bitcoin/database/impl/query/optional.ipp \
193194
include/bitcoin/database/impl/query/query.ipp \
194195
include/bitcoin/database/impl/query/translate.ipp \
195-
include/bitcoin/database/impl/query/validate.ipp
196+
include/bitcoin/database/impl/query/validate.ipp \
197+
include/bitcoin/database/impl/query/wire.ipp
196198

197199
include_bitcoin_database_locksdir = ${includedir}/bitcoin/database/locks
198200
include_bitcoin_database_locks_HEADERS = \

builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@
164164
<ClCompile Include="..\..\..\..\test\query\optional.cpp" />
165165
<ClCompile Include="..\..\..\..\test\query\translate.cpp" />
166166
<ClCompile Include="..\..\..\..\test\query\validate.cpp" />
167+
<ClCompile Include="..\..\..\..\test\query\wire.cpp" />
167168
<ClCompile Include="..\..\..\..\test\settings.cpp" />
168169
<ClCompile Include="..\..\..\..\test\store.cpp" />
169170
<ClCompile Include="..\..\..\..\test\tables\archives\header.cpp" />

builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@
153153
<ClCompile Include="..\..\..\..\test\query\validate.cpp">
154154
<Filter>src\query</Filter>
155155
</ClCompile>
156+
<ClCompile Include="..\..\..\..\test\query\wire.cpp">
157+
<Filter>src\query</Filter>
158+
</ClCompile>
156159
<ClCompile Include="..\..\..\..\test\settings.cpp">
157160
<Filter>src</Filter>
158161
</ClCompile>

builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@
230230
<None Include="..\..\..\..\include\bitcoin\database\impl\query\query.ipp" />
231231
<None Include="..\..\..\..\include\bitcoin\database\impl\query\translate.ipp" />
232232
<None Include="..\..\..\..\include\bitcoin\database\impl\query\validate.ipp" />
233+
<None Include="..\..\..\..\include\bitcoin\database\impl\query\wire.ipp" />
233234
<None Include="..\..\..\..\include\bitcoin\database\impl\store.ipp" />
234235
<None Include="packages.config" />
235236
</ItemGroup>

builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,9 @@
379379
<None Include="..\..\..\..\include\bitcoin\database\impl\query\validate.ipp">
380380
<Filter>include\bitcoin\database\impl\query</Filter>
381381
</None>
382+
<None Include="..\..\..\..\include\bitcoin\database\impl\query\wire.ipp">
383+
<Filter>include\bitcoin\database\impl\query</Filter>
384+
</None>
382385
<None Include="..\..\..\..\include\bitcoin\database\impl\store.ipp">
383386
<Filter>include\bitcoin\database\impl</Filter>
384387
</None>
Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
/**
2+
* Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS)
3+
*
4+
* This file is part of libbitcoin.
5+
*
6+
* This program is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU Affero General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Affero General Public License
17+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
18+
*/
19+
#ifndef LIBBITCOIN_DATABASE_QUERY_WIRE_IPP
20+
#define LIBBITCOIN_DATABASE_QUERY_WIRE_IPP
21+
22+
#include <algorithm>
23+
#include <utility>
24+
#include <bitcoin/database/define.hpp>
25+
26+
namespace libbitcoin {
27+
namespace database {
28+
29+
// Wire serialized objects.
30+
// ----------------------------------------------------------------------------
31+
// Due to the idiotic segwit serialization of witness after output there is are
32+
// duplicated navigations to store_.ins and store_.input by the witness reader.
33+
// This normalized approach is also the most efficient.
34+
35+
TEMPLATE
36+
bool CLASS::get_wire_header(byteflipper& flipper,
37+
const header_link& link) const NOEXCEPT
38+
{
39+
const auto start = flipper.get_write_position();
40+
table::header::wire_header header{ {}, flipper };
41+
if (!store_.header.get(link, header))
42+
{
43+
flipper.invalidate();
44+
return false;
45+
}
46+
47+
// Genesis header parent is defaulted.
48+
if (header.parent_fk == schema::header::link::terminal)
49+
return true;
50+
51+
flipper.set_position(start);
52+
table::header::wire_key key{ {}, flipper };
53+
if (!store_.header.get(header.parent_fk, key))
54+
{
55+
flipper.invalidate();
56+
return false;
57+
}
58+
59+
return true;
60+
}
61+
62+
TEMPLATE
63+
bool CLASS::get_wire_input(byteflipper& flipper,
64+
const point_link& link) const NOEXCEPT
65+
{
66+
// [point]
67+
table::point::wire_point point{ {}, flipper };
68+
if (!store_.point.get(link, point))
69+
{
70+
flipper.invalidate();
71+
return false;
72+
}
73+
74+
// [[size]script]
75+
table::ins::get_input ins{};
76+
table::input::wire_script script{ {}, flipper };
77+
if (!store_.ins.get(link, ins) ||
78+
!store_.input.get(ins.input_fk, script))
79+
{
80+
flipper.invalidate();
81+
return false;
82+
}
83+
84+
// [sequence]
85+
flipper.write_4_bytes_little_endian(ins.sequence);
86+
return true;
87+
}
88+
89+
TEMPLATE
90+
bool CLASS::get_wire_output(byteflipper& flipper,
91+
const output_link& link) const NOEXCEPT
92+
{
93+
// [value][[[size]script]]
94+
table::output::wire_script out{ {}, flipper };
95+
if (!store_.output.get(link, out))
96+
{
97+
flipper.invalidate();
98+
return false;
99+
}
100+
101+
return true;
102+
}
103+
104+
TEMPLATE
105+
bool CLASS::get_wire_witness(byteflipper& flipper,
106+
const point_link& link) const NOEXCEPT
107+
{
108+
// [count][[[size]element]]
109+
table::ins::get_input ins{};
110+
table::input::wire_witness wire{ {}, flipper };
111+
if (!store_.ins.get(link, ins) ||
112+
!store_.input.get(ins.input_fk, wire))
113+
{
114+
flipper.invalidate();
115+
return false;
116+
}
117+
118+
return true;
119+
}
120+
121+
TEMPLATE
122+
bool CLASS::get_wire_tx(byteflipper& flipper, const tx_link& link,
123+
bool witness) const NOEXCEPT
124+
{
125+
table::transaction::record tx{};
126+
if (!store_.tx.get(link, tx))
127+
{
128+
flipper.invalidate();
129+
return false;
130+
}
131+
132+
table::outs::record outs{};
133+
outs.out_fks.resize(tx.outs_count);
134+
if (!store_.outs.get(tx.outs_fk, outs))
135+
{
136+
flipper.invalidate();
137+
return false;
138+
}
139+
140+
// Point links are contiguous (computed).
141+
const auto ins_begin = tx.point_fk;
142+
const auto ins_count = tx.ins_count;
143+
const auto ins_final = ins_begin + ins_count;
144+
const auto witnessed = witness && (tx.heavy != tx.light);
145+
146+
flipper.write_4_bytes_little_endian(tx.version);
147+
148+
if (witnessed)
149+
{
150+
flipper.write_byte(system::chain::witness_marker);
151+
flipper.write_byte(system::chain::witness_enabled);
152+
}
153+
154+
flipper.write_variable(ins_count);
155+
for (auto fk = ins_begin; fk < ins_final; ++fk)
156+
if (!get_wire_input(flipper, fk))
157+
return false;
158+
159+
flipper.write_variable(outs.out_fks.size());
160+
for (const auto& fk: outs.out_fks)
161+
if (!get_wire_output(flipper, fk))
162+
return false;
163+
164+
if (witnessed)
165+
{
166+
for (auto fk = ins_begin; fk < ins_final; ++fk)
167+
if (!get_wire_witness(flipper, fk))
168+
return false;
169+
}
170+
171+
flipper.write_4_bytes_little_endian(tx.locktime);
172+
return true;
173+
}
174+
175+
TEMPLATE
176+
bool CLASS::get_wire_block(byteflipper& flipper, const header_link& link,
177+
bool witness) const NOEXCEPT
178+
{
179+
if (!get_wire_header(flipper, link))
180+
return false;
181+
182+
const auto txs = to_transactions(link);
183+
if (txs.empty())
184+
{
185+
flipper.invalidate();
186+
return false;
187+
}
188+
189+
flipper.write_variable(txs.size());
190+
for (const auto& tx_link: txs)
191+
if (!get_wire_tx(flipper, tx_link, witness))
192+
return false;
193+
194+
return true;
195+
}
196+
197+
// These convenience wrappers are made practical by size caching for block and
198+
// tx for both nominal and witness wire encodings (and fixed size headers).
199+
// Intermediate objects (input, output, witness) have a size prefix
200+
201+
TEMPLATE
202+
data_chunk CLASS::get_wire_header(const header_link& link) const NOEXCEPT
203+
{
204+
using namespace system;
205+
data_chunk data(chain::header::serialized_size());
206+
207+
stream::flip::fast ostream(data);
208+
flip::bytes::fast out(ostream);
209+
if (!get_wire_header(out, link) || !out)
210+
return {};
211+
212+
return data;
213+
}
214+
215+
TEMPLATE
216+
data_chunk CLASS::get_wire_tx(const tx_link& link, bool witness) const NOEXCEPT
217+
{
218+
using namespace system;
219+
size_t size{};
220+
if (!get_tx_size(size, link, witness))
221+
return {};
222+
223+
data_chunk data(size);
224+
stream::flip::fast ostream(data);
225+
flip::bytes::fast out(ostream);
226+
if (!get_wire_tx(out, link, witness) || !out)
227+
return {};
228+
229+
return data;
230+
}
231+
232+
TEMPLATE
233+
data_chunk CLASS::get_wire_block(const header_link& link,
234+
bool witness) const NOEXCEPT
235+
{
236+
using namespace system;
237+
size_t size{};
238+
if (!get_block_size(size, link, witness))
239+
return {};
240+
241+
data_chunk data(size);
242+
stream::flip::fast ostream(data);
243+
flip::bytes::fast out(ostream);
244+
if (!get_wire_block(out, link, witness) || !out)
245+
return {};
246+
247+
return data;
248+
}
249+
250+
} // namespace database
251+
} // namespace libbitcoin
252+
253+
#endif

include/bitcoin/database/query.hpp

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -350,14 +350,10 @@ class query
350350
size_t position) const NOEXCEPT;
351351

352352
/// Sizes.
353-
bool get_tx_size(size_t& out, const tx_link& link,
354-
bool witness) const NOEXCEPT;
355-
bool get_block_size(size_t& out, const header_link& link,
356-
bool witness) const NOEXCEPT;
357-
bool get_block_sizes(size_t& light, size_t& heavy,
358-
const header_link& link) const NOEXCEPT;
359-
bool get_tx_sizes(size_t& light, size_t& heavy,
360-
const tx_link& link) const NOEXCEPT;
353+
bool get_tx_size(size_t& out, const tx_link& link, bool witness) const NOEXCEPT;
354+
bool get_block_size(size_t& out, const header_link& link, bool witness) const NOEXCEPT;
355+
bool get_block_sizes(size_t& light, size_t& heavy, const header_link& link) const NOEXCEPT;
356+
bool get_tx_sizes(size_t& light, size_t& heavy, const tx_link& link) const NOEXCEPT;
361357

362358
/// Heights.
363359
height_link get_height(const hash_digest& key) const NOEXCEPT;
@@ -374,6 +370,21 @@ class query
374370
bool get_block_spend(uint64_t& out, const header_link& link) const NOEXCEPT;
375371
bool get_block_fee(uint64_t& out, const header_link& link) const NOEXCEPT;
376372

373+
/// Wire.
374+
/// -----------------------------------------------------------------------
375+
376+
bool get_wire_input(byteflipper& flipper, const point_link& link) const NOEXCEPT;
377+
bool get_wire_output(byteflipper& flipper, const output_link& link) const NOEXCEPT;
378+
bool get_wire_witness(byteflipper& flipper, const point_link& link) const NOEXCEPT;
379+
bool get_wire_header(byteflipper& flipper, const header_link& link) const NOEXCEPT;
380+
bool get_wire_tx(byteflipper& flipper, const tx_link& link, bool witness) const NOEXCEPT;
381+
bool get_wire_block(byteflipper& flipper, const header_link& link,
382+
bool witness) const NOEXCEPT;
383+
384+
data_chunk get_wire_header(const header_link& link) const NOEXCEPT;
385+
data_chunk get_wire_tx(const tx_link& link, bool witness) const NOEXCEPT;
386+
data_chunk get_wire_block(const header_link& link, bool witness) const NOEXCEPT;
387+
377388
/// Objects.
378389
/// -----------------------------------------------------------------------
379390

@@ -836,6 +847,7 @@ BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT)
836847
#include <bitcoin/database/impl/query/merkle.ipp>
837848
#include <bitcoin/database/impl/query/translate.ipp>
838849
#include <bitcoin/database/impl/query/validate.ipp>
850+
#include <bitcoin/database/impl/query/wire.ipp>
839851

840852
BC_POP_WARNING()
841853

0 commit comments

Comments
 (0)