From 424ae982fb5eb9eb46cfb9863e041ef602d44094 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 5 Jan 2026 10:31:42 +0100 Subject: [PATCH 01/65] rename leveldb dictionary, optimize, update usage --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 48 ++-- src/blockchain/FileBased.h | 13 +- src/blockchain/FileBasedProvider.cpp | 16 +- src/blockchain/FileBasedProvider.h | 22 +- src/cache/BlockIndex.cpp | 392 ++++----------------------- src/cache/BlockIndex.h | 128 ++++++--- src/cache/Dictionary.cpp | 138 ---------- src/cache/Dictionary.h | 78 ------ src/lib/PersistentDictionary.h | 131 +++++++++ src/model/files/BlockIndex.cpp | 30 +- src/model/files/BlockIndex.h | 37 ++- src/model/files/LevelDBWrapper.cpp | 57 ++-- src/model/files/LevelDBWrapper.h | 7 +- src/serialization/String.h | 66 +++++ 15 files changed, 457 insertions(+), 708 deletions(-) delete mode 100644 src/cache/Dictionary.cpp delete mode 100644 src/cache/Dictionary.h create mode 100644 src/lib/PersistentDictionary.h create mode 100644 src/serialization/String.h diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 03dcbdf..4b6609b 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 03dcbdf4b31deefaf79a7ef08eb05c046931f2fa +Subproject commit 4b6609b3b2f137f716c571807eb23e8b3873d618 diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index e4f07af..8b087df 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -24,28 +24,34 @@ #include using namespace cache; +using std::string, std::string_view, std::vector; +using std::shared_ptr, std::make_shared; +using client::hiero::ConsensusClient; +using controller::SimpleOrderingManager; namespace gradido { + using data::LedgerAnchor; + using namespace interaction; namespace blockchain { FileBased::FileBased( Private, - std::string_view communityId, + string_view communityId, const hiero::TopicId& topicId, - std::string_view alias, - std::string_view folder, - std::vector>&& hieroClients) + string_view alias, + string_view folder, + vector>&& hieroClients) : Abstract(communityId), mExitCalled(false), mHieroTopicId(topicId), mAlias(alias), mFolderPath(folder), mTaskObserver(std::make_shared()), - mOrderingManager(std::make_shared(communityId)), + mOrderingManager(std::make_shared(communityId)), // mIotaMessageListener(new iota::MessageListener(communityId, alias)), - mPublicKeysIndex(std::make_shared(std::string(folder).append("/pubkeysCache"))), - mBlockchainState(std::string(folder).append("/.state")), - mMessageIdsCache(std::string(folder).append("/messageIdCache")), + mPublicKeysIndex((string(folder).append("/pubkeysCache"))), + mBlockchainState(string(folder).append("/.state")), + mMessageIdsCache(string(folder).append("/messageIdCache")), mTransactionTriggerEventsCache(std::string(folder).append("/transactionTriggerEventCache")), mCachedBlocks(ServerGlobals::g_CacheTimeout), mTransactionHashCache(communityId), @@ -66,11 +72,11 @@ namespace gradido { { assert(!mExitCalled); std::lock_guard _lock(mWorkMutex); - if (!mPublicKeysIndex->init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { + if (!mPublicKeysIndex.init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { // remove index files for regenration LOG_F(WARNING, "reset the public key index file"); // mCachedBlocks.clear(); - mPublicKeysIndex->reset(); + mPublicKeysIndex.reset(); resetBlockIndices = true; } @@ -200,13 +206,13 @@ namespace gradido { mOrderingManager->exit(); mCachedBlocks.clear(); mBlockchainState.exit(); - mPublicKeysIndex->exit(); + mPublicKeysIndex.exit(); mMessageIdsCache.exit(); mTransactionTriggerEventsCache.exit(); } bool FileBased::createAndAddConfirmedTransaction( data::ConstGradidoTransactionPtr gradidoTransaction, - memory::ConstBlockPtr messageId, + const data::LedgerAnchor& ledgerAnchor, data::Timestamp confirmedAt ) { if (!gradidoTransaction) { @@ -234,13 +240,13 @@ namespace gradido { } role->runPastAddToBlockchain(confirmedTransaction, getptr()); mBlockchainState.updateState(DefaultStateKeys::LAST_TRANSACTION_ID, confirmedTransaction->getId()); - mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex->getLastIndex()); + mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); mTransactionHashCache.push(*nodeTransactionEntry->getConfirmedTransaction()); mMessageIdsCache.add(confirmedTransaction->getMessageId(), confirmedTransaction->getId()); // add public keys to index auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); for (const auto& address : involvedAddresses) { - mPublicKeysIndex->getOrAddIndexForString(address->copyAsString()); + mPublicKeysIndex.getOrAddIndexForData(address); } if (mCommunityServer) { task::TaskPtr notifyClientTask = std::make_shared(mCommunityServer, confirmedTransaction); @@ -285,7 +291,7 @@ namespace gradido { Filter filterCopy(filter); bool stopped = false; iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, *mPublicKeysIndex); + auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, mPublicKeysIndex); for (auto transactionNr : transactionNrs) { if (!filter.pagination.hasCapacityLeft(result.size())) { return false; @@ -318,7 +324,7 @@ namespace gradido { // if pagination is used, filterCopy contain count of still to find transactions Filter filterCopy(filter); iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, *mPublicKeysIndex); + auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, mPublicKeysIndex); result.insert(result.end(), transactionNrs.begin(), transactionNrs.end()); if (filter.pagination.size) { filterCopy.pagination.size = filter.pagination.size - result.size(); @@ -334,7 +340,7 @@ namespace gradido { { size_t count = 0; iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - count += block.getBlockIndex().countTransactions(filter, *mPublicKeysIndex); + count += block.getBlockIndex().countTransactions(filter, mPublicKeysIndex); return true; }); return count; @@ -354,9 +360,9 @@ namespace gradido { } while (blockNr > 0); return nullptr; } - std::shared_ptr FileBased::findByMessageId( - memory::ConstBlockPtr messageId, - const Filter& filter/* = Filter::ALL_TRANSACTIONS */ + std::shared_ptr FileBased::findByLedgerAnchor( + const data::LedgerAnchor& ledgerAnchor, + const Filter& filter = Filter::ALL_TRANSACTIONS ) const { auto transactionNr = mMessageIdsCache.has(messageId); @@ -374,7 +380,7 @@ namespace gradido { void FileBased::loadStateFromBlockCache() { Profiler timeUsed; - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex->getLastIndex()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); auto lastBlockNr = model::files::Block::findLastBlockFileInFolder(mFolderPath); mBlockchainState.updateState(cache::DefaultStateKeys::LAST_BLOCK_NR, lastBlockNr); auto& block = getBlock(lastBlockNr); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 22c3443..31846ef 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -2,7 +2,6 @@ #define __GRADIDO_NODE_BLOCKCHAIN_FILE_BASED_H #include "../cache/Block.h" -#include "../cache/Dictionary.h" #include "../cache/HieroTransactionId.h" #include "../cache/State.h" #include "../cache/TransactionHash.h" @@ -10,6 +9,7 @@ #include "../client/Base.h" #include "../controller/TaskObserver.h" #include "../controller/SimpleOrderingManager.h" +#include "../lib/PersistentDictionary.h" #include "gradido_blockchain/blockchain/Abstract.h" #include "gradido_blockchain/data/hiero/TopicId.h" @@ -107,7 +107,7 @@ namespace gradido { //! \return false if transaction already exist virtual bool createAndAddConfirmedTransaction( data::ConstGradidoTransactionPtr gradidoTransaction, - memory::ConstBlockPtr messageId, + const data::LedgerAnchor& ledgerAnchor, data::Timestamp confirmedAt ) override; void updateLastKnownSequenceNumber(uint64_t newSequenceNumber); @@ -133,8 +133,9 @@ namespace gradido { size_t findAllResultCount(const Filter& filter) const; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; - virtual std::shared_ptr findByMessageId( - memory::ConstBlockPtr messageId, + //! \param filter use to speed up search if infos exist to narrow down search transactions range + virtual ConstTransactionEntryPtr findByLedgerAnchor( + const data::LedgerAnchor& ledgerAnchor, const Filter& filter = Filter::ALL_TRANSACTIONS ) const override; virtual AbstractProvider* getProvider() const override; @@ -143,7 +144,7 @@ namespace gradido { inline std::shared_ptr getListeningCommunityServer() const; inline uint32_t getOrAddIndexForPublicKey(memory::ConstBlockPtr publicKey) const { - return mPublicKeysIndex->getOrAddIndexForString(publicKey->copyAsString()); + return mPublicKeysIndex.getOrAddIndexForData(publicKey); } inline const hiero::TopicId& getHieroTopicId() const { return mHieroTopicId; } inline const std::string& getFolderPath() const { return mFolderPath; } @@ -178,7 +179,7 @@ namespace gradido { std::shared_ptr mHieroMessageListener; //! contain indices for every public key address, used overall for optimisation - mutable std::shared_ptr mPublicKeysIndex; + mutable PersistentDictionary mPublicKeysIndex; // level db to store state values like last transaction mutable cache::State mBlockchainState; diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index b5f8466..fe41e7f 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -15,7 +15,7 @@ namespace gradido { namespace blockchain { FileBasedProvider::FileBasedProvider() - :mGroupIndex(nullptr), mCommunityIdIndex(ServerGlobals::g_FilesPath + "/communityIdsCache"), mInitalized(false) + :mGroupIndex(nullptr), mInitalized(false) { } @@ -75,10 +75,6 @@ namespace gradido { LOG_F(ERROR, "more hiero clients per community as hiero clients"); return false; } - if (!mCommunityIdIndex.init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_ID_INDEX_CACHE_SIZE_MBYTE * 1024 * 1024)) { - mCommunityIdIndex.reset(); - resetAllCommunityIndices = true; - } mGroupIndex = new cache::GroupIndex(communityConfigFile); mGroupIndex->update(); auto communitiesIds = mGroupIndex->listCommunitiesIds(); @@ -88,7 +84,7 @@ namespace gradido { // exit if at least one blockchain from config couldn't be loaded // should only occure with invalid config const auto& details = mGroupIndex->getCommunityDetails(communityId); - if (!addCommunity(communityId, hiero::TopicId(details.topicId), details.alias, resetAllCommunityIndices)) { + if (!addCommunity(communityId, hiero::TopicId(details.topicId), details.alias)) { LOG_F(ERROR, "error adding community %s in folder: %s", details.alias.data(), details.folderName.data()); return false; } @@ -111,7 +107,6 @@ namespace gradido { blockchain.second->exit(); } mBlockchainsPerGroup.clear(); - mCommunityIdIndex.exit(); } int FileBasedProvider::reloadConfig() @@ -127,7 +122,7 @@ namespace gradido { const auto& details = mGroupIndex->getCommunityDetails(communityId); auto it = mBlockchainsPerGroup.find(communityId); if (it == mBlockchainsPerGroup.end()) { - if(addCommunity(communityId, hiero::TopicId(details.topicId), details.alias, false)) { + if(addCommunity(communityId, hiero::TopicId(details.topicId), details.alias)) { addedBlockchainsCount++; } } @@ -141,8 +136,7 @@ namespace gradido { std::shared_ptr FileBasedProvider::addCommunity( const std::string& communityId, const hiero::TopicId& topicId, - const std::string& alias, - bool resetIndices + const std::string& alias ) { try { auto folder = mGroupIndex->getFolder(communityId); @@ -164,7 +158,7 @@ namespace gradido { mBlockchainsPerGroup.erase(communityId); return nullptr; } - mCommunityIdIndex.getOrAddIndexForString(communityId); + mCommunityIdDicitionary.getIndexForData(communityId); return blockchain; } catch (GradidoBlockchainException& ex) { diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index e14f386..c3243ce 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -43,11 +43,6 @@ namespace gradido { //! \return count of added blockchain int reloadConfig(); - // access community id index - inline uint32_t getCommunityIdIndex(const std::string& communityId); - inline uint32_t getCommunityIdIndex(std::string_view communityId); - inline const std::string getCommunityIdString(uint32_t index); - //! list all known communities inline std::vector listCommunityIds() const; protected: @@ -67,31 +62,16 @@ namespace gradido { std::shared_ptr addCommunity( const std::string& communityId, const hiero::TopicId& topicId, - const std::string& alias, - bool resetIndices + const std::string& alias ); void updateListenerCommunity(const std::string& communityId, const std::string& alias, std::shared_ptr blockchain); cache::GroupIndex* mGroupIndex; - cache::Dictionary mCommunityIdIndex; std::vector> mHieroClients; uint8_t mHieroClientsPerCommunity; bool mInitalized; }; - uint32_t FileBasedProvider::getCommunityIdIndex(const std::string& communityId) - { - return mCommunityIdIndex.getIndexForString(communityId); - } - uint32_t FileBasedProvider::getCommunityIdIndex(std::string_view communityId) - { - return getCommunityIdIndex(std::string(communityId)); - } - const std::string FileBasedProvider::getCommunityIdString(uint32_t index) - { - return mCommunityIdIndex.getStringForIndex(index); - } - std::vector FileBasedProvider::listCommunityIds() const { return mGroupIndex->listCommunitiesIds(); diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index e03fdb5..01d6c0c 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -9,15 +9,14 @@ #include "loguru/loguru.hpp" -using namespace gradido::blockchain; -using namespace gradido::data; using namespace rapidjson; +using gradido::blockchain::TransactionsIndex, gradido::blockchain::AbstractProvider; namespace cache { - BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr) - : mFolderPath(groupFolderPath), mBlockNr(blockNr), mMaxTransactionNr(0), mMinTransactionNr(0), + BlockIndex::BlockIndex(AbstractProvider* blockchainProvider, std::string_view groupFolderPath, uint32_t blockNr) + : TransactionsIndex(blockchainProvider), mFolderPath(groupFolderPath), mBlockNr(blockNr), mDirty(false) { @@ -30,7 +29,6 @@ namespace cache { bool BlockIndex::init() { - std::lock_guard _lock(mRecursiveMutex); if (loadFromFile()) { return true; } @@ -44,8 +42,6 @@ namespace cache { writeIntoFile(); clearIndexEntries(); mTransactionNrsFileCursors.clear(); - mMaxTransactionNr = 0; - mMinTransactionNr = 0; } void BlockIndex::reset() @@ -54,7 +50,7 @@ namespace cache { clearIndexEntries(); mTransactionNrsFileCursors.clear(); model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr); - LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().data()); + LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().c_str()); blockIndexFile.reset(); mMaxTransactionNr = 0; mMinTransactionNr = 0; @@ -71,7 +67,6 @@ namespace cache { std::unique_ptr BlockIndex::serialize() { - std::lock_guard _lock(mRecursiveMutex); if (!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size() && !mMaxTransactionNr && !mMinTransactionNr) { // we haven't anything to save return nullptr; @@ -88,69 +83,32 @@ namespace cache { for (auto itMonth = itYear->second.begin(); itMonth != itYear->second.end(); itMonth++) { blockIndexFile->addMonthBlock(itMonth->first); - for (const auto& blockIndexEntry : itMonth->second) { - auto fileCursorIt = mTransactionNrsFileCursors.find(blockIndexEntry.transactionNr); - if (fileCursorIt == mTransactionNrsFileCursors.end()) { - throw GradidoNodeInvalidDataException("missing file cursor for transaction"); - } - publicKeyIndicesTemp.clear(); - for (auto i = 0; i < blockIndexEntry.addressIndiceCount; i++) { - publicKeyIndicesTemp.push_back(blockIndexEntry.addressIndices[i]); - } - blockIndexFile->addDataBlock( - blockIndexEntry.transactionNr, - fileCursorIt->second, - blockIndexEntry.transactionType, - blockIndexEntry.coinCommunityIdIndex, - publicKeyIndicesTemp - ); - } - } - } - // finally write down to file - return std::move(blockIndexFile); - } - - Value BlockIndex::serializeToJson(Document::AllocatorType& alloc) const - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size() && !mMaxTransactionNr && !mMinTransactionNr) { - // we haven't anything to show - return Value(kNullType); - } - Value rootJson(kObjectType); - for (auto itYear = mYearMonthAddressIndexEntries.begin(); itYear != mYearMonthAddressIndexEntries.end(); itYear++) - { - Value yearEntry(kObjectType); - for (auto itMonth = itYear->second.begin(); itMonth != itYear->second.end(); itMonth++) - { - Value monthEntry(kArrayType); - for (const auto& blockIndexEntry : itMonth->second) + for (const auto& transactionsIndexEntryVectors : itMonth->second) { - Value entry(kObjectType); - entry.AddMember("transactionNr", blockIndexEntry.transactionNr, alloc); - entry.AddMember("transactionType", serialization::toJson(blockIndexEntry.transactionType, alloc), alloc); - if (blockIndexEntry.coinCommunityIdIndex) { - entry.AddMember("coinCommunityIdIndex", blockIndexEntry.coinCommunityIdIndex, alloc); - } - if (blockIndexEntry.addressIndiceCount) { - Value addressIndices(kArrayType); - for (int i = 0; i < blockIndexEntry.addressIndiceCount; i++) { - addressIndices.PushBack(blockIndexEntry.addressIndices[i], alloc); + for (const auto& blockIndexEntry : transactionsIndexEntryVectors) + { + auto fileCursorIt = mTransactionNrsFileCursors.find(blockIndexEntry.transactionNr); + if (fileCursorIt == mTransactionNrsFileCursors.end()) { + throw GradidoNodeInvalidDataException("missing file cursor for transaction"); } - entry.AddMember("addressIndices", addressIndices, alloc); - } - auto fileCursorIt = mTransactionNrsFileCursors.find(blockIndexEntry.transactionNr); - if (fileCursorIt != mTransactionNrsFileCursors.end()) { - entry.AddMember("fileCursor", fileCursorIt->second, alloc); + publicKeyIndicesTemp.clear(); + for (auto i = 0; i < blockIndexEntry.addressIndiceCount; i++) { + publicKeyIndicesTemp.push_back(blockIndexEntry.addressIndices[i]); + } + blockIndexFile->addDataBlock( + blockIndexEntry.transactionNr, + fileCursorIt->second, + blockIndexEntry.transactionType, + blockIndexEntry.coinCommunityIdIndex, + publicKeyIndicesTemp, + blockIndexEntry.isBalanceChanging + ); } - monthEntry.PushBack(entry, alloc); } - yearEntry.AddMember(serialization::toJson(itMonth->first, alloc), monthEntry, alloc); } - rootJson.AddMember(serialization::toJson(itYear->first, alloc), yearEntry, alloc); } - return rootJson; + // finally write down to file + return std::move(blockIndexFile); } bool BlockIndex::writeIntoFile() @@ -165,90 +123,43 @@ namespace cache { bool BlockIndex::addIndicesForTransaction( gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, - date::year year, + std::optional coinCommunityIdIndex, + date::year year, date::month month, - uint64_t transactionNr, - int32_t fileCursor, - const uint32_t* addressIndices, - uint16_t addressIndiceCount + uint64_t transactionNr, + int32_t fileCursor, + const uint32_t* addressIndices, + uint16_t addressIndiceCount, + uint8_t isBalanceChanging ) { std::lock_guard _lock(mRecursiveMutex); mDirty = true; - if (transactionNr > mMaxTransactionNr) { - mMaxTransactionNr = transactionNr; - } - if (!mMinTransactionNr || transactionNr < mMinTransactionNr) { - mMinTransactionNr = transactionNr; - } - // year - auto yearIt = mYearMonthAddressIndexEntries.find(year); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - auto result = mYearMonthAddressIndexEntries.insert({ year, {} }); - yearIt = result.first; - } - // month - auto monthIt = yearIt->second.find(month); - if (monthIt == yearIt->second.end()) { - auto result = yearIt->second.insert({ month, std::list() }); - monthIt = result.first; - } - BlockIndexEntry entry; - entry.transactionNr = transactionNr; - entry.coinCommunityIdIndex = coinCommunityIdIndex; - entry.transactionType = transactionType; - entry.addressIndiceCount = addressIndiceCount; - if (addressIndiceCount > 256) { - throw GradidoNodeInvalidDataException("addressIndiceCount is bigger than 256, that cannot be"); - } - entry.addressIndices = new uint32_t[addressIndiceCount]; - memcpy(entry.addressIndices, addressIndices, sizeof(uint32_t) * addressIndiceCount); - - if (monthIt->second.size() && monthIt->second.back().transactionNr >= entry.transactionNr) { - throw BlockIndexException("try to add new transaction to block index with same or lesser transaction nr!"); - } - monthIt->second.push_back(entry); + TransactionsIndex::addIndicesForTransaction( + transactionType, + coinCommunityIdIndex, + year, + month, + transactionNr, + addressIndices, + addressIndiceCount, + isBalanceChanging + ); addFileCursorForTransaction(transactionNr, fileCursor); return true; } - bool BlockIndex::addIndicesForTransaction(std::shared_ptr transactionEntry) + bool BlockIndex::addIndicesForTransaction( + std::shared_ptr transactionEntry, + IMutableDictionary& publicKeyDictionary + ) { - auto fileCursor = transactionEntry->getFileCursor(); - auto transactionNr = transactionEntry->getTransactionNr(); - - if (transactionNr > mMaxTransactionNr) { - mMaxTransactionNr = transactionNr; - } - if (!mMinTransactionNr || transactionNr < mMinTransactionNr) { - mMinTransactionNr = transactionNr; - } - - // transaction nr - file cursor map - if (fileCursor >= 0) { - addFileCursorForTransaction(transactionNr, fileCursor); - } - auto fbp = gradido::blockchain::FileBasedProvider::getInstance(); - - uint32_t coinCommunityIndex = 0; - if (!transactionEntry->getCoinCommunityId().empty()) { - coinCommunityIndex = fbp->getCommunityIdIndex(transactionEntry->getCoinCommunityId()); - } - auto& publicKeyIndices = transactionEntry->getAddressIndices(); - return addIndicesForTransaction( - transactionEntry->getTransactionType(), - coinCommunityIndex, - transactionEntry->getYear(), - transactionEntry->getMonth(), - transactionNr, - transactionEntry->getFileCursor(), - publicKeyIndices.data(), - static_cast(publicKeyIndices.size()) - ); - + std::lock_guard _lock(mRecursiveMutex); + TransactionsIndex::addIndicesForTransaction(transactionEntry, publicKeyDictionary); + addFileCursorForTransaction(transactionEntry->getTransactionNr(), transactionEntry->getFileCursor()); + return true; } bool BlockIndex::addFileCursorForTransaction(uint64_t transactionNr, int32_t fileCursor) @@ -285,134 +196,6 @@ namespace cache { return false; } - std::vector BlockIndex::findTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const - { - std::lock_guard _lock(mRecursiveMutex); - // prefilter - if ((filter.minTransactionNr && filter.minTransactionNr > mMaxTransactionNr) || - (filter.maxTransactionNr && filter.maxTransactionNr < mMinTransactionNr)) { - return {}; - } - - uint32_t publicKeyIndex = 0; - if (filter.involvedPublicKey && !filter.involvedPublicKey->isEmpty()) { - auto involvedPublicKeyCopy = filter.involvedPublicKey->copyAsString(); - if (publicKeysDictionary.hasString(involvedPublicKeyCopy)) { - publicKeyIndex = publicKeysDictionary.getIndexForString(involvedPublicKeyCopy); - } else { - // if public key not exist, no transaction can match - return {}; - } - } - - std::vector result; - if (filter.pagination.size) { - result.reserve(filter.pagination.size); - } - - auto interval = filteredTimepointInterval(filter); - int paginationCursor = 0; - iterateRangeInOrder(interval.begin(), interval.end(), filter.searchDirection, - [&](const date::year_month& timepoint) -> bool - { - // if for a year/month combination no entries exist, return true, so continue the loop - auto yearIt = mYearMonthAddressIndexEntries.find(timepoint.year()); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - return true; - } - auto monthIt = yearIt->second.find(timepoint.month()); - if (monthIt == yearIt->second.end()) { - return true; - } - iterateRangeInOrder( - monthIt->second.begin(), - monthIt->second.end(), - filter.searchDirection, - [&filter, publicKeyIndex, &result, &paginationCursor](const BlockIndexEntry& entry) - { - auto filterResult = entry.isMatchingFilter(filter, publicKeyIndex); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - if (paginationCursor >= filter.pagination.skipEntriesCount()) { - result.push_back(entry.transactionNr); - } - paginationCursor++; - } - if (!filter.pagination.hasCapacityLeft(result.size()) || (filterResult & FilterResult::STOP) == FilterResult::STOP) { - return false; - } - return true; - } - ); - return true; - } - ); - return result; - } - - size_t BlockIndex::countTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const - { - std::lock_guard _lock(mRecursiveMutex); - // prefilter, early exit - if ((filter.minTransactionNr && filter.minTransactionNr > mMaxTransactionNr) || - (filter.maxTransactionNr && filter.maxTransactionNr < mMinTransactionNr)) { - return 0; - } - - uint32_t publicKeyIndex = 0; - if (filter.involvedPublicKey && !filter.involvedPublicKey->isEmpty()) { - auto involvedPublicKeyCopy = filter.involvedPublicKey->copyAsString(); - if (publicKeysDictionary.hasString(involvedPublicKeyCopy)) { - publicKeyIndex = publicKeysDictionary.getIndexForString(involvedPublicKeyCopy); - } else { - // if public key not exist, no transaction can match - return 0; - } - } - - auto interval = filteredTimepointInterval(filter); - size_t result = 0; - iterateRangeInOrder(interval.begin(), interval.end(), filter.searchDirection, - [&](const date::year_month& intervalIt) -> bool - { - auto yearIt = mYearMonthAddressIndexEntries.find(intervalIt.year()); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - return true; - } - auto monthIt = yearIt->second.find(intervalIt.month()); - if (monthIt == yearIt->second.end()) { - return true; - } - // iterate over entries in the month - iterateRangeInOrder(monthIt->second.begin(), monthIt->second.end(), SearchDirection::ASC, - [&](const BlockIndexEntry& entry) -> bool - { - auto filterResult = entry.isMatchingFilter(filter, publicKeyIndex); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - ++result; - } - return true; // keep going - } - ); - return true; - } - ); - return result; - } - - std::pair BlockIndex::findTransactionsForMonthYear(date::year year, date::month month) const - { - std::lock_guard _lock(mRecursiveMutex); - auto yearIt = mYearMonthAddressIndexEntries.find(year); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - return { 0, 0 }; - } - auto monthIt = yearIt->second.find(month); - if (monthIt == yearIt->second.end()) { - return { 0, 0 }; - } - return { monthIt->second.front().transactionNr, monthIt->second.back().transactionNr }; - } - bool BlockIndex::getFileCursorForTransactionNr(uint64_t transactionNr, int32_t& fileCursor) const { std::lock_guard _lock(mRecursiveMutex); @@ -450,80 +233,5 @@ namespace cache { return false; } - date::year_month BlockIndex::getOldestYearMonth() const - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size()) { - return { date::year(0), date::month(0) }; - } - auto firstEntry = mYearMonthAddressIndexEntries.begin(); - assert(firstEntry->second.size()); - return { firstEntry->first, firstEntry->second.begin()->first }; - } - date::year_month BlockIndex::getNewestYearMonth() const - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size()) { - return { date::year(0), date::month(0) }; - } - auto lastEntry = std::prev(mYearMonthAddressIndexEntries.end()); - assert(lastEntry->second.size()); - auto lastMonthEntry = std::prev(lastEntry->second.end()); - return { lastEntry->first, lastMonthEntry->first }; - } - - void BlockIndex::clearIndexEntries() - { - std::lock_guard _lock(mRecursiveMutex); - for (auto& yearBlock : mYearMonthAddressIndexEntries) { - for (auto& monthBlock : yearBlock.second) { - for (auto& blockIndexEntry : monthBlock.second) { - delete blockIndexEntry.addressIndices; - } - } - } - mYearMonthAddressIndexEntries.clear(); - } - - FilterResult BlockIndex::BlockIndexEntry::isMatchingFilter(const gradido::blockchain::Filter& filter, const uint32_t publicKeyIndex) const - { - if (filter.transactionType != TransactionType::NONE - && filter.transactionType != transactionType) { - return FilterResult::DISMISS; - } - uint32_t coinCommunityKeyIndex = 0; - if (!filter.coinCommunityId.empty()) { - coinCommunityKeyIndex = FileBasedProvider::getInstance()->getCommunityIdIndex(filter.coinCommunityId); - } - if (coinCommunityKeyIndex && coinCommunityKeyIndex != coinCommunityIdIndex) { - return FilterResult::DISMISS; - } - if (filter.minTransactionNr && filter.minTransactionNr > transactionNr) { - return FilterResult::DISMISS; - } - if (filter.maxTransactionNr && filter.maxTransactionNr < transactionNr) { - return FilterResult::DISMISS; - } - /*uint32_t publicKeyIndex = 0; - if (filter.involvedPublicKey && !filter.involvedPublicKey->isEmpty()) { - auto involvedPublicKeyCopy = filter.involvedPublicKey->copyAsString(); - if (publicKeysDictionary.hasString(involvedPublicKeyCopy)) { - publicKeyIndex = publicKeysDictionary.getIndexForString(involvedPublicKeyCopy); - } - }*/ - if (publicKeyIndex) { - bool found = false; - for (int iPublicKeyIndices = 0; iPublicKeyIndices < addressIndiceCount; iPublicKeyIndices++) { - if (publicKeyIndex == addressIndices[iPublicKeyIndices]) { - found = true; - break; - } - } - if (!found) { - return FilterResult::DISMISS; - } - } - return FilterResult::USE; - } - + } \ No newline at end of file diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index e6ad1c5..69ab80d 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -2,17 +2,24 @@ #define __GRADIDO_NODE_CONTROLLER_BLOCK_INDEX_H #include "gradido_blockchain/blockchain/Filter.h" +#include "gradido_blockchain/blockchain/TransactionsIndex.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" #include "../blockchain/NodeTransactionEntry.h" #include "../model/files/BlockIndex.h" #include "../task/CPUTask.h" -#include "Dictionary.h" - #include "rapidjson/document.h" #include #include +#include + +namespace gradido { + namespace blockchain { + class AbstractProvider; + } +} namespace cache { @@ -27,11 +34,11 @@ namespace cache { TODO: Auto-Recover if missing, and maybe check with saved block on startup */ - class BlockIndex : public model::files::IBlockIndexReceiver + class BlockIndex : public gradido::blockchain::TransactionsIndex, public model::files::IBlockIndexReceiver { - //friend model::files::BlockIndex; + // friend model::files::BlockIndex; public: - BlockIndex(std::string_view groupFolderPath, uint32_t blockNr); + BlockIndex(gradido::blockchain::AbstractProvider* blockchainProvider, std::string_view groupFolderPath, uint32_t blockNr); ~BlockIndex(); bool init(); @@ -43,23 +50,27 @@ namespace cache { //! \brief write block index into files std::unique_ptr serialize(); - rapidjson::Value serializeToJson(rapidjson::Document::AllocatorType& alloc) const; + inline rapidjson::Value serializeToJson(rapidjson::Document::AllocatorType& alloc) const; //! \brief //! \return true if there was something to write into file, after writing it to file bool writeIntoFile(); - bool addIndicesForTransaction(std::shared_ptr transactionEntry); + bool addIndicesForTransaction( + std::shared_ptr transactionEntry, + IMutableDictionary& publicKeyDictionary + ); //! implement from model::files::IBlockIndexReceiver, called by loading block index from file bool addIndicesForTransaction( gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, + std::optional coinCommunityIdIndex, date::year year, date::month month, uint64_t transactionNr, int32_t fileCursor, const uint32_t* addressIndices, - uint16_t addressIndiceCount + uint16_t addressIndiceCount, + uint8_t isBalanceChanging ); //! \brief add transactionNr - fileCursor pair to map if not already exist @@ -68,10 +79,10 @@ namespace cache { //! \brief search transaction nrs for search criteria in filter, ignore filter function //! \return transaction nrs - std::vector findTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const; + inline std::vector findTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; //! count all, ignore pagination - size_t countTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const; + inline size_t countTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; //! \brief find transaction nrs from specific month and year //! \return {0, 0} if nothing found @@ -82,43 +93,51 @@ namespace cache { bool getFileCursorForTransactionNr(uint64_t transactionNr, int32_t& fileCursor) const; inline bool hasTransactionNr(uint64_t transactionNr) const; - inline uint64_t getMaxTransactionNr() const { std::lock_guard _lock(mRecursiveMutex); return mMaxTransactionNr; } - inline uint64_t getMinTransactionNr() const { std::lock_guard _lock(mRecursiveMutex); return mMinTransactionNr; } + inline uint64_t getMaxTransactionNr() const; + inline uint64_t getMinTransactionNr() const; inline uint64_t getTransactionsCount() const; - date::year_month getOldestYearMonth() const; - date::year_month getNewestYearMonth() const; + inline date::year_month getOldestYearMonth() const; + inline date::year_month getNewestYearMonth() const; inline TimepointInterval filteredTimepointInterval(const gradido::blockchain::Filter& filter) const; protected: - void clearIndexEntries(); - //! \brief called from model::files::BlockIndex while reading file std::string mFolderPath; uint32_t mBlockNr; - uint64_t mMaxTransactionNr; - uint64_t mMinTransactionNr; - + std::map mTransactionNrsFileCursors; typedef std::pair TransactionNrsFileCursorsPair; - struct BlockIndexEntry - { - uint64_t transactionNr; - uint32_t* addressIndices; - uint32_t coinCommunityIdIndex; - gradido::data::TransactionType transactionType; - uint8_t addressIndiceCount; - gradido::blockchain::FilterResult isMatchingFilter(const gradido::blockchain::Filter& filter, const uint32_t publicKeyIndex) const; - }; - - std::map>> mYearMonthAddressIndexEntries; - mutable std::recursive_mutex mRecursiveMutex; bool mDirty; }; + rapidjson::Value BlockIndex::serializeToJson(rapidjson::Document::AllocatorType& alloc) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::serializeToJson(alloc); + } + + std::vector BlockIndex::findTransactions( + const gradido::blockchain::Filter& filter, + const IDictionary& publicKeysDictionary + ) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::findTransactions(filter, publicKeysDictionary); + } + + size_t BlockIndex::countTransactions( + const gradido::blockchain::Filter& filter, + const IDictionary& publicKeysDictionary + ) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::countTransactions(filter, publicKeysDictionary); + } + bool BlockIndex::hasTransactionNr(uint64_t transactionNr) const { std::lock_guard _lock(mRecursiveMutex); @@ -126,25 +145,44 @@ namespace cache { && transactionNr <= mMaxTransactionNr; } + uint64_t BlockIndex::getMaxTransactionNr() const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::getMaxTransactionNr(); + } + uint64_t BlockIndex::getMinTransactionNr() const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::getMinTransactionNr(); + } + uint64_t BlockIndex::getTransactionsCount() const { std::lock_guard _lock(mRecursiveMutex); - if (!mMaxTransactionNr && !mMinTransactionNr) return 0; - return mMaxTransactionNr - mMinTransactionNr + 1; + return gradido::blockchain::TransactionsIndex::getTransactionsCount(); } - TimepointInterval BlockIndex::filteredTimepointInterval(const gradido::blockchain::Filter& filter) const + date::year_month BlockIndex::getOldestYearMonth() const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::getOldestYearMonth(); + } + date::year_month BlockIndex::getNewestYearMonth() const { - TimepointInterval interval(getOldestYearMonth(), getNewestYearMonth()); - if (!filter.timepointInterval.isEmpty()) { - if (interval.getStartDate() < filter.timepointInterval.getStartDate()) { - interval.setStartDate(filter.timepointInterval.getStartDate()); - } - if (interval.getEndDate() > filter.timepointInterval.getEndDate()) { - interval.setEndDate(std::max(interval.getStartDate(), filter.timepointInterval.getEndDate())); - } - } - return interval; + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::getNewestYearMonth(); + } + + TimepointInterval BlockIndex::filteredTimepointInterval(const gradido::blockchain::Filter& filter) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::filteredTimepointInterval(filter); + } + + std::pair BlockIndex::findTransactionsForMonthYear(date::year year, date::month month) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::findTransactionsForMonthYear(year, month); } } diff --git a/src/cache/Dictionary.cpp b/src/cache/Dictionary.cpp deleted file mode 100644 index ece73dc..0000000 --- a/src/cache/Dictionary.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "Dictionary.h" -#include "Exceptions.h" -#include "../ServerGlobals.h" -#include "../SystemExceptions.h" -#include "../model/files/FileExceptions.h" - -#include "loguru/loguru.hpp" - -namespace cache { - - Dictionary::Dictionary(std::string_view fileName) - : mLastIndex(0), - mDictionaryFile(fileName) - { - } - - Dictionary::~Dictionary() - { - } - - bool Dictionary::init(size_t cacheInBytes) - { - std::lock_guard _lock(mFastMutex); - if (!mDictionaryFile.init(cacheInBytes)) { - return false; - } - mDictionaryFile.iterate([&](leveldb::Slice key, leveldb::Slice value) -> void { - char* end = nullptr; - mValueKeyReverseLookup.insert({ SignatureOctet((const uint8_t*)value.data(), value.size()), strtoul(key.data(), &end, 10) }); - }); - mLastIndex = mValueKeyReverseLookup.size(); - return true; - } - - void Dictionary::exit() - { - mDictionaryFile.exit(); - } - - void Dictionary::reset() - { - mDictionaryFile.reset(); - mLastIndex = 0; - } - - bool Dictionary::addStringIndex(const std::string& string, uint32_t index) - { - if (hasStringIndexPair(string, index)) { - return false; - } - std::lock_guard _lock(mFastMutex); - if (mLastIndex + 1 != index) { - throw DictionaryInvalidNewKeyException( - "addValueKeyPair called with invalid new key", - mDictionaryFile.getFolderName().data(), - index, - mLastIndex - ); - } - mLastIndex = index; - mDictionaryFile.setKeyValue(std::to_string(index).c_str(), string); - mValueKeyReverseLookup.insert({ string, index }); - - return true; - } - - uint32_t Dictionary::getIndexForString(const std::string& string) const - { - auto result = findIndexForString(string); - if (!result) { - auto hex = DataTypeConverter::binToHex(string); - throw DictionaryNotFoundException("string not found in dictionary", mDictionaryFile.getFolderName().data(), hex.data()); - } - return result; - } - - std::string Dictionary::getStringForIndex(uint32_t index) const - { - std::string value; - if(!mDictionaryFile.getValueForKey(std::to_string(index).c_str(), &value)) { - throw DictionaryNotFoundException( - "index not found in dictionary", - mDictionaryFile.getFolderName().data(), - std::to_string(index).c_str() - ); - } - return value; - } - bool Dictionary::hasIndex(uint32_t index) const - { - std::string value; - return mDictionaryFile.getValueForKey(std::to_string(index).c_str(), &value); - } - - uint32_t Dictionary::getOrAddIndexForString(const std::string& string) - { - auto index = findIndexForString(string); - if (!index) { - std::lock_guard _lock(mFastMutex); - index = ++mLastIndex; - mDictionaryFile.setKeyValue(std::to_string(index).c_str(), string); - mValueKeyReverseLookup.insert({ string, index }); - } - return index; - } - - bool Dictionary::hasStringIndexPair(const std::string& string, uint32_t index) const - { - std::lock_guard _lock(mFastMutex); - auto itRange = mValueKeyReverseLookup.equal_range(string); - if (itRange.first == mValueKeyReverseLookup.end() && itRange.second == mValueKeyReverseLookup.end()) { - return false; - } - for (auto& it = itRange.first; it != itRange.second; ++it) { - return it->second == index; - } - return false; - } - - uint32_t Dictionary::findIndexForString(const std::string& string) const - { - if (string.empty()) { - throw GradidoNodeInvalidDataException("string is empty"); - } - std::lock_guard _lock(mFastMutex); - auto itRange = mValueKeyReverseLookup.equal_range(string); - for (auto& it = itRange.first; it != itRange.second; ++it) { - std::string key = std::to_string(it->second); - std::string value; - if (mDictionaryFile.getValueForKey(key.c_str(), &value)) { - if (string == value) { - return it->second; - } - } - } - return 0; - } -} diff --git a/src/cache/Dictionary.h b/src/cache/Dictionary.h deleted file mode 100644 index 3ca3c3c..0000000 --- a/src/cache/Dictionary.h +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef __GRADIDO_NODE_CACHE_DICTIONARY_H -#define __GRADIDO_NODE_CACHE_DICTIONARY_H - -#include "../model/files/LevelDBWrapper.h" -#include "gradido_blockchain/crypto/SignatureOctet.h" - -#include - -namespace gradido { - namespace blockchain { - class FileBased; - } -} - -namespace cache { - - - /*! - * @author Dario Rekowski - * @date 2020-02-06 - * - * @brief Create Indices for strings, for more efficient working with strings in memory - * So basically every one use only the uint32_t index value for comparisation and stuff and only if the real value is needed, - * they will ask the Dictionary for the real value - * - */ - class Dictionary - { - public: - Dictionary(std::string_view fileName); - virtual ~Dictionary(); - - //! \param cacheInBytes level db cache in bytes, 0 for no cache - //! \return true if address indices could be loaded - //! \return false if address index file was corrupted - bool init(size_t cacheInBytes); - void exit(); - //! delete dictionary file instance and also file on disk - void reset(); - - //! \brief Get index from cache or if not in cache, loading file, maybe I/O read. - //! \param address public key as binary string - //! \return Index or throw exception if index not exist - virtual uint32_t getIndexForString(const std::string& string) const; - virtual inline bool hasString(const std::string& string) const { - return findIndexForString(string) != 0; - } - - virtual std::string getStringForIndex(uint32_t index) const; - virtual bool hasIndex(uint32_t index) const; - - //! \brief Get or add index if not exist in cache or file, maybe I/O read. - //! \param address User public key as binary string - //! \param lastIndex Last knowing index for group. - //! \return Index for address. - virtual uint32_t getOrAddIndexForString(const std::string& string); - - //! \brief Add index - //! \param address User public key as binary string - //! \param index Index for address - //! \return False if index address already exist, else true. - virtual bool addStringIndex(const std::string& string, uint32_t index); - - inline uint32_t getLastIndex() { std::lock_guard _lock(mFastMutex); return mLastIndex; } - - protected: - bool hasStringIndexPair(const std::string& string, uint32_t index) const; - uint32_t findIndexForString(const std::string& string) const; - - mutable std::mutex mFastMutex; - uint32_t mLastIndex; - // key is index, value is string - mutable model::files::LevelDBWrapper mDictionaryFile; - std::unordered_map mValueKeyReverseLookup; - }; -} - -#endif //__GRADIDO_NODE_CACHE_DICTIONARY_H diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h new file mode 100644 index 0000000..6efbb53 --- /dev/null +++ b/src/lib/PersistentDictionary.h @@ -0,0 +1,131 @@ +#ifndef __GRADIDO_NODE_PERSISTENT_DICTIONARY_H +#define __GRADIDO_NODE_PERSISTENT_DICTIONARY_H + +#include "../model/files/LevelDBWrapper.h" +#include "../serialization/String.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" +#include "gradido_blockchain/lib/DictionaryExceptions.h" + +#include "loguru/loguru.hpp" + +#include +#include +#include +#include + + +// TODO: remove bi-directionality, update whole code for not using getDataForIndex at all! +template +requires serialization::HasString +class PersistentDictionary: public IMutableDictionary +{ +public: + explicit PersistentDictionary(const std::string& directory) : mDictionaryFile(directory) {} + ~PersistentDictionary() {} + + bool init(size_t cacheInBytes); + void exit(); + void reset(); + uint32_t getLastIndex(); + + virtual std::optional getIndexForData(const DataType& data) const override; + virtual std::optional getDataForIndex(uint32_t index) const override; + virtual uint32_t getOrAddIndexForData(const DataType& data) override; + +private: + // LevelDB reads are logically const but mutate internal state + mutable model::files::LevelDBWrapper mDictionaryFile; + mutable std::shared_mutex mWorkingMutex; + std::unordered_map mIndexDataReverseLookup; +}; + + +template +requires serialization::HasString +bool PersistentDictionary::init(size_t cacheInBytes) +{ + std::unique_lock _lock(mWorkingMutex); + if (!mDictionaryFile.init(cacheInBytes)) { + return false; + } + + // key is DataType, value is uint32 + mDictionaryFile.iterate([&](leveldb::Slice key, leveldb::Slice value) -> void { + mIndexDataReverseLookup.insert({ + serialization::fromString(value.data(), value.size()), + serialization::fromString(key.data(), key.size()) + }); + }); + return true; +} + +template +requires serialization::HasString +void PersistentDictionary::exit() +{ + std::unique_lock _lock(mWorkingMutex); + mDictionaryFile.exit(); +} + +template +requires serialization::HasString +void PersistentDictionary::reset() +{ + std::unique_lock _lock(mWorkingMutex); + mDictionaryFile.reset(); +} + +template +requires serialization::HasString +uint32_t PersistentDictionary::getLastIndex() +{ + std::unique_lock _lock(mWorkingMutex); + mIndexDataReverseLookup.size() - 1; +} + +template +requires serialization::HasString +std::optional PersistentDictionary::getIndexForData(const DataType& data) const +{ + std::shared_lock _lock(mWorkingMutex); + auto result = mDictionaryFile.getValueForKey(serialization::toString(data).c_str()); + if (result.has_value()) { + const auto& value = result.value(); + return serialization::fromString(value.data(), value.size()); + } + return std::nullopt; +} + +template +requires serialization::HasString +std::optional PersistentDictionary::getDataForIndex(uint32_t index) const +{ + std::shared_lock _lock(mWorkingMutex); + auto it = mIndexDataReverseLookup.find(index); + if (it == mIndexDataReverseLookup.end()) { + return std::nullopt; + } + return it->second; +} + +template +requires serialization::HasString +uint32_t PersistentDictionary::getOrAddIndexForData(const DataType& data) +{ + auto dataString = serialization::toString(data); + std::unique_lock _lock(mWorkingMutex); + auto result = mDictionaryFile.getValueForKey(dataString); + if (result.has_value()) { + const auto& value = result.value(); + return serialization::fromString(value.data(), value.size()); + } + if (mIndexDataReverseLookup.size() >= static_cast(std::numeric_limits::max())) { + throw DictionaryOverflowException("try to add more index data set's as uint32_t as index can handle", mDictionaryFile.getFolderName()); + } + uint32_t index = static_cast(mIndexDataReverseLookup.size()); + mDictionaryFile.setKeyValue(dataString, serialization::toString(index)); + mIndexDataReverseLookup.insert({ index, data }); + return index; +} + +#endif //__GRADIDO_NODE_PERSISTENT_DICTIONARY_H \ No newline at end of file diff --git a/src/model/files/BlockIndex.cpp b/src/model/files/BlockIndex.cpp index 10474df..4f90c2d 100644 --- a/src/model/files/BlockIndex.cpp +++ b/src/model/files/BlockIndex.cpp @@ -30,7 +30,12 @@ namespace model { vFile->write(&transactionNr, sizeof(uint64_t)); vFile->write(&fileCursor, sizeof(int32_t)); vFile->write(&transactionType, sizeof(TransactionType)); - vFile->write(&coinCommunityIdIndex, sizeof(uint32_t)); + uint8_t hasValue = coinCommunityIdIndex.has_value(); + vFile->write(&hasValue, sizeof(uint8_t)); + if (hasValue) { + vFile->write(&coinCommunityIdIndex.value(), sizeof(uint32_t)); + } + vFile->write(&isBalanceChanging, sizeof(uint8_t)); vFile->write(&addressIndicesCount, sizeof(uint8_t)); //vFile->write(this, sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint16_t)); @@ -52,8 +57,13 @@ namespace model { std::to_string((uint8_t)transactionType).data() ); } - if(!vFile->read(&coinCommunityIdIndex, sizeof(uint32_t))) return false; - if(!vFile->read(&addressIndicesCount, sizeof(uint8_t))) return false; + uint8_t hasValue = 0; + if (!vFile->read(&hasValue, sizeof(uint8_t))) return false; + if (hasValue) { + if (!vFile->read(&coinCommunityIdIndex.value(), sizeof(uint32_t))) return false; + } + if (!vFile->read(&isBalanceChanging, sizeof(uint8_t))) return false; + if (!vFile->read(&addressIndicesCount, sizeof(uint8_t))) return false; auto addressIndexSize = sizeof(uint32_t) * addressIndicesCount; addressIndices = (uint32_t*)malloc(addressIndexSize); @@ -78,10 +88,15 @@ namespace model { std::shared_ptr BlockIndex::DataBlock::createTransactionEntry(date::month month, date::year year) { - auto coinCommunityId = FileBasedProvider::getInstance()->getCommunityIdString(coinCommunityIdIndex); - // TransactionEntry(uint64_t transactionNr, int32_t fileCursor, uint8_t month, uint16_t year, uint32_t* addressIndices, uint8_t addressIndiceCount); + std::string coinCommunityString = ""; + if (coinCommunityIdIndex.has_value()) { + auto res = FileBasedProvider::getInstance()->getCommunityIdString(coinCommunityIdIndex.value()); + if (res.has_value()) { + coinCommunityString = res.value(); + } + } auto transactionEntry = std::make_shared( - transactionNr, month, year, transactionType, coinCommunityId, addressIndices, addressIndicesCount + transactionNr, month, year, transactionType, coinCommunityString, addressIndices, addressIndicesCount ); transactionEntry->setFileCursor(fileCursor); return transactionEntry; @@ -249,7 +264,8 @@ namespace model { dataBlock->coinCommunityIdIndex, yearCursor, monthCursor, dataBlock->transactionNr, dataBlock->fileCursor, - dataBlock->addressIndices, dataBlock->addressIndicesCount + dataBlock->addressIndices, dataBlock->addressIndicesCount, + dataBlock->isBalanceChanging ); } diff --git a/src/model/files/BlockIndex.h b/src/model/files/BlockIndex.h index af5f681..f9985c9 100644 --- a/src/model/files/BlockIndex.h +++ b/src/model/files/BlockIndex.h @@ -9,6 +9,7 @@ #include #include +#include namespace cache { class BlockIndex; @@ -28,13 +29,14 @@ namespace model { public: virtual bool addIndicesForTransaction( gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, + std::optional coinCommunityIdIndex, date::year year, date::month month, uint64_t transactionNr, int32_t fileCursor, const uint32_t* addressIndices, - uint16_t addressIndiceCount + uint16_t addressIndiceCount, + uint8_t isBalanceChanging ) = 0; }; @@ -70,10 +72,11 @@ namespace model { uint64_t transactionNr, int32_t fileCursor, gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, + std::optional coinCommunityIdIndex, + uint8_t isBalanceChanging, const std::vector& addressIndices ) { - mDataBlocks.push(new DataBlock(transactionNr, fileCursor, transactionType, coinCommunityIdIndex, addressIndices)); + mDataBlocks.push(new DataBlock(transactionNr, fileCursor, transactionType, coinCommunityIdIndex, isBalanceChanging, addressIndices)); mDataBlockSumSize += mDataBlocks.back()->size(); } @@ -99,9 +102,6 @@ namespace model { protected: //! \brief replace Index File with new one, clear blocks after writing into file - - - enum BlockTypes { YEAR_BLOCK = 0xad, MONTH_BLOCK = 0x50, @@ -177,16 +177,18 @@ namespace model { uint64_t _transactionNr, int32_t _fileCursor, gradido::data::TransactionType _transactionType, - uint32_t _coinCommunityIdIndex, - const std::vector& _addressIndices + std::optional _coinCommunityIdIndex, + uint8_t _isBalanceChanging, + const std::vector& _addressIndices ) : Block(DATA_BLOCK), transactionNr(_transactionNr), fileCursor(_fileCursor), transactionType(_transactionType), coinCommunityIdIndex(_coinCommunityIdIndex), + isBalanceChanging(_isBalanceChanging), addressIndices(nullptr), - addressIndicesCount(_addressIndices.size()) + addressIndicesCount(_addressIndices.size()) { addressIndices = (uint32_t*)malloc(addressIndicesCount * sizeof(uint32_t)); assert(addressIndices); @@ -198,9 +200,10 @@ namespace model { transactionNr(0), fileCursor(-10), transactionType(gradido::data::TransactionType::NONE), - coinCommunityIdIndex(0), + coinCommunityIdIndex(std::nullopt), + isBalanceChanging(0), addressIndices(nullptr), - addressIndicesCount(0) + addressIndicesCount(0) { } @@ -211,13 +214,15 @@ namespace model { addressIndices = nullptr; addressIndicesCount = 0; fileCursor = 0; + isBalanceChanging = 0; } uint64_t transactionNr; int32_t fileCursor; gradido::data::TransactionType transactionType; - uint32_t coinCommunityIdIndex; + std::optional coinCommunityIdIndex; + uint8_t isBalanceChanging; uint8_t addressIndicesCount; - uint32_t* addressIndices; + uint32_t* addressIndices; size_t size() { return sizeof(uint8_t) // Block Type @@ -225,7 +230,9 @@ namespace model { + sizeof(int32_t) // fileCursor + sizeof(gradido::data::TransactionType) // transaction type + sizeof(uint32_t) // coin community id index size - + sizeof(uint8_t) + sizeof(uint32_t) * addressIndicesCount; // address index count, address indices array + + sizeof(uint8_t) // isBalanceChanging + + sizeof(uint8_t) + sizeof(uint32_t) * addressIndicesCount // address index count, address indices array + ; } virtual void writeIntoFile(VirtualFile* vFile); diff --git a/src/model/files/LevelDBWrapper.cpp b/src/model/files/LevelDBWrapper.cpp index dea8030..9f8564b 100644 --- a/src/model/files/LevelDBWrapper.cpp +++ b/src/model/files/LevelDBWrapper.cpp @@ -1,5 +1,6 @@ #include "LevelDBWrapper.h" #include "../../lib/LevelDBExceptions.h" +#include "../../SingletonManager/FileLockManager.h" #include "loguru/loguru.hpp" #include "leveldb/cache.h" @@ -9,13 +10,15 @@ #include #include +using std::chrono::milliseconds, std::this_thread::sleep_for; +using std::filesystem::remove_all; +using std::optional, std::nullopt, std::string, std::string_view, std::function; +using leveldb::Status, leveldb::DB, leveldb::Slice, leveldb::ReadOptions, leveldb::WriteOptions, leveldb::NewLRUCache; + namespace model { namespace files { - - // use this global mutex to prevent an error with level db if one group is deconstructed while the same group is created new at the same time - std::mutex g_StateMutex; - LevelDBWrapper::LevelDBWrapper(std::string_view folderName) + LevelDBWrapper::LevelDBWrapper(string_view folderName) : mFolderName(folderName), mLevelDB(nullptr) { } @@ -27,22 +30,27 @@ namespace model { bool LevelDBWrapper::init(size_t cacheInByte/* = 0*/) { - std::lock_guard _lock(g_StateMutex); + auto fm = FileLockManager::getInstance(); + if (!fm->tryLockTimeout(mFolderName, 100)) { + LOG_F(ERROR, "path: %s couldn't locked, another process still use this folder?", mFolderName.c_str()); + return false; + } mOptions.create_if_missing = true; mOptions.paranoid_checks = true; if (cacheInByte) { - mOptions.block_cache = leveldb::NewLRUCache(cacheInByte); + mOptions.block_cache = NewLRUCache(cacheInByte); } - leveldb::Status status = leveldb::DB::Open(mOptions, mFolderName, &mLevelDB); + Status status = DB::Open(mOptions, mFolderName, &mLevelDB); // if blockchain::FileBased is removed from cache and created new at the same time, the lock file from other level db instance is maybe still there // and trigger an io error, so give it same time an try it again, maximal 100 times. // TODO: Maybe use the FileLockManager for this int maxTry = 100; while (status.IsIOError() && maxTry > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - status = leveldb::DB::Open(mOptions, mFolderName, &mLevelDB); + sleep_for(milliseconds(100)); + status = DB::Open(mOptions, mFolderName, &mLevelDB); maxTry--; } + fm->unlock(mFolderName); if (!status.ok()) { LOG_F(ERROR, "path: %s, state: %s, ioError: %d", mFolderName.data(), status.ToString().data(), status.IsIOError()); } @@ -50,7 +58,11 @@ namespace model { } void LevelDBWrapper::exit() { - std::lock_guard _lock(g_StateMutex); + auto fm = FileLockManager::getInstance(); + if (!fm->tryLockTimeout(mFolderName, 1000)) { + LOG_F(FATAL, "on exit: path: %s couldn't locked, another process still use this folder, data maybe corrupted, please refresh!", mFolderName.c_str()); + return; + } if (mLevelDB) { delete mLevelDB; mLevelDB = nullptr; @@ -59,38 +71,43 @@ namespace model { mOptions.block_cache = nullptr; } } + fm->unlock(mFolderName); } void LevelDBWrapper::reset() { exit(); - std::filesystem::remove_all(mFolderName); + remove_all(mFolderName); init(); } - bool LevelDBWrapper::getValueForKey(const char* key, std::string* value) + optional LevelDBWrapper::getValueForKey(const std::string& key) { - leveldb::Status s = mLevelDB->Get(leveldb::ReadOptions(), key, value); - return s.ok(); + string value; + Status s = mLevelDB->Get(ReadOptions(), key, &value); + if (!s.ok()) { return nullopt; } + return value; } - void LevelDBWrapper::setKeyValue(const char* key, const std::string& value) + void LevelDBWrapper::setKeyValue(const std::string& key, const std::string& value) { - leveldb::Status s = mLevelDB->Put(leveldb::WriteOptions(), key, value); + WriteOptions writeOptions; + writeOptions.sync = true; + Status s = mLevelDB->Put(writeOptions, key, value); if (!s.ok()) { throw LevelDBStatusException("cannot put to level db", s); } } - void LevelDBWrapper::removeKey(const char* key) + void LevelDBWrapper::removeKey(const std::string& key) { - mLevelDB->Delete(leveldb::WriteOptions(), key); + mLevelDB->Delete(WriteOptions(), key); } - void LevelDBWrapper::iterate(std::function callback) + void LevelDBWrapper::iterate(function callback) { //! \brief get iterator for looping over every entry - leveldb::ReadOptions options; + ReadOptions options; options.fill_cache = false; options.verify_checksums = true; auto it = mLevelDB->NewIterator(options); diff --git a/src/model/files/LevelDBWrapper.h b/src/model/files/LevelDBWrapper.h index 10071b4..45ef9d1 100644 --- a/src/model/files/LevelDBWrapper.h +++ b/src/model/files/LevelDBWrapper.h @@ -5,6 +5,7 @@ #include #include +#include /*! * @author Dario Rekowski @@ -33,13 +34,13 @@ namespace model { //! read value for key from leveldb //! \param value pointer to string in which result will be write //! \return true if value could be found - bool getValueForKey(const char* key, std::string* value); + std::optional getValueForKey(const std::string& key); //! add new value key pair or update value if key exist //! \return true, throw exception on error - void setKeyValue(const char* key, const std::string& value); + void setKeyValue(const std::string& key, const std::string& value); - void removeKey(const char* key); + void removeKey(const std::string& key); //! go through all entries and call callback for each with key, value //! don't fill level db cache, verify checksum diff --git a/src/serialization/String.h b/src/serialization/String.h new file mode 100644 index 0000000..7cf7c08 --- /dev/null +++ b/src/serialization/String.h @@ -0,0 +1,66 @@ +#ifndef __GRADIDO_NODE_SERIALIZATION_STRING_H +#define __GRADIDO_NODE_SERIALIZATION_STRING_H + +#include +#include +#include "gradido_blockchain/memory/Block.h" + +namespace serialization { + template + std::string toString(const T& value); + + template + T fromString(const char* data, size_t size); + + template + concept HasString = + requires(const T& t, const char* c, size_t size) { + { toString(t) } -> std::same_as; + { fromString(c, size) } -> std::same_as; + }; + + // for string, it's only need's to copy + template<> + inline std::string toString(const std::string& s) { + return s; + } + + template<> + inline std::string fromString(const char* data, size_t size) { + return std::string(data, size); + } + + template<> + inline std::string toString(const uint32_t& v) { + return std::to_string(v); + } + + template<> + inline uint32_t fromString(const char* data, size_t size) { + return static_cast(strtoul(data, nullptr, 0)); + } + + template<> + inline std::string toString(const size_t& v) { + return std::to_string(v); + } + + template<> + inline size_t fromString(const char* data, size_t size) { + return strtoull(data, nullptr, 0); + } + + template<> + inline std::string toString(const memory::ConstBlockPtr& ptr) + { + return ptr->copyAsString(); + } + + template<> + inline memory::ConstBlockPtr fromString(const char* data, size_t size) + { + return std::make_shared(data, size); + } +} + +#endif //__GRADIDO_NODE_SERIALIZATION_STRING_H \ No newline at end of file From 1225fd081d85e5c0c185a463dff89079bcb0405c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 6 Jan 2026 17:46:43 +0100 Subject: [PATCH 02/65] speed up rebuilding block index --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 2 +- src/blockchain/FileBased.cpp | 30 ++--- src/blockchain/FileBased.h | 4 +- src/blockchain/NodeTransactionEntry.cpp | 12 +- src/blockchain/NodeTransactionEntry.h | 7 ++ src/cache/Block.cpp | 39 +++--- src/cache/Block.h | 15 ++- src/cache/BlockIndex.cpp | 4 +- src/cache/BlockIndex.h | 2 +- src/cache/HieroTransactionId.cpp | 105 ---------------- src/cache/LedgerAnchor.cpp | 119 ++++++++++++++++++ .../{HieroTransactionId.h => LedgerAnchor.h} | 30 +++-- src/cache/MessageId.cpp | 94 -------------- src/cache/MessageId.h | 40 ------ src/cache/State.cpp | 18 +-- src/controller/SimpleOrderingManager.cpp | 32 ++--- src/controller/SimpleOrderingManager.h | 9 +- src/lib/PersistentDictionary.h | 4 +- src/model/files/Block.cpp | 56 ++------- src/model/files/Block.h | 30 ++--- src/model/files/BlockIndex.cpp | 16 ++- src/model/files/LevelDBWrapper.cpp | 2 +- src/serialization/String.cpp | 22 ++++ src/serialization/String.h | 13 -- src/server/json-rpc/ApiHandler.cpp | 7 +- ...tchDeserializeConfirmedTransactionTask.cpp | 48 +++++++ ...BatchDeserializeConfirmedTransactionTask.h | 38 ++++++ src/task/CPUSheduler.cpp | 2 +- .../DeserializeConfirmedTransactionTask.cpp | 40 ++++++ .../DeserializeConfirmedTransactionTask.h | 35 ++++++ src/task/RebuildBlockIndexTask.cpp | 94 ++++++++++++++ src/task/RebuildBlockIndexTask.h | 84 +++++++++++++ src/task/Task.h | 1 + src/task/WriteTransactionsToBlockTask.cpp | 8 +- src/task/WriteTransactionsToBlockTask.h | 13 +- 36 files changed, 663 insertions(+), 414 deletions(-) delete mode 100644 src/cache/HieroTransactionId.cpp create mode 100644 src/cache/LedgerAnchor.cpp rename src/cache/{HieroTransactionId.h => LedgerAnchor.h} (50%) delete mode 100644 src/cache/MessageId.cpp delete mode 100644 src/cache/MessageId.h create mode 100644 src/serialization/String.cpp create mode 100644 src/task/BatchDeserializeConfirmedTransactionTask.cpp create mode 100644 src/task/BatchDeserializeConfirmedTransactionTask.h create mode 100644 src/task/DeserializeConfirmedTransactionTask.cpp create mode 100644 src/task/DeserializeConfirmedTransactionTask.h create mode 100644 src/task/RebuildBlockIndexTask.cpp create mode 100644 src/task/RebuildBlockIndexTask.h diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 4b6609b..443ee65 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 4b6609b3b2f137f716c571807eb23e8b3873d618 +Subproject commit 443ee65afcbefc9430d604fc46506e0e5b6d296f diff --git a/src/MainServer.cpp b/src/MainServer.cpp index bf004ff..2b8fac4 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -75,7 +75,7 @@ bool MainServer::init() // start cpu scheduler // std::thread::hardware_concurrency() sometime return 0 if number couldn't be determined - uint8_t worker_count = 2; // std::max(2, (int)std::thread::hardware_concurrency() * 2); + uint8_t worker_count = std::max(2, (int)std::thread::hardware_concurrency() * 2); // I think 1 or 2 by HDD is ok, more by SSD, but should be profiled on work load uint8_t io_worker_count = config.getInt("io.worker_count", 2); ServerGlobals::g_CPUScheduler = new task::CPUSheduler(worker_count, "Default Worker"); diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 8b087df..34386e1 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -51,7 +51,7 @@ namespace gradido { // mIotaMessageListener(new iota::MessageListener(communityId, alias)), mPublicKeysIndex((string(folder).append("/pubkeysCache"))), mBlockchainState(string(folder).append("/.state")), - mMessageIdsCache(string(folder).append("/messageIdCache")), + mLedgerAnchorCache(string(folder).append("/messageIdCache")), mTransactionTriggerEventsCache(std::string(folder).append("/transactionTriggerEventCache")), mCachedBlocks(ServerGlobals::g_CacheTimeout), mTransactionHashCache(communityId), @@ -95,16 +95,16 @@ namespace gradido { mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0); mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); - if (!mMessageIdsCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { - mMessageIdsCache.reset(); - if (!mMessageIdsCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { + if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { + mLedgerAnchorCache.reset(); + if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { throw ClassNotInitalizedException("cannot initalize message id cache", "cache::MessageId"); } // load last 20 message ids into cache FilterBuilder filterBuilder; auto transactions = findAll(filterBuilder.setPagination({20}).setSearchDirection(SearchDirection::DESC).build()); for (auto& transaction : transactions) { - mMessageIdsCache.add(transaction->getConfirmedTransaction()->getMessageId(), transaction->getTransactionNr()); + mLedgerAnchorCache.add(transaction->getConfirmedTransaction()->getLedgerAnchor(), transaction->getTransactionNr()); } } @@ -207,7 +207,7 @@ namespace gradido { mCachedBlocks.clear(); mBlockchainState.exit(); mPublicKeysIndex.exit(); - mMessageIdsCache.exit(); + mLedgerAnchorCache.exit(); mTransactionTriggerEventsCache.exit(); } bool FileBased::createAndAddConfirmedTransaction( @@ -218,13 +218,13 @@ namespace gradido { if (!gradidoTransaction) { throw GradidoNullPointerException("missing transaction", "GradidoTransactionPtr", __FUNCTION__); } - if (!messageId) { - throw GradidoNullPointerException("missing messageId", "memory::ConstBlockPtr", __FUNCTION__); + if (ledgerAnchor.empty()) { + throw GradidoNullPointerException("empty ledger anchor", "gradido::data::LedgerAnchor", __FUNCTION__); } std::lock_guard _lock(mWorkMutex); if (mExitCalled) { return false;} confirmTransaction::Context confirmTransactionContext(getptr()); - auto role = confirmTransactionContext.createRole(gradidoTransaction, messageId, confirmedAt); + auto role = confirmTransactionContext.createRole(gradidoTransaction, ledgerAnchor, confirmedAt); auto confirmedTransaction = confirmTransactionContext.run(role); // will occure if transaction already exist if (!confirmedTransaction) { @@ -233,7 +233,7 @@ namespace gradido { auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); auto& block = getBlock(blockNr); auto nodeTransactionEntry = std::make_shared(confirmedTransaction, getptr()); - if (!block.pushTransaction(nodeTransactionEntry)) { + if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex)) { // block was already stopped, so we can stop here also LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); return false; @@ -242,7 +242,7 @@ namespace gradido { mBlockchainState.updateState(DefaultStateKeys::LAST_TRANSACTION_ID, confirmedTransaction->getId()); mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); mTransactionHashCache.push(*nodeTransactionEntry->getConfirmedTransaction()); - mMessageIdsCache.add(confirmedTransaction->getMessageId(), confirmedTransaction->getId()); + mLedgerAnchorCache.add(confirmedTransaction->getLedgerAnchor(), confirmedTransaction->getId()); // add public keys to index auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); for (const auto& address : involvedAddresses) { @@ -362,14 +362,14 @@ namespace gradido { } std::shared_ptr FileBased::findByLedgerAnchor( const data::LedgerAnchor& ledgerAnchor, - const Filter& filter = Filter::ALL_TRANSACTIONS + const Filter& filter/* = Filter::ALL_TRANSACTIONS*/ ) const { - auto transactionNr = mMessageIdsCache.has(messageId); + auto transactionNr = mLedgerAnchorCache.has(ledgerAnchor); if (transactionNr) { return getTransactionForId(transactionNr); } - return Abstract::findByMessageId(messageId, filter); + return Abstract::findByLedgerAnchor(ledgerAnchor, filter); } AbstractProvider* FileBased::getProvider() const { @@ -463,7 +463,7 @@ namespace gradido { if (!block) { auto block = std::make_shared(blockNr, getptr()); // return false if block not exist and will be created - if (!block->init()) { + if (!block->init(mPublicKeysIndex)) { if (blockNr > mBlockchainState.readInt32State(DefaultStateKeys::LAST_BLOCK_NR, 1)) { mBlockchainState.updateState(DefaultStateKeys::LAST_BLOCK_NR, blockNr); } diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 31846ef..8ceb7e8 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -2,7 +2,7 @@ #define __GRADIDO_NODE_BLOCKCHAIN_FILE_BASED_H #include "../cache/Block.h" -#include "../cache/HieroTransactionId.h" +#include "../cache/LedgerAnchor.h" #include "../cache/State.h" #include "../cache/TransactionHash.h" #include "../cache/TransactionTriggerEvent.h" @@ -183,7 +183,7 @@ namespace gradido { // level db to store state values like last transaction mutable cache::State mBlockchainState; - mutable cache::HieroTransactionId mMessageIdsCache; + mutable cache::LedgerAnchor mLedgerAnchorCache; cache::TransactionTriggerEvent mTransactionTriggerEventsCache; diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index 137f70a..ea7e91d 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -15,7 +15,7 @@ namespace gradido { mPublicKeyIndices.reserve(involvedPublicKeys.size()); for (auto& publicKey : involvedPublicKeys) { mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(publicKey)); - } + } } @@ -49,5 +49,15 @@ namespace gradido { } } + NodeTransactionEntry::NodeTransactionEntry( + gradido::data::ConstConfirmedTransactionPtr transaction, + memory::ConstBlockPtr serializedTransaction, + std::shared_ptr blockchain, + int32_t fileCursor/* = -10 */ + ) : TransactionEntry(serializedTransaction, transaction), mFileCursor(fileCursor) + { + + } + } } diff --git a/src/blockchain/NodeTransactionEntry.h b/src/blockchain/NodeTransactionEntry.h index e5445fa..3b5e269 100644 --- a/src/blockchain/NodeTransactionEntry.h +++ b/src/blockchain/NodeTransactionEntry.h @@ -36,6 +36,13 @@ namespace gradido { int32_t fileCursor = -10 ); + NodeTransactionEntry( + gradido::data::ConstConfirmedTransactionPtr transaction, + memory::ConstBlockPtr serializedTransaction, + std::shared_ptr blockchain, + int32_t fileCursor = -10 + ); + //! \brief init entry object from details e.g. by loading from file NodeTransactionEntry( uint64_t transactionNr, diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 21a5890..9979223 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -8,11 +8,13 @@ #include "../controller/TaskObserver.h" #include "../model/files/Block.h" #include "../model/files/FileExceptions.h" +#include "../task/RebuildBlockIndexTask.h" #include "../SingletonManager/CacheManager.h" #include "gradido_blockchain/Application.h" #include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_blockchain/lib/Profiler.h" @@ -23,12 +25,13 @@ using namespace gradido::blockchain; using namespace gradido::interaction; +using task::RebuildBlockIndexTask; namespace cache { Block::Block(uint32_t blockNr, std::shared_ptr blockchain) : mBlockNr(blockNr), mSerializedTransactions(ServerGlobals::g_CacheTimeout), - mBlockIndex(std::make_shared(blockchain->getFolderPath(), blockNr)), + mBlockIndex(std::make_shared(blockchain->getProvider(), blockchain->getFolderPath(), blockNr)), mBlockFile(std::make_shared(blockchain->getFolderPath(), blockNr)), mBlockchain(blockchain), mExitCalled(false) @@ -48,39 +51,32 @@ namespace cache { mSerializedTransactions.clear(); } - bool Block::init() + bool Block::init(IMutableDictionary& publicKeyDictionary) { std::lock_guard lock(mFastMutex); if (!mBlockIndex->loadFromFile()) { // check if Block exist if (mBlockFile->getCurrentFileSize()) { Profiler timeUsed; - auto rebuildBlockIndexTask = mBlockFile->rebuildBlockIndex(mBlockchain); + mBlockIndex->reset(); + std::shared_ptr rebuildBlockIndexTask = std::make_shared( + mBlockchain, + mBlockIndex, + publicKeyDictionary + ); + mBlockFile->fillRebuildBlockIndexTask(rebuildBlockIndexTask); if (!rebuildBlockIndexTask) { throw GradidoNullPointerException("missing rebuild block index task", "RebuildBlockIndexTask", __FUNCTION__); } - rebuildBlockIndexTask->scheduleTask(rebuildBlockIndexTask); int sumWaited = 0; - while (!rebuildBlockIndexTask->isPendingQueueEmpty() && sumWaited < 1000) { + while (!rebuildBlockIndexTask->isPendingQueueEmpty() && sumWaited < GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); sumWaited += 100; } if (!rebuildBlockIndexTask->isPendingQueueEmpty()) { - LOG_F(FATAL, "rebuildBlockIndex Task isn't finished after waiting a whole second"); + LOG_F(FATAL, "rebuildBlockIndex Task isn't finished after waiting a whole minute"); Application::terminate(); } - auto transactionEntries = rebuildBlockIndexTask->getTransactionEntries(); - mBlockIndex->reset(); - uint64_t prevId = 0; - std::for_each(transactionEntries.begin(), transactionEntries.end(), - [&](const std::shared_ptr& transactionEntry) { - if (transactionEntry->getTransactionNr() <= prevId) { - throw BlockchainOrderException("transaction nrs aren't in ascending order"); - } - prevId = transactionEntry->getTransactionNr(); - mBlockIndex->addIndicesForTransaction(transactionEntry); - } - ); LOG_F(INFO, "time for rebuilding block index for block %s: %s", mBlockFile->getBlockPath().data(), timeUsed.string().data()); } else { @@ -101,7 +97,10 @@ namespace cache { } //bool Block::pushTransaction(const std::string& serializedTransaction, uint64_t transactionNr) - bool Block::pushTransaction(std::shared_ptr transaction) + bool Block::pushTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary + ) { std::lock_guard lock(mFastMutex); if (mExitCalled) return false; @@ -110,7 +109,7 @@ namespace cache { // std::shared_ptr blockFile, std::shared_ptr blockIndex mTransactionWriteTask = std::make_shared(mBlockFile, mBlockIndex); } - mTransactionWriteTask->addSerializedTransaction(transaction); + mTransactionWriteTask->addSerializedTransaction(transaction, publicKeyDictionary); mSerializedTransactions.add(transaction->getTransactionNr(), transaction); return true; diff --git a/src/cache/Block.h b/src/cache/Block.h index 5668b75..cb0e425 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -8,9 +8,11 @@ #include "../task/WriteTransactionsToBlockTask.h" #include "gradido_blockchain/lib/AccessExpireCache.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" #include "gradido_blockchain/blockchain/TransactionEntry.h" #include +#include namespace model { namespace files { @@ -25,8 +27,14 @@ namespace gradido { } } +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr; +} + // TODO: move into config #define GRADIDO_NODE_CACHE_BLOCK_MAX_FILE_SIZE_BYTE 128 * 1024 * 1024 +#define GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS 1000 * 60 namespace cache { class Group; @@ -44,11 +52,14 @@ namespace cache { ~Block(); //! \return false if block not exist - bool init(); + bool init(IMutableDictionary& publicKeyDictionary); void exit(); //! \brief put new transaction to cache and file system - bool pushTransaction(std::shared_ptr transaction); + bool pushTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary + ); //! \brief load transaction from cache or file system std::shared_ptr getTransaction(uint64_t transactionNr) const; diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index 01d6c0c..8117713 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -100,8 +100,8 @@ namespace cache { fileCursorIt->second, blockIndexEntry.transactionType, blockIndexEntry.coinCommunityIdIndex, - publicKeyIndicesTemp, - blockIndexEntry.isBalanceChanging + blockIndexEntry.isBalanceChanging, + publicKeyIndicesTemp ); } } diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index 69ab80d..ca5630b 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -86,7 +86,7 @@ namespace cache { //! \brief find transaction nrs from specific month and year //! \return {0, 0} if nothing found - std::pair findTransactionsForMonthYear(date::year year, date::month month) const; + inline std::pair findTransactionsForMonthYear(date::year year, date::month month) const; //! \param fileCursor reference to be filled with fileCursor //! \return true if transaction nr was found and fileCursor was set, else return false diff --git a/src/cache/HieroTransactionId.cpp b/src/cache/HieroTransactionId.cpp deleted file mode 100644 index 2952c5b..0000000 --- a/src/cache/HieroTransactionId.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include "HieroTransactionId.h" -#include "../ServerGlobals.h" -#include "../SystemExceptions.h" -#include "../lib/LevelDBExceptions.h" -#include "gradido_blockchain/interaction/deserialize/Context.h" - -#include "loguru/loguru.hpp" - -namespace cache { - HieroTransactionId::HieroTransactionId(std::string_view folder) - : mInitalized(false), - mLevelDBFile(folder), - mHieroTransactionIdTransactionNrs(ServerGlobals::g_CacheTimeout) - { - - } - HieroTransactionId::~HieroTransactionId() - { - - } - - // try to open db - bool HieroTransactionId::init(size_t cacheInBytes) - { - if (mInitalized) { - throw ClassAlreadyInitalizedException("init was already called", "cache::MessageId"); - } - if (!mLevelDBFile.init(cacheInBytes)) { - return false; - } - mInitalized = true; - return true; - } - - void HieroTransactionId::exit() - { - if (!mInitalized) { - LOG_F(WARNING, "init wasn't called, cache::MessageId aren't stored in leveldb file, or exit was called more than once"); - } - mInitalized = false; - mLevelDBFile.exit(); - } - - //! remove state level db folder, clear maps - void HieroTransactionId::reset() - { - mLevelDBFile.reset(); - mHieroTransactionIdTransactionNrs.clear(); - } - - void HieroTransactionId::add(memory::ConstBlockPtr transactionId, uint64_t transactionNr) - { - auto hieroTransactionId = fromProtobuf(transactionId); - if (mHieroTransactionIdTransactionNrs.get(hieroTransactionId).has_value()) { - throw GradidoAlreadyExist("cache::HieroTransactionId already has key!"); - } - mHieroTransactionIdTransactionNrs.add(hieroTransactionId, transactionNr); - if (mInitalized) { - mLevelDBFile.setKeyValue(transactionId->convertToBase64().data(), std::to_string(transactionNr).data()); - } - } - - bool HieroTransactionId::has(memory::ConstBlockPtr transactionId) - { - auto hieroTransactionId = fromProtobuf(transactionId); - if (mHieroTransactionIdTransactionNrs.get(hieroTransactionId).has_value()) { - return true; - } - return readFromLevelDb(transactionId, hieroTransactionId) != 0; - } - - uint64_t HieroTransactionId::getTransactionNrForHieroTransactionId(memory::ConstBlockPtr transactionId) - { - auto hieroTransactionId = fromProtobuf(transactionId); - auto result = mHieroTransactionIdTransactionNrs.get(hieroTransactionId); - if (result.has_value()) { - return result.value(); - } - return readFromLevelDb(transactionId, hieroTransactionId); - } - - uint64_t HieroTransactionId::readFromLevelDb(memory::ConstBlockPtr transactionId, hiero::TransactionId transactionIdObj) - { - if (mInitalized) { - std::string value; - if (mLevelDBFile.getValueForKey(transactionId->convertToBase64().data(), &value)) { - auto transactionNr = std::stoull(value); - mHieroTransactionIdTransactionNrs.add(transactionIdObj, transactionNr); - return transactionNr; - } - } - return 0; - } - - hiero::TransactionId HieroTransactionId::fromProtobuf(memory::ConstBlockPtr transactionId) const - { - gradido::interaction::deserialize::Context deserializer(transactionId, gradido::interaction::deserialize::Type::HIERO_TRANSACTION_ID); - deserializer.run(); - if (!deserializer.isHieroTransactionId()) { - throw GradidoNodeInvalidDataException("[cache::HieroTransactionId::fromProtobuf] transactionId cannot be deserialized as hiero::TransactionId"); - } - return deserializer.getHieroTransactionId(); - } - -} \ No newline at end of file diff --git a/src/cache/LedgerAnchor.cpp b/src/cache/LedgerAnchor.cpp new file mode 100644 index 0000000..080a7d6 --- /dev/null +++ b/src/cache/LedgerAnchor.cpp @@ -0,0 +1,119 @@ +#include "LedgerAnchor.h" +#include "../ServerGlobals.h" +#include "../SystemExceptions.h" +#include "../lib/LevelDBExceptions.h" +#include "gradido_blockchain/data/LedgerAnchor.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/interaction/serialize/Context.h" +#include "gradido_blockchain/memory/Block.h" + +#include "loguru/loguru.hpp" +using memory::Block; +using std::shared_ptr, std::make_shared; + +namespace cache { + LedgerAnchor::LedgerAnchor(std::string_view folder) + : mInitalized(false), + mLevelDBFile(folder), + mLedgerAnchorTransactionNrs(ServerGlobals::g_CacheTimeout) + { + + } + LedgerAnchor::~LedgerAnchor() + { + + } + + // try to open db + bool LedgerAnchor::init(size_t cacheInBytes) + { + if (mInitalized) { + throw ClassAlreadyInitalizedException("init was already called", "cache::MessageId"); + } + if (!mLevelDBFile.init(cacheInBytes)) { + return false; + } + mInitalized = true; + return true; + } + + void LedgerAnchor::exit() + { + if (!mInitalized) { + LOG_F(WARNING, "init wasn't called, cache::MessageId aren't stored in leveldb file, or exit was called more than once"); + } + mInitalized = false; + mLevelDBFile.exit(); + } + + //! remove state level db folder, clear maps + void LedgerAnchor::reset() + { + mLevelDBFile.reset(); + mLedgerAnchorTransactionNrs.clear(); + } + + void LedgerAnchor::add(const gradido::data::LedgerAnchor& transactionId, uint64_t transactionNr) + { + auto ledgerAnchorSerialized = toProtobuf(transactionId); + + if (mLedgerAnchorTransactionNrs.get(ledgerAnchorSerialized).has_value()) { + throw GradidoAlreadyExist("cache::LedgerAnchor already has key!"); + } + mLedgerAnchorTransactionNrs.add(ledgerAnchorSerialized, transactionNr); + if (mInitalized) { + mLevelDBFile.setKeyValue(ledgerAnchorSerialized, std::to_string(transactionNr).data()); + } + } + + bool LedgerAnchor::has(const gradido::data::LedgerAnchor& transactionId) + { + auto ledgerAnchorSerialized = toProtobuf(transactionId); + if (mLedgerAnchorTransactionNrs.get(ledgerAnchorSerialized).has_value()) { + return true; + } + return readFromLevelDb(ledgerAnchorSerialized) != 0; + } + + uint64_t LedgerAnchor::getTransactionNrForLedgerAnchor(const gradido::data::LedgerAnchor& transactionId) + { + auto ledgerAnchorSerialized = toProtobuf(transactionId); + auto result = mLedgerAnchorTransactionNrs.get(ledgerAnchorSerialized); + if (result.has_value()) { + return result.value(); + } + return readFromLevelDb(ledgerAnchorSerialized); + } + + uint64_t LedgerAnchor::readFromLevelDb(const std::string& ledgerAnchorSerialized) + { + if (mInitalized) { + auto result = mLevelDBFile.getValueForKey(ledgerAnchorSerialized); + if (result.has_value()) + { + auto transactionNr = std::stoull(result.value()); + mLedgerAnchorTransactionNrs.add(ledgerAnchorSerialized, transactionNr); + return transactionNr; + } + } + return 0; + } + + gradido::data::LedgerAnchor LedgerAnchor::fromProtobuf(const std::string transactionIdString) const + { + auto binTransactionId = make_shared(transactionIdString); + gradido::interaction::deserialize::Context deserializer(binTransactionId, gradido::interaction::deserialize::Type::LEDGER_ANCHOR); + deserializer.run(); + if (!deserializer.isLedgerAnchor()) { + throw GradidoNodeInvalidDataException("[cache::LedgerAnchor::fromProtobuf] transactionId cannot be deserialized as gradido::data::LedgerAnchor"); + } + return deserializer.getLedgerAnchor(); + } + + std::string LedgerAnchor::toProtobuf(const gradido::data::LedgerAnchor& transactionId) const + { + gradido::interaction::serialize::Context serializer(transactionId); + return serializer.run()->copyAsString(); + } + +} \ No newline at end of file diff --git a/src/cache/HieroTransactionId.h b/src/cache/LedgerAnchor.h similarity index 50% rename from src/cache/HieroTransactionId.h rename to src/cache/LedgerAnchor.h index 4317179..2c62d42 100644 --- a/src/cache/HieroTransactionId.h +++ b/src/cache/LedgerAnchor.h @@ -5,13 +5,20 @@ #include "gradido_blockchain/lib/ExpireCache.h" #include "gradido_blockchain/data/hiero/TransactionId.h" -namespace cache +namespace gradido { + namespace data { + class LedgerAnchor; + } +} + +namespace cache { - class HieroTransactionId + // TODO: optimize + class LedgerAnchor { public: - HieroTransactionId(std::string_view folder); - ~HieroTransactionId(); + LedgerAnchor(std::string_view folder); + ~LedgerAnchor(); // try to open db //! \param cacheInBytes level db cache in bytes, 0 for no cache @@ -20,21 +27,22 @@ namespace cache //! remove state level db folder, clear maps void reset(); - void add(memory::ConstBlockPtr transactionId, uint64_t transactionNr); - bool has(memory::ConstBlockPtr transactionId); + void add(const gradido::data::LedgerAnchor& transactionId, uint64_t transactionNr); + bool has(const gradido::data::LedgerAnchor& transactionId); //! \return 0 if not found, else transaction nr for message id - uint64_t getTransactionNrForHieroTransactionId(memory::ConstBlockPtr messageId); + uint64_t getTransactionNrForLedgerAnchor(const gradido::data::LedgerAnchor& transactionId); protected: //! read message id as key from level db and put into mMessageIdTransactionNrs if found //! \return 0 if not found or else transactionNr for message Id - uint64_t readFromLevelDb(memory::ConstBlockPtr transactionId, hiero::TransactionId transactionIdObj); - hiero::TransactionId fromProtobuf(memory::ConstBlockPtr transactionId) const; + uint64_t readFromLevelDb(const std::string& ledgerAnchorSerialized); + gradido::data::LedgerAnchor fromProtobuf(const std::string transactionIdString) const; + std::string toProtobuf(const gradido::data::LedgerAnchor& transactionId) const; bool mInitalized; model::files::LevelDBWrapper mLevelDBFile; - //! key is iota message id, value is transaction nr - ExpireCache mHieroTransactionIdTransactionNrs; + //! key is ledger anchor serialized with protopuf, value is transaction nr + ExpireCache mLedgerAnchorTransactionNrs; }; } diff --git a/src/cache/MessageId.cpp b/src/cache/MessageId.cpp deleted file mode 100644 index 5584175..0000000 --- a/src/cache/MessageId.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include "MessageId.h" -#include "../ServerGlobals.h" -#include "../SystemExceptions.h" -#include "../lib/LevelDBExceptions.h" - -#include "loguru/loguru.hpp" - -namespace cache { - MessageId::MessageId(std::string_view folder) - : mInitalized(false), - mLevelDBFile(folder), - mMessageIdTransactionNrs(ServerGlobals::g_CacheTimeout) - { - - } - MessageId::~MessageId() - { - - } - - // try to open db - bool MessageId::init(size_t cacheInBytes) - { - if (mInitalized) { - throw ClassAlreadyInitalizedException("init was already called", "cache::MessageId"); - } - if (!mLevelDBFile.init(cacheInBytes)) { - return false; - } - mInitalized = true; - return true; - } - - void MessageId::exit() - { - if (!mInitalized) { - LOG_F(WARNING, "init wasn't called, cache::MessageId aren't stored in leveldb file, or exit was called more than once"); - } - mInitalized = false; - mLevelDBFile.exit(); - } - - //! remove state level db folder, clear maps - void MessageId::reset() - { - mLevelDBFile.reset(); - mMessageIdTransactionNrs.clear(); - } - - void MessageId::add(memory::ConstBlockPtr messageId, uint64_t transactionNr) - { - iota::MessageId messageIdObj(*messageId); - if (mMessageIdTransactionNrs.get(messageIdObj).has_value()) { - throw GradidoAlreadyExist("cache::MessageId already has key!"); - } - mMessageIdTransactionNrs.add(messageIdObj, transactionNr); - if (mInitalized) { - mLevelDBFile.setKeyValue(messageId->convertToHex().data(), std::to_string(transactionNr).data()); - } - } - - bool MessageId::has(memory::ConstBlockPtr messageId) - { - iota::MessageId messageIdObj(*messageId); - if (mMessageIdTransactionNrs.get(messageIdObj).has_value()) { - return true; - } - return readFromLevelDb(messageIdObj) != 0; - } - - uint64_t MessageId::getTransactionNrForMessageId(memory::ConstBlockPtr messageId) - { - iota::MessageId messageIdObj(*messageId); - auto result = mMessageIdTransactionNrs.get(messageIdObj); - if (result.has_value()) { - return result.value(); - } - return readFromLevelDb(messageIdObj); - } - - uint64_t MessageId::readFromLevelDb(const iota::MessageId& iotaMessageIdObj) - { - if (mInitalized) { - std::string value; - if (mLevelDBFile.getValueForKey(iotaMessageIdObj.toHex().data(), &value)) { - auto transactionNr = std::stoull(value); - mMessageIdTransactionNrs.add(iotaMessageIdObj, transactionNr); - return transactionNr; - } - } - return 0; - } - -} \ No newline at end of file diff --git a/src/cache/MessageId.h b/src/cache/MessageId.h deleted file mode 100644 index a0ecf06..0000000 --- a/src/cache/MessageId.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef __GRADIDO_NODE_CACHE_MESSAGE_ID_H -#define __GRADIDO_NODE_CACHE_MESSAGE_ID_H - -#include "../model/files/LevelDBWrapper.h" -#include "gradido_blockchain/lib/ExpireCache.h" -#include "gradido_blockchain/data/iota/MessageId.h" - -namespace cache -{ - class MessageId - { - public: - MessageId(std::string_view folder); - ~MessageId(); - - // try to open db - //! \param cacheInBytes level db cache in bytes, 0 for no cache - bool init(size_t cacheInBytes); - void exit(); - //! remove state level db folder, clear maps - void reset(); - - void add(memory::ConstBlockPtr messageId, uint64_t transactionNr); - bool has(memory::ConstBlockPtr messageId); - //! \return 0 if not found, else transaction nr for message id - uint64_t getTransactionNrForMessageId(memory::ConstBlockPtr messageId); - - protected: - //! read message id as key from level db and put into mMessageIdTransactionNrs if found - //! \return 0 if not found or else transactionNr for message Id - uint64_t readFromLevelDb(const iota::MessageId& iotaMessageIdObj); - - bool mInitalized; - model::files::LevelDBWrapper mLevelDBFile; - //! key is iota message id, value is transaction nr - ExpireCache mMessageIdTransactionNrs; - }; -} - -#endif //__GRADIDO_NODE_CACHE_MESSAGE_ID_H \ No newline at end of file diff --git a/src/cache/State.cpp b/src/cache/State.cpp index 7ad7e1c..33fada7 100644 --- a/src/cache/State.cpp +++ b/src/cache/State.cpp @@ -101,9 +101,9 @@ namespace cache { LOG_F(WARNING, "init wasn't called, leveldb file couldn't be used"); return defaultValue; } - std::string tmp; - if (mStateFile.getValueForKey(key, &tmp)) { - return tmp; + auto result = mStateFile.getValueForKey(key); + if (result.has_value()) { + return result.value(); } return defaultValue; } @@ -114,9 +114,9 @@ namespace cache { LOG_F(WARNING, "init wasn't called, leveldb file couldn't be used"); return defaultValue; } - std::string tmp; - if (mStateFile.getValueForKey(key, &tmp)) { - return atoi(tmp.data()); + auto result = mStateFile.getValueForKey(key); + if (result.has_value()) { + return atoi(result.value().data()); } return defaultValue; } @@ -127,9 +127,9 @@ namespace cache { LOG_F(WARNING, "init wasn't called, leveldb file couldn't be used"); return defaultValue; } - std::string tmp; - if (mStateFile.getValueForKey(key, &tmp)) { - return strtoll(tmp.data(), nullptr, 10); + auto result = mStateFile.getValueForKey(key); + if (result.has_value()) { + return strtoll(result.value().data(), nullptr, 10); } return defaultValue; } diff --git a/src/controller/SimpleOrderingManager.cpp b/src/controller/SimpleOrderingManager.cpp index db4bd81..17b9902 100644 --- a/src/controller/SimpleOrderingManager.cpp +++ b/src/controller/SimpleOrderingManager.cpp @@ -3,8 +3,7 @@ #include "../blockchain/FileBasedProvider.h" #include "../blockchain/Exceptions.h" #include "../task/HieroMessageToTransactionTask.h" -#include "gradido_blockchain/interaction/serialize/Context.h" -#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/data/LedgerAnchor.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_blockchain/const.h" @@ -14,7 +13,8 @@ using namespace gradido; using namespace blockchain; -using namespace interaction; + +using gradido::data::LedgerAnchor; namespace controller { @@ -94,17 +94,15 @@ namespace controller { updateSequenceNumber(currentSequenceNumber); continue; } - deserialize::Context topicIdDeserializer(gradidoTransaction->getParingMessageId(), deserialize::Type::HIERO_TRANSACTION_ID); - topicIdDeserializer.run(); - if (!topicIdDeserializer.isHieroTransactionId()) { - task->notificateFailedTransaction(blockchain, "Transaction skipped (pairing transactionId invalid)"); + if (gradidoTransaction->getPairingLedgerAnchor().empty()) { + task->notificateFailedTransaction(blockchain, "Transaction skipped (pairing ledger anchor empty)"); mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); continue; } auto pairTask = static_cast(otherBlockchain.get()) ->getOrderingManager() - ->findCrossGroupTransactionPair(topicIdDeserializer.getHieroTransactionId()) + ->findCrossGroupTransactionPair(gradidoTransaction->getPairingLedgerAnchor()) ; if (!pairTask || !pairTask->isTaskFinished()) { // not found? maybe it need some more time? @@ -136,17 +134,16 @@ namespace controller { throw CommunityNotFoundExceptions("couldn't find group", mCommunityId); } auto transaction = gradidoTransactionWorkData.deserializeTask->getGradidoTransaction(); - const auto& transactionId = gradidoTransactionWorkData.consensusTopicResponse.getChunkInfo().getInitialTransactionId(); + auto transactionId = LedgerAnchor(gradidoTransactionWorkData.consensusTopicResponse.getChunkInfo().getInitialTransactionId()); const auto& confirmedAt = gradidoTransactionWorkData.consensusTopicResponse.getConsensusTimestamp(); if (transactionId.empty()) { throw GradidoNodeInvalidDataException("missing transaction id in hiero response"); } auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); try { - serialize::Context serializer(transactionId); bool result = blockchain->createAndAddConfirmedTransaction( transaction, - serializer.run(), + transactionId, confirmedAt ); fileBasedBlockchain->updateLastKnownSequenceNumber(mLastSequenceNumber); @@ -207,13 +204,20 @@ namespace controller { return PushResult::ADDED; } - std::shared_ptr SimpleOrderingManager::findCrossGroupTransactionPair(const hiero::TransactionId& transactionId) const + std::shared_ptr SimpleOrderingManager::findCrossGroupTransactionPair(const LedgerAnchor& transactionId) const { if (isExitCalled()) { return nullptr; } std::lock_guard _lock(mTransactionsMutex); for (auto it = mTransactions.begin(); it != mTransactions.end(); it++) { - if (it->second.consensusTopicResponse.getChunkInfo().getInitialTransactionId() == transactionId) { - return it->second.deserializeTask; + if (transactionId.isHieroTransactionId()) { + if (it->second.consensusTopicResponse.getChunkInfo().getInitialTransactionId() == transactionId.getHieroTransactionId()) { + return it->second.deserializeTask; + } + } + else { + if (it->second.deserializeTask->isSuccess() && it->second.deserializeTask->getGradidoTransaction()->getPairingLedgerAnchor() == transactionId) { + return it->second.deserializeTask; + } } } return nullptr; diff --git a/src/controller/SimpleOrderingManager.h b/src/controller/SimpleOrderingManager.h index c248004..2994e86 100644 --- a/src/controller/SimpleOrderingManager.h +++ b/src/controller/SimpleOrderingManager.h @@ -2,7 +2,6 @@ #define __GRADIDO_NODE_SINGLETON_MANAGER_ORDERING_MANAGER #include "gradido_blockchain/data/GradidoTransaction.h" -#include "gradido_blockchain/data/hiero/TransactionId.h" #include "gradido_blockchain/crypto/SignatureOctet.h" #include "gradido_blockchain/lib/ExpireCache.h" #include "../task/Thread.h" @@ -16,6 +15,12 @@ namespace task { class HieroMessageToTransactionTask; } +namespace gradido { + namespace data { + class LedgerAnchor; + } +} + namespace controller { /*! @@ -43,7 +48,7 @@ namespace controller { IN_SHUTDOWN }; PushResult pushTransaction(hiero::ConsensusTopicResponse&& consensusTopicResponse); - std::shared_ptr findCrossGroupTransactionPair(const hiero::TransactionId& transactionId) const; + std::shared_ptr findCrossGroupTransactionPair(const gradido::data::LedgerAnchor& transactionId) const; protected: diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h index 6efbb53..2e666c3 100644 --- a/src/lib/PersistentDictionary.h +++ b/src/lib/PersistentDictionary.h @@ -80,7 +80,7 @@ requires serialization::HasString uint32_t PersistentDictionary::getLastIndex() { std::unique_lock _lock(mWorkingMutex); - mIndexDataReverseLookup.size() - 1; + return mIndexDataReverseLookup.size() - 1; } template @@ -88,7 +88,7 @@ requires serialization::HasString std::optional PersistentDictionary::getIndexForData(const DataType& data) const { std::shared_lock _lock(mWorkingMutex); - auto result = mDictionaryFile.getValueForKey(serialization::toString(data).c_str()); + auto result = mDictionaryFile.getValueForKey(serialization::toString(data)); if (result.has_value()) { const auto& value = result.value(); return serialization::fromString(value.data(), value.size()); diff --git a/src/model/files/Block.cpp b/src/model/files/Block.cpp index 1952e35..0d2fe42 100644 --- a/src/model/files/Block.cpp +++ b/src/model/files/Block.cpp @@ -6,6 +6,7 @@ #include "../../ServerGlobals.h" #include "../../SingletonManager/FileLockManager.h" #include "../../SingletonManager/CacheManager.h" +#include "../../task/RebuildBlockIndexTask.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" #include "gradido_blockchain/interaction/deserialize/Context.h" @@ -268,11 +269,10 @@ namespace model { return result; } - std::shared_ptr Block::rebuildBlockIndex(std::shared_ptr blockchain) - { + void Block::fillRebuildBlockIndexTask(std::shared_ptr rebuildTask) + { + Profiler timeUsed; auto fl = FileLockManager::getInstance(); - std::shared_ptr rebuildTask = std::make_shared(blockchain); - int32_t fileCursor = 0; std::shared_ptr readBuffer; unsigned char hash[crypto_generichash_KEYBYTES]; @@ -281,10 +281,12 @@ namespace model { // read in every line while (fileCursor + sizeof(uint16_t) + MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE <= mCurrentFileSize) { auto lineSize = readLine(fileCursor, &readBuffer); - rebuildTask->pushLine(fileCursor, readBuffer); + rebuildTask->pushLine(fileCursor, readBuffer, rebuildTask); calculateOneHashStep(hash, (const unsigned char*)readBuffer->data(), readBuffer->size()); fileCursor += lineSize + sizeof(uint16_t); } + rebuildTask->flush(rebuildTask); + LOG_F(INFO, "%s time for read block file (%u) into rebuild task", timeUsed.string().c_str(), mCurrentFileSize); unsigned char hash2[crypto_generichash_KEYBYTES]; if (!fl->tryLockTimeout(mBlockPath, 100)) { @@ -299,10 +301,8 @@ namespace model { result = true; } - if (result) { - return rebuildTask; - } - else { + if (!result) + { throw HashMismatchException( "block hash mismatch", memory::Block(sizeof hash, hash), @@ -358,43 +358,5 @@ namespace model { return 0; } - - - RebuildBlockIndexTask::RebuildBlockIndexTask(std::shared_ptr blockchain) - : task::CPUTask(ServerGlobals::g_CPUScheduler), mBlockchain(blockchain) - { - - } - - int RebuildBlockIndexTask::run() - { - while (!mPendingFileCursorLine.empty()) - { - std::pair> fileCursorLine; - if (!mPendingFileCursorLine.pop(fileCursorLine)) { - throw std::runtime_error("don't get next file cursor line"); - } - auto& serializedTransaction = fileCursorLine.second; - deserialize::Context deserializer(serializedTransaction, deserialize::Type::CONFIRMED_TRANSACTION); - deserializer.run(); - if (!deserializer.isConfirmedTransaction()) { - throw InvalidGradidoTransaction("invalid transaction from block file while rebuilding block index", serializedTransaction); - } - lock(); - std::shared_ptr transactionEntry = std::make_shared( - deserializer.getConfirmedTransaction(), - mBlockchain, - fileCursorLine.first - ); - mTransactionEntries.push_back(transactionEntry); - unlock(); - } - return 0; - } - - void RebuildBlockIndexTask::pushLine(int32_t fileCursor, std::shared_ptr line) - { - mPendingFileCursorLine.push({ fileCursor, line }); - } } } diff --git a/src/model/files/Block.h b/src/model/files/Block.h index f44fe3e..814cbbc 100644 --- a/src/model/files/Block.h +++ b/src/model/files/Block.h @@ -23,6 +23,10 @@ namespace gradido { } } +namespace task { + class RebuildBlockIndexTask; +} + namespace model { namespace files { class RebuildBlockIndexTask; @@ -59,7 +63,9 @@ namespace model { bool validateHash(); // read whole file, validate hash - std::shared_ptr rebuildBlockIndex(std::shared_ptr blockchain); + // put lines of serialized transactions into task + // TODO: maybe make it more abstract so different tasks can use this + void fillRebuildBlockIndexTask(std::shared_ptr rebuildTask); static uint32_t findLastBlockFileInFolder(std::string_view groupFolderPath); @@ -100,27 +106,7 @@ namespace model { std::vector mCursorPositions; }; - //! TODO: update for able to start with first line, while calling function is still loading more and more lines from file - //! gives the additional option to prevent task for storing to many lines at once - //! use ability of Task Object for resheduling - class RebuildBlockIndexTask : public task::CPUTask - { - public: - RebuildBlockIndexTask(std::shared_ptr blockchain); - const char* getResourceType() const { return "RebuildBlockIndexTask"; }; - - int run(); - //! \param line will be moved - void pushLine(int32_t fileCursor, std::shared_ptr line); - const std::list>& getTransactionEntries() const { return mTransactionEntries; } - - inline bool isPendingQueueEmpty() { return mPendingFileCursorLine.empty(); } - - protected: - std::shared_ptr mBlockchain; - std::list> mTransactionEntries; - MultithreadQueue>> mPendingFileCursorLine; - }; + } } diff --git a/src/model/files/BlockIndex.cpp b/src/model/files/BlockIndex.cpp index 4f90c2d..e5bca2d 100644 --- a/src/model/files/BlockIndex.cpp +++ b/src/model/files/BlockIndex.cpp @@ -60,7 +60,12 @@ namespace model { uint8_t hasValue = 0; if (!vFile->read(&hasValue, sizeof(uint8_t))) return false; if (hasValue) { - if (!vFile->read(&coinCommunityIdIndex.value(), sizeof(uint32_t))) return false; + uint32_t coinCommnityIdIndexRaw = 0; + if (!vFile->read(&coinCommnityIdIndexRaw, sizeof(uint32_t))) return false; + coinCommunityIdIndex = coinCommnityIdIndexRaw; + } + else { + coinCommunityIdIndex = nullopt; } if (!vFile->read(&isBalanceChanging, sizeof(uint8_t))) return false; if (!vFile->read(&addressIndicesCount, sizeof(uint8_t))) return false; @@ -79,8 +84,11 @@ namespace model { //crypto_generichash_update(state, (const unsigned char*)this, sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint16_t)); crypto_generichash_update(state, (const unsigned char*)&transactionNr, sizeof(uint64_t)); crypto_generichash_update(state, (const unsigned char*)&fileCursor, sizeof(int32_t)); - crypto_generichash_update(state, (const unsigned char*)&coinCommunityIdIndex, sizeof(uint32_t)); + if (coinCommunityIdIndex.has_value()) { + crypto_generichash_update(state, (const unsigned char*)&coinCommunityIdIndex.value(), sizeof(uint32_t)); + } crypto_generichash_update(state, (const unsigned char*)&addressIndicesCount, sizeof(uint8_t)); + crypto_generichash_update(state, (const unsigned char*)&isBalanceChanging, sizeof(uint8_t)); // second part crypto_generichash_update(state, (const unsigned char*)addressIndices, sizeof(uint32_t) * addressIndicesCount); @@ -210,8 +218,8 @@ namespace model { return false; } - unsigned char hashFromFile[crypto_generichash_BYTES]; - unsigned char hashCalculated[crypto_generichash_BYTES]; + unsigned char hashFromFile[crypto_generichash_BYTES]; memset(hashFromFile, 0, crypto_generichash_BYTES); + unsigned char hashCalculated[crypto_generichash_BYTES]; memset(hashCalculated, 0, crypto_generichash_BYTES); crypto_generichash_state state; crypto_generichash_init(&state, nullptr, 0, crypto_generichash_BYTES); diff --git a/src/model/files/LevelDBWrapper.cpp b/src/model/files/LevelDBWrapper.cpp index 9f8564b..8a1599a 100644 --- a/src/model/files/LevelDBWrapper.cpp +++ b/src/model/files/LevelDBWrapper.cpp @@ -92,7 +92,7 @@ namespace model { void LevelDBWrapper::setKeyValue(const std::string& key, const std::string& value) { WriteOptions writeOptions; - writeOptions.sync = true; + writeOptions.sync = false; Status s = mLevelDB->Put(writeOptions, key, value); if (!s.ok()) { throw LevelDBStatusException("cannot put to level db", s); diff --git a/src/serialization/String.cpp b/src/serialization/String.cpp new file mode 100644 index 0000000..5e03136 --- /dev/null +++ b/src/serialization/String.cpp @@ -0,0 +1,22 @@ +#include "String.h" +#include "gradido_blockchain/memory/Block.h" + +#include + +using memory::ConstBlockPtr, memory::Block; +using std::string; +using std::make_shared; + +namespace serialization { + template<> + string toString(const ConstBlockPtr& ptr) + { + return ptr->copyAsString(); + } + + template<> + ConstBlockPtr fromString(const char* data, size_t size) + { + return make_shared(size, reinterpret_cast(data)); + } +} \ No newline at end of file diff --git a/src/serialization/String.h b/src/serialization/String.h index 7cf7c08..c1f5343 100644 --- a/src/serialization/String.h +++ b/src/serialization/String.h @@ -3,7 +3,6 @@ #include #include -#include "gradido_blockchain/memory/Block.h" namespace serialization { template @@ -49,18 +48,6 @@ namespace serialization { inline size_t fromString(const char* data, size_t size) { return strtoull(data, nullptr, 0); } - - template<> - inline std::string toString(const memory::ConstBlockPtr& ptr) - { - return ptr->copyAsString(); - } - - template<> - inline memory::ConstBlockPtr fromString(const char* data, size_t size) - { - return std::make_shared(data, size); - } } #endif //__GRADIDO_NODE_SERIALIZATION_STRING_H \ No newline at end of file diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index a62715e..1590d78 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -6,6 +6,7 @@ #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/interaction/calculateAccountBalance/Context.h" #include "gradido_blockchain/interaction/calculateCreationSum/Context.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/interaction/serialize/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" #include "gradido_blockchain/serialization/toJson.h" @@ -346,7 +347,11 @@ namespace server { transactionEntry = blockchain->getTransactionForId(transactionId); } else { - transactionEntry = blockchain->findByMessageId(iotaMessageId); + deserialize::Context deserializer(iotaMessageId); + deserializer.run(); + if (deserializer.isLedgerAnchor()) { + transactionEntry = blockchain->findByLedgerAnchor(deserializer.getLedgerAnchor()); + } } if (!transactionEntry) { error(responseJson, JSON_RPC_ERROR_TRANSACTION_NOT_FOUND, "transaction not found"); diff --git a/src/task/BatchDeserializeConfirmedTransactionTask.cpp b/src/task/BatchDeserializeConfirmedTransactionTask.cpp new file mode 100644 index 0000000..e38ca4e --- /dev/null +++ b/src/task/BatchDeserializeConfirmedTransactionTask.cpp @@ -0,0 +1,48 @@ +#include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/interaction/deserialize/Type.h" +#include "gradido_blockchain/GradidoBlockchainException.h" + +#include "BatchDeserializeConfirmedTransactionTask.h" +#include "../ServerGlobals.h" + +#include "loguru/loguru.hpp" + +using gradido::data::ConfirmedTransaction; +using std::shared_ptr; +using Deserializer = gradido::interaction::deserialize::Context; +using DeserializerType = gradido::interaction::deserialize::Type; + +namespace task { + BatchDeserializeConfirmedTransactionTask::BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions) + : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransactions(std::move(rawTransactions)) + { + } + + BatchDeserializeConfirmedTransactionTask::~BatchDeserializeConfirmedTransactionTask() + { + } + + int BatchDeserializeConfirmedTransactionTask::run() + { + mConfirmedTransactions.clear(); + mConfirmedTransactions.resize(mRawTransactions.size(), nullptr); + for (size_t i = 0; i < mRawTransactions.size(); i++) { + Deserializer deserializer(mRawTransactions[i], DeserializerType::CONFIRMED_TRANSACTION); + try { + deserializer.run(); + } + catch (std::exception& e) { + int zahl = 0; + LOG_F(ERROR, "error on deserialize: %s", e.what()); + return -1; + } + if (!deserializer.isConfirmedTransaction()) { + throw InvalidGradidoTransaction("invalid confirmed transaction", mRawTransactions[i]); + } + mConfirmedTransactions[i] = deserializer.getConfirmedTransaction(); + // printf("%d ", deserializer.getConfirmedTransaction()->getId()); + } + return 0; + } +} diff --git a/src/task/BatchDeserializeConfirmedTransactionTask.h b/src/task/BatchDeserializeConfirmedTransactionTask.h new file mode 100644 index 0000000..3551618 --- /dev/null +++ b/src/task/BatchDeserializeConfirmedTransactionTask.h @@ -0,0 +1,38 @@ +#ifndef __GRADIDO_NODE_TASK_BATCH_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H +#define __GRADIDO_NODE_TASK_BATCH_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H + +#include "CPUTask.h" +#include +#include + +namespace gradido { + namespace data { + class ConfirmedTransaction; + } +} + +namespace memory { + class Block; + typedef std::shared_ptr ConstBlockPtr; +} + +namespace task { + + class BatchDeserializeConfirmedTransactionTask : public CPUTask { + public: + BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions); + virtual ~BatchDeserializeConfirmedTransactionTask(); + + const char* getResourceType() const override { return "BatchDeserializeConfirmedTransactionTask"; }; + int run() override; + + inline const std::vector>& getConfirmedTransactions() const { return mConfirmedTransactions; } + inline const std::vector& getRawTransactions() const { return mRawTransactions; } + + private: + std::vector mRawTransactions; + std::vector> mConfirmedTransactions; + }; +} + +#endif // __GRADIDO_NODE_TASK_BATCH_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H \ No newline at end of file diff --git a/src/task/CPUSheduler.cpp b/src/task/CPUSheduler.cpp index c2e7674..1a8524a 100644 --- a/src/task/CPUSheduler.cpp +++ b/src/task/CPUSheduler.cpp @@ -56,7 +56,7 @@ namespace task { // else put task to pending queue // printf("[CPUSheduler::sheduleTask] all %d threads in use \n", getThreadCount()); if (mFreeWorkerThreads.empty()) { - LOG_F(INFO, "sheduleTask in %s all %u threads in use, add to pending task list", mName.data(), getThreadCount()); + // LOG_F(INFO, "sheduleTask in %s all %u threads in use, add to pending task list", mName.data(), getThreadCount()); } { std::lock_guard _lock(mPendingTasksMutex); diff --git a/src/task/DeserializeConfirmedTransactionTask.cpp b/src/task/DeserializeConfirmedTransactionTask.cpp new file mode 100644 index 0000000..b9c87fe --- /dev/null +++ b/src/task/DeserializeConfirmedTransactionTask.cpp @@ -0,0 +1,40 @@ +#include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/interaction/deserialize/Type.h" +#include "gradido_blockchain/GradidoBlockchainException.h" + +#include "DeserializeConfirmedTransactionTask.h" +#include "../ServerGlobals.h" + +#include + +using gradido::data::ConfirmedTransaction; +using std::shared_ptr; +using Deserializer = gradido::interaction::deserialize::Context; +using DeserializerType = gradido::interaction::deserialize::Type; + +namespace task { + DeserializeConfirmedTransactionTask::DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction) + : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransaction(rawTransaction) { + } + + DeserializeConfirmedTransactionTask::~DeserializeConfirmedTransactionTask() { + } + + int DeserializeConfirmedTransactionTask::run() { + Deserializer deserializer(mRawTransaction, DeserializerType::CONFIRMED_TRANSACTION); + try { + deserializer.run(); + } + catch (std::exception& e) { + int zahl = 0; + LOG_F(ERROR, "error on deserialize: %s", e.what()); + return -1; + } + if (!deserializer.isConfirmedTransaction()) { + throw InvalidGradidoTransaction("invalid confirmed transaction", mRawTransaction); + } + mConfirmedTransaction = deserializer.getConfirmedTransaction(); + return 0; + } +} diff --git a/src/task/DeserializeConfirmedTransactionTask.h b/src/task/DeserializeConfirmedTransactionTask.h new file mode 100644 index 0000000..fc7e8b9 --- /dev/null +++ b/src/task/DeserializeConfirmedTransactionTask.h @@ -0,0 +1,35 @@ +#ifndef __GRADIDO_NODE_TASK_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H +#define __GRADIDO_NODE_TASK_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H + +#include "CPUTask.h" +#include + +namespace gradido { + namespace data { + class ConfirmedTransaction; + } +} + +namespace memory { + class Block; + typedef std::shared_ptr ConstBlockPtr; +} + +namespace task { + class DeserializeConfirmedTransactionTask : public CPUTask { + public: + DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction); + virtual ~DeserializeConfirmedTransactionTask(); + + const char* getResourceType() const override { return "DeserializeConfirmedTransactionTask"; }; + int run() override; + + inline std::shared_ptr getConfirmedTransaction() const { return mConfirmedTransaction; } + + private: + memory::ConstBlockPtr mRawTransaction; + std::shared_ptr mConfirmedTransaction; + }; +} + +#endif // __GRADIDO_NODE_TASK_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H \ No newline at end of file diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp new file mode 100644 index 0000000..dde66a8 --- /dev/null +++ b/src/task/RebuildBlockIndexTask.cpp @@ -0,0 +1,94 @@ +#include "RebuildBlockIndexTask.h" +#include "BatchDeserializeConfirmedTransactionTask.h" +#include "../ServerGlobals.h" +#include "../blockchain/FileBased.h" + +#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/memory/Block.h" +#include "gradido_blockchain/serialization/toJsonString.h" + +using memory::Block; +using gradido::blockchain::FileBased; +using std::shared_ptr, std::make_shared; +using ServerGlobals::g_CPUScheduler; + +namespace task { + RebuildBlockIndexTask::RebuildBlockIndexTask( + std::shared_ptr blockchain, + std::shared_ptr blockIndex, + IMutableDictionary& publicKeyDictionary + ): task::CPUTask(g_CPUScheduler), mBlockchain(blockchain), mBlockIndex(blockIndex), mPublicKeyIndex(publicKeyDictionary) + { + mRawTransactionsBulk.reserve(REBUILD_BLOCK_INDEX_TASK_BULK_SIZE); + mActiveDeserializerTasks = 0; + } + + int RebuildBlockIndexTask::run() + { + LOG_F(WARNING, "not supposed to sheduled as regular task"); + return 0; + } + + void RebuildBlockIndexTask::pushLine(int32_t fileCursor, memory::ConstBlockPtr line, std::shared_ptr ownPtr) + { + std::lock_guard _lock(mFinishLineMutex); + mFileCursorsQueue.push_back(fileCursor); + mRawTransactionsBulk.push_back(line); + if (mRawTransactionsBulk.size() >= REBUILD_BLOCK_INDEX_TASK_BULK_SIZE) { + flush(ownPtr, false); + } + } + + void RebuildBlockIndexTask::flush(std::shared_ptr ownPtr, bool last/* = true, */) + { + std::lock_guard _lock(mFinishLineMutex); + if (!mRawTransactionsBulk.size()) { return; } + auto deserializeTask = make_shared(std::move(mRawTransactionsBulk)); + // deserializeTask->setFinishCommand(new FinishedDeserializeForRebuildBlockIndexCommand(ownPtr)); + mBulkDeserializerTasks.push(deserializeTask); + deserializeTask->scheduleTask(deserializeTask); + + if (!last) { + mRawTransactionsBulk.reserve(REBUILD_BLOCK_INDEX_TASK_BULK_SIZE); + } + } + + void RebuildBlockIndexTask::finishBulk() + { + std::lock_guard lock(mFinishLineMutex); + while (!mBulkDeserializerTasks.empty() && mBulkDeserializerTasks.front()->isTaskFinished()) + { + auto& task = mBulkDeserializerTasks.front(); + const auto& confirmedTransactions = task->getConfirmedTransactions(); + const auto& rawTransactions = task->getRawTransactions(); + for (int i = 0; i < confirmedTransactions.size(); i++) { + const auto& confirmedTransaction = confirmedTransactions[i]; + auto fileCursor = mFileCursorsQueue.front(); + + std::shared_ptr transactionEntry = std::make_shared( + confirmedTransaction, + rawTransactions[i], + mBlockchain, + fileCursor + ); + try { + mBlockIndex->addIndicesForTransaction(transactionEntry, mPublicKeyIndex); + } + catch (std::exception& e) { + LOG_F(FATAL, "%s, couldn't add indices for transaction: %s", + e.what(), + serialization::toJsonString(*transactionEntry->getConfirmedTransaction(), true).c_str() + ); + } + mFileCursorsQueue.pop_front(); + } + mBulkDeserializerTasks.pop(); + } + } + + bool RebuildBlockIndexTask::isPendingQueueEmpty() + { + finishBulk(); + return mBulkDeserializerTasks.empty(); + } +} \ No newline at end of file diff --git a/src/task/RebuildBlockIndexTask.h b/src/task/RebuildBlockIndexTask.h new file mode 100644 index 0000000..7072d90 --- /dev/null +++ b/src/task/RebuildBlockIndexTask.h @@ -0,0 +1,84 @@ +#ifndef __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H +#define __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H + +#include "CPUTask.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" + +#include +#include +#include +#include + +namespace gradido { + namespace blockchain { + class FileBased; + class NodeTransactionEntry; + } +} + +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr; +} + +namespace cache { + class BlockIndex; +} + +// TODO: maybe put into config +#define REBUILD_BLOCK_INDEX_TASK_BULK_SIZE 1000 + +namespace task { + class BatchDeserializeConfirmedTransactionTask; + + + //! remove dependencie to CPUTask because this isn't really a cpu task, more are result storage, + //! because it will start subsequent tasks of it own which will call back via command if finished + class RebuildBlockIndexTask : public CPUTask + { + public: + RebuildBlockIndexTask( + std::shared_ptr blockchain, + std::shared_ptr blockIndex, + IMutableDictionary& publicKeyDictionary + ); + const char* getResourceType() const { return "RebuildBlockIndexTask"; }; + + int run(); + //! \param line will be moved + void pushLine(int32_t fileCursor, memory::ConstBlockPtr line, std::shared_ptr ownPtr); + // flush batch buffer + void flush(std::shared_ptr ownPtr, bool last = true); + // called from DeserializeConfirmedTransactionTask, will process queue from begin as long current entry was already deserialized + void finishBulk(); + + bool isPendingQueueEmpty(); + + protected: + std::shared_ptr mBlockchain; + std::shared_ptr mBlockIndex; + IMutableDictionary& mPublicKeyIndex; + std::queue> mBulkDeserializerTasks; + std::deque mFileCursorsQueue; + std::vector mRawTransactionsBulk; + std::recursive_mutex mFinishLineMutex; + std::atomic mActiveDeserializerTasks; + }; + + + class FinishedDeserializeForRebuildBlockIndexCommand : public Command + { + public: + FinishedDeserializeForRebuildBlockIndexCommand(std::shared_ptr parent) : mParent(parent) {} + int taskFinished(Task* task) override + { + mParent->finishBulk(); + return 0; + } + + protected: + std::shared_ptr mParent; + }; +} + +#endif // __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H \ No newline at end of file diff --git a/src/task/Task.h b/src/task/Task.h index 44d6438..d38a940 100644 --- a/src/task/Task.h +++ b/src/task/Task.h @@ -46,6 +46,7 @@ namespace task { class Command { public: virtual ~Command() = default; + // TODO: check if int as return is needed and can be used or better void or better enum virtual int taskFinished(Task* task) = 0; }; diff --git a/src/task/WriteTransactionsToBlockTask.cpp b/src/task/WriteTransactionsToBlockTask.cpp index 0cd63a8..aaaee13 100644 --- a/src/task/WriteTransactionsToBlockTask.cpp +++ b/src/task/WriteTransactionsToBlockTask.cpp @@ -5,6 +5,7 @@ #include "../ServerGlobals.h" #include "gradido_blockchain/GradidoBlockchainException.h" +#include "gradido_blockchain/memory/Block.h" #include "loguru/loguru.hpp" #include @@ -61,13 +62,16 @@ namespace task { return 0; } - void WriteTransactionsToBlockTask::addSerializedTransaction(std::shared_ptr transaction) + void WriteTransactionsToBlockTask::addSerializedTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary + ) { assert(!isTaskSheduled()); assert(!isTaskFinished()); std::lock_guard lock(mFastMutex); mTransactions.insert({transaction->getTransactionNr(), transaction}); - mBlockIndex->addIndicesForTransaction(transaction); + mBlockIndex->addIndicesForTransaction(transaction, publicKeyDictionary); } std::shared_ptr WriteTransactionsToBlockTask::getTransaction(uint64_t nr) diff --git a/src/task/WriteTransactionsToBlockTask.h b/src/task/WriteTransactionsToBlockTask.h index 89af9ff..0443db4 100644 --- a/src/task/WriteTransactionsToBlockTask.h +++ b/src/task/WriteTransactionsToBlockTask.h @@ -9,6 +9,9 @@ #include "CPUTask.h" #include "gradido_blockchain/lib/MultithreadQueue.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" + +#include namespace cache { class BlockIndex; @@ -26,6 +29,11 @@ namespace gradido { } } +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr; +} + /*! * @author Dario Rekowski * @date 202-03-11 @@ -47,7 +55,10 @@ namespace task { //! no mutex lock, value doesn't change, set in WriteTransactionsToBlockTask() inline Timepoint getCreationDate() { return mCreationDate; } - void addSerializedTransaction(std::shared_ptr transaction); + void addSerializedTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary + ); //! return transaction by nr std::shared_ptr getTransaction(uint64_t nr); From 36b3bd0073d86580ba265a93e5f7d3bab6861ad2 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 7 Jan 2026 15:07:20 +0100 Subject: [PATCH 03/65] check all transactions on startup, make TransactionHash collision resistend --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 6 ++-- src/cache/TransactionHash.cpp | 58 +++++++++++++++++++++--------- src/cache/TransactionHash.h | 8 +++-- src/task/RebuildBlockIndexTask.cpp | 3 +- 5 files changed, 54 insertions(+), 23 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 443ee65..d02cb6c 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 443ee65afcbefc9430d604fc46506e0e5b6d296f +Subproject commit d02cb6cc9f5b1771189626f3da16f66971fc4890 diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 34386e1..1902645 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -30,7 +30,7 @@ using client::hiero::ConsensusClient; using controller::SimpleOrderingManager; namespace gradido { - using data::LedgerAnchor; + using data::LedgerAnchor, data::AddressType; using namespace interaction; namespace blockchain { @@ -120,7 +120,7 @@ namespace gradido { int count = 0; findAll(builder .setSearchDirection(SearchDirection::DESC) - .setPagination({ GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE }) + //.setPagination({ GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE }) .setFilterFunction([this, &count](const TransactionEntry& transactionEntry) -> FilterResult { auto previousTransactionNr = transactionEntry.getTransactionNr() - 1; data::ConstConfirmedTransactionPtr previousConfirmedTransaction; @@ -143,6 +143,7 @@ namespace gradido { validator.run(validationLevel, getptr()); mTransactionHashCache.push(*transactionEntry.getConfirmedTransaction()); count++; + printf("\r%d", count); return FilterResult::DISMISS; }) .build() @@ -371,6 +372,7 @@ namespace gradido { } return Abstract::findByLedgerAnchor(ledgerAnchor, filter); } + AbstractProvider* FileBased::getProvider() const { return FileBasedProvider::getInstance(); diff --git a/src/cache/TransactionHash.cpp b/src/cache/TransactionHash.cpp index c23b8ca..5acf3cc 100644 --- a/src/cache/TransactionHash.cpp +++ b/src/cache/TransactionHash.cpp @@ -4,6 +4,8 @@ #include "../blockchain/FileBased.h" #include "../blockchain/NodeTransactionEntry.h" #include "gradido_blockchain/data/GradidoTransaction.h" +#include "gradido_blockchain/serialization/toJsonString.h" + #include "loguru/loguru.hpp" @@ -11,47 +13,69 @@ using namespace gradido; using namespace blockchain; using namespace data; +using serialization::toJsonString; + namespace cache { TransactionHash::TransactionHash(std::string_view communityId) - : mSignaturePartTransactionNrs(MAGIC_NUMBER_SIGNATURE_CACHE), mCommunityId(communityId) + : //mSignaturePartTransactionNrs(MAGIC_NUMBER_SIGNATURE_CACHE), + mCommunityId(communityId) { } TransactionHash::~TransactionHash() { - mSignaturePartTransactionNrs.clear(); + // mSignaturePartTransactionNrs.clear(); + mBodyBytesTransactionNrs.clear(); } void TransactionHash::push(const gradido::data::ConfirmedTransaction& confirmedTransaction) { - auto gradidoTransaction = confirmedTransaction.getGradidoTransaction(); - auto firstSignature = gradidoTransaction->getSignatureMap().getSignaturePairs().front().getSignature(); - auto pair = mSignaturePartTransactionNrs.get(*firstSignature); - if (pair.has_value()) { + SignatureOctet hash = deriveHash(*confirmedTransaction.getGradidoTransaction()); + // auto pair = mSignaturePartTransactionNrs.get(hash); + auto it = mBodyBytesTransactionNrs.find(confirmedTransaction.getGradidoTransaction()->getBodyBytes()); + //if (pair.has_value()) { + if (it != mBodyBytesTransactionNrs.end()) { + auto storedTransaction = FileBasedProvider::getInstance()->findBlockchain(mCommunityId)->getTransactionForId(it->second); + LOG_F(ERROR, "Hash collision detected, %s and %s have the same hash", + toJsonString(confirmedTransaction, true).c_str(), + toJsonString(*storedTransaction->getConfirmedTransaction(), true).c_str() + ); throw GradidoAlreadyExist("key already exist"); } - mSignaturePartTransactionNrs.add(*firstSignature, confirmedTransaction.getId()); + //mSignaturePartTransactionNrs.add(hash, confirmedTransaction.getId()); + mBodyBytesTransactionNrs.insert({ confirmedTransaction.getGradidoTransaction()->getBodyBytes(), confirmedTransaction.getId() }); } bool TransactionHash::has(const data::GradidoTransaction& transaction) const { // get first signature from transaction - auto firstSignature = transaction.getSignatureMap().getSignaturePairs().front().getSignature(); - auto pair = mSignaturePartTransactionNrs.get(SignatureOctet(*firstSignature)); - if (pair.has_value()) { + auto hash = deriveHash(transaction); + // auto pair = mSignaturePartTransactionNrs.get(hash); + auto it = mBodyBytesTransactionNrs.find(transaction.getBodyBytes()); + //if (pair.has_value()) + if (it != mBodyBytesTransactionNrs.end()) + { // hash collision check - auto transaction = FileBasedProvider::getInstance()->findBlockchain(mCommunityId)->getTransactionForId(pair.value()); - auto firstSignatureFromFoundedTransaction = transaction->getConfirmedTransaction()->getGradidoTransaction()->getSignatureMap().getSignaturePairs().front().getSignature(); - if (firstSignatureFromFoundedTransaction->isTheSame(firstSignature)) { + auto storedTransaction = FileBasedProvider::getInstance()->findBlockchain(mCommunityId)->getTransactionForId(it->second); + if (storedTransaction->getConfirmedTransaction()->getGradidoTransaction()->isTheSame(transaction)) { return true; - } - else { - LOG_F(INFO, "Hash collision detected, %s and %s have the same first 8 Bytes", - firstSignature->convertToHex().data(), firstSignatureFromFoundedTransaction->convertToHex().data() + } else { + LOG_F(WARNING, "Hash collision detected, %s and %s have the same hash", + toJsonString(transaction, true).c_str(), + toJsonString(*storedTransaction->getConfirmedTransaction(), true).c_str() ); } } return false; } + + SignatureOctet TransactionHash::deriveHash(const gradido::data::GradidoTransaction& transaction) const + { + if (transaction.getSignatureMap().getSignaturePairs().size()) { + return transaction.getSignatureMap().getSignaturePairs().front().getSignature()->calculateHash(); + } else { + return transaction.getBodyBytes()->calculateHash(); + } + } } \ No newline at end of file diff --git a/src/cache/TransactionHash.h b/src/cache/TransactionHash.h index 49f3400..6d43bc9 100644 --- a/src/cache/TransactionHash.h +++ b/src/cache/TransactionHash.h @@ -2,10 +2,12 @@ #define __GRADIDO_NODE_CACHE_TRANSACTION_HASH_H #include "gradido_blockchain/crypto/SignatureOctet.h" -#include "gradido_blockchain/lib/ExpireCache.h" +//#include "gradido_blockchain/lib/AccessExpireCache.h" #include "gradido_blockchain/const.h" +#include "gradido_blockchain/memory/Block.h" #include +#include namespace gradido { namespace data { @@ -31,9 +33,11 @@ namespace cache { void push(const gradido::data::ConfirmedTransaction& confirmedTransaction); bool has(const gradido::data::GradidoTransaction& gradidoTransaction) const; protected: + SignatureOctet deriveHash(const gradido::data::GradidoTransaction& transaction) const; //! key is first 8 Byte from Transaction Signature, the distribution on ed25519 signatures should be good enough even by using only the first 8 Bytes //! data are transaction nr - ExpireCache mSignaturePartTransactionNrs; + // AccessExpireCache mSignaturePartTransactionNrs; + std::unordered_map mBodyBytesTransactionNrs; std::string mCommunityId; }; } diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp index dde66a8..161db87 100644 --- a/src/task/RebuildBlockIndexTask.cpp +++ b/src/task/RebuildBlockIndexTask.cpp @@ -83,7 +83,8 @@ namespace task { mFileCursorsQueue.pop_front(); } mBulkDeserializerTasks.pop(); - } + } + int zahl = 0; } bool RebuildBlockIndexTask::isPendingQueueEmpty() From d2a727a3b5f25e344e25fbbf24eb9f701ad55ee0 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 7 Jan 2026 15:08:48 +0100 Subject: [PATCH 04/65] validate only last 1,5k --- src/blockchain/FileBased.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 1902645..0f4b19e 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -120,7 +120,7 @@ namespace gradido { int count = 0; findAll(builder .setSearchDirection(SearchDirection::DESC) - //.setPagination({ GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE }) + .setPagination({ GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE }) .setFilterFunction([this, &count](const TransactionEntry& transactionEntry) -> FilterResult { auto previousTransactionNr = transactionEntry.getTransactionNr() - 1; data::ConstConfirmedTransactionPtr previousConfirmedTransaction; @@ -143,7 +143,6 @@ namespace gradido { validator.run(validationLevel, getptr()); mTransactionHashCache.push(*transactionEntry.getConfirmedTransaction()); count++; - printf("\r%d", count); return FilterResult::DISMISS; }) .build() From efa3de63772a140624b570ea8c90855617067cee Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 11 Jan 2026 09:04:47 +0100 Subject: [PATCH 05/65] fix some ui errors, debug findOne error --- dependencies/gradido_blockchain | 2 +- src/lib/PersistentDictionary.h | 2 +- src/model/Apollo/TransactionList.cpp | 49 +++++++++++++++---- .../RedeemDeferredTransferTransactionRole.cpp | 6 --- ...TimeoutDeferredTransferTransactionRole.cpp | 11 ++++- src/server/json-rpc/ApiHandler.cpp | 6 +-- 6 files changed, 54 insertions(+), 22 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index d02cb6c..1749d08 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit d02cb6cc9f5b1771189626f3da16f66971fc4890 +Subproject commit 1749d0814ca0b7c956417025bc6d23d055467751 diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h index 2e666c3..c805b3c 100644 --- a/src/lib/PersistentDictionary.h +++ b/src/lib/PersistentDictionary.h @@ -25,7 +25,7 @@ class PersistentDictionary: public IMutableDictionary bool init(size_t cacheInBytes); void exit(); - void reset(); + void reset() override; uint32_t getLastIndex(); virtual std::optional getIndexForData(const DataType& data) const override; diff --git a/src/model/Apollo/TransactionList.cpp b/src/model/Apollo/TransactionList.cpp index acde4db..4699694 100644 --- a/src/model/Apollo/TransactionList.cpp +++ b/src/model/Apollo/TransactionList.cpp @@ -1,6 +1,8 @@ #include "TransactionList.h" #include "createTransaction/Context.h" #include "gradido_blockchain/blockchain/Filter.h" +#include "gradido_blockchain/data/Timestamp.h" +#include "gradido_blockchain/serialization/toJsonString.h" #include "../../blockchain/FileBased.h" #include "../../blockchain/NodeTransactionEntry.h" @@ -11,7 +13,8 @@ using namespace rapidjson; using namespace gradido::interaction; using namespace gradido::blockchain; using namespace magic_enum; - +using gradido::data::Timestamp; +using serialization::toJsonString; namespace model { namespace Apollo { @@ -41,12 +44,13 @@ namespace model { auto filterOutNotForWallet = [&filter](const TransactionEntry& entry) -> FilterResult { // filter out creation transactions which this user has signed as moderator, and isn't the benefitor - if (entry.isCreation()) { + // shouldn't be needed any longer, because of Transaction Index change, using updatedBalancePublicKey instead of involvedPublicKey + /*if (entry.isCreation()) { auto creation = entry.getTransactionBody()->getCreation(); if (!creation->getRecipient().getPublicKey()->isTheSame(filter.involvedPublicKey)) { return FilterResult::DISMISS; } - } + }*/ // filter out register address transaction, because this won't show in wallet view if (entry.isRegisterAddress()) { return FilterResult::DISMISS; @@ -68,7 +72,7 @@ namespace model { }; fileBasedBlockchain->findAll(countFilter); - auto addressType = mBlockchain->getAddressType(Filter(0,0,filter.involvedPublicKey)); + auto addressType = mBlockchain->getAddressType(Filter(0,0,filter.updatedBalancePublicKey)); transactionList.AddMember("addressType", Value(enum_name(addressType).data(), alloc), alloc); Filter filterCopy = filter; @@ -81,18 +85,43 @@ namespace model { return std::move(transactionList); } - // copy into vector to make reversing and loop through faster (cache-hit) - std::vector> allTransactionsVector(allTransactions.begin(), allTransactions.end()); - allTransactions.clear(); if (filter.searchDirection == SearchDirection::DESC) { - std::reverse(allTransactionsVector.begin(), allTransactionsVector.end()); + std::reverse(allTransactions.begin(), allTransactions.end()); } // all transaction is always sorted ASC, regardless of filter.searchDirection value GradidoUnit previousBalance(GradidoUnit::zero()); + // load previous balance before first transaction for decay Timepoint previousDate = mBlockchain->getStartDate(); + auto firstTransactionNr = allTransactions.front()->getTransactionNr(); + if (firstTransactionNr > 1) { + const auto& previousTransactionDate = allTransactions.front()->getConfirmedTransaction()->getConfirmedAt(); + auto beforePreviousTransactionDate = Timestamp( + previousTransactionDate.getSeconds(), + previousTransactionDate.getNanos() - 1000 + ); + + Filter previousTransactionFilter = Filter::LAST_TRANSACTION; + previousTransactionFilter.maxTransactionNr = firstTransactionNr - 1; + previousTransactionFilter.updatedBalancePublicKey = filter.updatedBalancePublicKey; + previousTransactionFilter.timepointInterval = TimepointInterval(previousDate, beforePreviousTransactionDate); + auto previousTransaction = mBlockchain->findOne(previousTransactionFilter); + if (previousTransaction) { + auto accountBalance = previousTransaction->getConfirmedTransaction()->getAccountBalance( + mPubkey, + filter.coinCommunityId + ); + printf("filter: %s\n", toJsonString(previousTransactionFilter, true).c_str()); + printf("previous transaction: %s\n", toJsonString(*previousTransaction->getConfirmedTransaction(), true).c_str()); + if (accountBalance.getBalance() > GradidoUnit::zero()) { + previousBalance = accountBalance.getBalance(); + previousDate = previousTransaction->getConfirmedTransaction()->getConfirmedAt(); + } + } + } + createTransaction::Context createTransactionContext(mBlockchain, addressType); - for (auto& entry: allTransactionsVector) + for (auto& entry: allTransactions) { auto confirmedTransaction = entry->getConfirmedTransaction(); auto transactions = createTransactionContext.run(*confirmedTransaction, mPubkey); @@ -116,7 +145,7 @@ namespace model { } } } - allTransactionsVector.clear(); + allTransactions.clear(); if (transactionsVector.empty()) { transactionList.AddMember("transactions", Value(kArrayType), alloc); return std::move(transactionList); diff --git a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp index 8b79805..b87627c 100644 --- a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp @@ -45,12 +45,6 @@ namespace model { confirmedTransaction.getConfirmedAt(), transfer.getSender() ); - printf( - "RedeemDeferredTransferTransactionRole::createTransaction setting change amount: %s, negated: %s, pubkey: %s\n", - change.getBalance().toString().data(), - change.getBalance().negated().toString().data(), - change.getPublicKey()->convertToHex().data() - ); result.setChange(change.getBalance().negated(), change.getPublicKey()); } result.setAmount(amount); diff --git a/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp index 20f964d..c68d9a9 100644 --- a/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp @@ -1,5 +1,6 @@ #include "TimeoutDeferredTransferTransactionRole.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/blockchain/Abstract.h" using namespace gradido; @@ -21,9 +22,17 @@ namespace model { timeoutDeferredTransfer->getDeferredTransferTransactionNr(), confirmedTransaction.getConfirmedAt() ); + const auto& deferredTransferEntry = mBlockchain->getTransactionForId(timeoutDeferredTransfer->getDeferredTransferTransactionNr()); + const auto& deferredTransferBody = deferredTransferEntry->getTransactionBody(); + assert(deferredTransferBody->isDeferredTransfer()); + const auto& deferredTransfer = deferredTransferBody->getDeferredTransfer(); result.setType(TransactionType::LINK_TIMEOUT); - result.setAmount(changeAccountBalance.getBalance()); + auto balance = changeAccountBalance.getBalance(); + if (pubkey->isTheSame(deferredTransfer->getRecipientPublicKey())) { + balance.negate(); + } + result.setAmount(balance); result.setPubkey(changeAccountBalance.getPublicKey()); return result; } diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 1590d78..4d70d3c 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -205,7 +205,7 @@ namespace server { else if (method == "listTransactions") { Filter f; f.pagination = Pagination(25, 1); - f.involvedPublicKey = pubkey; + f.updatedBalancePublicKey = pubkey; if (params.HasMember("currentPage") && params["currentPage"].IsInt()) { f.pagination.page = params["currentPage"].GetInt(); } @@ -466,14 +466,14 @@ namespace server { Profiler timeUsed; auto& alloc = mRootJson.GetAllocator(); - model::Apollo::TransactionList transactionList(blockchain, filter.involvedPublicKey); + model::Apollo::TransactionList transactionList(blockchain, filter.updatedBalancePublicKey); Timepoint now = std::chrono::system_clock::now(); auto transactionListValue = transactionList.generateList(now, filter, mRootJson); calculateAccountBalance::Context calculateAddressBalance(blockchain); // TODO: add balances from another communities - auto balance = calculateAddressBalance.fromEnd(filter.involvedPublicKey, now, ""); + auto balance = calculateAddressBalance.fromEnd(filter.updatedBalancePublicKey, now, ""); std::string balanceString = balance.toString(); transactionListValue.AddMember("balance", Value(balanceString.data(), balanceString.size(), alloc), alloc); resultJson.AddMember("transactionList", transactionListValue, alloc); From 1606367caed3a7f93d05d81ed5d34745f501fb15 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 14 Jan 2026 16:33:24 +0100 Subject: [PATCH 06/65] calculate decay with subraction and addition only, set precision for debugging higher --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 51 ++++++++++++++++--- src/blockchain/FileBased.h | 6 ++- src/cache/Block.cpp | 14 +++-- src/cache/Block.h | 11 +++- src/model/Apollo/Decay.cpp | 34 ++++++++++--- src/model/Apollo/Decay.h | 14 ++++- src/model/Apollo/Transaction.cpp | 26 +++++++--- src/model/Apollo/Transaction.h | 12 +++-- src/model/Apollo/TransactionList.cpp | 36 +++++-------- .../AbstractTransactionRole.h | 1 + .../CreationTransactionRole.cpp | 10 ++-- .../DeferredTransferTransactionRole.cpp | 21 ++++---- .../RedeemDeferredTransferTransactionRole.cpp | 17 +++---- ...TimeoutDeferredTransferTransactionRole.cpp | 8 +-- .../TransferTransactionRole.cpp | 21 ++++---- src/server/json-rpc/ApiHandler.cpp | 7 +-- 17 files changed, 185 insertions(+), 106 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 1749d08..6279f1b 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 1749d0814ca0b7c956417025bc6d23d055467751 +Subproject commit 6279f1bc8719eb77a7271a60c811b96323629ec1 diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 0f4b19e..a523d9e 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -16,6 +16,7 @@ #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/interaction/confirmTransaction/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" +#include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_blockchain/lib/Profiler.h" #include "loguru/loguru.hpp" @@ -28,6 +29,7 @@ using std::string, std::string_view, std::vector; using std::shared_ptr, std::make_shared; using client::hiero::ConsensusClient; using controller::SimpleOrderingManager; +using serialization::toJsonString; namespace gradido { using data::LedgerAnchor, data::AddressType; @@ -296,7 +298,7 @@ namespace gradido { if (!filter.pagination.hasCapacityLeft(result.size())) { return false; } - auto transaction = block.getTransaction(transactionNr); + auto transaction = block.getTransaction(transactionNr, mPublicKeysIndex); auto filterResult = filter.matches(transaction, FilterCriteria::FILTER_FUNCTION); if ((filterResult & FilterResult::USE) == FilterResult::USE) { result.push_back(transaction); @@ -318,8 +320,36 @@ namespace gradido { return result; } + size_t FileBased::countAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS*/) const + { + size_t count = 0; + // check if filter has fields which aren't checked by index + if (!gradido::blockchain::TransactionsIndex::canMatchWithoutDeserialize(filter)) { + LOG_F( + WARNING, + "slow count, detect fields in Filter which aren't covered by index: %s", + toJsonString(filter).c_str() + ); + return findAll(filter).size(); + } + iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { + count += block.getBlockIndex().countTransactions(filter, mPublicKeysIndex); + return true; + }); + return count; + } + std::vector FileBased::findAllFast(const Filter& filter) const { + // check if filter has fields which aren't checked by index + if (!gradido::blockchain::TransactionsIndex::canMatchWithoutDeserialize(filter)) { + LOG_F( + ERROR, + "findAllFast call with invalid filter not covered by index: %s", + toJsonString(filter).c_str() + ); + return {}; + } std::vector result; // if pagination is used, filterCopy contain count of still to find transactions Filter filterCopy(filter); @@ -336,17 +366,22 @@ namespace gradido { return result; } - size_t FileBased::findAllResultCount(const Filter& filter) const + data::AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const { - size_t count = 0; + data::AddressType result = data::AddressType::NONE; iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - count += block.getBlockIndex().countTransactions(filter, mPublicKeysIndex); - return true; + result = block.getBlockIndex().getAddressType(filter.involvedPublicKey, mPublicKeysIndex); + if (data::AddressType::NONE == result) { + return true; + } + return false; }); - return count; + if (data::AddressType::NONE == result) { + result = getAddressTypeSlow(filter); + } + return result; } - std::shared_ptr FileBased::getTransactionForId(uint64_t transactionId) const { std::lock_guard _lock(mWorkMutex); @@ -354,7 +389,7 @@ namespace gradido { do { auto& block = getBlock(blockNr); if (block.getBlockIndex().hasTransactionNr(transactionId)) { - return block.getTransaction(transactionId); + return block.getTransaction(transactionId, mPublicKeysIndex); } blockNr--; } while (blockNr > 0); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 8ceb7e8..84fe1b4 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -125,12 +125,14 @@ namespace gradido { //! main search function, do all the work, reference from other functions virtual TransactionEntries findAll(const Filter& filter) const override; + // find all optimized for counting transaction nrs, better not use the filter.function for that, because this would slow down + virtual size_t countAll(const Filter& filter = Filter::ALL_TRANSACTIONS) const override; + //! use only index for searching, ignore filter function //! \return vector with transaction nrs std::vector findAllFast(const Filter& filter) const; - //! count results for a specific filter, using only the index, ignore filter function - size_t findAllResultCount(const Filter& filter) const; + virtual data::AddressType getAddressType(const Filter& filter = Filter::LAST_TRANSACTION) const override; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; //! \param filter use to speed up search if infos exist to narrow down search transactions range diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 9979223..cf10bb0 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -115,14 +115,22 @@ namespace cache { } - void Block::addTransaction(memory::ConstBlockPtr serializedTransaction, int32_t fileCursor) const + void Block::addTransaction( + memory::ConstBlockPtr serializedTransaction, + int32_t fileCursor, + IMutableDictionary& publicKeyDictionary + ) const { auto transactionEntry = std::make_shared(serializedTransaction, mBlockchain, fileCursor); if (mExitCalled) return; mSerializedTransactions.add(transactionEntry->getTransactionNr(), transactionEntry); + // mBlockIndex->updateAddressIndex(transactionEntry, publicKeyDictionary); } - std::shared_ptr Block::getTransaction(uint64_t transactionNr) const + std::shared_ptr Block::getTransaction( + uint64_t transactionNr, + IMutableDictionary& publicKeyDictionary + ) const { assert(transactionNr); std::lock_guard lock(mFastMutex); @@ -162,7 +170,7 @@ namespace cache { } try { auto blockLine = mBlockFile->readLine(fileCursor); - addTransaction(blockLine, fileCursor); + addTransaction(blockLine, fileCursor, publicKeyDictionary); } catch (model::files::EndReachingException& ex) { LOG_F(ERROR, "%s", ex.getFullString().data()); diff --git a/src/cache/Block.h b/src/cache/Block.h index cb0e425..e44825a 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -62,7 +62,10 @@ namespace cache { ); //! \brief load transaction from cache or file system - std::shared_ptr getTransaction(uint64_t transactionNr) const; + std::shared_ptr getTransaction( + uint64_t transactionNr, + IMutableDictionary& publicKeyDictionary + ) const; inline BlockIndex& getBlockIndex() { return *mBlockIndex; } inline const BlockIndex& getBlockIndex() const { return *mBlockIndex; } @@ -76,7 +79,11 @@ namespace cache { protected: //! \brief add transaction from Block File, called by Block File, adding to cache and index //! not locking mutex! - void addTransaction(memory::ConstBlockPtr serializedTransaction, int32_t fileCursor) const; + void addTransaction( + memory::ConstBlockPtr serializedTransaction, + int32_t fileCursor, + IMutableDictionary& publicKeyDictionary + ) const; mutable std::mutex mFastMutex; uint32_t mBlockNr; diff --git a/src/model/Apollo/Decay.cpp b/src/model/Apollo/Decay.cpp index eff2706..0464885 100644 --- a/src/model/Apollo/Decay.cpp +++ b/src/model/Apollo/Decay.cpp @@ -1,10 +1,13 @@ #include "Decay.h" #include "gradido_blockchain/lib/DataTypeConverter.h" +#include "magic_enum/magic_enum.hpp" + #include using namespace rapidjson; using namespace std::chrono; +using namespace magic_enum; namespace model { namespace Apollo { @@ -14,19 +17,26 @@ namespace model { std::string formatJsCompatible(Timepoint date) { auto dateString = DataTypeConverter::timePointToString(date, jsDateTimeFormat); - return dateString; + // add Z to say it is UTC + return dateString + "Z"; // return dateString.substr(0, dateString.find_last_of('.')); } Decay::Decay(Decay* parent) - : mDecayStart(parent->mDecayStart), mDecayEnd(parent->mDecayEnd), mDecayAmount(parent->mDecayAmount) + : mDecayStart(parent->mDecayStart), mDecayEnd(parent->mDecayEnd), mDecayAmount(parent->mDecayAmount), mDecayType(parent->getDecayType()) { } - Decay::Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) - : mDecayStart(decayStart), mDecayEnd(decayEnd) + + Decay::Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance, GradidoUnit decayAmount) + : mDecayStart(decayStart), mDecayEnd(decayEnd), mDecayAmount(decayAmount), mDecayType(decideDecayType(decayStart, decayEnd)) { - mDecayAmount = startBalance.calculateDecay(mDecayStart, mDecayEnd) - startBalance; + if (mDecayStart < DECAY_START_TIME) { + mDecayStart = DECAY_START_TIME; + } + if (mDecayEnd < mDecayStart) { + mDecayEnd = mDecayStart; + } } Decay::~Decay() @@ -37,14 +47,26 @@ namespace model { Value Decay::toJson(Document::AllocatorType& alloc) { Value decay(kObjectType); - decay.AddMember("decay", Value(mDecayAmount.toString(2).data(), alloc), alloc); + decay.AddMember("decay", Value(mDecayAmount.toString().data(), alloc), alloc); decay.AddMember("start", Value(formatJsCompatible(mDecayStart).data(), alloc), alloc); decay.AddMember("end", Value(formatJsCompatible(mDecayEnd).data(), alloc), alloc); decay.AddMember("duration", static_cast( duration_cast(mDecayEnd - mDecayStart).count() ), alloc); + decay.AddMember("type", Value(enum_name(mDecayType).data(), alloc), alloc); decay.AddMember("__typename", "Decay", alloc); return std::move(decay); } + + DecayType Decay::decideDecayType(Timepoint decayStart, Timepoint decayEnd) + { + if (decayEnd <= DECAY_START_TIME) { + return DecayType::BEFORE_START_BLOCK; + } + if (decayStart >= DECAY_START_TIME) { + return DecayType::AFTER_START_BLOCK; + } + return DecayType::START_BLOCK_INSIDE; + } } } \ No newline at end of file diff --git a/src/model/Apollo/Decay.h b/src/model/Apollo/Decay.h index 7bae548..17d7247 100644 --- a/src/model/Apollo/Decay.h +++ b/src/model/Apollo/Decay.h @@ -10,22 +10,32 @@ namespace model { std::string formatJsCompatible(Timepoint date); + enum DecayType + { + BEFORE_START_BLOCK, + START_BLOCK_INSIDE, + AFTER_START_BLOCK + }; + class Decay { public: Decay(Decay* parent); - Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); + Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance, GradidoUnit decayAmount); ~Decay(); rapidjson::Value toJson(rapidjson::Document::AllocatorType& alloc); inline GradidoUnit getDecayAmount() const { return mDecayAmount; } - + inline DecayType getDecayType() const { return mDecayType; } protected: + static DecayType decideDecayType(Timepoint decayStart, Timepoint decayEnd); + Timepoint mDecayStart; Timepoint mDecayEnd; GradidoUnit mDecayAmount; + DecayType mDecayType; }; } } diff --git a/src/model/Apollo/Transaction.cpp b/src/model/Apollo/Transaction.cpp index 10130dd..a3c5df7 100644 --- a/src/model/Apollo/Transaction.cpp +++ b/src/model/Apollo/Transaction.cpp @@ -39,14 +39,14 @@ namespace model { } Transaction::Transaction(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) - : mType(TransactionType::DECAY), mId(-1), mDate(decayEnd), mDecay(nullptr) + : mType(TransactionType::DECAY), mId(-1), mDate(decayEnd), mDecay(nullptr), mHasChange(false) { calculateDecay(decayStart, decayEnd, startBalance); mAmount = mDecay->getDecayAmount(); mBalance = startBalance + mDecay->getDecayAmount(); mPreviousBalance = startBalance; } - + /* * TransactionType mType; mpfr_ptr mAmount; @@ -141,28 +141,38 @@ namespace model { return *this; } + void Transaction::setDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) + { + if (mDecay) { + delete mDecay; + mDecay = nullptr; + } + + mDecay = new Decay(decayStart, decayEnd, startBalance, (startBalance - mBalance + mAmount).negate()); + } void Transaction::calculateDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) { if (mDecay) { delete mDecay; mDecay = nullptr; } - mDecay = new Decay(decayStart, decayEnd, startBalance); + mDecay = new Decay(decayStart, decayEnd, startBalance, startBalance.calculateDecay(decayStart, decayEnd) - startBalance); } - void Transaction::setBalance(GradidoUnit balance) + /* void Transaction::setBalance(GradidoUnit balance) { mBalance = balance; } + */ Value Transaction::toJson(Document::AllocatorType& alloc) { Value transaction(kObjectType); transaction.AddMember("id", mId, alloc); transaction.AddMember("typeId", Value(enum_name(mType).data(), alloc), alloc); - transaction.AddMember("amount", Value(mAmount.toString(2).data(), alloc), alloc); - transaction.AddMember("balance", Value(mBalance.toString(2).data(), alloc), alloc); - transaction.AddMember("previousBalance", Value(mPreviousBalance.toString(2).data(), alloc), alloc); + transaction.AddMember("amount", Value(mAmount.toString().data(), alloc), alloc); + transaction.AddMember("balance", Value(mBalance.toString().data(), alloc), alloc); + transaction.AddMember("previousBalance", Value(mPreviousBalance.toString().data(), alloc), alloc); transaction.AddMember("memo", Value(mMemo.data(), alloc), alloc); if (!mPubkey.empty() || !mFirstName.empty() || !mLastName.empty()) { @@ -178,7 +188,7 @@ namespace model { if(mHasChange) { Value changeObj(kObjectType); printf("Transaction::toJson adding change amount: %s, pubkey: %s\n", mChangeAmount.toString().data(), mChangePubkey.data()); - changeObj.AddMember("amount", Value(mChangeAmount.toString(2).data(), alloc), alloc); + changeObj.AddMember("amount", Value(mChangeAmount.toString().data(), alloc), alloc); changeObj.AddMember("pubkey", Value(mChangePubkey.data(), alloc), alloc); changeObj.AddMember("__typename", "Change", alloc); transaction.AddMember("change", changeObj, alloc); diff --git a/src/model/Apollo/Transaction.h b/src/model/Apollo/Transaction.h index fd13e47..df41a6f 100644 --- a/src/model/Apollo/Transaction.h +++ b/src/model/Apollo/Transaction.h @@ -38,6 +38,7 @@ namespace model { const gradido::data::ConfirmedTransaction& confirmedTransaction, memory::ConstBlockPtr pubkey ); + // constructor for last decay to now transaction Transaction(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); // Move constrcutor @@ -48,8 +49,9 @@ namespace model { Transaction& operator=(const Transaction& other); // copy + void setDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); void calculateDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); - void setBalance(GradidoUnit balance); + // void setBalance(GradidoUnit balance); inline GradidoUnit getBalance() const {return mBalance;} inline void setPreviousBalance(GradidoUnit previousBalance) {mPreviousBalance = previousBalance;} inline void setChange(const GradidoUnit& changeAmount, memory::ConstBlockPtr changePubkey); @@ -90,10 +92,10 @@ namespace model { }; void Transaction::setChange(const GradidoUnit& changeAmount, memory::ConstBlockPtr changePubkey) { - mHasChange = true; - mChangeAmount = changeAmount; - mChangePubkey = changePubkey->convertToHex(); - } + mHasChange = true; + mChangeAmount = changeAmount; + mChangePubkey = changePubkey->convertToHex(); + } } } diff --git a/src/model/Apollo/TransactionList.cpp b/src/model/Apollo/TransactionList.cpp index 4699694..331b105 100644 --- a/src/model/Apollo/TransactionList.cpp +++ b/src/model/Apollo/TransactionList.cpp @@ -2,6 +2,7 @@ #include "createTransaction/Context.h" #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/data/Timestamp.h" +#include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "../../blockchain/FileBased.h" @@ -15,6 +16,7 @@ using namespace gradido::blockchain; using namespace magic_enum; using gradido::data::Timestamp; using serialization::toJsonString; + namespace model { namespace Apollo { @@ -58,19 +60,15 @@ namespace model { return FilterResult::USE; }; - int countTransactions = 0; + size_t countTransactions = 0; Filter countFilter = filter; countFilter.pagination = Pagination(0, 0); - countFilter.filterFunction = [&filterOutNotForWallet, &countTransactions](const TransactionEntry& entry) -> FilterResult - { - auto result = filterOutNotForWallet(entry); - if ((result & FilterResult::USE) == FilterResult::USE) { - countTransactions++; - return FilterResult::DISMISS; - } - return result; - }; - fileBasedBlockchain->findAll(countFilter); + auto allTransactionsCount = mBlockchain->countAll(countFilter); + countFilter.transactionType = gradido::data::TransactionType::REGISTER_ADDRESS; + auto registerAddressTransactionsCount = mBlockchain->countAll(countFilter); + if (registerAddressTransactionsCount < allTransactionsCount) { + countTransactions = allTransactionsCount - registerAddressTransactionsCount; + } auto addressType = mBlockchain->getAddressType(Filter(0,0,filter.updatedBalancePublicKey)); transactionList.AddMember("addressType", Value(enum_name(addressType).data(), alloc), alloc); @@ -111,8 +109,6 @@ namespace model { mPubkey, filter.coinCommunityId ); - printf("filter: %s\n", toJsonString(previousTransactionFilter, true).c_str()); - printf("previous transaction: %s\n", toJsonString(*previousTransaction->getConfirmedTransaction(), true).c_str()); if (accountBalance.getBalance() > GradidoUnit::zero()) { previousBalance = accountBalance.getBalance(); previousDate = previousTransaction->getConfirmedTransaction()->getConfirmedAt(); @@ -127,22 +123,16 @@ namespace model { auto transactions = createTransactionContext.run(*confirmedTransaction, mPubkey); for (auto& transaction: transactions) { transactionsVector.push_back(transaction); + // TODO: choose correct coin color + auto balance = confirmedTransaction->getAccountBalance(mPubkey, ""); transactionsVector.back().setPreviousBalance( previousBalance ); if (previousBalance > GradidoUnit::zero()) { - transactionsVector.back().calculateDecay(previousDate, confirmedTransaction->getConfirmedAt(), previousBalance); + transactionsVector.back().setDecay(previousDate, confirmedTransaction->getConfirmedAt(), previousBalance); } previousDate = confirmedTransaction->getConfirmedAt(); - auto& balances = confirmedTransaction->getAccountBalances(); - previousBalance = GradidoUnit::zero(); - for (auto& balance : balances) { - // calculate sum of all balances belonging to this user, of all coin color - // TODO: choose correct coin color - if (balance.getPublicKey()->isTheSame(mPubkey)) { - previousBalance += balance.getBalance(); - } - } + previousBalance = transactionsVector.back().getBalance(); } } allTransactions.clear(); diff --git a/src/model/Apollo/createTransaction/AbstractTransactionRole.h b/src/model/Apollo/createTransaction/AbstractTransactionRole.h index c2e6af7..ad81235 100644 --- a/src/model/Apollo/createTransaction/AbstractTransactionRole.h +++ b/src/model/Apollo/createTransaction/AbstractTransactionRole.h @@ -29,6 +29,7 @@ namespace model { ) = 0; protected: + std::shared_ptr mBlockchain; }; } diff --git a/src/model/Apollo/createTransaction/CreationTransactionRole.cpp b/src/model/Apollo/createTransaction/CreationTransactionRole.cpp index a616d48..de8e15e 100644 --- a/src/model/Apollo/createTransaction/CreationTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/CreationTransactionRole.cpp @@ -16,11 +16,11 @@ namespace model { Transaction result(confirmedTransaction, pubkey); result.setType(TransactionType::CREATE); - auto creation = transactionBody->getCreation(); - result.setAmount(creation->getRecipient().getAmount()); - result.setFirstName("Gradido"); - result.setLastName("Akademie"); - result.setPubkey(gradidoTransaction->getSignatureMap().getSignaturePairs().front().getPublicKey()); + auto creation = transactionBody->getCreation(); + result.setAmount(creation->getRecipient().getAmount()); + result.setFirstName("Gradido"); + result.setLastName("Akademie"); + result.setPubkey(gradidoTransaction->getSignatureMap().getSignaturePairs().front().getPublicKey()); return result; } } diff --git a/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp index b89c9c8..2f02e58 100644 --- a/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp @@ -15,22 +15,21 @@ namespace model { Transaction result(confirmedTransaction, pubkey); auto deferredTransfer = transactionBody->getDeferredTransfer(); - auto transfer = deferredTransfer->getTransfer(); + const auto& transfer = deferredTransfer->getTransfer(); auto amount = transfer.getSender().getAmount(); if (transfer.getRecipient()->isTheSame(pubkey)) { - result.setType(TransactionType::LINK_CHARGE); - result.setPubkey(transfer.getSender().getPublicKey()); - } - else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { - result.setType(TransactionType::LINK_SEND); - result.setPubkey(transfer.getRecipient()); - amount.negate(); - } else { + result.setType(TransactionType::LINK_CHARGE); + result.setPubkey(transfer.getSender().getPublicKey()); + } else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { + result.setType(TransactionType::LINK_SEND); + result.setPubkey(transfer.getRecipient()); + amount.negate(); + } else { throw GradidoNodeInvalidDataException("unhandled case in model::Apollo::createTransaction::DeferredTransferTransactionRole if pubkey is neither sender or recipient"); - } + } - result.setAmount(amount); + result.setAmount(amount); return result; } } diff --git a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp index b87627c..f900f38 100644 --- a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp @@ -19,18 +19,17 @@ namespace model { Transaction result(confirmedTransaction, pubkey); auto redeemDeferredTransfer = transactionBody->getRedeemDeferredTransfer(); - auto transfer = redeemDeferredTransfer->getTransfer(); + const auto& transfer = redeemDeferredTransfer->getTransfer(); auto amount = transfer.getSender().getAmount(); if (transfer.getRecipient()->isTheSame(pubkey)) { result.setType(TransactionType::LINK_RECEIVE); result.setPubkey(transfer.getSender().getPublicKey()); - } - else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { - result.setType(TransactionType::SEND); - result.setPubkey(transfer.getRecipient()); - amount.negate(); - } else { + } else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { + result.setType(TransactionType::SEND); + result.setPubkey(transfer.getRecipient()); + amount.negate(); + } else { amount = calculateChange( redeemDeferredTransfer->getDeferredTransferTransactionNr(), confirmedTransaction.getConfirmedAt(), @@ -38,7 +37,7 @@ namespace model { ).getBalance(); result.setType(TransactionType::LINK_CHANGE); result.setPubkey(transfer.getSender().getPublicKey()); - } + } if (data::AddressType::DEFERRED_TRANSFER == mAddressType) { auto change = calculateChange( redeemDeferredTransfer->getDeferredTransferTransactionNr(), @@ -47,7 +46,7 @@ namespace model { ); result.setChange(change.getBalance().negated(), change.getPublicKey()); } - result.setAmount(amount); + result.setAmount(amount); return result; } diff --git a/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp index c68d9a9..e40494d 100644 --- a/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp @@ -10,10 +10,10 @@ namespace model { Transaction TimeoutDeferredTransferTransactionRole::createTransaction( const data::ConfirmedTransaction& confirmedTransaction, - memory::ConstBlockPtr pubkey + memory::ConstBlockPtr pubkey ) { auto gradidoTransaction = confirmedTransaction.getGradidoTransaction(); - auto transactionBody = gradidoTransaction->getTransactionBody(); + auto transactionBody = gradidoTransaction->getTransactionBody(); assert(transactionBody->isTimeoutDeferredTransfer()); Transaction result(confirmedTransaction, pubkey); @@ -32,8 +32,8 @@ namespace model { if (pubkey->isTheSame(deferredTransfer->getRecipientPublicKey())) { balance.negate(); } - result.setAmount(balance); - result.setPubkey(changeAccountBalance.getPublicKey()); + result.setAmount(balance); + result.setPubkey(changeAccountBalance.getPublicKey()); return result; } } diff --git a/src/model/Apollo/createTransaction/TransferTransactionRole.cpp b/src/model/Apollo/createTransaction/TransferTransactionRole.cpp index b400d7d..db03457 100644 --- a/src/model/Apollo/createTransaction/TransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/TransferTransactionRole.cpp @@ -14,21 +14,20 @@ namespace model { assert(transactionBody->isTransfer()); Transaction result(confirmedTransaction, pubkey); - auto transfer = transactionBody->getTransfer(); + const auto& transfer = transactionBody->getTransfer(); auto amount = transfer->getSender().getAmount(); if (transfer->getRecipient()->isTheSame(pubkey)) { - result.setType(TransactionType::RECEIVE); - result.setPubkey(transfer->getSender().getPublicKey()); - } - else if (transfer->getSender().getPublicKey()->isTheSame(pubkey)) { - result.setType(TransactionType::SEND); - result.setPubkey(transfer->getRecipient()); - amount.negate(); - } else { + result.setType(TransactionType::RECEIVE); + result.setPubkey(transfer->getSender().getPublicKey()); + } else if (transfer->getSender().getPublicKey()->isTheSame(pubkey)) { + result.setType(TransactionType::SEND); + result.setPubkey(transfer->getRecipient()); + amount.negate(); + } else { throw GradidoNodeInvalidDataException("unhandled case in model::Apollo::createTransaction::TransferTransactionRole if pubkey is neither sender or recipient"); - } - result.setAmount(amount); + } + result.setAmount(amount); return result; } } diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 4d70d3c..239c320 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -273,16 +273,11 @@ namespace server { auto alloc = mRootJson.GetAllocator(); // count for pagination - uint64_t totalCount = 0; Filter countFilter = filter; countFilter.pagination = Pagination(); // remove pagination for count countFilter.minTransactionNr = 0; // remove minTransactionNr for count countFilter.maxTransactionNr = 0; // remove maxTransactionNr for count - countFilter.filterFunction = [&totalCount](const TransactionEntry& transactionEntry) { - totalCount++; - return FilterResult::DISMISS; - }; - blockchain->findAll(countFilter); + auto totalCount = blockchain->countAll(countFilter); resultJson.AddMember("totalCount", totalCount, alloc); auto transactions = blockchain->findAll(filter); From 38ffec565b40b24ea39d9d1581b03d0b4fc8de0a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 20 Jan 2026 16:49:32 +0100 Subject: [PATCH 07/65] adapt to community id changes --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 61 ++++++---- src/blockchain/FileBased.cpp | 10 +- src/blockchain/FileBased.h | 11 +- src/blockchain/FileBasedProvider.cpp | 111 +++++++++++------- src/blockchain/FileBasedProvider.h | 14 ++- src/blockchain/NodeTransactionEntry.cpp | 12 +- src/blockchain/NodeTransactionEntry.h | 3 +- src/cache/Block.cpp | 2 +- src/cache/BlockIndex.cpp | 13 +- src/cache/BlockIndex.h | 8 +- src/cache/GroupIndex.cpp | 89 ++++++++++---- src/cache/GroupIndex.h | 20 +++- src/controller/ControllerExceptions.cpp | 29 +++-- src/controller/ControllerExceptions.h | 1 + src/controller/RemoteGroup.cpp | 4 +- src/controller/RemoteGroup.h | 2 +- src/controller/SimpleOrderingManager.cpp | 2 +- src/lib/PersistentDictionary.h | 32 ++--- src/model/Apollo/Transaction.cpp | 2 +- src/model/Apollo/TransactionList.cpp | 12 +- .../RedeemDeferredTransferTransactionRole.cpp | 4 +- src/model/files/BlockIndex.cpp | 38 ++---- src/model/files/BlockIndex.h | 16 +-- src/model/files/LevelDBWrapper.cpp | 1 + src/server/json-rpc/ApiHandler.cpp | 22 ++-- src/server/json-rpc/ApiHandler.h | 2 +- src/task/SyncTopicOnStartup.cpp | 10 +- 28 files changed, 321 insertions(+), 212 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 6279f1b..5d92950 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 6279f1bc8719eb77a7271a60c811b96323629ec1 +Subproject commit 5d92950a7716afef7ec632e61d24667870d74c9d diff --git a/src/MainServer.cpp b/src/MainServer.cpp index 2b8fac4..e2c75d3 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -9,21 +9,29 @@ #include "hiero/Addressbook.h" #include "client/hiero/const.h" +#include "lib/PersistentDictionary.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/http/ServerConfig.h" #include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include #include "loguru.hpp" -using namespace gradido; -using namespace blockchain; -namespace fs = std::filesystem; +using gradido::blockchain::FileBasedProvider; +using gradido::g_appContext, gradido::AppContext; +using std::filesystem::create_directories, std::filesystem::exists, std::filesystem::is_regular_file, std::filesystem::path; +using std::shared_ptr, std::make_unique; +using std::string; +using std::vector; MainServer::MainServer() : mHttpServer(nullptr) @@ -38,21 +46,21 @@ bool MainServer::init() { Profiler usedTime; ServerGlobals::g_FilesPath = getHomeDir() + "/.gradido"; - fs::create_directories(ServerGlobals::g_FilesPath); - + create_directories(ServerGlobals::g_FilesPath); + // ********** logging ************************************ - std::string logPath = ServerGlobals::g_FilesPath + "/logs"; - fs::create_directories(logPath); + string logPath = ServerGlobals::g_FilesPath + "/logs"; + create_directories(logPath); // beware, no logrotation in loguru // TODO: add config options to choose which to use // TODO: check switching to https://github.com/gabime/spdlog - std::string errorLogFile = logPath + "/errors.log"; + string errorLogFile = logPath + "/errors.log"; loguru::add_file(errorLogFile.data(), loguru::Append, loguru::Verbosity_WARNING); // info - std::string infoLogFile = logPath + "/infos.log"; + string infoLogFile = logPath + "/infos.log"; loguru::add_file(infoLogFile.data(), loguru::Append, loguru::Verbosity_INFO); // infos and above - std::string debugLogFile = logPath + "/debug.log"; + string debugLogFile = logPath + "/debug.log"; loguru::add_file(debugLogFile.data(), loguru::Truncate, loguru::Verbosity_MAX); #if defined(__linux__) || defined(__unix__) // use syslog on linux, this has logrotation build in @@ -67,6 +75,10 @@ bool MainServer::init() unsigned short jsonrpc_port = (unsigned short)config.getInt("server.json_rpc", 8340); + auto communityDictionary = make_unique>(ServerGlobals::g_FilesPath + "/communityIdsCache"); + communityDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); + g_appContext = make_unique(std::move(communityDictionary)); + // timeouts ServerGlobals::loadTimeouts(config); ServerGlobals::g_LogTransactions = config.getBool("logging.log_transactions", ServerGlobals::g_LogTransactions); @@ -83,16 +95,16 @@ bool MainServer::init() CacheManager::getInstance()->getFuzzyTimer()->addTimer("mainCPUScheduler", ServerGlobals::g_CPUScheduler, std::chrono::milliseconds(100)); ServerGlobals::g_WriteFileCPUScheduler = new task::CPUSheduler(io_worker_count, "IO Worker"); // ServerGlobals::g_IotaRequestCPUScheduler = new task::CPUSheduler(2, "Iota Worker"); - std::string hieroNetworkType = config.getString("clients.hiero.networkType", "testnet"); + string hieroNetworkType = config.getString("clients.hiero.networkType", "testnet"); ServerGlobals::initHiero(hieroNetworkType); uint8_t hieroNodeCount = config.getInt("clients.hiero.nodeCount", 3); uint8_t hieroNodeCountPerCommunity = config.getInt("clients.hiero.nodeCountPerCommunity", 3); - std::vector> hieroClients; + vector> hieroClients; if (!ServerGlobals::g_isOfflineMode) { //iota::MqttClientWrapper::getInstance()->init(); - std::string grpcAddressesFile = ServerGlobals::g_FilesPath + "/addressbook/" + hieroNetworkType + ".pb"; + string grpcAddressesFile = ServerGlobals::g_FilesPath + "/addressbook/" + hieroNetworkType + ".pb"; if (!hieroNodeCount || !hieroNodeCountPerCommunity) { LOG_F(ERROR, "clients.hiero.nodeCountPerCommunity and clients.hiero.nodeCount need to be both >0"); @@ -124,6 +136,7 @@ bool MainServer::init() hieroClients.push_back(hieroClient); } } + if (!FileBasedProvider::getInstance()->init(ServerGlobals::g_FilesPath + "/communities.json", std::move(hieroClients), hieroNodeCountPerCommunity)) { LOG_F(ERROR, "Error loading communities, please try to delete communities folders and try again!"); return false; @@ -160,15 +173,15 @@ void MainServer::exit() FileBasedProvider::getInstance()->exit(); } -bool MainServer::configExists(const std::string& fileName) { - return fs::exists(fileName) && fs::is_regular_file(fileName); +bool MainServer::configExists(const string& fileName) { + return exists(fileName) && is_regular_file(fileName); } -std::string MainServer::findConfigFile() +string MainServer::findConfigFile() { // possible paths - fs::path currentPath = "gradido.yaml"; // current location - fs::path homePath = fs::path(getHomeDir()) / ".gradido" / "gradido.yaml"; + path currentPath = "gradido.yaml"; // current location + path homePath = path(getHomeDir()) / ".gradido" / "gradido.yaml"; // check paths if (configExists(currentPath.string())) { @@ -183,12 +196,12 @@ std::string MainServer::findConfigFile() return ""; } -std::string MainServer::getHomeDir() +string MainServer::getHomeDir() { #if defined(_WIN32) || defined(_WIN64) - return fs::path(getenv("USERPROFILE")).string(); // windows + return path(getenv("USERPROFILE")).string(); // windows #else - return fs::path(getenv("HOME")).string(); // linux + return path(getenv("HOME")).string(); // linux #endif } diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index a523d9e..b1c1d76 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -1,3 +1,4 @@ +#include "gradido_blockchain/AppContext.h" #include "FileBased.h" #include "../client/hiero/ConsensusClient.h" #include "FileBasedProvider.h" @@ -38,16 +39,17 @@ namespace gradido { namespace blockchain { FileBased::FileBased( Private, - string_view communityId, + const string& communityId, const hiero::TopicId& topicId, string_view alias, string_view folder, vector>&& hieroClients) - : Abstract(communityId), + : Abstract(g_appContext->getOrAddCommunityIdIndex(communityId)), mExitCalled(false), mHieroTopicId(topicId), mAlias(alias), - mFolderPath(folder), + mFolderPath(folder), + mCommunityId(communityId), mTaskObserver(std::make_shared()), mOrderingManager(std::make_shared(communityId)), // mIotaMessageListener(new iota::MessageListener(communityId, alias)), @@ -234,7 +236,7 @@ namespace gradido { } auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); auto& block = getBlock(blockNr); - auto nodeTransactionEntry = std::make_shared(confirmedTransaction, getptr()); + auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex)) { // block was already stopped, so we can stop here also LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 84fe1b4..46dec63 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -23,6 +23,7 @@ //! TODO: Test and Profile different values, or create dynamic algorithmus #define GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES 10 #define GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES 1 +#define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES 400 #define GRADIDO_NODE_MAGIC_NUMBER_TRANSACTION_TRIGGER_EVENTS_CACHE_MEGA_BTYES 1 #include @@ -62,7 +63,7 @@ namespace gradido { // Constructor is only usable by this class FileBased( Private, - std::string_view communityId, + const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, std::string_view folder, @@ -70,7 +71,7 @@ namespace gradido { ); // make sure that all shared_ptr from FileBased Blockchain know each other static inline std::shared_ptr create( - std::string_view communityId, + const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, std::string_view folder, @@ -149,8 +150,9 @@ namespace gradido { return mPublicKeysIndex.getOrAddIndexForData(publicKey); } inline const hiero::TopicId& getHieroTopicId() const { return mHieroTopicId; } - inline const std::string& getFolderPath() const { return mFolderPath; } inline const std::string& getAlias() const { return mAlias; } + inline const std::string& getFolderPath() const { return mFolderPath; } + inline const std::string& getCommunityId() const { return mCommunityId; } inline TaskObserver& getTaskObserver() const { return *mTaskObserver; } inline std::shared_ptr pickHieroClient() const { return mHieroClients[std::rand() % mHieroClients.size()]; } std::shared_ptr getOrderingManager() { return mOrderingManager; } @@ -172,6 +174,7 @@ namespace gradido { hiero::TopicId mHieroTopicId; std::string mAlias; std::string mFolderPath; + std::string mCommunityId; //! observe write to file tasks from block, mayber later more mutable std::shared_ptr mTaskObserver; @@ -200,7 +203,7 @@ namespace gradido { }; std::shared_ptr FileBased::create( - std::string_view communityId, + const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, std::string_view folder, diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index fe41e7f..99a5635 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -6,11 +6,22 @@ #include "../ServerGlobals.h" #include "../task/SyncTopicOnStartup.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/data/hiero/TopicId.h" +#include "gradido_blockchain/lib/DictionaryExceptions.h" #include "loguru/loguru.hpp" +#include #include +#include +#include +#include + +using std::shared_lock, std::unique_lock; +using std::shared_ptr, std::make_shared; +using std::string; +using std::vector; namespace gradido { namespace blockchain { @@ -22,14 +33,12 @@ namespace gradido { FileBasedProvider::~FileBasedProvider() { - std::lock_guard _lock(mWorkMutex); + unique_lock _lock(mWorkMutex); if (mGroupIndex) { delete mGroupIndex; mGroupIndex = nullptr; } - } - - + } FileBasedProvider* FileBasedProvider::getInstance() { @@ -37,36 +46,57 @@ namespace gradido { return &one; } - std::shared_ptr FileBasedProvider::findBlockchain(std::string_view communityId) + shared_ptr FileBasedProvider::findBlockchain(uint32_t communityIdIndex) { - std::lock_guard _lock(mWorkMutex); + shared_lock _lock(mWorkMutex); if (!mInitalized) { throw ClassNotInitalizedException("please call init before", "blockchain::FileBasedProvider"); } - auto it = mBlockchainsPerGroup.find(communityId); + auto it = mBlockchainsPerGroup.find(communityIdIndex); if (it != mBlockchainsPerGroup.end()) { return it->second; } - // go manual + + + return nullptr; + } + + shared_ptr FileBasedProvider::findBlockchain(const string& communityId) + { + auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(communityId); + if (!communityIdIndex.has_value()) { + LOG_F(WARNING, "no community id index for %s", communityId.c_str()); + } + else { + return findBlockchain(communityIdIndex.value()); + } + return nullptr; + } + + shared_ptr FileBasedProvider::findBlockchain(hiero::TopicId& topicId) + { try { - const auto& groupIndexEntry = mGroupIndex->getCommunityDetails(hiero::TopicId(std::string(communityId))); - auto it = mBlockchainsPerGroup.find(groupIndexEntry.communityId); - if (it != mBlockchainsPerGroup.end()) { - return it->second; + const auto& groupIndexEntry = mGroupIndex->getCommunityDetails(topicId); + auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(groupIndexEntry.communityId); + if (!communityIdIndex.has_value()) { + LOG_F(WARNING, "no community id index for %s", groupIndexEntry.communityId.c_str()); } - } + else { + return findBlockchain(communityIdIndex.value()); + } + } catch (GradidoBlockchainException& ex) { LOG_F(WARNING, "%s", ex.getFullString().data()); } - return nullptr; } + bool FileBasedProvider::init( - const std::string& communityConfigFile, - std::vector>&& hieroClients, + const string& communityConfigFile, + vector>&& hieroClients, uint8_t hieroClientsPerCommunity/* = 3 */ ) { - std::lock_guard _lock(mWorkMutex); + unique_lock _lock(mWorkMutex); mInitalized = true; bool resetAllCommunityIndices = false; mHieroClientsPerCommunity = hieroClientsPerCommunity; @@ -101,7 +131,7 @@ namespace gradido { } void FileBasedProvider::exit() { - std::lock_guard _lock(mWorkMutex); + unique_lock _lock(mWorkMutex); mInitalized = false; for (auto blockchain : mBlockchainsPerGroup) { blockchain.second->exit(); @@ -111,38 +141,38 @@ namespace gradido { int FileBasedProvider::reloadConfig() { - std::lock_guard _lock(mWorkMutex); + unique_lock _lock(mWorkMutex); if (!mInitalized) { throw ClassNotInitalizedException("please call init before", "blockchain::FileBasedProvider"); } mGroupIndex->update(); - auto communitiesIds = mGroupIndex->listCommunitiesIds(); int addedBlockchainsCount = 0; - for (auto& communityId : communitiesIds) { - const auto& details = mGroupIndex->getCommunityDetails(communityId); - auto it = mBlockchainsPerGroup.find(communityId); + mGroupIndex->iterate([&](const cache::CommunityIndexEntry& details) -> bool { + auto it = mBlockchainsPerGroup.find(details.communityIdIndex); if (it == mBlockchainsPerGroup.end()) { - if(addCommunity(communityId, hiero::TopicId(details.topicId), details.alias)) { + if (addCommunity(details.communityId, hiero::TopicId(details.topicId), details.alias)) { addedBlockchainsCount++; } } else { - updateListenerCommunity(communityId, details.alias, it->second); + updateListenerCommunity(details.communityIdIndex, details.alias, it->second); } - } + return true; + }); return addedBlockchainsCount; } - std::shared_ptr FileBasedProvider::addCommunity( - const std::string& communityId, + shared_ptr FileBasedProvider::addCommunity( + const string& communityId, const hiero::TopicId& topicId, - const std::string& alias + const string& alias ) { try { - auto folder = mGroupIndex->getFolder(communityId); + auto communityIdIndex = g_appContext->getOrAddCommunityIdIndex(communityId); + auto folder = mGroupIndex->getFolder(communityIdIndex); // with more hiero clients as per community needed, we make sure we not take always the first mHieroClientsPerCommunity from them - std::vector> hieroClients = mHieroClients; // copy + vector> hieroClients = mHieroClients; // copy if (hieroClients.size() > mHieroClientsPerCommunity) { std::shuffle(hieroClients.begin(), hieroClients.end(), std::mt19937{ std::random_device{}() }); hieroClients.resize(mHieroClientsPerCommunity); @@ -150,15 +180,14 @@ namespace gradido { // with that call community will be initialized and start listening auto blockchain = FileBased::create(communityId, topicId, alias, folder, std::move(hieroClients)); - updateListenerCommunity(communityId, alias, blockchain); + updateListenerCommunity(communityIdIndex, alias, blockchain); // need to have blockchain in map for init able to work - mBlockchainsPerGroup.insert({ communityId, blockchain }); + mBlockchainsPerGroup.insert({ communityIdIndex, blockchain }); if (!blockchain->init(false)) { LOG_F(ERROR, "error initalizing blockchain: %s", communityId.data()); - mBlockchainsPerGroup.erase(communityId); + mBlockchainsPerGroup.erase(communityIdIndex); return nullptr; } - mCommunityIdDicitionary.getIndexForData(communityId); return blockchain; } catch (GradidoBlockchainException& ex) { @@ -169,19 +198,19 @@ namespace gradido { return nullptr; } } - void FileBasedProvider::updateListenerCommunity(const std::string& communityId, const std::string& alias, std::shared_ptr blockchain) + void FileBasedProvider::updateListenerCommunity(uint32_t communityIdIndex, const string& alias, shared_ptr blockchain) { - const auto& communityConfig = mGroupIndex->getCommunityDetails(communityId); + const auto& communityConfig = mGroupIndex->getCommunityDetails(communityIdIndex); // for notification of community server by new transaction // deprecated, will be replaced with mqtt in future if (!communityConfig.newBlockUri.empty()) { - std::shared_ptr clientBase; - auto uri = std::string(communityConfig.newBlockUri); + shared_ptr clientBase; + auto uri = string(communityConfig.newBlockUri); if (communityConfig.blockUriType == "json") { - clientBase = std::make_shared(uri); + clientBase = make_shared(uri); } else if (communityConfig.blockUriType == "graphql") { - clientBase = std::make_shared(uri); + clientBase = make_shared(uri); } else { LOG_F(ERROR, "unknown new block uri type: %s", communityConfig.blockUriType.data()); diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index c3243ce..721f52c 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -6,6 +6,9 @@ #include "FileBased.h" #include "../cache/GroupIndex.h" +#include +#include + #define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_ID_INDEX_CACHE_SIZE_MBYTE 1 namespace hiero { @@ -30,7 +33,10 @@ namespace gradido { public: static FileBasedProvider* getInstance(); - std::shared_ptr findBlockchain(std::string_view communityId); + std::shared_ptr findBlockchain(uint32_t communityIdIndex) override; + std::shared_ptr findBlockchain(const std::string& communityId) override; + + std::shared_ptr findBlockchain(hiero::TopicId& topicId); //! \return true if successfully else return false bool init( const std::string& communityConfigFile, @@ -47,8 +53,8 @@ namespace gradido { inline std::vector listCommunityIds() const; protected: - std::map, StringViewCompare> mBlockchainsPerGroup; - std::recursive_mutex mWorkMutex; + std::unordered_map> mBlockchainsPerGroup; + std::shared_mutex mWorkMutex; private: FileBasedProvider(); @@ -64,7 +70,7 @@ namespace gradido { const hiero::TopicId& topicId, const std::string& alias ); - void updateListenerCommunity(const std::string& communityId, const std::string& alias, std::shared_ptr blockchain); + void updateListenerCommunity(uint32_t communityIdIndex, const std::string& alias, std::shared_ptr blockchain); cache::GroupIndex* mGroupIndex; std::vector> mHieroClients; diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index ea7e91d..16e7233 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -9,7 +9,7 @@ namespace gradido { gradido::data::ConstConfirmedTransactionPtr transaction, std::shared_ptr blockchain, int32_t fileCursor /*= -10*/ - ) : TransactionEntry(transaction), mFileCursor(fileCursor) + ) : TransactionEntry(transaction, blockchain->getCommunityIdIndex()), mFileCursor(fileCursor) { auto involvedPublicKeys = transaction->getInvolvedAddresses(); mPublicKeyIndices.reserve(involvedPublicKeys.size()); @@ -24,11 +24,12 @@ namespace gradido { date::month month, date::year year, gradido::data::TransactionType transactionType, - const std::string& coinGroupId, + std::optional coinCommunityIdIndex, const uint32_t* addressIndices, uint8_t addressIndiceCount, + uint32_t blockchainCommunityIdIndex, int32_t fileCursor /*= -10*/ - ) : TransactionEntry(transactionNr, month, year, transactionType, coinGroupId), mFileCursor(fileCursor) + ) : TransactionEntry(transactionNr, month, year, transactionType, coinCommunityIdIndex, blockchainCommunityIdIndex), mFileCursor(fileCursor) { mPublicKeyIndices.reserve(addressIndiceCount); for (int i = 0; i < addressIndiceCount; i++) { @@ -40,7 +41,7 @@ namespace gradido { memory::ConstBlockPtr serializedTransaction, std::shared_ptr blockchain, int32_t fileCursor/* = -10 */ - ) : TransactionEntry(serializedTransaction), mFileCursor(fileCursor) + ) : TransactionEntry(serializedTransaction, blockchain->getCommunityIdIndex()), mFileCursor(fileCursor) { auto involvedPublicKeys = getConfirmedTransaction()->getInvolvedAddresses(); mPublicKeyIndices.reserve(involvedPublicKeys.size()); @@ -54,10 +55,9 @@ namespace gradido { memory::ConstBlockPtr serializedTransaction, std::shared_ptr blockchain, int32_t fileCursor/* = -10 */ - ) : TransactionEntry(serializedTransaction, transaction), mFileCursor(fileCursor) + ) : TransactionEntry(serializedTransaction, transaction, blockchain->getCommunityIdIndex()), mFileCursor(fileCursor) { } - } } diff --git a/src/blockchain/NodeTransactionEntry.h b/src/blockchain/NodeTransactionEntry.h index 3b5e269..ea222b2 100644 --- a/src/blockchain/NodeTransactionEntry.h +++ b/src/blockchain/NodeTransactionEntry.h @@ -49,8 +49,9 @@ namespace gradido { date::month month, date::year year, gradido::data::TransactionType transactionType, - const std::string& coinGroupId, + std::optional coinCommunityIdIndex, const uint32_t* addressIndices, uint8_t addressIndiceCount, + uint32_t blockchainCommunityIdIndex, int32_t fileCursor = -10 ); diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index cf10bb0..4cd988d 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -31,7 +31,7 @@ namespace cache { Block::Block(uint32_t blockNr, std::shared_ptr blockchain) : mBlockNr(blockNr), mSerializedTransactions(ServerGlobals::g_CacheTimeout), - mBlockIndex(std::make_shared(blockchain->getProvider(), blockchain->getFolderPath(), blockNr)), + mBlockIndex(std::make_shared(blockchain->getFolderPath(), blockNr, blockchain->getCommunityIdIndex())), mBlockFile(std::make_shared(blockchain->getFolderPath(), blockNr)), mBlockchain(blockchain), mExitCalled(false) diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index 8117713..e1fa0e5 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -15,9 +15,8 @@ using gradido::blockchain::TransactionsIndex, gradido::blockchain::AbstractProvi namespace cache { - BlockIndex::BlockIndex(AbstractProvider* blockchainProvider, std::string_view groupFolderPath, uint32_t blockNr) - : TransactionsIndex(blockchainProvider), mFolderPath(groupFolderPath), mBlockNr(blockNr), - mDirty(false) + BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) + : mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) { } @@ -49,7 +48,7 @@ namespace cache { std::lock_guard _lock(mRecursiveMutex); clearIndexEntries(); mTransactionNrsFileCursors.clear(); - model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr); + model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().c_str()); blockIndexFile.reset(); mMaxTransactionNr = 0; @@ -61,7 +60,7 @@ namespace cache { std::lock_guard _lock(mRecursiveMutex); assert(!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size()); - model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr); + model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); return blockIndexFile.readFromFile(this); } @@ -73,7 +72,7 @@ namespace cache { } assert(mYearMonthAddressIndexEntries.size() && mTransactionNrsFileCursors.size()); - auto blockIndexFile = std::make_unique(mFolderPath, mBlockNr); + auto blockIndexFile = std::make_unique(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); std::vector publicKeyIndicesTemp; publicKeyIndicesTemp.reserve(10); @@ -123,7 +122,7 @@ namespace cache { bool BlockIndex::addIndicesForTransaction( gradido::data::TransactionType transactionType, - std::optional coinCommunityIdIndex, + uint32_t coinCommunityIdIndex, date::year year, date::month month, uint64_t transactionNr, diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index ca5630b..08e457f 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -38,7 +38,7 @@ namespace cache { { // friend model::files::BlockIndex; public: - BlockIndex(gradido::blockchain::AbstractProvider* blockchainProvider, std::string_view groupFolderPath, uint32_t blockNr); + BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex); ~BlockIndex(); bool init(); @@ -63,7 +63,7 @@ namespace cache { //! implement from model::files::IBlockIndexReceiver, called by loading block index from file bool addIndicesForTransaction( gradido::data::TransactionType transactionType, - std::optional coinCommunityIdIndex, + uint32_t coinCommunityIdIndex, date::year year, date::month month, uint64_t transactionNr, @@ -105,13 +105,15 @@ namespace cache { //! \brief called from model::files::BlockIndex while reading file std::string mFolderPath; - uint32_t mBlockNr; + uint32_t mBlockNr; + uint32_t mBlockchainCommunityIdIndex; std::map mTransactionNrsFileCursors; typedef std::pair TransactionNrsFileCursorsPair; mutable std::recursive_mutex mRecursiveMutex; bool mDirty; + }; rapidjson::Value BlockIndex::serializeToJson(rapidjson::Document::AllocatorType& alloc) const diff --git a/src/cache/GroupIndex.cpp b/src/cache/GroupIndex.cpp index ec6f778..2b28326 100644 --- a/src/cache/GroupIndex.cpp +++ b/src/cache/GroupIndex.cpp @@ -2,15 +2,26 @@ #include "../controller/ControllerExceptions.h" #include "../ServerGlobals.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/lib/RapidjsonHelper.h" #include "gradido_blockchain/data/hiero/TopicId.h" #include "loguru/loguru.hpp" #include +#include +#include +#include +#include + +using gradido::g_appContext; +using std::function; +using std::string; +using std::shared_lock, std::unique_lock; +using std::vector; namespace cache { - GroupIndex::GroupIndex(const std::string& jsonConfigFileName) + GroupIndex::GroupIndex(const string& jsonConfigFileName) : mConfig(jsonConfigFileName) { @@ -28,7 +39,7 @@ namespace cache { size_t GroupIndex::update() { - std::scoped_lock _lock(mWorkMutex); + unique_lock _lock(mWorkMutex); clear(); try { @@ -44,6 +55,7 @@ namespace cache { rapidjson_helper::checkMember(communityEntry, "hieroTopicId", rapidjson_helper::MemberType::STRING); entry.alias = communityEntry["alias"].GetString(); entry.communityId = communityEntry["communityId"].GetString(); + entry.communityIdIndex = g_appContext->getOrAddCommunityIdIndex(entry.communityId); entry.topicId = communityEntry["hieroTopicId"].GetString(); entry.folderName = communityEntry["folder"].GetString(); if (communityEntry.HasMember("newBlockUri")) { @@ -52,7 +64,7 @@ namespace cache { if (communityEntry.HasMember("blockUriType")) { entry.blockUriType = communityEntry["blockUriType"].GetString(); } - mCommunities.insert({ entry.communityId, entry }); + mCommunities.insert({ entry.communityIdIndex, entry }); } } else { @@ -67,13 +79,12 @@ namespace cache { return mCommunities.size(); } - std::string GroupIndex::getFolder(const std::string& communityId) + string GroupIndex::getFolder(uint32_t communityIdIndex) const { - std::scoped_lock _lock(mWorkMutex); - - auto it = mCommunities.find(communityId); + shared_lock _lock(mWorkMutex); + auto it = mCommunities.find(communityIdIndex); if(it != mCommunities.end()) { - std::string folder = ServerGlobals::g_FilesPath + '/'; + string folder = ServerGlobals::g_FilesPath + '/'; folder += it->second.folderName; if(!std::filesystem::exists(folder)) { std::filesystem::create_directories(folder); @@ -82,19 +93,21 @@ namespace cache { } return ""; } - const CommunityIndexEntry& GroupIndex::getCommunityDetails(const std::string& communityId) const - { - std::scoped_lock _lock(mWorkMutex); - auto it = mCommunities.find(communityId); - if (it != mCommunities.end()) { - return it->second; + const CommunityIndexEntry& GroupIndex::getCommunityDetails(const string& communityId) const + { + shared_lock _lock(mWorkMutex); + for (auto& it : mCommunities) { + if (it.second.communityId == communityId) { + return it.second; + } } throw controller::GroupNotFoundException("couldn't found config details for community", communityId); } + const CommunityIndexEntry& GroupIndex::getCommunityDetails(const hiero::TopicId& topicId) const { - std::scoped_lock _lock(mWorkMutex); + shared_lock _lock(mWorkMutex); for (auto& it : mCommunities) { if (topicId == hiero::TopicId(it.second.topicId)) { return it.second; @@ -102,18 +115,50 @@ namespace cache { } throw controller::GroupNotFoundException("couldn't found config details for community by topic id", topicId.toString()); } - bool GroupIndex::isCommunityInConfig(const std::string& communityId) const + + const CommunityIndexEntry& GroupIndex::getCommunityDetails(uint32_t communityIdIndex) const { - std::scoped_lock _lock(mWorkMutex); - return mCommunities.find(communityId) != mCommunities.end(); + shared_lock _lock(mWorkMutex); + auto it = mCommunities.find(communityIdIndex); + if (it != mCommunities.end()) { + return it->second; + } + throw controller::GroupNotFoundException("couldn't found config details for community", communityIdIndex); + } + + bool GroupIndex::isCommunityInConfig(uint32_t communityIdIndex) const + { + shared_lock _lock(mWorkMutex); + return mCommunities.find(communityIdIndex) != mCommunities.end(); + } + + void GroupIndex::iterate(function callback) const + { + shared_lock _lock(mWorkMutex); + for (const auto& it : mCommunities) { + auto result = callback(it.second); + if (!result) break; + } + } + + vector GroupIndex::listCommunitiesIds() const + { + shared_lock _lock(mWorkMutex); + vector result; + result.reserve(mCommunities.size()); + for (auto it = mCommunities.begin(); it != mCommunities.end(); it++) { + result.emplace_back(it->second.communityId); + } + return result; } - std::vector GroupIndex::listCommunitiesIds() + vector GroupIndex::listCommunitiesIdIndices() const { - std::scoped_lock _lock(mWorkMutex); - std::vector result; + shared_lock _lock(mWorkMutex); + vector result; + result.reserve(mCommunities.size()); for (auto it = mCommunities.begin(); it != mCommunities.end(); it++) { - result.push_back(it->first); + result.emplace_back(it->second.communityIdIndex); } return result; } diff --git a/src/cache/GroupIndex.h b/src/cache/GroupIndex.h index cc62458..202227a 100644 --- a/src/cache/GroupIndex.h +++ b/src/cache/GroupIndex.h @@ -4,8 +4,10 @@ #include "gradido_blockchain/lib/DRHash.h" #include "../model/files/JsonFile.h" +#include #include #include +#include #include namespace hiero { @@ -22,6 +24,7 @@ namespace cache { std::string folderName; std::string newBlockUri; std::string blockUriType; + uint32_t communityIdIndex; DHASH makeHash() { return DRMakeStringHash(alias.data(), alias.size()); @@ -55,21 +58,26 @@ namespace cache { //! \brief get full folder path for group public key //! if the hash from groupPublicKey exist multiple time, getting folder from mConfig, else from mHashList (faster) //! \return complete path to group folder or empty path if not group not found - std::string getFolder(const std::string& communityId); + std::string getFolder(uint32_t communityIdIndex) const; //! throw GroupNotFoundException Exception of community don't exist in config const CommunityIndexEntry& getCommunityDetails(const std::string& communityId) const; const CommunityIndexEntry& getCommunityDetails(const hiero::TopicId& topicId) const; - bool isCommunityInConfig(const std::string& communityId) const; + const CommunityIndexEntry& getCommunityDetails(uint32_t communityIdIndex) const; + bool isCommunityInConfig(uint32_t communityIdIndex) const; + + // callback for each community, stop if return false + void iterate(std::function callback) const; //! \brief collect all group aliases from unordered map (not the fastest operation from unordered map) //! \return vector with group aliases registered to the node server - std::vector listCommunitiesIds(); - + std::vector listCommunitiesIds() const; + std::vector listCommunitiesIdIndices() const; + protected: - mutable std::mutex mWorkMutex; + mutable std::shared_mutex mWorkMutex; model::files::JsonFile mConfig; - std::unordered_map mCommunities; + std::unordered_map mCommunities; //! \brief clear hash list and doublet's vector void clear(); diff --git a/src/controller/ControllerExceptions.cpp b/src/controller/ControllerExceptions.cpp index 9aebc27..7bc5e81 100644 --- a/src/controller/ControllerExceptions.cpp +++ b/src/controller/ControllerExceptions.cpp @@ -4,17 +4,26 @@ #include +using std::string, std::to_string; + using namespace gradido::data; namespace controller { - GroupNotFoundException::GroupNotFoundException(const char* what, const std::string& groupAlias) noexcept + GroupNotFoundException::GroupNotFoundException(const char* what, const string& groupAlias) noexcept : GradidoBlockchainException(what), mGroupAlias(groupAlias) { } - std::string GroupNotFoundException::getFullString() const + + GroupNotFoundException::GroupNotFoundException(const char* what, uint32_t communityIdIndex) noexcept + : GradidoBlockchainException(what), mGroupAlias(to_string(communityIdIndex)) + { + + } + + string GroupNotFoundException::getFullString() const { - std::string resultString; + string resultString; size_t resultSize = strlen(what()) + mGroupAlias.size() + 2 + 15; resultString.reserve(resultSize); resultString = what(); @@ -23,16 +32,16 @@ namespace controller { } // ******************* BlockNotLoadedException ***************************** - BlockNotLoadedException::BlockNotLoadedException(const char* what, const std::string& groupAlias, int blockNr) noexcept + BlockNotLoadedException::BlockNotLoadedException(const char* what, const string& groupAlias, int blockNr) noexcept : GradidoBlockchainException(what), mGroupAlias(groupAlias), mBlockNr(blockNr) { } - std::string BlockNotLoadedException::getFullString() const + string BlockNotLoadedException::getFullString() const { - std::string resultString; - std::string blockNrString = std::to_string(mBlockNr); + string resultString; + string blockNrString = to_string(mBlockNr); size_t resultSize = strlen(what()) + 2 + 14 + 13 + mGroupAlias.size() + blockNrString.size(); resultString = what(); resultString += ", with group: " + mGroupAlias; @@ -54,10 +63,10 @@ namespace controller { } - std::string WrongTransactionTypeException::getFullString() const + string WrongTransactionTypeException::getFullString() const { - std::string resultString; - std::string transactionTypeString(magic_enum::enum_name(mType)); + string resultString; + string transactionTypeString(magic_enum::enum_name(mType)); size_t resultSize = strlen(what()) + 2 + 20 + transactionTypeString.size() + mPubkeyHex.size() + 10; resultString.reserve(resultSize); resultString = what(); diff --git a/src/controller/ControllerExceptions.h b/src/controller/ControllerExceptions.h index ee6b145..772dcb7 100644 --- a/src/controller/ControllerExceptions.h +++ b/src/controller/ControllerExceptions.h @@ -10,6 +10,7 @@ namespace controller { { public: explicit GroupNotFoundException(const char* what, const std::string& groupAlias) noexcept; + explicit GroupNotFoundException(const char* what, uint32_t communityIdIndex) noexcept; std::string getFullString() const; protected: std::string mGroupAlias; diff --git a/src/controller/RemoteGroup.cpp b/src/controller/RemoteGroup.cpp index 0f37aed..76b7943 100644 --- a/src/controller/RemoteGroup.cpp +++ b/src/controller/RemoteGroup.cpp @@ -4,8 +4,8 @@ using namespace gradido::blockchain; using namespace gradido::data; namespace controller { - RemoteGroup::RemoteGroup(const std::string& groupAlias) - : Abstract(groupAlias) + RemoteGroup::RemoteGroup(uint32_t communityIdIndex) + : Abstract(communityIdIndex) { // get coin color on first connect to remote group } diff --git a/src/controller/RemoteGroup.h b/src/controller/RemoteGroup.h index e2e2585..b387fdd 100644 --- a/src/controller/RemoteGroup.h +++ b/src/controller/RemoteGroup.h @@ -16,7 +16,7 @@ namespace controller { class RemoteGroup : public gradido::blockchain::Abstract { public: - RemoteGroup(const std::string& groupAlias); + RemoteGroup(uint32_t communityIdIndex); //! validate and generate confirmed transaction //! throw if gradido transaction isn't valid diff --git a/src/controller/SimpleOrderingManager.cpp b/src/controller/SimpleOrderingManager.cpp index 17b9902..a699d8c 100644 --- a/src/controller/SimpleOrderingManager.cpp +++ b/src/controller/SimpleOrderingManager.cpp @@ -87,7 +87,7 @@ namespace controller { updateSequenceNumber(currentSequenceNumber); continue; } - auto otherBlockchain = blockchainProvider->findBlockchain(body->getOtherGroup()); + auto otherBlockchain = blockchainProvider->findBlockchain(body->getOtherCommunityIdIndex().value()); if (!otherBlockchain) { task->notificateFailedTransaction(blockchain, "Transaction skipped (target community unknown)"); mTransactions.erase(it); diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h index c805b3c..735dab1 100644 --- a/src/lib/PersistentDictionary.h +++ b/src/lib/PersistentDictionary.h @@ -26,17 +26,17 @@ class PersistentDictionary: public IMutableDictionary bool init(size_t cacheInBytes); void exit(); void reset() override; - uint32_t getLastIndex(); + size_t getLastIndex(); - virtual std::optional getIndexForData(const DataType& data) const override; - virtual std::optional getDataForIndex(uint32_t index) const override; - virtual uint32_t getOrAddIndexForData(const DataType& data) override; + virtual std::optional getIndexForData(const DataType& data) const override; + virtual std::optional getDataForIndex(size_t index) const override; + virtual size_t getOrAddIndexForData(const DataType& data) override; private: // LevelDB reads are logically const but mutate internal state mutable model::files::LevelDBWrapper mDictionaryFile; mutable std::shared_mutex mWorkingMutex; - std::unordered_map mIndexDataReverseLookup; + std::unordered_map mIndexDataReverseLookup; }; @@ -49,10 +49,10 @@ bool PersistentDictionary::init(size_t cacheInBytes) return false; } - // key is DataType, value is uint32 + // key is DataType, value is size_t mDictionaryFile.iterate([&](leveldb::Slice key, leveldb::Slice value) -> void { mIndexDataReverseLookup.insert({ - serialization::fromString(value.data(), value.size()), + serialization::fromString(value.data(), value.size()), serialization::fromString(key.data(), key.size()) }); }); @@ -77,7 +77,7 @@ void PersistentDictionary::reset() template requires serialization::HasString -uint32_t PersistentDictionary::getLastIndex() +size_t PersistentDictionary::getLastIndex() { std::unique_lock _lock(mWorkingMutex); return mIndexDataReverseLookup.size() - 1; @@ -85,20 +85,20 @@ uint32_t PersistentDictionary::getLastIndex() template requires serialization::HasString -std::optional PersistentDictionary::getIndexForData(const DataType& data) const +std::optional PersistentDictionary::getIndexForData(const DataType& data) const { std::shared_lock _lock(mWorkingMutex); auto result = mDictionaryFile.getValueForKey(serialization::toString(data)); if (result.has_value()) { const auto& value = result.value(); - return serialization::fromString(value.data(), value.size()); + return serialization::fromString(value.data(), value.size()); } return std::nullopt; } template requires serialization::HasString -std::optional PersistentDictionary::getDataForIndex(uint32_t index) const +std::optional PersistentDictionary::getDataForIndex(size_t index) const { std::shared_lock _lock(mWorkingMutex); auto it = mIndexDataReverseLookup.find(index); @@ -110,7 +110,7 @@ std::optional PersistentDictionary::getDataForIndex(uint32_t template requires serialization::HasString -uint32_t PersistentDictionary::getOrAddIndexForData(const DataType& data) +size_t PersistentDictionary::getOrAddIndexForData(const DataType& data) { auto dataString = serialization::toString(data); std::unique_lock _lock(mWorkingMutex); @@ -119,11 +119,11 @@ uint32_t PersistentDictionary::getOrAddIndexForData(const DataType& da const auto& value = result.value(); return serialization::fromString(value.data(), value.size()); } - if (mIndexDataReverseLookup.size() >= static_cast(std::numeric_limits::max())) { - throw DictionaryOverflowException("try to add more index data set's as uint32_t as index can handle", mDictionaryFile.getFolderName()); + if (mIndexDataReverseLookup.size() >= static_cast(std::numeric_limits::max())) { + throw DictionaryOverflowException("try to add more index data set's as size_t as index can handle", mDictionaryFile.getFolderName()); } - uint32_t index = static_cast(mIndexDataReverseLookup.size()); - mDictionaryFile.setKeyValue(dataString, serialization::toString(index)); + size_t index = mIndexDataReverseLookup.size(); + mDictionaryFile.setKeyValue(dataString, serialization::toString(index)); mIndexDataReverseLookup.insert({ index, data }); return index; } diff --git a/src/model/Apollo/Transaction.cpp b/src/model/Apollo/Transaction.cpp index a3c5df7..0d77526 100644 --- a/src/model/Apollo/Transaction.cpp +++ b/src/model/Apollo/Transaction.cpp @@ -35,7 +35,7 @@ namespace model { } mId = confirmedTransaction.getId(); mDate = confirmedTransaction.getConfirmedAt(); - mBalance = confirmedTransaction.getAccountBalance(pubkey, "").getBalance(); + mBalance = confirmedTransaction.getAccountBalance(pubkey, std::nullopt).getBalance(); } Transaction::Transaction(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) diff --git a/src/model/Apollo/TransactionList.cpp b/src/model/Apollo/TransactionList.cpp index 331b105..c6e5d0b 100644 --- a/src/model/Apollo/TransactionList.cpp +++ b/src/model/Apollo/TransactionList.cpp @@ -31,6 +31,10 @@ namespace model { Value TransactionList::generateList(Timepoint now, const Filter& filter, Document& root) { auto fileBasedBlockchain = std::dynamic_pointer_cast(mBlockchain); + uint32_t coinCommunityId = mBlockchain->getCommunityIdIndex(); + if (filter.coinCommunityIdIndex.has_value()) { + coinCommunityId = filter.coinCommunityIdIndex.value(); + } assert(fileBasedBlockchain); auto& alloc = root.GetAllocator(); @@ -104,11 +108,9 @@ namespace model { previousTransactionFilter.updatedBalancePublicKey = filter.updatedBalancePublicKey; previousTransactionFilter.timepointInterval = TimepointInterval(previousDate, beforePreviousTransactionDate); auto previousTransaction = mBlockchain->findOne(previousTransactionFilter); + if (previousTransaction) { - auto accountBalance = previousTransaction->getConfirmedTransaction()->getAccountBalance( - mPubkey, - filter.coinCommunityId - ); + auto accountBalance = previousTransaction->getConfirmedTransaction()->getAccountBalance(mPubkey, coinCommunityId); if (accountBalance.getBalance() > GradidoUnit::zero()) { previousBalance = accountBalance.getBalance(); previousDate = previousTransaction->getConfirmedTransaction()->getConfirmedAt(); @@ -124,7 +126,7 @@ namespace model { for (auto& transaction: transactions) { transactionsVector.push_back(transaction); // TODO: choose correct coin color - auto balance = confirmedTransaction->getAccountBalance(mPubkey, ""); + auto balance = confirmedTransaction->getAccountBalance(mPubkey, coinCommunityId); transactionsVector.back().setPreviousBalance( previousBalance ); diff --git a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp index f900f38..3a99efa 100644 --- a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp @@ -62,7 +62,7 @@ namespace model { return { decayedAccountBalance.getPublicKey(), decayedAccountBalance.getBalance() - transferAmount.getAmount(), - decayedAccountBalance.getCommunityId() + decayedAccountBalance.getCoinCommunityIdIndex() }; } @@ -83,7 +83,7 @@ namespace model { targetDate ); - return data::AccountBalance(transferAmount.getPublicKey(), decayed, transferAmount.getCommunityId()); + return data::AccountBalance(transferAmount.getPublicKey(), decayed, transferAmount.getCoinCommunityIdIndex()); } } } diff --git a/src/model/files/BlockIndex.cpp b/src/model/files/BlockIndex.cpp index e5bca2d..6b22cc1 100644 --- a/src/model/files/BlockIndex.cpp +++ b/src/model/files/BlockIndex.cpp @@ -30,11 +30,7 @@ namespace model { vFile->write(&transactionNr, sizeof(uint64_t)); vFile->write(&fileCursor, sizeof(int32_t)); vFile->write(&transactionType, sizeof(TransactionType)); - uint8_t hasValue = coinCommunityIdIndex.has_value(); - vFile->write(&hasValue, sizeof(uint8_t)); - if (hasValue) { - vFile->write(&coinCommunityIdIndex.value(), sizeof(uint32_t)); - } + vFile->write(&coinCommunityIdIndex, sizeof(uint32_t)); vFile->write(&isBalanceChanging, sizeof(uint8_t)); vFile->write(&addressIndicesCount, sizeof(uint8_t)); //vFile->write(this, sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint16_t)); @@ -56,17 +52,8 @@ namespace model { "gradido::data::TransactionType", std::to_string((uint8_t)transactionType).data() ); - } - uint8_t hasValue = 0; - if (!vFile->read(&hasValue, sizeof(uint8_t))) return false; - if (hasValue) { - uint32_t coinCommnityIdIndexRaw = 0; - if (!vFile->read(&coinCommnityIdIndexRaw, sizeof(uint32_t))) return false; - coinCommunityIdIndex = coinCommnityIdIndexRaw; - } - else { - coinCommunityIdIndex = nullopt; - } + } + if (!vFile->read(&coinCommunityIdIndex, sizeof(uint32_t))) return false; if (!vFile->read(&isBalanceChanging, sizeof(uint8_t))) return false; if (!vFile->read(&addressIndicesCount, sizeof(uint8_t))) return false; @@ -84,9 +71,7 @@ namespace model { //crypto_generichash_update(state, (const unsigned char*)this, sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint16_t)); crypto_generichash_update(state, (const unsigned char*)&transactionNr, sizeof(uint64_t)); crypto_generichash_update(state, (const unsigned char*)&fileCursor, sizeof(int32_t)); - if (coinCommunityIdIndex.has_value()) { - crypto_generichash_update(state, (const unsigned char*)&coinCommunityIdIndex.value(), sizeof(uint32_t)); - } + crypto_generichash_update(state, (const unsigned char*)&coinCommunityIdIndex, sizeof(uint32_t)); crypto_generichash_update(state, (const unsigned char*)&addressIndicesCount, sizeof(uint8_t)); crypto_generichash_update(state, (const unsigned char*)&isBalanceChanging, sizeof(uint8_t)); @@ -94,17 +79,10 @@ namespace model { crypto_generichash_update(state, (const unsigned char*)addressIndices, sizeof(uint32_t) * addressIndicesCount); } - std::shared_ptr BlockIndex::DataBlock::createTransactionEntry(date::month month, date::year year) + std::shared_ptr BlockIndex::DataBlock::createTransactionEntry(date::month month, date::year year, uint32_t blockchainCommunityIdIndex) { - std::string coinCommunityString = ""; - if (coinCommunityIdIndex.has_value()) { - auto res = FileBasedProvider::getInstance()->getCommunityIdString(coinCommunityIdIndex.value()); - if (res.has_value()) { - coinCommunityString = res.value(); - } - } auto transactionEntry = std::make_shared( - transactionNr, month, year, transactionType, coinCommunityString, addressIndices, addressIndicesCount + transactionNr, month, year, transactionType, coinCommunityIdIndex, addressIndices, addressIndicesCount, blockchainCommunityIdIndex ); transactionEntry->setFileCursor(fileCursor); return transactionEntry; @@ -112,8 +90,8 @@ namespace model { // ************************************************************************** - BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr) - : mDataBlockSumSize(0), mFileName(groupFolderPath) + BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) + : mDataBlockSumSize(0), mFileName(groupFolderPath), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex) { std::stringstream fileNameStream; fileNameStream << "/blk" << std::setw(8) << std::setfill('0') << blockNr << ".index"; diff --git a/src/model/files/BlockIndex.h b/src/model/files/BlockIndex.h index f9985c9..1960b41 100644 --- a/src/model/files/BlockIndex.h +++ b/src/model/files/BlockIndex.h @@ -9,7 +9,6 @@ #include #include -#include namespace cache { class BlockIndex; @@ -29,7 +28,7 @@ namespace model { public: virtual bool addIndicesForTransaction( gradido::data::TransactionType transactionType, - std::optional coinCommunityIdIndex, + uint32_t coinCommunityIdIndex, date::year year, date::month month, uint64_t transactionNr, @@ -55,7 +54,7 @@ namespace model { { public: //! create filename from path and blocknr - BlockIndex(std::string_view groupFolderPath, uint32_t blockNr); + BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex); //! use full filename which includes also the block nr BlockIndex(std::string_view filename); ~BlockIndex(); @@ -72,7 +71,7 @@ namespace model { uint64_t transactionNr, int32_t fileCursor, gradido::data::TransactionType transactionType, - std::optional coinCommunityIdIndex, + uint32_t coinCommunityIdIndex, uint8_t isBalanceChanging, const std::vector& addressIndices ) { @@ -177,7 +176,7 @@ namespace model { uint64_t _transactionNr, int32_t _fileCursor, gradido::data::TransactionType _transactionType, - std::optional _coinCommunityIdIndex, + uint32_t _coinCommunityIdIndex, uint8_t _isBalanceChanging, const std::vector& _addressIndices ) : @@ -200,7 +199,7 @@ namespace model { transactionNr(0), fileCursor(-10), transactionType(gradido::data::TransactionType::NONE), - coinCommunityIdIndex(std::nullopt), + coinCommunityIdIndex(0), isBalanceChanging(0), addressIndices(nullptr), addressIndicesCount(0) @@ -219,7 +218,7 @@ namespace model { uint64_t transactionNr; int32_t fileCursor; gradido::data::TransactionType transactionType; - std::optional coinCommunityIdIndex; + uint32_t coinCommunityIdIndex; uint8_t isBalanceChanging; uint8_t addressIndicesCount; uint32_t* addressIndices; @@ -239,10 +238,11 @@ namespace model { virtual bool readFromFile(VirtualFile* vFile); virtual void updateHash(crypto_generichash_state* state); - std::shared_ptr createTransactionEntry(date::month month, date::year year); + std::shared_ptr createTransactionEntry(date::month month, date::year year, uint32_t blockchainCommunityIdIndex); }; std::string mFileName; + uint32_t mBlockchainCommunityIdIndex; std::queue mDataBlocks; size_t mDataBlockSumSize; }; diff --git a/src/model/files/LevelDBWrapper.cpp b/src/model/files/LevelDBWrapper.cpp index 8a1599a..f6fb305 100644 --- a/src/model/files/LevelDBWrapper.cpp +++ b/src/model/files/LevelDBWrapper.cpp @@ -56,6 +56,7 @@ namespace model { } return status.ok(); } + void LevelDBWrapper::exit() { auto fm = FileLockManager::getInstance(); diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 239c320..7ce3ea1 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -3,6 +3,7 @@ // need to be here, else it produce a linker error, or more precisly the member function generateList // TODO: fix the reason #include "../../model/Apollo/TransactionList.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/interaction/calculateAccountBalance/Context.h" #include "gradido_blockchain/interaction/calculateCreationSum/Context.h" @@ -23,6 +24,7 @@ #include "magic_enum/magic_enum.hpp" #include "loguru/loguru.hpp" +#include #include using namespace rapidjson; @@ -33,6 +35,9 @@ using namespace serialization; using namespace data; using namespace magic_enum; +using std::optional, std::nullopt; +using gradido::g_appContext; + namespace server { namespace json_rpc { @@ -146,12 +151,11 @@ namespace server { } auto date = DataTypeConverter::dateTimeStringToTimePoint(date_string); - std::string coinCommunityId = ""; + optional coinCommunityId = nullopt; if (params.HasMember("coinCommunityId") && params["coinCommunityId"].IsString()) { - coinCommunityId = params["coinCommunityId"].GetString(); + coinCommunityId = g_appContext->getCommunityIds().getIndexForData(params["coinCommunityId"].GetString()); } getAddressBalance(resultJson, pubkey, date, blockchain, coinCommunityId); - } else if (method == "getAddressType") { getAddressType(resultJson, pubkey, blockchain); @@ -312,8 +316,8 @@ namespace server { auto communityRoot = communityRootBody->getCommunityRoot(); auto gmwAddress = communityRoot->getGmwPubkey(); auto aufAddress = communityRoot->getAufPubkey(); - auto gmwBalance = calculateAddressBalance.fromEnd(gmwAddress, now, ""); - auto aufBalance = calculateAddressBalance.fromEnd(aufAddress, now, ""); + auto gmwBalance = calculateAddressBalance.fromEnd(gmwAddress, now, blockchain->getCommunityIdIndex()); + auto aufBalance = calculateAddressBalance.fromEnd(aufAddress, now, blockchain->getCommunityIdIndex()); resultJson.AddMember("gmwBalance", Value(gmwBalance.toString().data(), alloc), alloc); resultJson.AddMember("aufBalance", Value(aufBalance.toString().data(), alloc), alloc); } else { @@ -396,14 +400,13 @@ namespace server { memory::ConstBlockPtr pubkey, Timepoint date, std::shared_ptr blockchain, - const std::string& coinCommunityId /* = "" */ + optional coinCommunityIdIndex /* = nullopt */ ) { assert(blockchain); auto& alloc = mRootJson.GetAllocator(); calculateAccountBalance::Context calculateAccountBalance(blockchain); - // TODO: add coinCommunity�d Filter to calculateAccountBalance Context - auto balanceString = calculateAccountBalance.fromEnd(pubkey, date, coinCommunityId, 0).toString(); + auto balanceString = calculateAccountBalance.fromEnd(pubkey, date, coinCommunityIdIndex).toString(); resultJson.AddMember("balance", Value(balanceString.data(), balanceString.size(), alloc), alloc); } @@ -467,8 +470,7 @@ namespace server { auto transactionListValue = transactionList.generateList(now, filter, mRootJson); calculateAccountBalance::Context calculateAddressBalance(blockchain); - // TODO: add balances from another communities - auto balance = calculateAddressBalance.fromEnd(filter.updatedBalancePublicKey, now, ""); + auto balance = calculateAddressBalance.fromEnd(filter.updatedBalancePublicKey, now, filter.coinCommunityIdIndex); std::string balanceString = balance.toString(); transactionListValue.AddMember("balance", Value(balanceString.data(), balanceString.size(), alloc), alloc); resultJson.AddMember("transactionList", transactionListValue, alloc); diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index d599daa..3513993 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -61,7 +61,7 @@ namespace server { memory::ConstBlockPtr pubkey, Timepoint date, std::shared_ptr blockchain, - const std::string& coinCommunityId = "" + std::optional coinCommunityIdIndex = std::nullopt ); void getAddressType( rapidjson::Value& resultJson, diff --git a/src/task/SyncTopicOnStartup.cpp b/src/task/SyncTopicOnStartup.cpp index 675bd69..ba45665 100644 --- a/src/task/SyncTopicOnStartup.cpp +++ b/src/task/SyncTopicOnStartup.cpp @@ -4,6 +4,7 @@ #include "../client/hiero/MirrorClient.h" #include "../ServerGlobals.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/lib/DataTypeConverter.h" @@ -11,16 +12,23 @@ #include "loguru/loguru.hpp" +#include +#include + using namespace gradido; using namespace blockchain; using namespace interaction; using namespace serialization; +using gradido::blockchain::FileBased; +using std::shared_ptr; +using std::string, std::to_string; + namespace task { SyncTopicOnStartup::SyncTopicOnStartup( uint64_t lastKnownSequenceNumber, hiero::TopicId lastKnowTopicId, - std::shared_ptr blockchain + shared_ptr blockchain ) : CPUTaskGRPCReactor(ServerGlobals::g_CPUScheduler), mLastKnownSequenceNumber(lastKnownSequenceNumber), mLastKnowTopicId(lastKnowTopicId), From 64d36da688cf2febfc7055baca40e383164dd771 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 24 Jan 2026 14:18:02 +0100 Subject: [PATCH 08/65] update for changes blockchain::Abstract --- src/blockchain/FileBased.cpp | 57 +++++++++++++++++++++++++++- src/blockchain/FileBased.h | 9 ++++- src/blockchain/FileBasedProvider.cpp | 6 +-- src/blockchain/FileBasedProvider.h | 2 +- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index b1c1d76..9c417e9 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -214,6 +214,7 @@ namespace gradido { mLedgerAnchorCache.exit(); mTransactionTriggerEventsCache.exit(); } + bool FileBased::createAndAddConfirmedTransaction( data::ConstGradidoTransactionPtr gradidoTransaction, const data::LedgerAnchor& ledgerAnchor, @@ -238,7 +239,61 @@ namespace gradido { auto& block = getBlock(blockNr); auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex)) { - // block was already stopped, so we can stop here also + // block was already stopped, so we can stop here also + LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); + return false; + } + role->runPastAddToBlockchain(confirmedTransaction, getptr()); + mBlockchainState.updateState(DefaultStateKeys::LAST_TRANSACTION_ID, confirmedTransaction->getId()); + mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); + mTransactionHashCache.push(*nodeTransactionEntry->getConfirmedTransaction()); + mLedgerAnchorCache.add(confirmedTransaction->getLedgerAnchor(), confirmedTransaction->getId()); + // add public keys to index + auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); + for (const auto& address : involvedAddresses) { + mPublicKeysIndex.getOrAddIndexForData(address); + } + if (mCommunityServer) { + task::TaskPtr notifyClientTask = std::make_shared(mCommunityServer, confirmedTransaction); + notifyClientTask->scheduleTask(notifyClientTask); + } + return true; + } + + bool FileBased::createAndAddConfirmedTransactionExtern( + data::ConstGradidoTransactionPtr gradidoTransaction, + const data::LedgerAnchor& ledgerAnchor, + std::vector accountBalances + ) + { + if (!gradidoTransaction) { + throw GradidoNullPointerException("missing transaction", "GradidoTransactionPtr", __FUNCTION__); + } + if (ledgerAnchor.empty()) { + throw GradidoNullPointerException("empty ledger anchor", "gradido::data::LedgerAnchor", __FUNCTION__); + } + std::lock_guard _lock(mWorkMutex); + if (mExitCalled) { return false; } + confirmTransaction::Context confirmTransactionContext(getptr()); + auto role = confirmTransactionContext.createRole( + gradidoTransaction, + ledgerAnchor, + gradidoTransaction->getTransactionBody()->getCreatedAt() + ); + if (!role) { + throw GradidoNotImplementedException("missing role for gradido transaction"); + } + role->setAccountBalances(accountBalances); + auto confirmedTransaction = confirmTransactionContext.run(role); + // will occure if transaction already exist + if (!confirmedTransaction) { + return false; + } + auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); + auto& block = getBlock(blockNr); + auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); + if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex)) { + // block was already stopped, so we can stop here also LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); return false; } diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 46dec63..85093cd 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -110,7 +110,14 @@ namespace gradido { data::ConstGradidoTransactionPtr gradidoTransaction, const data::LedgerAnchor& ledgerAnchor, data::Timestamp confirmedAt - ) override; + ) override; + + virtual bool createAndAddConfirmedTransactionExtern( + data::ConstGradidoTransactionPtr gradidoTransaction, + const data::LedgerAnchor& ledgerAnchor, + std::vector accountBalances + ) override; + void updateLastKnownSequenceNumber(uint64_t newSequenceNumber); virtual void addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent) override; virtual void removeTransactionTriggerEvent(const data::TransactionTriggerEvent& transactionTriggerEvent) override; diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 99a5635..375cb50 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -38,7 +38,7 @@ namespace gradido { delete mGroupIndex; mGroupIndex = nullptr; } - } + } FileBasedProvider* FileBasedProvider::getInstance() { @@ -56,7 +56,7 @@ namespace gradido { if (it != mBlockchainsPerGroup.end()) { return it->second; } - + return nullptr; } @@ -84,7 +84,7 @@ namespace gradido { else { return findBlockchain(communityIdIndex.value()); } - } + } catch (GradidoBlockchainException& ex) { LOG_F(WARNING, "%s", ex.getFullString().data()); } diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index 721f52c..6103270 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -35,7 +35,7 @@ namespace gradido { std::shared_ptr findBlockchain(uint32_t communityIdIndex) override; std::shared_ptr findBlockchain(const std::string& communityId) override; - + std::shared_ptr findBlockchain(hiero::TopicId& topicId); //! \return true if successfully else return false bool init( From 169d4a23b6c108408f997605e94a575e07beb51a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 18 Feb 2026 15:56:55 +0100 Subject: [PATCH 09/65] refactor with new knowledge and gradido-blockchain changes --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 5 +- src/SingletonManager/FileLockManager.cpp | 19 ++ src/SingletonManager/FileLockManager.h | 1 + src/blockchain/FileBased.cpp | 168 ++++++++++++------ src/blockchain/FileBased.h | 32 +++- src/blockchain/FileBasedProvider.cpp | 29 ++- src/blockchain/FileBasedProvider.h | 5 +- src/blockchain/NodeTransactionEntry.cpp | 9 +- src/cache/Block.cpp | 54 +++--- src/cache/Block.h | 8 +- src/cache/BlockIndex.cpp | 75 ++++---- src/cache/BlockIndex.h | 24 ++- src/cache/TransactionTriggerEvent.cpp | 12 +- src/cache/TransactionTriggerEvent.h | 8 +- src/lib/PersistentDictionary.h | 27 ++- src/model/Apollo/Transaction.cpp | 1 + src/model/Apollo/Transaction.h | 2 + src/model/files/Block.cpp | 118 +++++++----- src/model/files/Block.h | 30 +++- src/model/files/BlockIndex.h | 1 + src/model/files/LMDBWrapper.cpp | 0 src/model/files/LMDBWrapper.h | 0 src/serialization/String.cpp | 18 +- src/server/json-rpc/ApiHandler.cpp | 32 ++-- src/server/json-rpc/ApiHandler.h | 5 + ...tchDeserializeConfirmedTransactionTask.cpp | 6 +- ...BatchDeserializeConfirmedTransactionTask.h | 5 +- .../DeserializeConfirmedTransactionTask.cpp | 6 +- .../DeserializeConfirmedTransactionTask.h | 3 +- src/task/HieroMessageToTransactionTask.cpp | 2 +- src/task/RebuildBlockIndexTask.cpp | 98 ++++------ src/task/RebuildBlockIndexTask.h | 62 +++---- src/task/WriteTransactionsToBlockTask.cpp | 2 +- src/task/WriteTransactionsToBlockTask.h | 3 +- 35 files changed, 553 insertions(+), 319 deletions(-) create mode 100644 src/model/files/LMDBWrapper.cpp create mode 100644 src/model/files/LMDBWrapper.h diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 5d92950..65c988b 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 5d92950a7716afef7ec632e61d24667870d74c9d +Subproject commit 65c988b8f793e4a45a092c9f4e432fcddc4b9064 diff --git a/src/MainServer.cpp b/src/MainServer.cpp index e2c75d3..778d61e 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -77,7 +77,10 @@ bool MainServer::init() auto communityDictionary = make_unique>(ServerGlobals::g_FilesPath + "/communityIdsCache"); communityDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); - g_appContext = make_unique(std::move(communityDictionary)); + auto nameHashDictionary = make_unique>(ServerGlobals::g_FilesPath + "/nameHashCache"); + nameHashDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); + g_appContext = make_unique(std::move(communityDictionary), std::move(nameHashDictionary)); + g_appContext->syncCommunityContextsWithCommunityIds(); // timeouts ServerGlobals::loadTimeouts(config); diff --git a/src/SingletonManager/FileLockManager.cpp b/src/SingletonManager/FileLockManager.cpp index 19d5c16..bc212af 100644 --- a/src/SingletonManager/FileLockManager.cpp +++ b/src/SingletonManager/FileLockManager.cpp @@ -7,12 +7,15 @@ FileLockManager::FileLockManager() + : mInitialized(true) { } FileLockManager::~FileLockManager() { + std::scoped_lock lock(mWorkingMutex); + mInitialized = false; for (auto it = mFiles.begin(); it != mFiles.end(); it++) { // printf("%s \n", it->first.data()); delete it->second; @@ -28,6 +31,10 @@ FileLockManager* FileLockManager::getInstance() bool FileLockManager::isLock(const std::string& file) { + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager, will return false regardless of real state\n", __FUNCTION__); + return false; + } std::scoped_lock lock(mWorkingMutex); auto it = mFiles.find(file); if (it != mFiles.end()) { @@ -40,6 +47,10 @@ bool FileLockManager::isLock(const std::string& file) bool FileLockManager::tryLock(const std::string& file) { std::scoped_lock lock(mWorkingMutex); + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager, will return true regardless of real state\n", __FUNCTION__); + return true; + } auto it = mFiles.find(file); if (it != mFiles.end()) { if (!*it->second) { @@ -54,6 +65,10 @@ bool FileLockManager::tryLock(const std::string& file) bool FileLockManager::tryLockTimeout(const std::string& file, int tryCount) { + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager, will return true regardless of real state\n", __FUNCTION__); + return true; + } int timeoutRounds = tryCount; bool fileLocked = false; while (!fileLocked && timeoutRounds > 0) { @@ -73,6 +88,10 @@ bool FileLockManager::tryLockTimeout(const std::string& file, int tryCount) void FileLockManager::unlock(const std::string& file) { std::scoped_lock lock(mWorkingMutex); + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager\n", __FUNCTION__); + return; + } auto it = mFiles.find(file); assert(it != mFiles.end()); diff --git a/src/SingletonManager/FileLockManager.h b/src/SingletonManager/FileLockManager.h index b3bbb78..2004bf3 100644 --- a/src/SingletonManager/FileLockManager.h +++ b/src/SingletonManager/FileLockManager.h @@ -27,6 +27,7 @@ class FileLockManager protected: FileLockManager(); + bool mInitialized; std::mutex mWorkingMutex; std::unordered_map mFiles; diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 9c417e9..95d8c22 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -14,7 +14,12 @@ #include "../client/hiero/MirrorClient.h" #include "gradido_blockchain/const.h" +#include "gradido_blockchain/blockchain/batch/signaturesVerify.h" +#include "gradido_blockchain/blockchain/batch/ThreadingPolicy.h" +#include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" +#include "gradido_blockchain/data/adapter/PublicKey.h" +#include "gradido_blockchain/data/Timestamp.h" #include "gradido_blockchain/interaction/confirmTransaction/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" #include "gradido_blockchain/serialization/toJsonString.h" @@ -33,10 +38,13 @@ using controller::SimpleOrderingManager; using serialization::toJsonString; namespace gradido { - using data::LedgerAnchor, data::AddressType; + using data::adapter::toPublicKey; + using data::AddressType, data::Timestamp, data::LedgerAnchor; using namespace interaction; namespace blockchain { + using batch::ThreadingPolicy, batch::verifySignatures; + FileBased::FileBased( Private, const string& communityId, @@ -98,7 +106,13 @@ namespace gradido { mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_TRANSACTION_ID, 0); mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0); mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); + + return true; + } + bool FileBased::startValidationTransactions() + { + auto lastBlockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { mLedgerAnchorCache.reset(); if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { @@ -106,7 +120,7 @@ namespace gradido { } // load last 20 message ids into cache FilterBuilder filterBuilder; - auto transactions = findAll(filterBuilder.setPagination({20}).setSearchDirection(SearchDirection::DESC).build()); + auto transactions = findAll(filterBuilder.setPagination({ 20 }).setSearchDirection(SearchDirection::DESC).build()); for (auto& transaction : transactions) { mLedgerAnchorCache.add(transaction->getConfirmedTransaction()->getLedgerAnchor(), transaction->getTransactionNr()); } @@ -118,45 +132,13 @@ namespace gradido { LOG_F(INFO, "rescan blockchain for transaction trigger events, time: %s", timeUsed.string().data()); } - // load first GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE transaction into cache and validate the transaction to check file integrity - Profiler timeUsed; - FilterBuilder builder; - int count = 0; - findAll(builder - .setSearchDirection(SearchDirection::DESC) - .setPagination({ GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE }) - .setFilterFunction([this, &count](const TransactionEntry& transactionEntry) -> FilterResult { - auto previousTransactionNr = transactionEntry.getTransactionNr() - 1; - data::ConstConfirmedTransactionPtr previousConfirmedTransaction; - auto transactionBody = transactionEntry.getTransactionBody(); - validate::Context validator(*transactionEntry.getConfirmedTransaction()); - if (previousTransactionNr >= 1) { - auto entry = getTransactionForId(previousTransactionNr); - if (entry) { - previousConfirmedTransaction = entry->getConfirmedTransaction(); - } - } - validate::Type validationLevel = validate::Type::SINGLE | validate::Type::ACCOUNT; - if (transactionBody->getType() != data::CrossGroupType::LOCAL) { - validationLevel = validationLevel | validate::Type::PAIRED; - } - if (previousConfirmedTransaction) { - validationLevel = validationLevel | validate::Type::PREVIOUS; - validator.setSenderPreviousConfirmedTransaction(previousConfirmedTransaction); - } - validator.run(validationLevel, getptr()); - mTransactionHashCache.push(*transactionEntry.getConfirmedTransaction()); - count++; - return FilterResult::DISMISS; - }) - .build() - ); - LOG_F(INFO, "time used for loading and validating last: %d/%d transactions: %s", - count, - GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE, - timeUsed.string().data() - ); - return true; + // if state was empty, we better validate full blockchain + if (!lastBlockNr) { + return validateLastTransactions(0); + } + else { + return validateLastTransactions(GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE); + } } std::shared_ptr FileBased::initOnline() @@ -251,7 +233,7 @@ namespace gradido { // add public keys to index auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); for (const auto& address : involvedAddresses) { - mPublicKeysIndex.getOrAddIndexForData(address); + mPublicKeysIndex.getOrAddIndexForData(toPublicKey(address)); } if (mCommunityServer) { task::TaskPtr notifyClientTask = std::make_shared(mCommunityServer, confirmedTransaction); @@ -305,7 +287,7 @@ namespace gradido { // add public keys to index auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); for (const auto& address : involvedAddresses) { - mPublicKeysIndex.getOrAddIndexForData(address); + mPublicKeysIndex.getOrAddIndexForData(toPublicKey(address)); } if (mCommunityServer) { task::TaskPtr notifyClientTask = std::make_shared(mCommunityServer, confirmedTransaction); @@ -331,16 +313,16 @@ namespace gradido { mTransactionTriggerEventsCache.removeTransactionTriggerEvent(transactionTriggerEvent); } - std::vector> FileBased::findTransactionTriggerEventsInRange(TimepointInterval range) + std::vector> FileBased::findTransactionTriggerEventsInRange(Timestamp startDate, Timestamp endDate) { std::lock_guard _lock(mWorkMutex); - return mTransactionTriggerEventsCache.findTransactionTriggerEventsInRange(range); + return mTransactionTriggerEventsCache.findTransactionTriggerEventsInRange(startDate, endDate); } - std::shared_ptr FileBased::findNextTransactionTriggerEventInRange(TimepointInterval range) + std::shared_ptr FileBased::findNextTransactionTriggerEventInRange(Timestamp startDate, Timestamp endDate) { std::lock_guard _lock(mWorkMutex); - return mTransactionTriggerEventsCache.findNextTransactionTriggerEventInRange(range); + return mTransactionTriggerEventsCache.findNextTransactionTriggerEventInRange(startDate, endDate); } TransactionEntries FileBased::findAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS */) const @@ -377,6 +359,11 @@ namespace gradido { return result; } + data::compact::ConfirmedTxs FileBased::findAll(const CompactFilter& filter) const + { + throw GradidoNotImplementedException("FileBased::findAll with compact filter not yet implemented"); + } + size_t FileBased::countAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS*/) const { size_t count = 0; @@ -425,17 +412,25 @@ namespace gradido { data::AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const { + // return getAddressTypeSlow(filter); + data::AddressType result = data::AddressType::NONE; iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - result = block.getBlockIndex().getAddressType(filter.involvedPublicKey, mPublicKeysIndex); + auto addressTypeStateChange = block.getBlockIndex().getAddressType(filter.involvedPublicKey, mPublicKeysIndex); + result = addressTypeStateChange.getValue(); + if (addressTypeStateChange.getTxId()) { + auto tx = getTransactionForId(addressTypeStateChange.getTxId()); + if (FilterResult::USE != (filter.matches(tx, FilterCriteria::MAX) & FilterResult::USE)) { + result = data::AddressType::NONE; + } + return false; //break iterateBlocks + } + // result if (data::AddressType::NONE == result) { return true; } return false; }); - if (data::AddressType::NONE == result) { - result = getAddressTypeSlow(filter); - } return result; } @@ -452,6 +447,12 @@ namespace gradido { } while (blockNr > 0); return nullptr; } + + std::optional> FileBased::getConfirmedTxForId(uint64_t transactionId) const + { + throw GradidoNotImplementedException("FileBased::getConfirmedTxForId not implemented yet"); + } + std::shared_ptr FileBased::findByLedgerAnchor( const data::LedgerAnchor& ledgerAnchor, const Filter& filter/* = Filter::ALL_TRANSACTIONS*/ @@ -566,5 +567,70 @@ namespace gradido { } return *block.value(); } + + bool FileBased::validateLastTransactions(uint64_t countToValidate) + { + // load first GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE transaction into cache and validate the transaction to check file integrity + Profiler timeUsed; + Profiler timeSinceLastPrint; + data::ConstConfirmedTransactionPtr previousConfirmedTransaction = nullptr; + auto lastTransaction = findOne(Filter::LAST_TRANSACTION); + if (!lastTransaction) { + // seems we have nothing todo here + LOG_F(WARNING, "startValidationTransactions called on empty blockchain"); + return true; + } + int count = 0; + Filter f; + f.searchDirection = SearchDirection::ASC; + int countTarget = countToValidate; + if (countToValidate) { + f.minTransactionNr = lastTransaction->getTransactionNr() - countToValidate; + } + else { + countTarget = lastTransaction->getTransactionNr(); + } + + f.filterFunction = + [&](const TransactionEntry& transactionEntry) -> FilterResult + { + auto transactionBody = transactionEntry.getTransactionBody(); + validate::Context validator(*transactionEntry.getConfirmedTransaction()); + validate::Type validationLevel = validate::Type::SINGLE | validate::Type::ACCOUNT; + if (transactionBody->getType() != data::CrossGroupType::LOCAL) { + validationLevel = validationLevel | validate::Type::PAIRED; + } + if (previousConfirmedTransaction) { + validationLevel = validationLevel | validate::Type::PREVIOUS; + validator.setSenderPreviousConfirmedTransaction(previousConfirmedTransaction); + } + validator.disableVerify(); + validator.run(validationLevel, getptr()); + if (transactionEntry.getTransactionNr() > (lastTransaction->getTransactionNr() - GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE)) { + mTransactionHashCache.push(*transactionEntry.getConfirmedTransaction()); + } + previousConfirmedTransaction = transactionEntry.getConfirmedTransaction(); + count++; + /*if (timeSinceLastPrint.millis() > 150) { + printf("\r%.2f%%", ((double)count / (double)countTarget) * 100.0); + timeSinceLastPrint.reset(); + }*/ + return FilterResult::DISMISS; + }; + findAll(f); + // printf("\r"); + f.filterFunction = nullptr; + Profiler batchVerifyTime; + auto invalidSignatures = verifySignatures(f, mCommunityId, ThreadingPolicy::ThreeQuarter); + LOG_F(INFO, "time used for loading and validating last: %d transactions: %s (%s for batch verify)", + count, + timeUsed.string().c_str(), + batchVerifyTime.string().c_str() + ); + if (!invalidSignatures.empty()) { + throw GradidoNodeInvalidDataException("verify from at least one transaction failed"); + } + return true; + } } } \ No newline at end of file diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 85093cd..20c2cca 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -12,6 +12,7 @@ #include "../lib/PersistentDictionary.h" #include "gradido_blockchain/blockchain/Abstract.h" +#include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/data/hiero/TopicId.h" #include "gradido_blockchain/lib/AccessExpireCache.h" @@ -22,7 +23,7 @@ #define GRADIDO_NODE_MAGIC_NUMBER_BLOCKCHAIN_STATE_CACHE_SIZE_BYTES 192 //! TODO: Test and Profile different values, or create dynamic algorithmus #define GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES 10 -#define GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES 1 +#define GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES 10 #define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES 400 #define GRADIDO_NODE_MAGIC_NUMBER_TRANSACTION_TRIGGER_EVENTS_CACHE_MEGA_BTYES 1 @@ -91,11 +92,16 @@ namespace gradido { bool init(bool resetBlockIndices); //! init 2 + //! check last GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE if they are valid + //! should be called, after all communities where initalized, because it would need other communities for cross group transaction validation + bool startValidationTransactions(); + + //! init 3 //! prepare task for syncronize with hiero topic //! all SyncTopicOnStartup for all communities should be started/scheduled at the same time because there could need each other for new cross group transactions std::shared_ptr initOnline(); - //! init 3 + //! init 4 //! start listening to topic, will be called from SyncTopicOnStartup at the end, will update last known TopicId void startListening(data::Timestamp lastTransactionConfirmedAt); @@ -122,17 +128,19 @@ namespace gradido { virtual void addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent) override; virtual void removeTransactionTriggerEvent(const data::TransactionTriggerEvent& transactionTriggerEvent) override; - virtual bool isTransactionExist(data::ConstGradidoTransactionPtr gradidoTransaction) const override { + virtual bool isTransactionExist(data::ConstGradidoTransactionPtr gradidoTransaction, data::Timestamp confirmedAt) const override { return mTransactionHashCache.has(*gradidoTransaction); } //! return events in asc order of targetDate - virtual std::vector> findTransactionTriggerEventsInRange(TimepointInterval range) override; - virtual std::shared_ptr findNextTransactionTriggerEventInRange(TimepointInterval range) override; + virtual std::vector> findTransactionTriggerEventsInRange(data::Timestamp startDate, data::Timestamp endDate) override; + virtual std::shared_ptr findNextTransactionTriggerEventInRange(data::Timestamp startDate, data::Timestamp endDate) override; //! main search function, do all the work, reference from other functions virtual TransactionEntries findAll(const Filter& filter) const override; + virtual data::compact::ConfirmedTxs findAll(const CompactFilter& filter) const override; + // find all optimized for counting transaction nrs, better not use the filter.function for that, because this would slow down virtual size_t countAll(const Filter& filter = Filter::ALL_TRANSACTIONS) const override; @@ -143,6 +151,7 @@ namespace gradido { virtual data::AddressType getAddressType(const Filter& filter = Filter::LAST_TRANSACTION) const override; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; + virtual std::optional> getConfirmedTxForId(uint64_t transactionId) const; //! \param filter use to speed up search if infos exist to narrow down search transactions range virtual ConstTransactionEntryPtr findByLedgerAnchor( const data::LedgerAnchor& ledgerAnchor, @@ -153,9 +162,15 @@ namespace gradido { inline void setListeningCommunityServer(std::shared_ptr client); inline std::shared_ptr getListeningCommunityServer() const; - inline uint32_t getOrAddIndexForPublicKey(memory::ConstBlockPtr publicKey) const { + inline virtual const IDictionary& getPublicKeyDictionary() const override { return mPublicKeysIndex; } + inline virtual uint32_t getOrAddPublicKey(const PublicKey& publicKey) override { return mPublicKeysIndex.getOrAddIndexForData(publicKey); } + + inline uint32_t getOrAddIndexForPublicKey(const PublicKey& publicKey) const { + return mPublicKeysIndex.getOrAddIndexForData(publicKey); + } + inline const hiero::TopicId& getHieroTopicId() const { return mHieroTopicId; } inline const std::string& getAlias() const { return mAlias; } inline const std::string& getFolderPath() const { return mFolderPath; } @@ -176,6 +191,9 @@ namespace gradido { cache::Block& getBlock(uint32_t blockNr) const; + //! \param countToValidate lastTransaction.nr - countToValidate = minTransaction.nr, 0 for all + bool validateLastTransactions(uint64_t countToValidate); + mutable std::recursive_mutex mWorkMutex; bool mExitCalled; hiero::TopicId mHieroTopicId; @@ -191,7 +209,7 @@ namespace gradido { std::shared_ptr mHieroMessageListener; //! contain indices for every public key address, used overall for optimisation - mutable PersistentDictionary mPublicKeysIndex; + mutable PersistentDictionary mPublicKeysIndex; // level db to store state values like last transaction mutable cache::State mBlockchainState; diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 375cb50..ae12617 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -18,13 +18,14 @@ #include #include -using std::shared_lock, std::unique_lock; +using std::lock_guard; using std::shared_ptr, std::make_shared; using std::string; using std::vector; namespace gradido { namespace blockchain { + FileBasedProvider::FileBasedProvider() :mGroupIndex(nullptr), mInitalized(false) { @@ -33,7 +34,7 @@ namespace gradido { FileBasedProvider::~FileBasedProvider() { - unique_lock _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); if (mGroupIndex) { delete mGroupIndex; mGroupIndex = nullptr; @@ -48,7 +49,7 @@ namespace gradido { shared_ptr FileBasedProvider::findBlockchain(uint32_t communityIdIndex) { - shared_lock _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); if (!mInitalized) { throw ClassNotInitalizedException("please call init before", "blockchain::FileBasedProvider"); } @@ -96,7 +97,7 @@ namespace gradido { vector>&& hieroClients, uint8_t hieroClientsPerCommunity/* = 3 */ ) { - unique_lock _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); mInitalized = true; bool resetAllCommunityIndices = false; mHieroClientsPerCommunity = hieroClientsPerCommunity; @@ -119,7 +120,19 @@ namespace gradido { return false; } } - // step 2: check for new transactions in hiero network, all blockchains at the same time + // step 2: validate last transactions + for (const auto& pair : mBlockchainsPerGroup) { + if (!pair.second->startValidationTransactions()) { + LOG_F( + ERROR, + "error validate last transactions from community: %s in folder: %s", + pair.second->getCommunityId().c_str(), + pair.second->getFolderPath().c_str() + ); + } + } + + // step 3: check for new transactions in hiero network, all blockchains at the same time for (const auto& pair : mBlockchainsPerGroup) { auto task = pair.second->initOnline(); auto hieroClient = pair.second->pickHieroClient(); @@ -131,7 +144,7 @@ namespace gradido { } void FileBasedProvider::exit() { - unique_lock _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); mInitalized = false; for (auto blockchain : mBlockchainsPerGroup) { blockchain.second->exit(); @@ -141,7 +154,7 @@ namespace gradido { int FileBasedProvider::reloadConfig() { - unique_lock _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); if (!mInitalized) { throw ClassNotInitalizedException("please call init before", "blockchain::FileBasedProvider"); } @@ -180,7 +193,9 @@ namespace gradido { // with that call community will be initialized and start listening auto blockchain = FileBased::create(communityId, topicId, alias, folder, std::move(hieroClients)); + g_appContext->addBlockchain(communityIdIndex, blockchain); updateListenerCommunity(communityIdIndex, alias, blockchain); + // need to have blockchain in map for init able to work mBlockchainsPerGroup.insert({ communityIdIndex, blockchain }); if (!blockchain->init(false)) { diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index 6103270..81b08a9 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -7,7 +7,7 @@ #include "../cache/GroupIndex.h" #include -#include +#include #define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_ID_INDEX_CACHE_SIZE_MBYTE 1 @@ -53,8 +53,9 @@ namespace gradido { inline std::vector listCommunityIds() const; protected: + // check if neccessary, or community context is enough std::unordered_map> mBlockchainsPerGroup; - std::shared_mutex mWorkMutex; + std::recursive_mutex mWorkMutex; private: FileBasedProvider(); diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index 16e7233..5be0265 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -1,8 +1,11 @@ #include "NodeTransactionEntry.h" - #include "FileBased.h" +#include "gradido_blockchain/data/adapter/PublicKey.h" + namespace gradido { + using data::adapter::toPublicKey; + namespace blockchain { NodeTransactionEntry::NodeTransactionEntry( @@ -14,7 +17,7 @@ namespace gradido { auto involvedPublicKeys = transaction->getInvolvedAddresses(); mPublicKeyIndices.reserve(involvedPublicKeys.size()); for (auto& publicKey : involvedPublicKeys) { - mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(publicKey)); + mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(toPublicKey(publicKey))); } } @@ -46,7 +49,7 @@ namespace gradido { auto involvedPublicKeys = getConfirmedTransaction()->getInvolvedAddresses(); mPublicKeyIndices.reserve(involvedPublicKeys.size()); for (auto& publicKey : involvedPublicKeys) { - mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(publicKey)); + mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(toPublicKey(publicKey))); } } diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 4cd988d..784684e 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -13,6 +13,7 @@ #include "../SingletonManager/CacheManager.h" #include "gradido_blockchain/Application.h" +#include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" @@ -21,12 +22,17 @@ #include "loguru/loguru.hpp" #include +#include +#include #include using namespace gradido::blockchain; +using gradido::data::TransactionType; using namespace gradido::interaction; +using std::shared_ptr, std::make_shared, std::lock_guard; using task::RebuildBlockIndexTask; + namespace cache { Block::Block(uint32_t blockNr, std::shared_ptr blockchain) : mBlockNr(blockNr), @@ -51,38 +57,46 @@ namespace cache { mSerializedTransactions.clear(); } - bool Block::init(IMutableDictionary& publicKeyDictionary) + bool Block::init(IMutableDictionary& publicKeyDictionary) { - std::lock_guard lock(mFastMutex); - if (!mBlockIndex->loadFromFile()) { + lock_guard lock(mFastMutex); + // todo: add data for address index in file, until then rebuild block index on each program start + // if (!mBlockIndex->loadFromFile(publicKeyDictionary)) + { // check if Block exist - if (mBlockFile->getCurrentFileSize()) { + if (mBlockFile->getCurrentFileSize()) + { Profiler timeUsed; mBlockIndex->reset(); - std::shared_ptr rebuildBlockIndexTask = std::make_shared( - mBlockchain, + auto rebuildBlockIndexTask = make_shared( mBlockIndex, - publicKeyDictionary + mBlockchain->getCommunityIdIndex() ); - mBlockFile->fillRebuildBlockIndexTask(rebuildBlockIndexTask); - if (!rebuildBlockIndexTask) { - throw GradidoNullPointerException("missing rebuild block index task", "RebuildBlockIndexTask", __FUNCTION__); - } + rebuildBlockIndexTask->scheduleTask(rebuildBlockIndexTask); + mBlockFile->readBuffered(rebuildBlockIndexTask->getAlloc(), rebuildBlockIndexTask.get()); + int sumWaited = 0; - while (!rebuildBlockIndexTask->isPendingQueueEmpty() && sumWaited < GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS) { + while (!rebuildBlockIndexTask->isTaskFinished() && sumWaited < GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); sumWaited += 100; } - if (!rebuildBlockIndexTask->isPendingQueueEmpty()) { + if (!rebuildBlockIndexTask->isTaskFinished()) { LOG_F(FATAL, "rebuildBlockIndex Task isn't finished after waiting a whole minute"); Application::terminate(); } LOG_F(INFO, "time for rebuilding block index for block %s: %s", mBlockFile->getBlockPath().data(), timeUsed.string().data()); + mBlockIndex->writeIntoFile(); } else { return false; } } + /*else { + // hot fix: init address index + // if block index was loaded from file, we load all transactions which change something in address index + // TODO: persistent storage for address index + } + */ return true; } @@ -99,10 +113,10 @@ namespace cache { //bool Block::pushTransaction(const std::string& serializedTransaction, uint64_t transactionNr) bool Block::pushTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) { - std::lock_guard lock(mFastMutex); + lock_guard lock(mFastMutex); if (mExitCalled) return false; if (!mTransactionWriteTask) { @@ -118,7 +132,7 @@ namespace cache { void Block::addTransaction( memory::ConstBlockPtr serializedTransaction, int32_t fileCursor, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) const { auto transactionEntry = std::make_shared(serializedTransaction, mBlockchain, fileCursor); @@ -127,13 +141,13 @@ namespace cache { // mBlockIndex->updateAddressIndex(transactionEntry, publicKeyDictionary); } - std::shared_ptr Block::getTransaction( + shared_ptr Block::getTransaction( uint64_t transactionNr, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) const { assert(transactionNr); - std::lock_guard lock(mFastMutex); + lock_guard lock(mFastMutex); auto transactionEntry = mSerializedTransactions.get(transactionNr); if (!transactionEntry) { @@ -182,7 +196,7 @@ namespace cache { LOG_F(ERROR, "fileCursor: %d", fileCursor); auto blockLine = mBlockFile->readLine(fileCursor); deserialize::Context deserializer(blockLine, deserialize::Type::CONFIRMED_TRANSACTION); - deserializer.run(); + deserializer.run(mBlockchain->getCommunityIdIndex()); if (deserializer.isConfirmedTransaction()) { LOG_F(ERROR, "block: %s", serialization::toJsonString(*deserializer.getConfirmedTransaction(), true).data()); } diff --git a/src/cache/Block.h b/src/cache/Block.h index e44825a..bba077f 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -52,19 +52,19 @@ namespace cache { ~Block(); //! \return false if block not exist - bool init(IMutableDictionary& publicKeyDictionary); + bool init(IMutableDictionary& publicKeyDictionary); void exit(); //! \brief put new transaction to cache and file system bool pushTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ); //! \brief load transaction from cache or file system std::shared_ptr getTransaction( uint64_t transactionNr, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) const; inline BlockIndex& getBlockIndex() { return *mBlockIndex; } @@ -82,7 +82,7 @@ namespace cache { void addTransaction( memory::ConstBlockPtr serializedTransaction, int32_t fileCursor, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) const; mutable std::mutex mFastMutex; diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index e1fa0e5..d2fc097 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -4,21 +4,23 @@ #include "../task/HddWriteBufferTask.h" #include "../ServerGlobals.h" #include "../blockchain/FileBasedProvider.h" +#include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/RangeUtils.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/serialization/toJson.h" #include "loguru/loguru.hpp" using namespace rapidjson; -using gradido::blockchain::TransactionsIndex, gradido::blockchain::AbstractProvider; +using gradido::blockchain::AbstractProvider, gradido::blockchain::Filter, gradido::blockchain::SearchDirection, gradido::blockchain::TransactionsIndex; +using gradido::data::TransactionType; namespace cache { - BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) - : mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) + : TransactionsIndex(), mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) { - } BlockIndex::~BlockIndex() @@ -26,9 +28,9 @@ namespace cache { exit(); } - bool BlockIndex::init() + bool BlockIndex::init(const IDictionary& publicKeysDictionary) { - if (loadFromFile()) { + if (loadFromFile(publicKeysDictionary)) { return true; } return false; @@ -55,7 +57,7 @@ namespace cache { mMinTransactionNr = 0; } - bool BlockIndex::loadFromFile() + bool BlockIndex::loadFromFile(const IDictionary& publicKeysDictionary) { std::lock_guard _lock(mRecursiveMutex); assert(!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size()); @@ -73,37 +75,35 @@ namespace cache { assert(mYearMonthAddressIndexEntries.size() && mTransactionNrsFileCursors.size()); auto blockIndexFile = std::make_unique(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); - + blockIndexFile->addYearBlock(mMinYearMonth.year()); + std::vector publicKeyIndicesTemp; publicKeyIndicesTemp.reserve(10); - for (auto itYear = mYearMonthAddressIndexEntries.begin(); itYear != mYearMonthAddressIndexEntries.end(); itYear++) + for (auto monthYearIndex = 0; monthYearIndex < mYearMonthAddressIndexEntries.size(); monthYearIndex++) { - blockIndexFile->addYearBlock(itYear->first); - for (auto itMonth = itYear->second.begin(); itMonth != itYear->second.end(); itMonth++) + auto monthYear = indexToYearMonth(monthYearIndex); + if (monthYear.month() <= date::month(1) && mMinYearMonth != monthYear) { + blockIndexFile->addYearBlock(monthYear.year()); + } + blockIndexFile->addMonthBlock(monthYear.month()); + for (const auto& itEntry : mYearMonthAddressIndexEntries[monthYearIndex]) { - blockIndexFile->addMonthBlock(itMonth->first); - for (const auto& transactionsIndexEntryVectors : itMonth->second) - { - for (const auto& blockIndexEntry : transactionsIndexEntryVectors) - { - auto fileCursorIt = mTransactionNrsFileCursors.find(blockIndexEntry.transactionNr); - if (fileCursorIt == mTransactionNrsFileCursors.end()) { - throw GradidoNodeInvalidDataException("missing file cursor for transaction"); - } - publicKeyIndicesTemp.clear(); - for (auto i = 0; i < blockIndexEntry.addressIndiceCount; i++) { - publicKeyIndicesTemp.push_back(blockIndexEntry.addressIndices[i]); - } - blockIndexFile->addDataBlock( - blockIndexEntry.transactionNr, - fileCursorIt->second, - blockIndexEntry.transactionType, - blockIndexEntry.coinCommunityIdIndex, - blockIndexEntry.isBalanceChanging, - publicKeyIndicesTemp - ); - } + auto fileCursorIt = mTransactionNrsFileCursors.find(itEntry.transactionNr); + if (fileCursorIt == mTransactionNrsFileCursors.end()) { + throw GradidoNodeInvalidDataException("missing file cursor for transaction"); + } + publicKeyIndicesTemp.clear(); + for (auto i = 0; i < itEntry.addressIndiceCount; i++) { + publicKeyIndicesTemp.push_back(itEntry.addressIndices[i]); } + blockIndexFile->addDataBlock( + itEntry.transactionNr, + fileCursorIt->second, + itEntry.transactionType, + itEntry.coinCommunityIdIndex, + itEntry.isBalanceChanging, + publicKeyIndicesTemp + ); } } // finally write down to file @@ -152,7 +152,7 @@ namespace cache { bool BlockIndex::addIndicesForTransaction( std::shared_ptr transactionEntry, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) { std::lock_guard _lock(mRecursiveMutex); @@ -161,6 +161,13 @@ namespace cache { return true; } + bool BlockIndex::addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx) + { + std::lock_guard _lock(mRecursiveMutex); + TransactionsIndex::addIndicesForTransaction(compactTx); + return true; + } + bool BlockIndex::addFileCursorForTransaction(uint64_t transactionNr, int32_t fileCursor) { if (fileCursor < 0) return false; diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index 08e457f..976e663 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -1,8 +1,10 @@ #ifndef __GRADIDO_NODE_CONTROLLER_BLOCK_INDEX_H #define __GRADIDO_NODE_CONTROLLER_BLOCK_INDEX_H +#include "gradido_blockchain/blockchain/CompactFilter.h" #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/TransactionsIndex.h" +#include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/lib/DictionaryInterface.h" #include "../blockchain/NodeTransactionEntry.h" @@ -16,6 +18,9 @@ #include namespace gradido { + namespace data::compact { + class ConfirmedGradidoTx; + } namespace blockchain { class AbstractProvider; } @@ -41,12 +46,12 @@ namespace cache { BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex); ~BlockIndex(); - bool init(); + bool init(const IDictionary& publicKeysDictionary); void exit(); void reset(); //! \brief loading block index from file (or at least try to load) - bool loadFromFile(); + bool loadFromFile(const IDictionary& publicKeysDictionary); //! \brief write block index into files std::unique_ptr serialize(); @@ -57,8 +62,9 @@ namespace cache { bool addIndicesForTransaction( std::shared_ptr transactionEntry, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ); + bool addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx); //! implement from model::files::IBlockIndexReceiver, called by loading block index from file bool addIndicesForTransaction( @@ -79,10 +85,10 @@ namespace cache { //! \brief search transaction nrs for search criteria in filter, ignore filter function //! \return transaction nrs - inline std::vector findTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; + inline std::vector findTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; //! count all, ignore pagination - inline size_t countTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; + inline size_t countTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; //! \brief find transaction nrs from specific month and year //! \return {0, 0} if nothing found @@ -99,7 +105,7 @@ namespace cache { inline date::year_month getOldestYearMonth() const; inline date::year_month getNewestYearMonth() const; - inline TimepointInterval filteredTimepointInterval(const gradido::blockchain::Filter& filter) const; + inline TimepointInterval filteredTimepointInterval(const gradido::blockchain::CompactFilter& filter) const; protected: @@ -124,7 +130,7 @@ namespace cache { std::vector BlockIndex::findTransactions( const gradido::blockchain::Filter& filter, - const IDictionary& publicKeysDictionary + const IDictionary& publicKeysDictionary ) const { std::lock_guard _lock(mRecursiveMutex); @@ -133,7 +139,7 @@ namespace cache { size_t BlockIndex::countTransactions( const gradido::blockchain::Filter& filter, - const IDictionary& publicKeysDictionary + const IDictionary& publicKeysDictionary ) const { std::lock_guard _lock(mRecursiveMutex); @@ -175,7 +181,7 @@ namespace cache { return gradido::blockchain::TransactionsIndex::getNewestYearMonth(); } - TimepointInterval BlockIndex::filteredTimepointInterval(const gradido::blockchain::Filter& filter) const + TimepointInterval BlockIndex::filteredTimepointInterval(const gradido::blockchain::CompactFilter& filter) const { std::lock_guard _lock(mRecursiveMutex); return gradido::blockchain::TransactionsIndex::filteredTimepointInterval(filter); diff --git a/src/cache/TransactionTriggerEvent.cpp b/src/cache/TransactionTriggerEvent.cpp index 574486b..606f72a 100644 --- a/src/cache/TransactionTriggerEvent.cpp +++ b/src/cache/TransactionTriggerEvent.cpp @@ -78,10 +78,10 @@ namespace cache { LOG_F(WARNING, "couldn't find transactionTriggerEvent for removal for transaction: %lu", transactionTriggerEvent.getLinkedTransactionId()); } - std::vector> TransactionTriggerEvent::findTransactionTriggerEventsInRange(TimepointInterval range) + std::vector> TransactionTriggerEvent::findTransactionTriggerEventsInRange(Timestamp startDate, Timestamp endDate) { - auto startIt = mTransactionTriggerEvents.lower_bound(range.getStartDate()); - auto endIt = mTransactionTriggerEvents.upper_bound(range.getEndDate()); + auto startIt = mTransactionTriggerEvents.lower_bound(startDate); + auto endIt = mTransactionTriggerEvents.upper_bound(endDate); std::vector> result; result.reserve(std::distance(startIt, endIt)); for (auto& it = startIt; it != endIt; it++) { @@ -90,10 +90,10 @@ namespace cache { return result; } - std::shared_ptr TransactionTriggerEvent::findNextTransactionTriggerEventInRange(TimepointInterval range) + std::shared_ptr TransactionTriggerEvent::findNextTransactionTriggerEventInRange(Timestamp startDate, Timestamp endDate) { - auto startIt = mTransactionTriggerEvents.lower_bound(range.getStartDate()); - if (startIt != mTransactionTriggerEvents.end() && startIt->first.getAsTimepoint() <= range.getEndDate()) { + auto startIt = mTransactionTriggerEvents.lower_bound(startDate); + if (startIt != mTransactionTriggerEvents.end() && startIt->first.getAsTimepoint() <= endDate.getAsTimepoint()) { return startIt->second; } return nullptr; diff --git a/src/cache/TransactionTriggerEvent.h b/src/cache/TransactionTriggerEvent.h index 0417a31..ded3783 100644 --- a/src/cache/TransactionTriggerEvent.h +++ b/src/cache/TransactionTriggerEvent.h @@ -25,8 +25,12 @@ namespace cache { void addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent); void removeTransactionTriggerEvent(const gradido::data::TransactionTriggerEvent& transactionTriggerEvent); - std::vector> findTransactionTriggerEventsInRange(TimepointInterval range); - std::shared_ptr findNextTransactionTriggerEventInRange(TimepointInterval range); + std::vector> findTransactionTriggerEventsInRange( + gradido::data::Timestamp startDate, gradido::data::Timestamp endDate + ); + std::shared_ptr findNextTransactionTriggerEventInRange( + gradido::data::Timestamp startDate, gradido::data::Timestamp endDate + ); protected: // store all transaction trigger events, belonging to the same target date as key,value pair in level db diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h index 735dab1..3a4e140 100644 --- a/src/lib/PersistentDictionary.h +++ b/src/lib/PersistentDictionary.h @@ -13,7 +13,10 @@ #include #include - +// TODO: check usage of LMDB +// it is magnitude faster but especially it is designed for prevent data loss on system failure, data can only be corrupted through hardware failure! +// - https://de.wikipedia.org/wiki/Lightning_Memory-Mapped_Database +// - https://github.com/LMDB/lmdb/tree/mdb.master/libraries/liblmdb // TODO: remove bi-directionality, update whole code for not using getDataForIndex at all! template requires serialization::HasString @@ -30,7 +33,9 @@ class PersistentDictionary: public IMutableDictionary virtual std::optional getIndexForData(const DataType& data) const override; virtual std::optional getDataForIndex(size_t index) const override; + virtual DataType getDataForIndexOrThrow(size_t index) const override; virtual size_t getOrAddIndexForData(const DataType& data) override; + virtual bool hasIndex(size_t index) const override; private: // LevelDB reads are logically const but mutate internal state @@ -108,6 +113,17 @@ std::optional PersistentDictionary::getDataForIndex(size_t i return it->second; } +template + requires serialization::HasString +DataType PersistentDictionary::getDataForIndexOrThrow(size_t index) const +{ + auto data = getDataForIndex(index); + if (!data) { + throw DictionaryMissingEntryException(mDictionaryFile.getFolderName().data(), std::to_string(index)); + } + return data.value(); +} + template requires serialization::HasString size_t PersistentDictionary::getOrAddIndexForData(const DataType& data) @@ -128,4 +144,13 @@ size_t PersistentDictionary::getOrAddIndexForData(const DataType& data return index; } +template +requires serialization::HasString +bool PersistentDictionary::hasIndex(size_t index) const +{ + std::shared_lock _lock(mWorkingMutex); + auto it = mIndexDataReverseLookup.find(index); + return it != mIndexDataReverseLookup.end(); +} + #endif //__GRADIDO_NODE_PERSISTENT_DICTIONARY_H \ No newline at end of file diff --git a/src/model/Apollo/Transaction.cpp b/src/model/Apollo/Transaction.cpp index 0d77526..130c69f 100644 --- a/src/model/Apollo/Transaction.cpp +++ b/src/model/Apollo/Transaction.cpp @@ -1,6 +1,7 @@ #include "Transaction.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "gradido_blockchain/blockchain/Abstract.h" +#include "gradido_blockchain/data/ConfirmedTransaction.h" #include "magic_enum/magic_enum.hpp" #include "loguru/loguru.hpp" diff --git a/src/model/Apollo/Transaction.h b/src/model/Apollo/Transaction.h index df41a6f..013e055 100644 --- a/src/model/Apollo/Transaction.h +++ b/src/model/Apollo/Transaction.h @@ -3,6 +3,8 @@ #include "Decay.h" +#include "gradido_blockchain/memory/Block.h" + #include namespace gradido { diff --git a/src/model/files/Block.cpp b/src/model/files/Block.cpp index 0d2fe42..4c30d6c 100644 --- a/src/model/files/Block.cpp +++ b/src/model/files/Block.cpp @@ -141,6 +141,80 @@ namespace model { auto transactionSize = readLine(startReading, &result); return result; } + bool Block::readBuffered(grdu_memory* alloc, IBlockBufferRead* callback) + { + assert(alloc && callback); + + auto fileStream = getOpenFile(); + if (fileStream->fail()) { + throw std::runtime_error("[model::files::Block::readBuffered] file stream is failing!"); + } + auto minimalFileSize = sizeof(uint16_t) + MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE; + if (mCurrentFileSize <= minimalFileSize) { + throw EndReachingException("file is smaller than minimal block size", mBlockPath.data(), 0, minimalFileSize); + } + auto fl = FileLockManager::getInstance(); + if (!fl->tryLockTimeout(mBlockPath, 100)) { + throw LockException("cannot lock file for reading", mBlockPath.data()); + } + + uint16_t transactionSize = 0; + // call seek only if it is really necessary + // for example if a block file is read in complete line by line on program startup + // https://stackoverflow.com/questions/2438953/how-is-fseek-implemented-in-the-filesystem + // "One observation I have made about fseek on Solaris, is that each call to it resets the read buffer of the FILE.The next read will then always read a full block(8K by default)." + // https://bytes.com/topic/c/answers/218188-fseek-speed + // "However, a side - effect of the fseek is the flushing of the buffer.Without + // the fseek(), your output will(actually, I suppose "can" is correct in the + // general sense) be buffered, and only written when the buffer fille.With + // the fseek(), you are forcing the buffer to be written for every character." + // https://stackoverflow.com/questions/9349470/whats-the-difference-between-fseek-lseek-seekg-seekp + // "The difference between the various seek functions is just the kind of file/stream objects on which they operate. + // On Linux, seekg and fseek are probably implemented in terms of lseek." + + fileStream->seekg(0, std::ios_base::beg); + int32_t readed = 0; + unsigned char hash[crypto_generichash_KEYBYTES]; + memset(hash, 0, sizeof hash); + + while (fileStream->good() && readed < mCurrentFileSize) { + auto fileCursor = readed; + fileStream->read((char*)&transactionSize, sizeof(uint16_t)); + readed += sizeof(uint16_t); + if (readed + transactionSize > mCurrentFileSize) { + fl->unlock(mBlockPath); + throw EndReachingException("file is to small for transaction size", mBlockPath.data(), readed, transactionSize); + } + if (transactionSize < MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE) { + fl->unlock(mBlockPath); + throw InvalidReadBlockSize("transactionSize is to small to contain a transaction", mBlockPath.data(), readed, transactionSize); + } + auto memStart = alloc->last_index; + auto buffer = grdu_memory_alloc(alloc, transactionSize); + if (alloc->out_of_memory_capacity) { + // TODO: own exception + throw GradidoNodeInvalidDataException("memory buffer to small"); + } + fileStream->read(reinterpret_cast(buffer), transactionSize); + readed += transactionSize; + calculateOneHashStep(hash, buffer, transactionSize); + callback->finishedLine(memStart, transactionSize, fileCursor); + } + callback->flush(); + unsigned char hash2[crypto_generichash_KEYBYTES]; + fileStream->read((char*)hash2, crypto_generichash_KEYBYTES); + int filePointer = fileStream->tellg(); + if (0 != sodium_memcmp(hash, hash2, crypto_generichash_KEYBYTES)) { + throw HashMismatchException( + "block hash mismatch", + memory::Block(sizeof hash, hash), + memory::Block(sizeof hash2, hash2) + ); + } + + fl->unlock(mBlockPath); + return transactionSize; + } int32_t Block::appendLine(memory::ConstBlockPtr line) @@ -268,49 +342,7 @@ namespace model { return result; } - - void Block::fillRebuildBlockIndexTask(std::shared_ptr rebuildTask) - { - Profiler timeUsed; - auto fl = FileLockManager::getInstance(); - int32_t fileCursor = 0; - std::shared_ptr readBuffer; - unsigned char hash[crypto_generichash_KEYBYTES]; - memset(hash, 0, sizeof hash); - - // read in every line - while (fileCursor + sizeof(uint16_t) + MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE <= mCurrentFileSize) { - auto lineSize = readLine(fileCursor, &readBuffer); - rebuildTask->pushLine(fileCursor, readBuffer, rebuildTask); - calculateOneHashStep(hash, (const unsigned char*)readBuffer->data(), readBuffer->size()); - fileCursor += lineSize + sizeof(uint16_t); - } - rebuildTask->flush(rebuildTask); - LOG_F(INFO, "%s time for read block file (%u) into rebuild task", timeUsed.string().c_str(), mCurrentFileSize); - - unsigned char hash2[crypto_generichash_KEYBYTES]; - if (!fl->tryLockTimeout(mBlockPath, 100)) { - throw LockException("couldn't lock file in time", mBlockPath.data()); - } - auto fileStream = getOpenFile(); - fileStream->read((char*)hash2, crypto_generichash_KEYBYTES); - fl->unlock(mBlockPath); - - bool result = false; - if (0 == sodium_memcmp(hash, hash2, crypto_generichash_KEYBYTES)) { - result = true; - } - - if (!result) - { - throw HashMismatchException( - "block hash mismatch", - memory::Block(sizeof hash, hash), - memory::Block(sizeof hash2, hash2) - ); - } - } - + uint32_t Block::findLastBlockFileInFolder(std::string_view groupFolderPath) { std::filesystem::path groupFolder(groupFolderPath); diff --git a/src/model/files/Block.h b/src/model/files/Block.h index 814cbbc..c00a6c9 100644 --- a/src/model/files/Block.h +++ b/src/model/files/Block.h @@ -7,11 +7,21 @@ #include "../../task/CPUTask.h" +#include "gradido_protobuf_zig.h" + +#include + #include +#include //! MAGIC NUMBER: use to check if a file is big enough to could contain a transaction #define MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE 25 +namespace cache { + class BlockIndex; +} + + namespace controller { class AddressIndex; } @@ -23,6 +33,11 @@ namespace gradido { } } +namespace memory { + class Block; + using BlockPtr = std::shared_ptr; +} + namespace task { class RebuildBlockIndexTask; } @@ -31,6 +46,14 @@ namespace model { namespace files { class RebuildBlockIndexTask; + class IBlockBufferRead + { + public: + virtual void finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) = 0; + // will be called after last line was finished + virtual void flush() = 0; + }; + class Block : public TimerCallback { public: @@ -48,6 +71,8 @@ namespace model { //! \return size of line (without size field in file) uint16_t readLine(uint32_t startReading, memory::BlockPtr* buffer); std::shared_ptr readLine(uint32_t startReading); + // read whole file, validate hash + bool readBuffered(grdu_memory* alloc, IBlockBufferRead* callback); //! \brief call appendLines //! \return file cursor pos at start from this line in file (0 at start of file) @@ -62,11 +87,6 @@ namespace model { // very expensive, read in whole file and calculate hash bool validateHash(); - // read whole file, validate hash - // put lines of serialized transactions into task - // TODO: maybe make it more abstract so different tasks can use this - void fillRebuildBlockIndexTask(std::shared_ptr rebuildTask); - static uint32_t findLastBlockFileInFolder(std::string_view groupFolderPath); protected: diff --git a/src/model/files/BlockIndex.h b/src/model/files/BlockIndex.h index 1960b41..f29d763 100644 --- a/src/model/files/BlockIndex.h +++ b/src/model/files/BlockIndex.h @@ -6,6 +6,7 @@ #include "../../lib/VirtualFile.h" #include "date/date.h" +#include #include #include diff --git a/src/model/files/LMDBWrapper.cpp b/src/model/files/LMDBWrapper.cpp new file mode 100644 index 0000000..e69de29 diff --git a/src/model/files/LMDBWrapper.h b/src/model/files/LMDBWrapper.h new file mode 100644 index 0000000..e69de29 diff --git a/src/serialization/String.cpp b/src/serialization/String.cpp index 5e03136..92a3e26 100644 --- a/src/serialization/String.cpp +++ b/src/serialization/String.cpp @@ -1,4 +1,5 @@ #include "String.h" +#include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/memory/Block.h" #include @@ -9,7 +10,7 @@ using std::make_shared; namespace serialization { template<> - string toString(const ConstBlockPtr& ptr) + string toString(const ConstBlockPtr& ptr) { return ptr->copyAsString(); } @@ -19,4 +20,19 @@ namespace serialization { { return make_shared(size, reinterpret_cast(data)); } + + template<> + string toString>(const ByteArray<32>& bytes) + { + return { (char*)bytes.data(), 32 }; + } + + template<> + ByteArray<32> fromString>(const char* data, size_t size) + { + if (size != 32) { + throw InvalidSizeException("fromString for ByteArray<32> called with non-32 sized string", 32, size); + } + return ByteArray<32>(reinterpret_cast(data)); + } } \ No newline at end of file diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 7ce3ea1..f7b1ad4 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -5,16 +5,20 @@ #include "../../model/Apollo/TransactionList.h" #include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" +#include "gradido_blockchain/data/adapter/byteArray.h" +#include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/data/hiero/TransactionId.h" #include "gradido_blockchain/interaction/calculateAccountBalance/Context.h" #include "gradido_blockchain/interaction/calculateCreationSum/Context.h" #include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/interaction/serialize/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" -#include "gradido_blockchain/serialization/toJson.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "gradido_blockchain/lib/Profiler.h" -#include "gradido_blockchain/data/ConfirmedTransaction.h" -#include "gradido_blockchain/data/hiero/TransactionId.h" +#include "gradido_blockchain/memory/Block.h" +#include "gradido_blockchain/serialization/toJson.h" #include "../../blockchain/FileBased.h" #include "../../blockchain/FileBasedProvider.h" @@ -153,7 +157,10 @@ namespace server { auto date = DataTypeConverter::dateTimeStringToTimePoint(date_string); optional coinCommunityId = nullopt; if (params.HasMember("coinCommunityId") && params["coinCommunityId"].IsString()) { - coinCommunityId = g_appContext->getCommunityIds().getIndexForData(params["coinCommunityId"].GetString()); + auto coinCommunityIdIndexOptional = g_appContext->getCommunityIds().getIndexForData(params["coinCommunityId"].GetString()); + if (coinCommunityIdIndexOptional) { + coinCommunityId = static_cast(coinCommunityIdIndexOptional.value()); + } } getAddressBalance(resultJson, pubkey, date, blockchain, coinCommunityId); } @@ -314,8 +321,8 @@ namespace server { auto communityRootBody = communityRootEntry->getTransactionBody(); assert(communityRootBody->isCommunityRoot()); auto communityRoot = communityRootBody->getCommunityRoot(); - auto gmwAddress = communityRoot->getGmwPubkey(); - auto aufAddress = communityRoot->getAufPubkey(); + auto gmwAddress = adapter::toConstBlockPtr(communityRoot->gmwPublicKeyIndex); + auto aufAddress = adapter::toConstBlockPtr(communityRoot->aufPublicKeyIndex); auto gmwBalance = calculateAddressBalance.fromEnd(gmwAddress, now, blockchain->getCommunityIdIndex()); auto aufBalance = calculateAddressBalance.fromEnd(aufAddress, now, blockchain->getCommunityIdIndex()); resultJson.AddMember("gmwBalance", Value(gmwBalance.toString().data(), alloc), alloc); @@ -510,13 +517,18 @@ namespace server { { Profiler timeUsed; Filter f; + auto nameHashId = g_appContext->getUserNameHashs().getIndexForData(adapter::toByteArray<32>(nameHash)); + if (!nameHashId) { + error(responseJson, JSON_RPC_ERROR_ADDRESS_NOT_FOUND, "user not found"); + return; + } f.transactionType = data::TransactionType::REGISTER_ADDRESS; // std::function filterFunction; - f.filterFunction = [nameHash](const TransactionEntry& entry) { + f.filterFunction = [nameHashId](const TransactionEntry& entry) { auto body = entry.getTransactionBody(); assert(body->isRegisterAddress()); auto registerAddress = body->getRegisterAddress(); - if (registerAddress->getNameHash()->isTheSame(nameHash)) { + if (nameHashId.value() == static_cast(registerAddress->nameHashIndex)) { return FilterResult::USE | FilterResult::STOP; } return FilterResult::DISMISS; @@ -530,9 +542,7 @@ namespace server { assert(body); auto registerAddress = body->getRegisterAddress(); assert(registerAddress); - auto accountPubkey = registerAddress->getAccountPublicKey(); - assert(accountPubkey); - resultJson.AddMember("pubkey", Value(accountPubkey->convertToHex().data(), alloc), alloc); + resultJson.AddMember("pubkey", toJson(registerAddress->accountPublicKeyIndex.getRawKey().convertToHex(), alloc), alloc); } else { error(responseJson, JSON_RPC_ERROR_ADDRESS_NOT_FOUND, "user not found"); diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index 3513993..89b4420 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -11,6 +11,11 @@ namespace gradido { } } +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr ; +} + namespace server { namespace json_rpc { diff --git a/src/task/BatchDeserializeConfirmedTransactionTask.cpp b/src/task/BatchDeserializeConfirmedTransactionTask.cpp index e38ca4e..fdddda9 100644 --- a/src/task/BatchDeserializeConfirmedTransactionTask.cpp +++ b/src/task/BatchDeserializeConfirmedTransactionTask.cpp @@ -14,8 +14,8 @@ using Deserializer = gradido::interaction::deserialize::Context; using DeserializerType = gradido::interaction::deserialize::Type; namespace task { - BatchDeserializeConfirmedTransactionTask::BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions) - : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransactions(std::move(rawTransactions)) + BatchDeserializeConfirmedTransactionTask::BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions, uint32_t communityIdIndex) + : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransactions(std::move(rawTransactions)), mCommunityIdIndex(communityIdIndex) { } @@ -30,7 +30,7 @@ namespace task { for (size_t i = 0; i < mRawTransactions.size(); i++) { Deserializer deserializer(mRawTransactions[i], DeserializerType::CONFIRMED_TRANSACTION); try { - deserializer.run(); + deserializer.run(mCommunityIdIndex); } catch (std::exception& e) { int zahl = 0; diff --git a/src/task/BatchDeserializeConfirmedTransactionTask.h b/src/task/BatchDeserializeConfirmedTransactionTask.h index 3551618..fd9e778 100644 --- a/src/task/BatchDeserializeConfirmedTransactionTask.h +++ b/src/task/BatchDeserializeConfirmedTransactionTask.h @@ -20,11 +20,11 @@ namespace task { class BatchDeserializeConfirmedTransactionTask : public CPUTask { public: - BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions); + BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions, uint32_t communityIdIndex); virtual ~BatchDeserializeConfirmedTransactionTask(); const char* getResourceType() const override { return "BatchDeserializeConfirmedTransactionTask"; }; - int run() override; + int run() override; inline const std::vector>& getConfirmedTransactions() const { return mConfirmedTransactions; } inline const std::vector& getRawTransactions() const { return mRawTransactions; } @@ -32,6 +32,7 @@ namespace task { private: std::vector mRawTransactions; std::vector> mConfirmedTransactions; + uint32_t mCommunityIdIndex; }; } diff --git a/src/task/DeserializeConfirmedTransactionTask.cpp b/src/task/DeserializeConfirmedTransactionTask.cpp index b9c87fe..3ffab56 100644 --- a/src/task/DeserializeConfirmedTransactionTask.cpp +++ b/src/task/DeserializeConfirmedTransactionTask.cpp @@ -14,8 +14,8 @@ using Deserializer = gradido::interaction::deserialize::Context; using DeserializerType = gradido::interaction::deserialize::Type; namespace task { - DeserializeConfirmedTransactionTask::DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction) - : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransaction(rawTransaction) { + DeserializeConfirmedTransactionTask::DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction, uint32_t communityIdIndex) + : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransaction(rawTransaction), mCommunityIdIndex(communityIdIndex) { } DeserializeConfirmedTransactionTask::~DeserializeConfirmedTransactionTask() { @@ -24,7 +24,7 @@ namespace task { int DeserializeConfirmedTransactionTask::run() { Deserializer deserializer(mRawTransaction, DeserializerType::CONFIRMED_TRANSACTION); try { - deserializer.run(); + deserializer.run(mCommunityIdIndex); } catch (std::exception& e) { int zahl = 0; diff --git a/src/task/DeserializeConfirmedTransactionTask.h b/src/task/DeserializeConfirmedTransactionTask.h index fc7e8b9..ec14fdc 100644 --- a/src/task/DeserializeConfirmedTransactionTask.h +++ b/src/task/DeserializeConfirmedTransactionTask.h @@ -18,7 +18,7 @@ namespace memory { namespace task { class DeserializeConfirmedTransactionTask : public CPUTask { public: - DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction); + DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction, uint32_t communityIdIndex); virtual ~DeserializeConfirmedTransactionTask(); const char* getResourceType() const override { return "DeserializeConfirmedTransactionTask"; }; @@ -29,6 +29,7 @@ namespace task { private: memory::ConstBlockPtr mRawTransaction; std::shared_ptr mConfirmedTransaction; + uint32_t mCommunityIdIndex; }; } diff --git a/src/task/HieroMessageToTransactionTask.cpp b/src/task/HieroMessageToTransactionTask.cpp index e135ec5..62a9a74 100644 --- a/src/task/HieroMessageToTransactionTask.cpp +++ b/src/task/HieroMessageToTransactionTask.cpp @@ -85,7 +85,7 @@ namespace task { // also if we already have this transaction auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); assert(fileBasedBlockchain); - if (fileBasedBlockchain->isTransactionExist(mTransaction)) { + if (fileBasedBlockchain->isTransactionExist(mTransaction, mConsensusTimestamp)) { LOG_F(INFO, "Transaction skipped (cached): %s", mConsensusTimestamp.toString().data()); return 0; } diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp index 161db87..7e26ab1 100644 --- a/src/task/RebuildBlockIndexTask.cpp +++ b/src/task/RebuildBlockIndexTask.cpp @@ -6,90 +6,66 @@ #include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" +#include "gradido_protobuf_zig.h" +#include "loguru/loguru.hpp" +#include "magic_enum/magic_enum.hpp" + +#include + +using namespace magic_enum; using memory::Block; using gradido::blockchain::FileBased; -using std::shared_ptr, std::make_shared; +using gradido::data::compact::ConfirmedGradidoTx; +using std::shared_ptr, std::make_shared, std::unique_lock; using ServerGlobals::g_CPUScheduler; namespace task { RebuildBlockIndexTask::RebuildBlockIndexTask( - std::shared_ptr blockchain, std::shared_ptr blockIndex, - IMutableDictionary& publicKeyDictionary - ): task::CPUTask(g_CPUScheduler), mBlockchain(blockchain), mBlockIndex(blockIndex), mPublicKeyIndex(publicKeyDictionary) + uint32_t communityIdIndex + ): task::CPUTask(g_CPUScheduler), mBlockIndex(blockIndex), mCommunityIdIndex(communityIdIndex), + mLastLineReaded(false) { - mRawTransactionsBulk.reserve(REBUILD_BLOCK_INDEX_TASK_BULK_SIZE); - mActiveDeserializerTasks = 0; + grdu_memory_init_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); } int RebuildBlockIndexTask::run() { - LOG_F(WARNING, "not supposed to sheduled as regular task"); + unique_lock lock(mWorkConfirmedMutex); + mConfirmedTxReadyCondition.wait(lock, [&] { return mLastLineReaded.load(); }); return 0; } - void RebuildBlockIndexTask::pushLine(int32_t fileCursor, memory::ConstBlockPtr line, std::shared_ptr ownPtr) + void RebuildBlockIndexTask::finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) { - std::lock_guard _lock(mFinishLineMutex); - mFileCursorsQueue.push_back(fileCursor); - mRawTransactionsBulk.push_back(line); - if (mRawTransactionsBulk.size() >= REBUILD_BLOCK_INDEX_TASK_BULK_SIZE) { - flush(ownPtr, false); + grdu_memory decodeMemoryAlloc; + grdu_memory_init_static(&decodeMemoryAlloc, mBuffers[1], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + grdw_confirmed_transaction tx{}; + auto encodeResult = grdw_confirmed_transaction_decode(&decodeMemoryAlloc, &tx, mBuffers[0], size); + if (GRDW_ENCODING_ERROR_SUCCESS != encodeResult.state) { + LOG_F(ERROR, "decode error: %s", enum_name(encodeResult.state).data()); + throw GradidoNodeInvalidDataException("error deserialize confirmed transaction"); } - } - - void RebuildBlockIndexTask::flush(std::shared_ptr ownPtr, bool last/* = true, */) - { - std::lock_guard _lock(mFinishLineMutex); - if (!mRawTransactionsBulk.size()) { return; } - auto deserializeTask = make_shared(std::move(mRawTransactionsBulk)); - // deserializeTask->setFinishCommand(new FinishedDeserializeForRebuildBlockIndexCommand(ownPtr)); - mBulkDeserializerTasks.push(deserializeTask); - deserializeTask->scheduleTask(deserializeTask); - if (!last) { - mRawTransactionsBulk.reserve(REBUILD_BLOCK_INDEX_TASK_BULK_SIZE); + grdw_transaction_body body{}; + grdu_memory_init_static(&decodeMemoryAlloc, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + encodeResult = grdw_transaction_body_decode(&decodeMemoryAlloc, &body, tx.transaction.body_bytes, tx.transaction.body_bytes_size); + if (GRDW_ENCODING_ERROR_SUCCESS != encodeResult.state) { + LOG_F(ERROR, "body decode error: %s", enum_name(encodeResult.state).data()); + throw GradidoNodeInvalidDataException("error deserialize transaction body"); } - } - void RebuildBlockIndexTask::finishBulk() - { - std::lock_guard lock(mFinishLineMutex); - while (!mBulkDeserializerTasks.empty() && mBulkDeserializerTasks.front()->isTaskFinished()) - { - auto& task = mBulkDeserializerTasks.front(); - const auto& confirmedTransactions = task->getConfirmedTransactions(); - const auto& rawTransactions = task->getRawTransactions(); - for (int i = 0; i < confirmedTransactions.size(); i++) { - const auto& confirmedTransaction = confirmedTransactions[i]; - auto fileCursor = mFileCursorsQueue.front(); + auto compactConfirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, &body, mCommunityIdIndex, false); + mBlockIndex->addIndicesForTransaction(compactConfirmedTx); + mBlockIndex->addFileCursorForTransaction(compactConfirmedTx.txNr, fileCursor); - std::shared_ptr transactionEntry = std::make_shared( - confirmedTransaction, - rawTransactions[i], - mBlockchain, - fileCursor - ); - try { - mBlockIndex->addIndicesForTransaction(transactionEntry, mPublicKeyIndex); - } - catch (std::exception& e) { - LOG_F(FATAL, "%s, couldn't add indices for transaction: %s", - e.what(), - serialization::toJsonString(*transactionEntry->getConfirmedTransaction(), true).c_str() - ); - } - mFileCursorsQueue.pop_front(); - } - mBulkDeserializerTasks.pop(); - } - int zahl = 0; + grdu_memory_init_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); } - bool RebuildBlockIndexTask::isPendingQueueEmpty() + void RebuildBlockIndexTask::flush() { - finishBulk(); - return mBulkDeserializerTasks.empty(); - } + mLastLineReaded = true; + mConfirmedTxReadyCondition.notify_one(); + } } \ No newline at end of file diff --git a/src/task/RebuildBlockIndexTask.h b/src/task/RebuildBlockIndexTask.h index 7072d90..1c14fa5 100644 --- a/src/task/RebuildBlockIndexTask.h +++ b/src/task/RebuildBlockIndexTask.h @@ -2,12 +2,16 @@ #define __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H #include "CPUTask.h" +#include "../model/files/Block.h" +#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/lib/DictionaryInterface.h" -#include -#include +#include #include +#include #include +#include namespace gradido { namespace blockchain { @@ -26,59 +30,41 @@ namespace cache { } // TODO: maybe put into config -#define REBUILD_BLOCK_INDEX_TASK_BULK_SIZE 1000 +#define REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE 1024 namespace task { - class BatchDeserializeConfirmedTransactionTask; - - + //! remove dependencie to CPUTask because this isn't really a cpu task, more are result storage, //! because it will start subsequent tasks of it own which will call back via command if finished - class RebuildBlockIndexTask : public CPUTask + class RebuildBlockIndexTask : public CPUTask, public model::files::IBlockBufferRead { public: RebuildBlockIndexTask( - std::shared_ptr blockchain, std::shared_ptr blockIndex, - IMutableDictionary& publicKeyDictionary + uint32_t communityIdIndex ); const char* getResourceType() const { return "RebuildBlockIndexTask"; }; int run(); - //! \param line will be moved - void pushLine(int32_t fileCursor, memory::ConstBlockPtr line, std::shared_ptr ownPtr); - // flush batch buffer - void flush(std::shared_ptr ownPtr, bool last = true); - // called from DeserializeConfirmedTransactionTask, will process queue from begin as long current entry was already deserialized - void finishBulk(); - - bool isPendingQueueEmpty(); + + virtual void finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) override; + virtual void flush() override; + + inline grdu_memory* getAlloc() { return &mReadInAllocator; } protected: - std::shared_ptr mBlockchain; + + uint8_t mBuffers[2][REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE]; + + grdu_memory mReadInAllocator; + std::mutex mWorkConfirmedMutex; + gradido::data::compact::ConfirmedGradidoTx mConfirmedTx; + std::condition_variable mConfirmedTxReadyCondition; std::shared_ptr mBlockIndex; - IMutableDictionary& mPublicKeyIndex; - std::queue> mBulkDeserializerTasks; - std::deque mFileCursorsQueue; - std::vector mRawTransactionsBulk; - std::recursive_mutex mFinishLineMutex; - std::atomic mActiveDeserializerTasks; + uint32_t mCommunityIdIndex; + std::atomic mLastLineReaded; }; - - class FinishedDeserializeForRebuildBlockIndexCommand : public Command - { - public: - FinishedDeserializeForRebuildBlockIndexCommand(std::shared_ptr parent) : mParent(parent) {} - int taskFinished(Task* task) override - { - mParent->finishBulk(); - return 0; - } - - protected: - std::shared_ptr mParent; - }; } #endif // __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H \ No newline at end of file diff --git a/src/task/WriteTransactionsToBlockTask.cpp b/src/task/WriteTransactionsToBlockTask.cpp index aaaee13..0ecb658 100644 --- a/src/task/WriteTransactionsToBlockTask.cpp +++ b/src/task/WriteTransactionsToBlockTask.cpp @@ -64,7 +64,7 @@ namespace task { void WriteTransactionsToBlockTask::addSerializedTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) { assert(!isTaskSheduled()); diff --git a/src/task/WriteTransactionsToBlockTask.h b/src/task/WriteTransactionsToBlockTask.h index 0443db4..43a0b8b 100644 --- a/src/task/WriteTransactionsToBlockTask.h +++ b/src/task/WriteTransactionsToBlockTask.h @@ -8,6 +8,7 @@ */ #include "CPUTask.h" +#include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/lib/MultithreadQueue.h" #include "gradido_blockchain/lib/DictionaryInterface.h" @@ -57,7 +58,7 @@ namespace task { void addSerializedTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ); //! return transaction by nr From 6efe6fe00bee0a9e16b49af4182baba9604f54fa Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 24 Feb 2026 16:47:13 +0100 Subject: [PATCH 10/65] update for changed gradido blockchain lib --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 126 ++++++++++++++++++++++++--- src/blockchain/FileBased.h | 2 +- src/blockchain/FileBasedProvider.cpp | 8 +- src/blockchain/FileBasedProvider.h | 1 + src/cache/Block.cpp | 61 ++++++++++--- src/cache/Block.h | 20 ++++- src/cache/BlockIndex.cpp | 2 +- src/cache/BlockIndex.h | 21 ++++- src/cache/GroupIndex.cpp | 9 -- src/cache/GroupIndex.h | 9 +- src/lib/PersistentDictionary.h | 23 ++--- src/server/json-rpc/ApiHandler.cpp | 54 ++++++++---- src/server/json-rpc/ApiHandler.h | 3 +- src/task/RebuildBlockIndexTask.cpp | 3 +- 15 files changed, 268 insertions(+), 76 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 65c988b..244928e 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 65c988b8f793e4a45a092c9f4e432fcddc4b9064 +Subproject commit 244928e47d3cbb512bd54700ca20e46adfb7c4db diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 95d8c22..554af98 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -19,6 +19,8 @@ #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/data/adapter/PublicKey.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" #include "gradido_blockchain/data/Timestamp.h" #include "gradido_blockchain/interaction/confirmTransaction/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" @@ -40,6 +42,7 @@ using serialization::toJsonString; namespace gradido { using data::adapter::toPublicKey; using data::AddressType, data::Timestamp, data::LedgerAnchor; + using data::compact::ConstConfirmedTxPtr, data::compact::ConfirmedTxs, data::compact::PublicKeyIndex; using namespace interaction; namespace blockchain { @@ -220,7 +223,7 @@ namespace gradido { auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); auto& block = getBlock(blockNr); auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); - if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex)) { + if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex, *g_appContext)) { // block was already stopped, so we can stop here also LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); return false; @@ -274,7 +277,7 @@ namespace gradido { auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); auto& block = getBlock(blockNr); auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); - if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex)) { + if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex, *g_appContext)) { // block was already stopped, so we can stop here also LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); return false; @@ -337,8 +340,8 @@ namespace gradido { if (!filter.pagination.hasCapacityLeft(result.size())) { return false; } - auto transaction = block.getTransaction(transactionNr, mPublicKeysIndex); - auto filterResult = filter.matches(transaction, FilterCriteria::FILTER_FUNCTION); + auto transaction = block.getTransaction(transactionNr, *g_appContext); + auto filterResult = filter.matches(transaction, FilterCriteria::FILTER_FUNCTION | FilterCriteria::TIMEPOINT_INTERVAL); if ((filterResult & FilterResult::USE) == FilterResult::USE) { result.push_back(transaction); } @@ -359,9 +362,88 @@ namespace gradido { return result; } - data::compact::ConfirmedTxs FileBased::findAll(const CompactFilter& filter) const + ConfirmedTxs FileBased::findAll(const CompactFilter& filter) const { - throw GradidoNotImplementedException("FileBased::findAll with compact filter not yet implemented"); + ConfirmedTxs results; + // if pagination is used, filterCopy contain count of still to find transactions + CompactFilter filterCopy(filter); + auto skipEntries = filter.pagination.skipEntriesCount(); + int paginationCursor = 0; + iterateBlocks(filterCopy.searchDirection, + [&](const cache::Block& block) -> bool + { + const auto& transactionIndex = block.getBlockIndex(); + if (PublicKeySearchType::BalanceChangingPublicKey == filterCopy.publicKeySearchType && filterCopy.publicKeyIndex.communityIdIndex == mCommunityIdIndex) + { + filterCopy.pagination.page = 1; + do { + auto balanceChangingTxsInRange = transactionIndex.findTransactionsBalanceChangingForPublicKey(filterCopy); + if (balanceChangingTxsInRange.empty()) { + break; + } + for (const auto& tx : balanceChangingTxsInRange) { + auto transaction = getConfirmedTxForId(tx); + if (!transaction) { + throw GradidoBlockchainTransactionNotFoundException("confirmed tx not found").setTransactionId(tx); + } + auto filterResult = filterCopy.matches(*transaction, FilterCriteria::TIMEPOINT_INTERVAL); + if ((filterResult & FilterResult::USE) == FilterResult::USE) { + if (paginationCursor >= skipEntries) { + results.push_back(transaction); + if (!filterCopy.pagination.hasCapacityLeft(results.size())) { + return false; + } + } + paginationCursor++; + } + if ((filterResult & FilterResult::STOP) == FilterResult::STOP) { + return false; + } + } + if (filterCopy.pagination.empty() || filter.pagination.size > balanceChangingTxsInRange.size()) { + break; + } + filterCopy.pagination.page++; + } while (filter.pagination.hasCapacityLeft(results.size())); + return true; + } + + transactionIndex.lock(); + try { + auto startIt = transactionIndex.begin(filter); + auto endIt = transactionIndex.end(filter); + auto it = startIt; + for (; it != endIt; ++it) + { + auto transaction = block.getCompactTransaction(*it, *g_appContext); + if (!transaction) { + throw GradidoBlockchainTransactionNotFoundException("confirmed tx not found").setTransactionId(*it); + } + auto filterResult = filter.matches(*transaction, FilterCriteria::TIMEPOINT_INTERVAL); + if ((filterResult & FilterResult::USE) == FilterResult::USE) { + if (paginationCursor >= skipEntries) { + results.push_back(transaction); + if (!filter.pagination.hasCapacityLeft(results.size())) { + transactionIndex.unlock(); + return false; + } + } + paginationCursor++; + } + if ((filterResult & FilterResult::STOP) == FilterResult::STOP) { + transactionIndex.unlock(); + return false; + } + } + transactionIndex.unlock(); + return true; + } + catch (...) { + transactionIndex.unlock(); + throw; + } + }); + return results; } size_t FileBased::countAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS*/) const @@ -413,10 +495,21 @@ namespace gradido { data::AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const { // return getAddressTypeSlow(filter); - + if (!filter.involvedPublicKey || filter.involvedPublicKey->isEmpty()) { + throw GradidoNodeInvalidDataException("missing public key, please use filter with involvedPublicKey set"); + } + auto publicKeyIndexOptional = mPublicKeysIndex.getIndexForData(toPublicKey(filter.involvedPublicKey)); + if (!publicKeyIndexOptional) { + return AddressType::NONE; + } + uint32_t publicKeyUint32 = (uint32_t)publicKeyIndexOptional; + if (publicKeyUint32 != publicKeyIndexOptional) { + throw GradidoNodeInvalidDataException("public key index overflow"); + } + PublicKeyIndex publicKeyIndex = { .communityIdIndex = mCommunityIdIndex, .publicKeyIndex = publicKeyUint32 }; data::AddressType result = data::AddressType::NONE; iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto addressTypeStateChange = block.getBlockIndex().getAddressType(filter.involvedPublicKey, mPublicKeysIndex); + auto addressTypeStateChange = block.getBlockIndex().getAddressType(publicKeyIndex); result = addressTypeStateChange.getValue(); if (addressTypeStateChange.getTxId()) { auto tx = getTransactionForId(addressTypeStateChange.getTxId()); @@ -441,16 +534,25 @@ namespace gradido { do { auto& block = getBlock(blockNr); if (block.getBlockIndex().hasTransactionNr(transactionId)) { - return block.getTransaction(transactionId, mPublicKeysIndex); + return block.getTransaction(transactionId, *g_appContext); } blockNr--; } while (blockNr > 0); return nullptr; } - std::optional> FileBased::getConfirmedTxForId(uint64_t transactionId) const + ConstConfirmedTxPtr FileBased::getConfirmedTxForId(uint64_t transactionId) const { - throw GradidoNotImplementedException("FileBased::getConfirmedTxForId not implemented yet"); + std::lock_guard _lock(mWorkMutex); + auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); + do { + auto& block = getBlock(blockNr); + if (block.getBlockIndex().hasTransactionNr(transactionId)) { + return block.getCompactTransaction(transactionId, *g_appContext); + } + blockNr--; + } while (blockNr > 0); + return nullptr; } std::shared_ptr FileBased::findByLedgerAnchor( @@ -557,7 +659,7 @@ namespace gradido { if (!block) { auto block = std::make_shared(blockNr, getptr()); // return false if block not exist and will be created - if (!block->init(mPublicKeysIndex)) { + if (!block->init()) { if (blockNr > mBlockchainState.readInt32State(DefaultStateKeys::LAST_BLOCK_NR, 1)) { mBlockchainState.updateState(DefaultStateKeys::LAST_BLOCK_NR, blockNr); } diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 20c2cca..e4c490d 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -151,7 +151,7 @@ namespace gradido { virtual data::AddressType getAddressType(const Filter& filter = Filter::LAST_TRANSACTION) const override; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; - virtual std::optional> getConfirmedTxForId(uint64_t transactionId) const; + virtual data::compact::ConstConfirmedTxPtr getConfirmedTxForId(uint64_t transactionId) const; //! \param filter use to speed up search if infos exist to narrow down search transactions range virtual ConstTransactionEntryPtr findByLedgerAnchor( const data::LedgerAnchor& ledgerAnchor, diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index ae12617..881b2c7 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -65,11 +65,11 @@ namespace gradido { shared_ptr FileBasedProvider::findBlockchain(const string& communityId) { auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(communityId); - if (!communityIdIndex.has_value()) { + if (!communityIdIndex) { LOG_F(WARNING, "no community id index for %s", communityId.c_str()); } else { - return findBlockchain(communityIdIndex.value()); + return findBlockchain(communityIdIndex); } return nullptr; } @@ -79,11 +79,11 @@ namespace gradido { try { const auto& groupIndexEntry = mGroupIndex->getCommunityDetails(topicId); auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(groupIndexEntry.communityId); - if (!communityIdIndex.has_value()) { + if (!communityIdIndex) { LOG_F(WARNING, "no community id index for %s", groupIndexEntry.communityId.c_str()); } else { - return findBlockchain(communityIdIndex.value()); + return findBlockchain(communityIdIndex); } } catch (GradidoBlockchainException& ex) { diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index 81b08a9..68ccaa3 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -51,6 +51,7 @@ namespace gradido { //! list all known communities inline std::vector listCommunityIds() const; + inline const cache::GroupIndex* getGroupIndex() const { return mGroupIndex; } protected: // check if neccessary, or community context is enough diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 784684e..012238e 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -13,11 +13,14 @@ #include "../SingletonManager/CacheManager.h" #include "gradido_blockchain/Application.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_blockchain/lib/Profiler.h" +#include "gradido_protobuf_zig.h" #include "loguru/loguru.hpp" @@ -26,8 +29,9 @@ #include #include +using gradido::AppContext; using namespace gradido::blockchain; -using gradido::data::TransactionType; +using gradido::data::TransactionType, gradido::data::compact::ConfirmedGradidoTx; using namespace gradido::interaction; using std::shared_ptr, std::make_shared, std::lock_guard; using task::RebuildBlockIndexTask; @@ -37,6 +41,7 @@ namespace cache { Block::Block(uint32_t blockNr, std::shared_ptr blockchain) : mBlockNr(blockNr), mSerializedTransactions(ServerGlobals::g_CacheTimeout), + mConfirmedTxByNr(ServerGlobals::g_CacheTimeout), mBlockIndex(std::make_shared(blockchain->getFolderPath(), blockNr, blockchain->getCommunityIdIndex())), mBlockFile(std::make_shared(blockchain->getFolderPath(), blockNr)), mBlockchain(blockchain), @@ -57,7 +62,7 @@ namespace cache { mSerializedTransactions.clear(); } - bool Block::init(IMutableDictionary& publicKeyDictionary) + bool Block::init() { lock_guard lock(mFastMutex); // todo: add data for address index in file, until then rebuild block index on each program start @@ -113,7 +118,8 @@ namespace cache { //bool Block::pushTransaction(const std::string& serializedTransaction, uint64_t transactionNr) bool Block::pushTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary, + AppContext& appContext ) { lock_guard lock(mFastMutex); @@ -125,6 +131,7 @@ namespace cache { } mTransactionWriteTask->addSerializedTransaction(transaction, publicKeyDictionary); mSerializedTransactions.add(transaction->getTransactionNr(), transaction); + addCompactTransaction(transaction, appContext); return true; } @@ -132,19 +139,39 @@ namespace cache { void Block::addTransaction( memory::ConstBlockPtr serializedTransaction, int32_t fileCursor, - IMutableDictionary& publicKeyDictionary + AppContext& appContext ) const { - auto transactionEntry = std::make_shared(serializedTransaction, mBlockchain, fileCursor); + auto transactionEntry = make_shared(serializedTransaction, mBlockchain, fileCursor); if (mExitCalled) return; mSerializedTransactions.add(transactionEntry->getTransactionNr(), transactionEntry); + addCompactTransaction(transactionEntry, appContext); // mBlockIndex->updateAddressIndex(transactionEntry, publicKeyDictionary); } - shared_ptr Block::getTransaction( - uint64_t transactionNr, - IMutableDictionary& publicKeyDictionary - ) const + void Block::addCompactTransaction(shared_ptr transactionEntry, AppContext& appContext) const + { + // create compact version + try { + uint8_t buffer[1024]; + grdu_memory alloc; + grdu_memory_init_static(&alloc, buffer, 1024); + grdw_confirmed_transaction tx{}; + auto communityIdIndex = mBlockchain->getCommunityIdIndex(); + transactionEntry->getConfirmedTransaction()->toGrdw(&alloc, &tx, communityIdIndex); + auto confirmedTxPtr = make_shared(ConfirmedGradidoTx::fromGrdw(&tx, communityIdIndex, appContext)); + alloc.last_index = 0; + grdw_transaction_body txBody{}; + transactionEntry->getTransactionBody()->toGrdw(&alloc, &txBody); + confirmedTxPtr->fillFromGrdwTransactionBody(&txBody, appContext); + mConfirmedTxByNr.add(confirmedTxPtr->txNr, confirmedTxPtr); + } + catch (GradidoBlockchainException& ex) { + LOG_F(WARNING, "%s on create compact", ex.getFullString().c_str()); + } + } + + shared_ptr Block::getTransaction(uint64_t transactionNr, AppContext& appContext) const { assert(transactionNr); lock_guard lock(mFastMutex); @@ -184,7 +211,7 @@ namespace cache { } try { auto blockLine = mBlockFile->readLine(fileCursor); - addTransaction(blockLine, fileCursor, publicKeyDictionary); + addTransaction(blockLine, fileCursor, appContext); } catch (model::files::EndReachingException& ex) { LOG_F(ERROR, "%s", ex.getFullString().data()); @@ -214,6 +241,20 @@ namespace cache { return transactionEntry.value(); } + std::shared_ptr Block::getCompactTransaction( + uint64_t transactionNr, + gradido::AppContext& appContext + ) const + { + auto confirmedTx = mConfirmedTxByNr.get(transactionNr); + if (!confirmedTx) { + // check write cache, else try to read from storage + getTransaction(transactionNr, appContext); + } + // should only don't work, if getTransaction failed, but this will throw an exception anyway + return mConfirmedTxByNr.get(transactionNr).value(); + } + bool Block::hasSpaceLeft() { return mBlockFile->getCurrentFileSize() + 32 < GRADIDO_NODE_CACHE_BLOCK_MAX_FILE_SIZE_BYTE; } diff --git a/src/cache/Block.h b/src/cache/Block.h index bba077f..dcfacac 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -21,10 +21,14 @@ namespace model { } namespace gradido { + class AppContext; namespace blockchain { class FileBased; class NodeTransactionEntry; } + namespace data::compact { + struct ConfirmedGradidoTx; + } } namespace memory { @@ -52,19 +56,25 @@ namespace cache { ~Block(); //! \return false if block not exist - bool init(IMutableDictionary& publicKeyDictionary); + bool init(); void exit(); //! \brief put new transaction to cache and file system bool pushTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary, + gradido::AppContext& appContext ); //! \brief load transaction from cache or file system std::shared_ptr getTransaction( uint64_t transactionNr, - IMutableDictionary& publicKeyDictionary + gradido::AppContext& appContext + ) const; + + std::shared_ptr getCompactTransaction( + uint64_t transactionNr, + gradido::AppContext& appContext ) const; inline BlockIndex& getBlockIndex() { return *mBlockIndex; } @@ -82,13 +92,15 @@ namespace cache { void addTransaction( memory::ConstBlockPtr serializedTransaction, int32_t fileCursor, - IMutableDictionary& publicKeyDictionary + gradido::AppContext& appContext ) const; + void addCompactTransaction(std::shared_ptr transactionEntry, gradido::AppContext& appContext) const; mutable std::mutex mFastMutex; uint32_t mBlockNr; mutable AccessExpireCache> mSerializedTransactions; + mutable AccessExpireCache> mConfirmedTxByNr; std::shared_ptr mBlockIndex; std::shared_ptr mBlockFile; diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index d2fc097..5e1803f 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -19,7 +19,7 @@ using gradido::data::TransactionType; namespace cache { BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) - : TransactionsIndex(), mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) + : TransactionsIndex(blockchainCommunityIdIndex), mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) { } diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index 976e663..9c9bfd9 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -23,6 +23,7 @@ namespace gradido { } namespace blockchain { class AbstractProvider; + class CompactFilter; } } @@ -87,6 +88,9 @@ namespace cache { //! \return transaction nrs inline std::vector findTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; + inline std::vector findTransactions(const gradido::blockchain::CompactFilter& filter) const; + inline std::vector findTransactionsBalanceChangingForPublicKey(const gradido::blockchain::CompactFilter& filter) const; + //! count all, ignore pagination inline size_t countTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; @@ -106,6 +110,8 @@ namespace cache { inline date::year_month getOldestYearMonth() const; inline date::year_month getNewestYearMonth() const; inline TimepointInterval filteredTimepointInterval(const gradido::blockchain::CompactFilter& filter) const; + inline void lock() const { mRecursiveMutex.lock(); } + inline void unlock() const { mRecursiveMutex.unlock(); } protected: @@ -134,7 +140,18 @@ namespace cache { ) const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::findTransactions(filter, publicKeysDictionary); + return gradido::blockchain::TransactionsIndex::findTransactions(filter, publicKeysDictionary, mBlockchainCommunityIdIndex); + } + + std::vector BlockIndex::findTransactions(const gradido::blockchain::CompactFilter& filter) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::findTransactions(filter); + } + std::vector BlockIndex::findTransactionsBalanceChangingForPublicKey(const gradido::blockchain::CompactFilter& filter) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::findTransactionsBalanceChangingForPublicKey(filter); } size_t BlockIndex::countTransactions( @@ -190,7 +207,7 @@ namespace cache { std::pair BlockIndex::findTransactionsForMonthYear(date::year year, date::month month) const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::findTransactionsForMonthYear(year, month); + return gradido::blockchain::TransactionsIndex::findTransactionsForMonthYear({ year, month }); } } diff --git a/src/cache/GroupIndex.cpp b/src/cache/GroupIndex.cpp index 2b28326..1bfbe70 100644 --- a/src/cache/GroupIndex.cpp +++ b/src/cache/GroupIndex.cpp @@ -132,15 +132,6 @@ namespace cache { return mCommunities.find(communityIdIndex) != mCommunities.end(); } - void GroupIndex::iterate(function callback) const - { - shared_lock _lock(mWorkMutex); - for (const auto& it : mCommunities) { - auto result = callback(it.second); - if (!result) break; - } - } - vector GroupIndex::listCommunitiesIds() const { shared_lock _lock(mWorkMutex); diff --git a/src/cache/GroupIndex.h b/src/cache/GroupIndex.h index 202227a..290c78c 100644 --- a/src/cache/GroupIndex.h +++ b/src/cache/GroupIndex.h @@ -66,7 +66,14 @@ namespace cache { bool isCommunityInConfig(uint32_t communityIdIndex) const; // callback for each community, stop if return false - void iterate(std::function callback) const; + template + void iterate(callbackFunc callback) const { + std::shared_lock _lock(mWorkMutex); + for (const auto& it : mCommunities) { + auto result = callback(it.second); + if (!result) break; + } + } //! \brief collect all group aliases from unordered map (not the fastest operation from unordered map) //! \return vector with group aliases registered to the node server diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h index 3a4e140..eed6e0a 100644 --- a/src/lib/PersistentDictionary.h +++ b/src/lib/PersistentDictionary.h @@ -31,7 +31,7 @@ class PersistentDictionary: public IMutableDictionary void reset() override; size_t getLastIndex(); - virtual std::optional getIndexForData(const DataType& data) const override; + virtual size_t getIndexForData(const DataType& data) const override; virtual std::optional getDataForIndex(size_t index) const override; virtual DataType getDataForIndexOrThrow(size_t index) const override; virtual size_t getOrAddIndexForData(const DataType& data) override; @@ -85,12 +85,12 @@ requires serialization::HasString size_t PersistentDictionary::getLastIndex() { std::unique_lock _lock(mWorkingMutex); - return mIndexDataReverseLookup.size() - 1; + return mIndexDataReverseLookup.size(); } template requires serialization::HasString -std::optional PersistentDictionary::getIndexForData(const DataType& data) const +size_t PersistentDictionary::getIndexForData(const DataType& data) const { std::shared_lock _lock(mWorkingMutex); auto result = mDictionaryFile.getValueForKey(serialization::toString(data)); @@ -98,19 +98,20 @@ std::optional PersistentDictionary::getIndexForData(const Data const auto& value = result.value(); return serialization::fromString(value.data(), value.size()); } - return std::nullopt; + return 0; } template requires serialization::HasString std::optional PersistentDictionary::getDataForIndex(size_t index) const { - std::shared_lock _lock(mWorkingMutex); - auto it = mIndexDataReverseLookup.find(index); - if (it == mIndexDataReverseLookup.end()) { - return std::nullopt; - } - return it->second; + std::shared_lock _lock(mWorkingMutex); + + auto it = mIndexDataReverseLookup.find(index); + if (it == mIndexDataReverseLookup.end()) { + return std::nullopt; + } + return it->second; } template @@ -138,7 +139,7 @@ size_t PersistentDictionary::getOrAddIndexForData(const DataType& data if (mIndexDataReverseLookup.size() >= static_cast(std::numeric_limits::max())) { throw DictionaryOverflowException("try to add more index data set's as size_t as index can handle", mDictionaryFile.getFolderName()); } - size_t index = mIndexDataReverseLookup.size(); + size_t index = mIndexDataReverseLookup.size() + 1; mDictionaryFile.setKeyValue(dataString, serialization::toString(index)); mIndexDataReverseLookup.insert({ index, data }); return index; diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index f7b1ad4..d89a937 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -4,6 +4,7 @@ // TODO: fix the reason #include "../../model/Apollo/TransactionList.h" #include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/data/adapter/byteArray.h" #include "gradido_blockchain/data/adapter/publicKey.h" @@ -41,6 +42,7 @@ using namespace magic_enum; using std::optional, std::nullopt; using gradido::g_appContext; +using gradido::data::compact::PublicKeyIndex; namespace server { namespace json_rpc { @@ -94,6 +96,8 @@ namespace server { // load public key for nearly all requests memory::BlockPtr pubkey; + PublicKeyIndex publickKeyIndex; + std::string pubkeyHex; std::set noNeedForPubkey = { "getLastTransaction", "getTransactions","getTransaction", "findUserByNameHash" @@ -103,7 +107,8 @@ namespace server { return; } pubkey = std::make_shared(memory::Block::fromHex(pubkeyHex)); - } + publickKeyIndex = adapter::toPublicKeyIndex(pubkey, blockchain->getCommunityIdIndex()); + } if (method == "getLastTransaction") { Profiler timeUsed; @@ -159,7 +164,7 @@ namespace server { if (params.HasMember("coinCommunityId") && params["coinCommunityId"].IsString()) { auto coinCommunityIdIndexOptional = g_appContext->getCommunityIds().getIndexForData(params["coinCommunityId"].GetString()); if (coinCommunityIdIndexOptional) { - coinCommunityId = static_cast(coinCommunityIdIndexOptional.value()); + coinCommunityId = static_cast(coinCommunityIdIndexOptional); } } getAddressBalance(resultJson, pubkey, date, blockchain, coinCommunityId); @@ -210,7 +215,7 @@ namespace server { } auto date = DataTypeConverter::dateTimeStringToTimePoint(date_string); - getCreationSumForMonth(resultJson, pubkey, targetDate, date, blockchain); + getCreationSumForMonth(resultJson, publickKeyIndex, targetDate, date, blockchain); } // TODO: think about better name, explain that this is extra formatted for the gradido frontend, to mimic current graphql backend response else if (method == "listTransactions") { @@ -267,9 +272,20 @@ namespace server { void ApiHandler::listCommunities(rapidjson::Value& resultJson) { Profiler timeUsed; - auto alloc = mRootJson.GetAllocator(); - auto groups = FileBasedProvider::getInstance()->listCommunityIds(); - resultJson.AddMember("communities", toJson(groups, alloc), alloc); + auto& alloc = mRootJson.GetAllocator(); + const auto& groupIndex = FileBasedProvider::getInstance()->getGroupIndex(); + Value communities(kArrayType); + groupIndex->iterate( + [&communities, &alloc](const cache::CommunityIndexEntry& comInfos) -> bool + { + Value community(kObjectType); + community.AddMember("communityId", toJson(comInfos.communityId, alloc), alloc); + community.AddMember("alias", toJson(comInfos.alias, alloc), alloc); + communities.PushBack(community, alloc); + return true; + } + ); + resultJson.AddMember("communities", communities, alloc); resultJson.AddMember("timeUsed", Value(timeUsed.string().data(), alloc).Move(), alloc); } @@ -316,15 +332,15 @@ namespace server { // read gmw and auf balance Timepoint now = std::chrono::system_clock::now(); calculateAccountBalance::Context calculateAddressBalance(blockchain); - auto communityRootEntry = blockchain->findOne(Filter::FIRST_TRANSACTION); + CompactFilter communityRootFindFilter; + communityRootFindFilter.searchDirection = SearchDirection::ASC; + communityRootFindFilter.pagination.size = 1; + auto communityRootEntry = blockchain->findOne(communityRootFindFilter); if (communityRootEntry) { - auto communityRootBody = communityRootEntry->getTransactionBody(); - assert(communityRootBody->isCommunityRoot()); - auto communityRoot = communityRootBody->getCommunityRoot(); - auto gmwAddress = adapter::toConstBlockPtr(communityRoot->gmwPublicKeyIndex); - auto aufAddress = adapter::toConstBlockPtr(communityRoot->aufPublicKeyIndex); - auto gmwBalance = calculateAddressBalance.fromEnd(gmwAddress, now, blockchain->getCommunityIdIndex()); - auto aufBalance = calculateAddressBalance.fromEnd(aufAddress, now, blockchain->getCommunityIdIndex()); + auto& tx = *communityRootEntry; + assert(tx.isCommunityRoot()); + auto gmwBalance = calculateAddressBalance.fromEnd(tx.getGmw(), now, blockchain->getCommunityIdIndex()); + auto aufBalance = calculateAddressBalance.fromEnd(tx.getAuf(), now, blockchain->getCommunityIdIndex()); resultJson.AddMember("gmwBalance", Value(gmwBalance.toString().data(), alloc), alloc); resultJson.AddMember("aufBalance", Value(aufBalance.toString().data(), alloc), alloc); } else { @@ -386,7 +402,7 @@ namespace server { void ApiHandler::getCreationSumForMonth( rapidjson::Value& resultJson, - memory::ConstBlockPtr pubkey, + gradido::data::compact::PublicKeyIndex publicKeyIndex, Timepoint targetDate, Timepoint transactionCreationDate, std::shared_ptr blockchain @@ -396,7 +412,7 @@ namespace server { auto& alloc = mRootJson.GetAllocator(); assert(blockchain); - calculateCreationSum::Context calculateCreationSum(transactionCreationDate, targetDate, pubkey); + calculateCreationSum::Context calculateCreationSum(transactionCreationDate, targetDate, publicKeyIndex); auto sumString = calculateCreationSum.run(*blockchain).toString(); resultJson.AddMember("sum", Value(sumString.data(), sumString.size(), alloc), alloc); resultJson.AddMember("timeUsed", Value(timeUsed.string().data(), alloc).Move(), alloc); @@ -528,7 +544,7 @@ namespace server { auto body = entry.getTransactionBody(); assert(body->isRegisterAddress()); auto registerAddress = body->getRegisterAddress(); - if (nameHashId.value() == static_cast(registerAddress->nameHashIndex)) { + if (nameHashId == static_cast(registerAddress->nameHashIndex)) { return FilterResult::USE | FilterResult::STOP; } return FilterResult::DISMISS; @@ -542,7 +558,9 @@ namespace server { assert(body); auto registerAddress = body->getRegisterAddress(); assert(registerAddress); - resultJson.AddMember("pubkey", toJson(registerAddress->accountPublicKeyIndex.getRawKey().convertToHex(), alloc), alloc); + const auto& dict = blockchain->getPublicKeyDictionary(); + + resultJson.AddMember("pubkey", toJson(dict.getDataForIndexOrThrow(registerAddress->accountPublicKeyIndex).convertToHex(), alloc), alloc); } else { error(responseJson, JSON_RPC_ERROR_ADDRESS_NOT_FOUND, "user not found"); diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index 89b4420..0c2dead 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -3,6 +3,7 @@ #include "RequestHandler.h" #include "gradido_blockchain/types.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" namespace gradido { namespace blockchain { @@ -56,7 +57,7 @@ namespace server { //! \param searchStartDate start date for reverse search for creation transactions range -2 month from there void getCreationSumForMonth( rapidjson::Value& resultJson, - memory::ConstBlockPtr pubkey, + gradido::data::compact::PublicKeyIndex publicKeyIndex, Timepoint targetDate, Timepoint transactionCreationDate, std::shared_ptr blockchain diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp index 7e26ab1..3b8093e 100644 --- a/src/task/RebuildBlockIndexTask.cpp +++ b/src/task/RebuildBlockIndexTask.cpp @@ -3,6 +3,7 @@ #include "../ServerGlobals.h" #include "../blockchain/FileBased.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" @@ -56,7 +57,7 @@ namespace task { throw GradidoNodeInvalidDataException("error deserialize transaction body"); } - auto compactConfirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, &body, mCommunityIdIndex, false); + auto compactConfirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, &body, mCommunityIdIndex, *gradido::g_appContext); mBlockIndex->addIndicesForTransaction(compactConfirmedTx); mBlockIndex->addFileCursorForTransaction(compactConfirmedTx.txNr, fileCursor); From 9bed252f1d4086dc0dd818b1801f6cb9c5b115f7 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 24 Feb 2026 18:00:59 +0100 Subject: [PATCH 11/65] linux build fix --- CMakeLists.txt | 8 ++++---- src/blockchain/FileBased.cpp | 2 +- src/blockchain/FileBased.h | 2 +- src/blockchain/NodeTransactionEntry.cpp | 2 +- src/task/RebuildBlockIndexTask.h | 13 ++++++------- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b4aced..301a360 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,8 +72,8 @@ FILE(GLOB SERVER_JSON_RPC "src/server/json-rpc/*.h" "src/server/json-rpc/*.cpp") FILE(GLOB LIB_SRC "src/lib/*.h" "src/lib/*.cpp" "src/lib/*.c") FILE(GLOB MODEL "src/model/*.h" "src/model/*.cpp") FILE(GLOB MODEL_APOLLO "src/model/Apollo/*.h" "src/model/Apollo/*.cpp") -FILE(GLOB MODEL_APOLLO_CREATE_TRANSACTION - "src/model/Apollo/createTransaction/*.h" +FILE(GLOB MODEL_APOLLO_CREATE_TRANSACTION + "src/model/Apollo/createTransaction/*.h" "src/model/Apollo/createTransaction/*.cpp" ) FILE(GLOB MODEL_FILES "src/model/files/*.h" "src/model/files/*.cpp") @@ -84,7 +84,7 @@ FILE(GLOB VIEW "src/view/*.h" "src/view/*.cpp") FILE(GLOB MAIN "src/*.cpp" "src/*.c" "src/*.h") -SET(LOCAL_SRCS +SET(LOCAL_SRCS ${BLOCKCHAIN} ${CACHE_SRC} ${CONTAINER} ${CONTROLLER} ${CLIENT} ${CLIENT_HIERO} ${SERIALIZATION} @@ -205,7 +205,7 @@ add_executable(GradidoNode ${LOCAL_SRCS}) if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") # needed for clang/zig - target_compile_options(GradidoNode PRIVATE -Wno-error=deprecated-literal-operator) + target_compile_options(GradidoNode PRIVATE -Wno-error=deprecated-literal-operator -Wno-error=deprecated-declarations) endif() #SUBDIRS("src/test") diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 554af98..83995aa 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -18,7 +18,7 @@ #include "gradido_blockchain/blockchain/batch/ThreadingPolicy.h" #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" -#include "gradido_blockchain/data/adapter/PublicKey.h" +#include "gradido_blockchain/data/adapter/publicKey.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" #include "gradido_blockchain/data/Timestamp.h" diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index e4c490d..49ea054 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -151,7 +151,7 @@ namespace gradido { virtual data::AddressType getAddressType(const Filter& filter = Filter::LAST_TRANSACTION) const override; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; - virtual data::compact::ConstConfirmedTxPtr getConfirmedTxForId(uint64_t transactionId) const; + virtual data::compact::ConstConfirmedTxPtr getConfirmedTxForId(uint64_t transactionId) const override; //! \param filter use to speed up search if infos exist to narrow down search transactions range virtual ConstTransactionEntryPtr findByLedgerAnchor( const data::LedgerAnchor& ledgerAnchor, diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index 5be0265..d3688e6 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -1,7 +1,7 @@ #include "NodeTransactionEntry.h" #include "FileBased.h" -#include "gradido_blockchain/data/adapter/PublicKey.h" +#include "gradido_blockchain/data/adapter/publicKey.h" namespace gradido { using data::adapter::toPublicKey; diff --git a/src/task/RebuildBlockIndexTask.h b/src/task/RebuildBlockIndexTask.h index 1c14fa5..015a10c 100644 --- a/src/task/RebuildBlockIndexTask.h +++ b/src/task/RebuildBlockIndexTask.h @@ -33,7 +33,7 @@ namespace cache { #define REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE 1024 namespace task { - + //! remove dependencie to CPUTask because this isn't really a cpu task, more are result storage, //! because it will start subsequent tasks of it own which will call back via command if finished class RebuildBlockIndexTask : public CPUTask, public model::files::IBlockBufferRead @@ -43,19 +43,18 @@ namespace task { std::shared_ptr blockIndex, uint32_t communityIdIndex ); - const char* getResourceType() const { return "RebuildBlockIndexTask"; }; + const char* getResourceType() const override { return "RebuildBlockIndexTask"; }; + + int run() override; - int run(); - virtual void finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) override; virtual void flush() override; - + inline grdu_memory* getAlloc() { return &mReadInAllocator; } protected: - uint8_t mBuffers[2][REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE]; - + grdu_memory mReadInAllocator; std::mutex mWorkConfirmedMutex; gradido::data::compact::ConfirmedGradidoTx mConfirmedTx; From 134d68ffe49a5c074b6494055985f6154c103221 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 11:33:03 +0100 Subject: [PATCH 12/65] make it work even without empty or missing hiero topic id in community config --- src/blockchain/FileBased.cpp | 52 +++++++++++++++---------- src/blockchain/FileBased.h | 27 +++++++++++-- src/blockchain/FileBasedProvider.cpp | 33 +++++++++------- src/blockchain/FileBasedProvider.h | 2 +- src/blockchain/NodeTransactionEntry.cpp | 2 +- src/cache/GroupIndex.cpp | 8 +++- src/cache/GroupIndex.h | 10 ++--- src/client/hiero/ConsensusClient.h | 6 +-- 8 files changed, 91 insertions(+), 49 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 83995aa..037d4fb 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -59,7 +59,7 @@ namespace gradido { mExitCalled(false), mHieroTopicId(topicId), mAlias(alias), - mFolderPath(folder), + mFolderPath(folder), mCommunityId(communityId), mTaskObserver(std::make_shared()), mOrderingManager(std::make_shared(communityId)), @@ -94,7 +94,7 @@ namespace gradido { mPublicKeysIndex.reset(); resetBlockIndices = true; } - + if (resetBlockIndices) { model::files::BlockIndex::removeAllBlockIndexFiles(mFolderPath); } @@ -109,7 +109,7 @@ namespace gradido { mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_TRANSACTION_ID, 0); mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0); mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); - + return true; } @@ -146,29 +146,39 @@ namespace gradido { std::shared_ptr FileBased::initOnline() { - return std::make_shared( - mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), - hiero::TopicId(mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString())), - getptr() - ); + auto hieroTopicId = hiero::TopicId(mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString())); + if (!hieroTopicId.empty()) { + return std::make_shared( + mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), + hieroTopicId, + getptr() + ); + } + LOG_F(WARNING, "init online called for community without hiero topic id"); + return nullptr; } void FileBased::startListening(data::Timestamp lastTransactionConfirmedAt) - { + { if (mHieroMessageListener) { LOG_F(WARNING, "called again, while listener where already existing"); } + auto hieroTopicId = hiero::TopicId(mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString())); + if (hieroTopicId.empty()) { + LOG_F(WARNING, "startListening called without valid hiero topic id"); + return; + } data::Timestamp listenFrom = { lastTransactionConfirmedAt.getSeconds(), lastTransactionConfirmedAt.getNanos() + 1 }; auto now = std::chrono::system_clock::now(); // TODO: restart after connection was closed because of timeout auto endTime = now + std::chrono::duration(std::chrono::years(10)); mHieroMessageListener = std::make_shared( - mHieroTopicId, - mCommunityId, - hiero::ConsensusTopicQuery( mHieroTopicId, listenFrom, endTime ) + hieroTopicId, + mCommunityId, + hiero::ConsensusTopicQuery( hieroTopicId, listenFrom, endTime ) ); ServerGlobals::g_HieroMirrorNode->subscribeTopic(mHieroMessageListener); - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, hieroTopicId.toString()); } void FileBased::exit() @@ -369,12 +379,12 @@ namespace gradido { CompactFilter filterCopy(filter); auto skipEntries = filter.pagination.skipEntriesCount(); int paginationCursor = 0; - iterateBlocks(filterCopy.searchDirection, - [&](const cache::Block& block) -> bool + iterateBlocks(filterCopy.searchDirection, + [&](const cache::Block& block) -> bool { const auto& transactionIndex = block.getBlockIndex(); if (PublicKeySearchType::BalanceChangingPublicKey == filterCopy.publicKeySearchType && filterCopy.publicKeyIndex.communityIdIndex == mCommunityIdIndex) - { + { filterCopy.pagination.page = 1; do { auto balanceChangingTxsInRange = transactionIndex.findTransactionsBalanceChangingForPublicKey(filterCopy); @@ -580,7 +590,7 @@ namespace gradido { auto lastBlockNr = model::files::Block::findLastBlockFileInFolder(mFolderPath); mBlockchainState.updateState(cache::DefaultStateKeys::LAST_BLOCK_NR, lastBlockNr); auto& block = getBlock(lastBlockNr); - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_TRANSACTION_ID, block.getBlockIndex().getMaxTransactionNr()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_TRANSACTION_ID, block.getBlockIndex().getMaxTransactionNr()); LOG_F(INFO, "timeUsed: %s", timeUsed.string().data()); } @@ -629,7 +639,7 @@ namespace gradido { }; findAll(f); } - + void FileBased::iterateBlocks(const SearchDirection& searchDir, std::function func) const { bool orderDesc = searchDir == SearchDirection::DESC; @@ -692,9 +702,9 @@ namespace gradido { else { countTarget = lastTransaction->getTransactionNr(); } - - f.filterFunction = - [&](const TransactionEntry& transactionEntry) -> FilterResult + + f.filterFunction = + [&](const TransactionEntry& transactionEntry) -> FilterResult { auto transactionBody = transactionEntry.getTransactionBody(); validate::Context validator(*transactionEntry.getConfirmedTransaction()); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 49ea054..4c48157 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -63,10 +63,10 @@ namespace gradido { public: // Constructor is only usable by this class FileBased( - Private, + Private, const std::string& communityId, const hiero::TopicId& topicId, - std::string_view alias, + std::string_view alias, std::string_view folder, std::vector>&& hieroClients ); @@ -78,6 +78,12 @@ namespace gradido { std::string_view folder, std::vector>&& hieroClients ); + // construct without valid HieroTopic Id. Make Blockchain txs available, but don't listen for new ones from hiero/hedera + static inline std::shared_ptr createWithoutHieroTopic( + const std::string& communityId, + std::string_view alias, + std::string_view folder + ); inline std::shared_ptr getptr(); inline std::shared_ptr getptr() const; @@ -170,7 +176,7 @@ namespace gradido { inline uint32_t getOrAddIndexForPublicKey(const PublicKey& publicKey) const { return mPublicKeysIndex.getOrAddIndexForData(publicKey); } - + inline const hiero::TopicId& getHieroTopicId() const { return mHieroTopicId; } inline const std::string& getAlias() const { return mAlias; } inline const std::string& getFolderPath() const { return mFolderPath; } @@ -237,6 +243,21 @@ namespace gradido { return std::make_shared(Private(), communityId, topicId, alias, folder, std::move(hieroClients)); } + std::shared_ptr FileBased::createWithoutHieroTopic( + const std::string& communityId, + std::string_view alias, + std::string_view folder + ) { + return std::make_shared( + Private(), + communityId, + hiero::TopicId(), + alias, + folder, + std::vector>() + ); + } + std::shared_ptr FileBased::getptr() { return shared_from_this(); diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 881b2c7..5fb6a66 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -124,16 +124,19 @@ namespace gradido { for (const auto& pair : mBlockchainsPerGroup) { if (!pair.second->startValidationTransactions()) { LOG_F( - ERROR, - "error validate last transactions from community: %s in folder: %s", - pair.second->getCommunityId().c_str(), + ERROR, + "error validate last transactions from community: %s in folder: %s", + pair.second->getCommunityId().c_str(), pair.second->getFolderPath().c_str() ); } } - + // step 3: check for new transactions in hiero network, all blockchains at the same time for (const auto& pair : mBlockchainsPerGroup) { + if (pair.second->getHieroTopicId().empty()) { + continue; + } auto task = pair.second->initOnline(); auto hieroClient = pair.second->pickHieroClient(); hieroClient->getTopicInfo(pair.second->getHieroTopicId(), task); @@ -176,23 +179,27 @@ namespace gradido { } shared_ptr FileBasedProvider::addCommunity( - const string& communityId, + const string& communityId, const hiero::TopicId& topicId, const string& alias ) { try { auto communityIdIndex = g_appContext->getOrAddCommunityIdIndex(communityId); auto folder = mGroupIndex->getFolder(communityIdIndex); + std::shared_ptr blockchain; + if (topicId.empty()) { + blockchain = FileBased::createWithoutHieroTopic(communityId, alias, folder); + } else { + // with more hiero clients as per community needed, we make sure we not take always the first mHieroClientsPerCommunity from them + vector> hieroClients = mHieroClients; // copy + if (hieroClients.size() > mHieroClientsPerCommunity) { + std::shuffle(hieroClients.begin(), hieroClients.end(), std::mt19937{ std::random_device{}() }); + hieroClients.resize(mHieroClientsPerCommunity); + } - // with more hiero clients as per community needed, we make sure we not take always the first mHieroClientsPerCommunity from them - vector> hieroClients = mHieroClients; // copy - if (hieroClients.size() > mHieroClientsPerCommunity) { - std::shuffle(hieroClients.begin(), hieroClients.end(), std::mt19937{ std::random_device{}() }); - hieroClients.resize(mHieroClientsPerCommunity); + // with that call community will be initialized and start listening + blockchain = FileBased::create(communityId, topicId, alias, folder, std::move(hieroClients)); } - - // with that call community will be initialized and start listening - auto blockchain = FileBased::create(communityId, topicId, alias, folder, std::move(hieroClients)); g_appContext->addBlockchain(communityIdIndex, blockchain); updateListenerCommunity(communityIdIndex, alias, blockchain); diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index 68ccaa3..eb73f07 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -54,7 +54,7 @@ namespace gradido { inline const cache::GroupIndex* getGroupIndex() const { return mGroupIndex; } protected: - // check if neccessary, or community context is enough + // check if necessary, or community context is enough std::unordered_map> mBlockchainsPerGroup; std::recursive_mutex mWorkMutex; diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index d3688e6..c0f44f4 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -18,7 +18,7 @@ namespace gradido { mPublicKeyIndices.reserve(involvedPublicKeys.size()); for (auto& publicKey : involvedPublicKeys) { mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(toPublicKey(publicKey))); - } + } } diff --git a/src/cache/GroupIndex.cpp b/src/cache/GroupIndex.cpp index 1bfbe70..68c6d1b 100644 --- a/src/cache/GroupIndex.cpp +++ b/src/cache/GroupIndex.cpp @@ -52,11 +52,15 @@ namespace cache { rapidjson_helper::checkMember(communityEntry, "alias", rapidjson_helper::MemberType::STRING); rapidjson_helper::checkMember(communityEntry, "communityId", rapidjson_helper::MemberType::STRING); rapidjson_helper::checkMember(communityEntry, "folder", rapidjson_helper::MemberType::STRING); - rapidjson_helper::checkMember(communityEntry, "hieroTopicId", rapidjson_helper::MemberType::STRING); + // rapidjson_helper::checkMember(communityEntry, "hieroTopicId", rapidjson_helper::MemberType::STRING); entry.alias = communityEntry["alias"].GetString(); entry.communityId = communityEntry["communityId"].GetString(); entry.communityIdIndex = g_appContext->getOrAddCommunityIdIndex(entry.communityId); - entry.topicId = communityEntry["hieroTopicId"].GetString(); + if (communityEntry.HasMember("hieroTopicId")) { + entry.topicId = communityEntry["hieroTopicId"].GetString(); + } else { + LOG_F(WARNING, "community entry %s doesn't have hieroTopicId, this community won't be listened for new transactions", entry.communityId.c_str()); + } entry.folderName = communityEntry["folder"].GetString(); if (communityEntry.HasMember("newBlockUri")) { entry.newBlockUri = communityEntry["newBlockUri"].GetString(); diff --git a/src/cache/GroupIndex.h b/src/cache/GroupIndex.h index 290c78c..3d0577c 100644 --- a/src/cache/GroupIndex.h +++ b/src/cache/GroupIndex.h @@ -33,12 +33,12 @@ namespace cache { /*! * @author Dario Rekowski - * + * * @date 2020-02-06 - * + * * @brief storing group folder name in memory in hash list for fast access * - * protected by FastMutex + * protected by FastMutex * * TODO: adding function to adding group folder pair and save changed group.index file */ @@ -77,9 +77,9 @@ namespace cache { //! \brief collect all group aliases from unordered map (not the fastest operation from unordered map) //! \return vector with group aliases registered to the node server - std::vector listCommunitiesIds() const; + std::vector listCommunitiesIds() const; std::vector listCommunitiesIdIndices() const; - + protected: mutable std::shared_mutex mWorkMutex; model::files::JsonFile mConfig; diff --git a/src/client/hiero/ConsensusClient.h b/src/client/hiero/ConsensusClient.h index 9fdf637..d6c4dae 100644 --- a/src/client/hiero/ConsensusClient.h +++ b/src/client/hiero/ConsensusClient.h @@ -12,7 +12,7 @@ namespace grpc { class ChannelCredentials; - class Client; + class Client; } namespace hiero { @@ -24,7 +24,7 @@ namespace client { namespace hiero { // Client for hiero/hedera Consensus Services - class ConsensusClient + class ConsensusClient { public: ~ConsensusClient(); @@ -47,7 +47,7 @@ namespace client { std::shared_ptr> responseListener ); - private: + private: ConsensusClient(std::shared_ptr channel); static std::shared_ptr getTlsChannelCredentials(memory::ConstBlockPtr certificateHash); From 46ff9f5b0ba7f4cb3949dd9bad55c6025d90a168 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 11:54:37 +0100 Subject: [PATCH 13/65] fix crash on empty hiero topic id --- src/blockchain/FileBased.cpp | 8 ++++---- src/blockchain/FileBased.h | 2 +- src/blockchain/FileBasedProvider.cpp | 6 +++++- src/cache/GroupIndex.cpp | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 037d4fb..ac03b9b 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -72,7 +72,7 @@ namespace gradido { mTransactionHashCache(communityId), mHieroClients(std::move(hieroClients)) { - assert(mHieroClients.size()); + assert(mHieroTopicId.empty() || mHieroClients.size()); } FileBased::~FileBased() @@ -146,11 +146,11 @@ namespace gradido { std::shared_ptr FileBased::initOnline() { - auto hieroTopicId = hiero::TopicId(mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString())); - if (!hieroTopicId.empty()) { + auto hieroTopicIdString = mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); + if (hieroTopicIdString.size()) { return std::make_shared( mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), - hieroTopicId, + hieroTopicIdString, getptr() ); } diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 4c48157..4bde22f 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -182,7 +182,7 @@ namespace gradido { inline const std::string& getFolderPath() const { return mFolderPath; } inline const std::string& getCommunityId() const { return mCommunityId; } inline TaskObserver& getTaskObserver() const { return *mTaskObserver; } - inline std::shared_ptr pickHieroClient() const { return mHieroClients[std::rand() % mHieroClients.size()]; } + inline std::shared_ptr pickHieroClient() const { return mHieroClients.size() ? mHieroClients[std::rand() % mHieroClients.size()] : nullptr; } std::shared_ptr getOrderingManager() { return mOrderingManager; } protected: diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 5fb6a66..16d2a67 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -115,7 +115,11 @@ namespace gradido { // exit if at least one blockchain from config couldn't be loaded // should only occure with invalid config const auto& details = mGroupIndex->getCommunityDetails(communityId); - if (!addCommunity(communityId, hiero::TopicId(details.topicId), details.alias)) { + hiero::TopicId topicId; + if (details.topicId.size()) { + topicId = details.topicId; + } + if (!addCommunity(communityId, topicId, details.alias)) { LOG_F(ERROR, "error adding community %s in folder: %s", details.alias.data(), details.folderName.data()); return false; } diff --git a/src/cache/GroupIndex.cpp b/src/cache/GroupIndex.cpp index 68c6d1b..1f2278e 100644 --- a/src/cache/GroupIndex.cpp +++ b/src/cache/GroupIndex.cpp @@ -56,7 +56,7 @@ namespace cache { entry.alias = communityEntry["alias"].GetString(); entry.communityId = communityEntry["communityId"].GetString(); entry.communityIdIndex = g_appContext->getOrAddCommunityIdIndex(entry.communityId); - if (communityEntry.HasMember("hieroTopicId")) { + if (communityEntry.HasMember("hieroTopicId") && communityEntry["hieroTopicId"].IsString()) { entry.topicId = communityEntry["hieroTopicId"].GetString(); } else { LOG_F(WARNING, "community entry %s doesn't have hieroTopicId, this community won't be listened for new transactions", entry.communityId.c_str()); From 2d67ed8f20c3ba43de18d7eafa0ec5e4c35e249d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 16:22:01 +0100 Subject: [PATCH 14/65] update gradido blockchain to tagged version --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 244928e..908764a 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 244928e47d3cbb512bd54700ca20e46adfb7c4db +Subproject commit 908764a2599f6a031c500731f0e414b5a589f09b From 64c3a784875bd7f0c7bc6d199924e93c972f4e29 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 27 Feb 2026 15:24:35 +0100 Subject: [PATCH 15/65] add restart of rpc listener loop like in hiero sdk --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 2 +- src/client/hiero/MirrorClient.cpp | 2 +- src/client/hiero/MirrorClient.h | 2 +- src/client/hiero/TopicMessageQuery.cpp | 28 ++++++++++++++++++-------- src/client/hiero/TopicMessageQuery.h | 8 ++++---- src/hiero/MessageListenerQuery.cpp | 6 +++--- src/hiero/MessageListenerQuery.h | 2 +- src/server/json-rpc/ApiHandler.cpp | 4 ++-- 9 files changed, 34 insertions(+), 22 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 908764a..1c039b9 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 908764a2599f6a031c500731f0e414b5a589f09b +Subproject commit 1c039b94bc84e2d587b066ba728eb3ae8ccd9924 diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index ac03b9b..c7f64ff 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -177,7 +177,7 @@ namespace gradido { mCommunityId, hiero::ConsensusTopicQuery( hieroTopicId, listenFrom, endTime ) ); - ServerGlobals::g_HieroMirrorNode->subscribeTopic(mHieroMessageListener); + ServerGlobals::g_HieroMirrorNode->subscribeTopic(mHieroMessageListener.get()); mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, hieroTopicId.toString()); } diff --git a/src/client/hiero/MirrorClient.cpp b/src/client/hiero/MirrorClient.cpp index 980d31b..8162ee6 100644 --- a/src/client/hiero/MirrorClient.cpp +++ b/src/client/hiero/MirrorClient.cpp @@ -102,7 +102,7 @@ namespace client { return ::hiero::ConsensusTopicResponse(resultJson); } - void MirrorClient::subscribeTopic(std::shared_ptr responseListener) { + void MirrorClient::subscribeTopic(TopicMessageQuery* responseListener) { if (!responseListener) { throw GradidoNullPointerException("missing response listener", "TopicMessageQuery", __FUNCTION__); } diff --git a/src/client/hiero/MirrorClient.h b/src/client/hiero/MirrorClient.h index 7965654..d0d2621 100644 --- a/src/client/hiero/MirrorClient.h +++ b/src/client/hiero/MirrorClient.h @@ -49,7 +49,7 @@ namespace client { ::hiero::ConsensusTopicResponse getTopicMessageByConsensusTimestamp(gradido::data::Timestamp confirmedAt); // gRPC API - void subscribeTopic(std::shared_ptr responseListener); + void subscribeTopic(TopicMessageQuery* responseListener); inline std::string getProtocolHost() const { return "https://" + std::string(getHost()); } inline const char* getHost() const { return MirrorNetworkEndpoints::getByEndpointName(mNetworkType.data()); } diff --git a/src/client/hiero/TopicMessageQuery.cpp b/src/client/hiero/TopicMessageQuery.cpp index d4f68f1..9e69886 100644 --- a/src/client/hiero/TopicMessageQuery.cpp +++ b/src/client/hiero/TopicMessageQuery.cpp @@ -1,8 +1,10 @@ #include "TopicMessageQuery.h" #include "const.h" #include "MemoryBlock.h" +#include "MirrorClient.h" #include "../../hiero/ConsensusTopicQuery.h" #include "../../lib/protopuf.h" +#include "../../ServerGlobals.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/lib/DataTypeConverter.h" @@ -14,6 +16,7 @@ #include using namespace magic_enum; +using std::make_unique; namespace client { namespace hiero { @@ -24,6 +27,8 @@ namespace client { mStartQuery(startQuery), mCallStatus(CallStatus::STATUS_CREATE) { + mCompletionQueues.push_back(make_unique()); + mClientContexts.push_back(make_unique()); mThread = std::thread(&TopicMessageQuery::ThreadFunction, this); } @@ -43,7 +48,7 @@ namespace client { { loguru::set_thread_name(mThreadName.data()); // copied most of the code from hiero cpp sdk from startSubscription from TopicMessageQuery.cc - ::hiero::ConsensusTopicQuery query; + // ::hiero::ConsensusTopicQuery query; // Declare needed variables. ::grpc::ByteBuffer grpcByteBuffer; ::hiero::ConsensusTopicResponse response; @@ -62,7 +67,7 @@ namespace client { while (!mExitCalled) { // Process based on the completion queue status. auto backOffFromNow = std::chrono::system_clock::now() + std::chrono::duration_cast(backoff); - auto nextStatus = mCompletionQueue.AsyncNext(&tag, &ok, backOffFromNow); + auto nextStatus = mCompletionQueues.back()->AsyncNext(&tag, &ok, backOffFromNow); LOG_F(2, "next status: %s", enum_name(nextStatus).data()); switch (nextStatus) { @@ -115,14 +120,14 @@ namespace client { if (!consensusTimestamp.empty()) { // Add one of the smallest denomination of time - query.setConsensusStartTime({ + mStartQuery.setConsensusStartTime({ consensusTimestamp.getSeconds(), consensusTimestamp.getNanos() + 1 }); } - if (query.getLimit() > 0ULL) + if (mStartQuery.getLimit() > 0ULL) { - query.setLimit(query.getLimit() - 1ULL); + mStartQuery.setLimit(mStartQuery.getLimit() - 1ULL); } // Process the received message. @@ -166,7 +171,7 @@ namespace client { LOG_F(INFO, "RPC subscription complete!"); // Shutdown the completion queue. - mCompletionQueue.Shutdown(); + mCompletionQueues.back()->Shutdown(); // Mark the RPC as complete. complete = true; @@ -175,8 +180,8 @@ namespace client { else { // An error occurred. Whether retrying or not, cancel the call and close the queue. - mClientContext.TryCancel(); - mCompletionQueue.Shutdown(); + mClientContexts.back()->TryCancel(); + mCompletionQueues.back()->Shutdown(); if (attempt >= ::hiero::DEFAULT_MAX_ATTEMPTS || !shouldRetry(grpcStatus)) { @@ -206,6 +211,7 @@ namespace client { { // Give a second for the queue to finish its processing. std::this_thread::sleep_for(std::chrono::seconds(1)); + LOG_F(INFO, "RPC Subscription for topic %s ended.", mStartQuery.getTopicId().toString().data()); return 0; } @@ -225,6 +231,12 @@ namespace client { reader = getConnectedMirrorNode(network)->getConsensusServiceStub()->AsyncsubscribeTopic( contexts.back().get(), query, queues.back().get(), callStatus.get()); */ + mCallStatus = CallStatus::STATUS_CREATE; + mCompletionQueues.push_back(make_unique()); + mClientContexts.push_back(make_unique()); + MemoryBlock memoryBuffer(protopuf::serialize<::hiero::ConsensusTopicQuery, ::hiero::ConsensusTopicQueryMessage>(mStartQuery)); + grpcByteBuffer = memoryBuffer.createGrpcBuffer(); + ServerGlobals::g_HieroMirrorNode->subscribeTopic(this); onConnectionClosed(); break; } diff --git a/src/client/hiero/TopicMessageQuery.h b/src/client/hiero/TopicMessageQuery.h index 588883f..3b6b404 100644 --- a/src/client/hiero/TopicMessageQuery.h +++ b/src/client/hiero/TopicMessageQuery.h @@ -39,8 +39,8 @@ namespace client { TopicMessageQuery(TopicMessageQuery&&) = delete; TopicMessageQuery& operator=(TopicMessageQuery&&) = delete; - grpc::CompletionQueue* getCompletionQueuePtr() { return &mCompletionQueue; } - grpc::ClientContext* getClientContextPtr() { return &mClientContext; } + grpc::CompletionQueue* getCompletionQueuePtr() { return mCompletionQueues.back().get(); } + grpc::ClientContext* getClientContextPtr() { return mClientContexts.back().get(); } void setResponseReader(std::unique_ptr>& responseReader); @@ -63,8 +63,8 @@ namespace client { ::hiero::ConsensusTopicQuery mStartQuery; - grpc::CompletionQueue mCompletionQueue; - grpc::ClientContext mClientContext; + std::vector> mCompletionQueues; + std::vector> mClientContexts; CallStatus mCallStatus; std::unique_ptr> mResponseReader; }; diff --git a/src/hiero/MessageListenerQuery.cpp b/src/hiero/MessageListenerQuery.cpp index d3ee344..db86395 100644 --- a/src/hiero/MessageListenerQuery.cpp +++ b/src/hiero/MessageListenerQuery.cpp @@ -41,8 +41,8 @@ namespace hiero { // will be called from grpc client if connection was closed void MessageListenerQuery::onConnectionClosed() - { - mIsClosed = true; - LOG_F(WARNING, "connection closed on topic: %s", mTopicId.toString().data()); + { + //mIsClosed = true; + LOG_F(WARNING, "connection closed on topic: %s, try reconnect", mTopicId.toString().data()); } } \ No newline at end of file diff --git a/src/hiero/MessageListenerQuery.h b/src/hiero/MessageListenerQuery.h index edbee66..83aa317 100644 --- a/src/hiero/MessageListenerQuery.h +++ b/src/hiero/MessageListenerQuery.h @@ -23,7 +23,7 @@ namespace hiero { void onConnectionClosed() override; inline bool isClosed() const { return mIsClosed; } - inline void cancelConnection() { mClientContext.TryCancel(); } + inline void cancelConnection() { getClientContextPtr()->TryCancel(); } protected: TopicId mTopicId; std::string mCommunityId; diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index d89a937..8c0e970 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -97,7 +97,7 @@ namespace server { // load public key for nearly all requests memory::BlockPtr pubkey; PublicKeyIndex publickKeyIndex; - + std::string pubkeyHex; std::set noNeedForPubkey = { "getLastTransaction", "getTransactions","getTransaction", "findUserByNameHash" @@ -108,7 +108,7 @@ namespace server { } pubkey = std::make_shared(memory::Block::fromHex(pubkeyHex)); publickKeyIndex = adapter::toPublicKeyIndex(pubkey, blockchain->getCommunityIdIndex()); - } + } if (method == "getLastTransaction") { Profiler timeUsed; From 3e7a4f39216314952a127908b22b966a5cb3013a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 27 Feb 2026 21:22:28 +0100 Subject: [PATCH 16/65] fix problems with counting --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 23 +++++++++++++++++++---- src/blockchain/FileBased.h | 2 ++ src/cache/BlockIndex.h | 9 ++++++++- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 908764a..962bf85 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 908764a2599f6a031c500731f0e414b5a589f09b +Subproject commit 962bf856571a28ab744b61ddff2265f3c4c07d62 diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index ac03b9b..c8ee49f 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -468,10 +468,25 @@ namespace gradido { ); return findAll(filter).size(); } - iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - count += block.getBlockIndex().countTransactions(filter, mPublicKeysIndex); - return true; - }); + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool { + count += block.getBlockIndex().countTransactions(filter, mPublicKeysIndex); + return true; + } + ); + return count; + } + + size_t FileBased::countAll(const CompactFilter& filter) const + { + size_t count = 0; + // check if filter has fields which aren't checked by index + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool { + count += block.getBlockIndex().countTransactions(filter); + return true; + } + ); return count; } diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 4bde22f..fe86e20 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -149,6 +149,7 @@ namespace gradido { // find all optimized for counting transaction nrs, better not use the filter.function for that, because this would slow down virtual size_t countAll(const Filter& filter = Filter::ALL_TRANSACTIONS) const override; + virtual size_t countAll(const CompactFilter& filter) const override; //! use only index for searching, ignore filter function //! \return vector with transaction nrs @@ -193,6 +194,7 @@ namespace gradido { void rescanForTransactionTriggerEvents(); //! \param func if function return false, stop iteration + //! TODO: make a template function without using std::function void iterateBlocks(const SearchDirection& searchDir, std::function func) const; cache::Block& getBlock(uint32_t blockNr) const; diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index 9c9bfd9..18f87d9 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -93,6 +93,7 @@ namespace cache { //! count all, ignore pagination inline size_t countTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; + inline size_t countTransactions(const gradido::blockchain::CompactFilter& filter) const; //! \brief find transaction nrs from specific month and year //! \return {0, 0} if nothing found @@ -160,7 +161,13 @@ namespace cache { ) const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::countTransactions(filter, publicKeysDictionary); + return gradido::blockchain::TransactionsIndex::countTransactions(gradido::blockchain::CompactFilter(filter, publicKeysDictionary, mBlockchainCommunityIdIndex)); + } + + size_t BlockIndex::countTransactions(const gradido::blockchain::CompactFilter& filter) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndex::countTransactions(filter); } bool BlockIndex::hasTransactionNr(uint64_t transactionNr) const From c55a8bd6fe33e02efe965e19ca2bc9f917be1641 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 28 Feb 2026 12:27:07 +0100 Subject: [PATCH 17/65] call correct run method with communit id index, catch exceptions with receiving txs --- src/client/hiero/TopicMessageQuery.cpp | 11 +++++++++++ src/task/HieroMessageToTransactionTask.cpp | 2 +- src/task/SyncTopicOnStartup.cpp | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/client/hiero/TopicMessageQuery.cpp b/src/client/hiero/TopicMessageQuery.cpp index 9e69886..586a3d2 100644 --- a/src/client/hiero/TopicMessageQuery.cpp +++ b/src/client/hiero/TopicMessageQuery.cpp @@ -134,7 +134,18 @@ namespace client { const auto& chunkInfo = response.getChunkInfo(); if (chunkInfo.empty() || chunkInfo.getTotal() == 1) { + try { onMessageArrived(std::move(response)); + } + catch (GradidoBlockchainException& ex) { + LOG_F(ERROR, "error calling onMessageArrived: %s", ex.getFullString().c_str()); + } + catch (std::exception& ex) { + LOG_F(ERROR, "std error calling onMessageArrived: %s", ex.what()); + } + catch (...) { + LOG_F(ERROR, "unknown error calling onMessageArrived"); + } } else { diff --git a/src/task/HieroMessageToTransactionTask.cpp b/src/task/HieroMessageToTransactionTask.cpp index 62a9a74..42fabfe 100644 --- a/src/task/HieroMessageToTransactionTask.cpp +++ b/src/task/HieroMessageToTransactionTask.cpp @@ -56,7 +56,7 @@ namespace task { // deserialize deserialize::Context deserializer(mTransactionRaw, deserialize::Type::GRADIDO_TRANSACTION); - deserializer.run(); + deserializer.run(blockchain->getCommunityIdIndex()); if (deserializer.isGradidoTransaction()) { mTransaction = deserializer.getGradidoTransaction(); } diff --git a/src/task/SyncTopicOnStartup.cpp b/src/task/SyncTopicOnStartup.cpp index ba45665..f90a1c3 100644 --- a/src/task/SyncTopicOnStartup.cpp +++ b/src/task/SyncTopicOnStartup.cpp @@ -116,7 +116,7 @@ namespace task { } deserialize::Context deserializer(lastTransactionMirrorRaw); - deserializer.run(); + deserializer.run(mBlockchain->getCommunityIdIndex()); if (!deserializer.isGradidoTransaction()) { LOG_F( ERROR, From 50ed6f5e400a6caeb17e6d8b6d7397d2b17bf8a8 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 28 Feb 2026 15:52:10 +0100 Subject: [PATCH 18/65] change json format for hiero tx id in gradido blockchain lib --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 962bf85..d8cd0b4 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 962bf856571a28ab744b61ddff2265f3c4c07d62 +Subproject commit d8cd0b496b9ad43d44f6240fd0a9bca05466bbe8 From c462c0436ddbe56709ec9b7b83dc01afbbe20795 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 1 Mar 2026 14:11:08 +0100 Subject: [PATCH 19/65] update blockchain lib with counting fix --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index d8cd0b4..5e05716 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit d8cd0b496b9ad43d44f6240fd0a9bca05466bbe8 +Subproject commit 5e0571628da0cf8a2b7ab6ba1bc66b8d3d7f2069 From 19324a06a93260534c8540dbe4446d0bc37c358e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 1 Mar 2026 15:26:06 +0100 Subject: [PATCH 20/65] fix find by ledger anchor --- src/blockchain/FileBased.cpp | 6 +++++- src/server/json-rpc/ApiHandler.cpp | 30 +++++++++++++++++------------- src/server/json-rpc/ApiHandler.h | 7 +++++-- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 171740c..3ea0383 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -589,7 +589,11 @@ namespace gradido { if (transactionNr) { return getTransactionForId(transactionNr); } - return Abstract::findByLedgerAnchor(ledgerAnchor, filter); + auto result = Abstract::findByLedgerAnchor(ledgerAnchor, filter); + if (result) { + mLedgerAnchorCache.add(ledgerAnchor, result->getTransactionNr()); + } + return result; } AbstractProvider* FileBased::getProvider() const diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 8c0e970..9fe80aa 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -10,6 +10,7 @@ #include "gradido_blockchain/data/adapter/publicKey.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/data/LedgerAnchor.h" #include "gradido_blockchain/data/hiero/TransactionId.h" #include "gradido_blockchain/interaction/calculateAccountBalance/Context.h" #include "gradido_blockchain/interaction/calculateCreationSum/Context.h" @@ -179,7 +180,7 @@ namespace server { std::string format; uint64_t transactionId = 0; std::string hieroTransactionIdString; - hiero::TransactionId hieroTransactionId; + LedgerAnchor ledgerAnchor; std::shared_ptr iotaMessageId; if (!getStringParameter(responseJson, params, "format", format)) { @@ -188,17 +189,18 @@ namespace server { getUInt64Parameter(responseJson, params, "transactionId", transactionId, true); getStringParameter(responseJson, params, "hieroTransactionId", hieroTransactionIdString, true); getBinaryFromHexStringParameter(responseJson, params, "iotaMessageId", iotaMessageId, true); - if (!iotaMessageId && !hieroTransactionIdString.empty()) { - hieroTransactionId = hiero::TransactionId(hieroTransactionIdString); - serialize::Context serializeContext(hieroTransactionId); - iotaMessageId = serializeContext.run(); + if (iotaMessageId) { + error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, "iotaMessageId is not longer supported"); } - if (!transactionId && !iotaMessageId && hieroTransactionId.empty()) { - error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, "transactionId, hieroTransactionId or iotaMessageId needed"); + if (!hieroTransactionIdString.empty()) { + ledgerAnchor = LedgerAnchor(hiero::TransactionId(hieroTransactionIdString)); + } + if (!transactionId && !iotaMessageId && ledgerAnchor.empty()) { + error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, "transactionId or hieroTransactionId needed"); return; } - getTransaction(resultJson, responseJson, blockchain, format, transactionId, iotaMessageId); + getTransaction(resultJson, responseJson, blockchain, format, transactionId, &ledgerAnchor); } else if (method == "getCreationSumForMonth") { int month, year; @@ -305,6 +307,9 @@ namespace server { countFilter.minTransactionNr = 0; // remove minTransactionNr for count countFilter.maxTransactionNr = 0; // remove maxTransactionNr for count auto totalCount = blockchain->countAll(countFilter); + auto microsForCounting = timeUsed.micros(); + printf("time used for counting: %.4f micro seconds\n", microsForCounting); + resultJson.AddMember("totalCount", totalCount, alloc); auto transactions = blockchain->findAll(filter); @@ -358,7 +363,7 @@ namespace server { std::shared_ptr blockchain, const std::string& format, uint64_t transactionId/* = 0*/, - std::shared_ptr iotaMessageId /* = nullptr */ + gradido::data::LedgerAnchor* ledgerAnchor/* = nullptr */ ) { Profiler timeUsed; @@ -369,13 +374,12 @@ namespace server { transactionEntry = blockchain->getTransactionForId(transactionId); } else { - deserialize::Context deserializer(iotaMessageId); - deserializer.run(); - if (deserializer.isLedgerAnchor()) { - transactionEntry = blockchain->findByLedgerAnchor(deserializer.getLedgerAnchor()); + if (ledgerAnchor && !ledgerAnchor->empty()) { + transactionEntry = blockchain->findByLedgerAnchor(*ledgerAnchor); } } if (!transactionEntry) { + printf("not found after: %s\n", timeUsed.string().c_str()); error(responseJson, JSON_RPC_ERROR_TRANSACTION_NOT_FOUND, "transaction not found"); return; } diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index 0c2dead..8b3faf5 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -10,6 +10,9 @@ namespace gradido { class Abstract; class Filter; } + namespace data { + class LedgerAnchor; + } } namespace memory { @@ -44,7 +47,7 @@ namespace server { * @param resultJson: for success result * @param responseJson: for the overall response, used for example for errors * @param transactionId: this parameter or - * @param iotaMessageId: this parameter for finding transaction + * @param ledgerAnchor: this parameter for finding transaction */ void getTransaction( rapidjson::Value& resultJson, @@ -52,7 +55,7 @@ namespace server { std::shared_ptr blockchain, const std::string& format, uint64_t transactionId = 0, - std::shared_ptr iotaMessageId = nullptr + gradido::data::LedgerAnchor* ledgerAnchor = nullptr ); //! \param searchStartDate start date for reverse search for creation transactions range -2 month from there void getCreationSumForMonth( From 555c0cec3716180d2e741f50e0b124d740ac573c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 1 Mar 2026 15:33:23 +0100 Subject: [PATCH 21/65] remove debug print --- src/server/json-rpc/ApiHandler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 9fe80aa..2cd0c02 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -307,8 +307,6 @@ namespace server { countFilter.minTransactionNr = 0; // remove minTransactionNr for count countFilter.maxTransactionNr = 0; // remove maxTransactionNr for count auto totalCount = blockchain->countAll(countFilter); - auto microsForCounting = timeUsed.micros(); - printf("time used for counting: %.4f micro seconds\n", microsForCounting); resultJson.AddMember("totalCount", totalCount, alloc); From c7c6d21b1916ac9cccb8e0bb9015ea45a972e9d2 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 2 Mar 2026 08:36:12 +0100 Subject: [PATCH 22/65] fix some bugs2 --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 5e05716..f03d7c7 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 5e0571628da0cf8a2b7ab6ba1bc66b8d3d7f2069 +Subproject commit f03d7c775a1edf88f0d3fdd08a773f88f1a36827 From c5974250e60b36d52a65b388ef6a50960ebb281c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 4 Mar 2026 13:02:28 +0100 Subject: [PATCH 23/65] use std::stop_token for more graceful shutdown --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 34 ++++++++++-------- src/MainServer.h | 1 + src/blockchain/FileBased.cpp | 53 ++++++++++++++++++---------- src/blockchain/FileBased.h | 15 ++++++-- src/blockchain/FileBasedProvider.cpp | 6 ++-- src/blockchain/FileBasedProvider.h | 6 +++- src/cache/Block.cpp | 4 +-- src/cache/Block.h | 3 +- src/cache/State.h | 1 - src/model/files/Block.cpp | 29 ++++++++------- src/model/files/Block.h | 3 +- 12 files changed, 100 insertions(+), 57 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index f03d7c7..1bf5f15 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit f03d7c775a1edf88f0d3fdd08a773f88f1a36827 +Subproject commit 1bf5f15e47c5585da3b3e2075ec9091ddd7e2621 diff --git a/src/MainServer.cpp b/src/MainServer.cpp index 778d61e..9a6546f 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -115,11 +115,12 @@ bool MainServer::init() if (hieroNodeCountPerCommunity > hieroNodeCount) { LOG_F(ERROR, "clients.hiero.nodeCountPerCommunity (%d) mustn't be greate than clients.hiero.nodeCount (%d)", hieroNodeCount, hieroNodeCountPerCommunity); } - hiero::Addressbook addressbook(grpcAddressesFile.data()); + hiero::Addressbook addressbook(grpcAddressesFile.c_str()); addressbook.load(); hieroClients.reserve(hieroNodeCount); for (int i = 0; i < hieroNodeCount; i++) { + if (mMasterStopSource.stop_requested()) break; const auto& hieroNode = addressbook.pickRandomNode(); const auto& endpoint = hieroNode.pickRandomEndpoint(); auto hieroServiceEndpointUrl = endpoint.getConnectionString(); @@ -129,37 +130,40 @@ bool MainServer::init() hieroNode.getNodeCertHash() ); if (!hieroClient) { - LOG_F(ERROR, "Error connecting with hiero network via service endpoint: %s", hieroServiceEndpointUrl.data()); + LOG_F(ERROR, "Error connecting with hiero network via service endpoint: %s", hieroServiceEndpointUrl.c_str()); return false; } LOG_F(INFO, "Hiero endpoint: %s (%s)", - hieroServiceEndpointUrl.data(), - hieroNode.getDescription().data() + hieroServiceEndpointUrl.c_str(), + hieroNode.getDescription().c_str() ); hieroClients.push_back(hieroClient); } } - if (!FileBasedProvider::getInstance()->init(ServerGlobals::g_FilesPath + "/communities.json", std::move(hieroClients), hieroNodeCountPerCommunity)) { + if (!FileBasedProvider::getInstance()->init(getStopToken(), ServerGlobals::g_FilesPath + "/communities.json", std::move(hieroClients), hieroNodeCountPerCommunity)) { LOG_F(ERROR, "Error loading communities, please try to delete communities folders and try again!"); return false; } - - // JSON Interface Server - mHttpServer = new Server("0.0.0.0", jsonrpc_port, "http-server"); - mHttpServer->init(); - mHttpServer->registerResponseHandler("/api", new server::json_rpc::ApiHandlerFactory()); - mHttpServer->run(); - LOG_F(INFO, "started in %s, json rpc port: %d", usedTime.string().data(), jsonrpc_port); - // start the json server - // doesn't return + if (!mMasterStopSource.stop_requested()) { + // start jsonrpc 2.0 server + mHttpServer = new Server("0.0.0.0", jsonrpc_port, "http-server"); + mHttpServer->init(); + mHttpServer->registerResponseHandler("/api", new server::json_rpc::ApiHandlerFactory()); + mHttpServer->run(); + LOG_F(INFO, "started in %s, json rpc port: %d", usedTime.string().c_str(), jsonrpc_port); + } + else { + LOG_F(INFO, "stopped before startup was finished in: %s", usedTime.string().c_str()); + return false; + } return true; } void MainServer::exit() { LOG_F(INFO, "Running Tasks Count on shutdown: %lu", ServerGlobals::g_NumberExistingTasks.load()); - + // stop worker scheduler // TODO: make sure that pending transaction are still write out to storage if (mHttpServer) { diff --git a/src/MainServer.h b/src/MainServer.h index 5ec3210..2d0a682 100644 --- a/src/MainServer.h +++ b/src/MainServer.h @@ -5,6 +5,7 @@ #include "gradido_blockchain/http/Server.h" #include +#include class MainServer : public Application { diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 3ea0383..a92bc9a 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -30,10 +30,12 @@ #include "loguru/loguru.hpp" #include +#include #include using namespace cache; -using std::string, std::string_view, std::vector; +using std::lock_guard; +using std::string, std::string_view, std::stop_token, std::vector; using std::shared_ptr, std::make_shared; using client::hiero::ConsensusClient; using controller::SimpleOrderingManager; @@ -50,13 +52,14 @@ namespace gradido { FileBased::FileBased( Private, + stop_token stopToken, const string& communityId, const hiero::TopicId& topicId, string_view alias, string_view folder, vector>&& hieroClients) : Abstract(g_appContext->getOrAddCommunityIdIndex(communityId)), - mExitCalled(false), + mStopToken(stopToken), mHieroTopicId(topicId), mAlias(alias), mFolderPath(folder), @@ -85,8 +88,10 @@ namespace gradido { } bool FileBased::init(bool resetBlockIndices) { - assert(!mExitCalled); - std::lock_guard _lock(mWorkMutex); + if (mStopToken.stop_requested()) { + return false; + } + lock_guard _lock(mWorkMutex); if (!mPublicKeysIndex.init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { // remove index files for regenration LOG_F(WARNING, "reset the public key index file"); @@ -115,6 +120,8 @@ namespace gradido { bool FileBased::startValidationTransactions() { + if (mStopToken.stop_requested()) return false; + lock_guard _lock(mWorkMutex); auto lastBlockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { mLedgerAnchorCache.reset(); @@ -146,6 +153,7 @@ namespace gradido { std::shared_ptr FileBased::initOnline() { + if (mStopToken.stop_requested()) return nullptr; auto hieroTopicIdString = mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); if (hieroTopicIdString.size()) { return std::make_shared( @@ -160,6 +168,7 @@ namespace gradido { void FileBased::startListening(data::Timestamp lastTransactionConfirmedAt) { + if (mStopToken.stop_requested()) return; if (mHieroMessageListener) { LOG_F(WARNING, "called again, while listener where already existing"); } @@ -183,8 +192,7 @@ namespace gradido { void FileBased::exit() { - std::lock_guard _lock(mWorkMutex); - mExitCalled = true; + lock_guard _lock(mWorkMutex); /*if (mIotaMessageListener) { delete mIotaMessageListener; mIotaMessageListener = nullptr; @@ -215,14 +223,15 @@ namespace gradido { const data::LedgerAnchor& ledgerAnchor, data::Timestamp confirmedAt ) { + if (mStopToken.stop_requested()) return false; if (!gradidoTransaction) { throw GradidoNullPointerException("missing transaction", "GradidoTransactionPtr", __FUNCTION__); } if (ledgerAnchor.empty()) { throw GradidoNullPointerException("empty ledger anchor", "gradido::data::LedgerAnchor", __FUNCTION__); } - std::lock_guard _lock(mWorkMutex); - if (mExitCalled) { return false;} + lock_guard _lock(mWorkMutex); + confirmTransaction::Context confirmTransactionContext(getptr()); auto role = confirmTransactionContext.createRole(gradidoTransaction, ledgerAnchor, confirmedAt); auto confirmedTransaction = confirmTransactionContext.run(role); @@ -261,14 +270,14 @@ namespace gradido { std::vector accountBalances ) { + if (mStopToken.stop_requested()) return false; if (!gradidoTransaction) { throw GradidoNullPointerException("missing transaction", "GradidoTransactionPtr", __FUNCTION__); } if (ledgerAnchor.empty()) { throw GradidoNullPointerException("empty ledger anchor", "gradido::data::LedgerAnchor", __FUNCTION__); } - std::lock_guard _lock(mWorkMutex); - if (mExitCalled) { return false; } + lock_guard _lock(mWorkMutex); confirmTransaction::Context confirmTransactionContext(getptr()); auto role = confirmTransactionContext.createRole( gradidoTransaction, @@ -316,25 +325,27 @@ namespace gradido { void FileBased::addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent) { - std::lock_guard _lock(mWorkMutex); + if (mStopToken.stop_requested()) return; + + lock_guard _lock(mWorkMutex); mTransactionTriggerEventsCache.addTransactionTriggerEvent(transactionTriggerEvent); } void FileBased::removeTransactionTriggerEvent(const data::TransactionTriggerEvent& transactionTriggerEvent) { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); mTransactionTriggerEventsCache.removeTransactionTriggerEvent(transactionTriggerEvent); } std::vector> FileBased::findTransactionTriggerEventsInRange(Timestamp startDate, Timestamp endDate) { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); return mTransactionTriggerEventsCache.findTransactionTriggerEventsInRange(startDate, endDate); } std::shared_ptr FileBased::findNextTransactionTriggerEventInRange(Timestamp startDate, Timestamp endDate) { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); return mTransactionTriggerEventsCache.findNextTransactionTriggerEventInRange(startDate, endDate); } @@ -517,7 +528,7 @@ namespace gradido { return result; } - data::AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const + AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const { // return getAddressTypeSlow(filter); if (!filter.involvedPublicKey || filter.involvedPublicKey->isEmpty()) { @@ -554,7 +565,7 @@ namespace gradido { std::shared_ptr FileBased::getTransactionForId(uint64_t transactionId) const { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); do { auto& block = getBlock(blockNr); @@ -568,7 +579,7 @@ namespace gradido { ConstConfirmedTxPtr FileBased::getConfirmedTxForId(uint64_t transactionId) const { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); do { auto& block = getBlock(blockNr); @@ -585,6 +596,7 @@ namespace gradido { const Filter& filter/* = Filter::ALL_TRANSACTIONS*/ ) const { + lock_guard _lock(mWorkMutex); auto transactionNr = mLedgerAnchorCache.has(ledgerAnchor); if (transactionNr) { return getTransactionForId(transactionNr); @@ -620,6 +632,7 @@ namespace gradido { Filter f = Filter::ALL_TRANSACTIONS; f.searchDirection = SearchDirection::ASC; f.filterFunction = [this](const TransactionEntry& entry) -> FilterResult { + if (mStopToken.stop_requested()) return FilterResult::STOP; if (entry.getTransactionType() == data::TransactionType::DEFERRED_TRANSFER) { auto confirmedTransaction = entry.getConfirmedTransaction(); auto body = entry.getTransactionBody(); @@ -630,7 +643,7 @@ namespace gradido { confirmedTransaction->getId(), targetDate, data::TransactionTriggerEventType::DEFERRED_TIMEOUT_REVERSAL - )); + )); } else if (entry.getTransactionType() == data::TransactionType::REDEEM_DEFERRED_TRANSFER) { // remove timeout transaction trigger event @@ -688,7 +701,7 @@ namespace gradido { if (!block) { auto block = std::make_shared(blockNr, getptr()); // return false if block not exist and will be created - if (!block->init()) { + if (!block->init(mStopToken)) { if (blockNr > mBlockchainState.readInt32State(DefaultStateKeys::LAST_BLOCK_NR, 1)) { mBlockchainState.updateState(DefaultStateKeys::LAST_BLOCK_NR, blockNr); } @@ -702,6 +715,7 @@ namespace gradido { bool FileBased::validateLastTransactions(uint64_t countToValidate) { // load first GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE transaction into cache and validate the transaction to check file integrity + if (mStopToken.stop_requested()) return false; Profiler timeUsed; Profiler timeSinceLastPrint; data::ConstConfirmedTransactionPtr previousConfirmedTransaction = nullptr; @@ -725,6 +739,7 @@ namespace gradido { f.filterFunction = [&](const TransactionEntry& transactionEntry) -> FilterResult { + if (mStopToken.stop_requested()) return FilterResult::STOP; auto transactionBody = transactionEntry.getTransactionBody(); validate::Context validator(*transactionEntry.getConfirmedTransaction()); validate::Type validationLevel = validate::Type::SINGLE | validate::Type::ACCOUNT; diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index fe86e20..2bf0636 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -28,6 +28,7 @@ #define GRADIDO_NODE_MAGIC_NUMBER_TRANSACTION_TRIGGER_EVENTS_CACHE_MEGA_BTYES 1 #include +#include namespace client { namespace hiero { @@ -64,6 +65,7 @@ namespace gradido { // Constructor is only usable by this class FileBased( Private, + std::stop_token stopToken, const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, @@ -72,6 +74,7 @@ namespace gradido { ); // make sure that all shared_ptr from FileBased Blockchain know each other static inline std::shared_ptr create( + std::stop_token stopToken, const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, @@ -80,6 +83,7 @@ namespace gradido { ); // construct without valid HieroTopic Id. Make Blockchain txs available, but don't listen for new ones from hiero/hedera static inline std::shared_ptr createWithoutHieroTopic( + std::stop_token stopToken, const std::string& communityId, std::string_view alias, std::string_view folder @@ -185,6 +189,7 @@ namespace gradido { inline TaskObserver& getTaskObserver() const { return *mTaskObserver; } inline std::shared_ptr pickHieroClient() const { return mHieroClients.size() ? mHieroClients[std::rand() % mHieroClients.size()] : nullptr; } std::shared_ptr getOrderingManager() { return mOrderingManager; } + std::stop_token getStopToken() const { return mStopToken; } protected: //! if state leveldb was invalid, recover values from block cache @@ -203,7 +208,7 @@ namespace gradido { bool validateLastTransactions(uint64_t countToValidate); mutable std::recursive_mutex mWorkMutex; - bool mExitCalled; + std::stop_token mStopToken; hiero::TopicId mHieroTopicId; std::string mAlias; std::string mFolderPath; @@ -219,6 +224,7 @@ namespace gradido { //! contain indices for every public key address, used overall for optimisation mutable PersistentDictionary mPublicKeysIndex; // level db to store state values like last transaction + // TODO: speedup with atcual struct, write out into leveldb/lmdb only on changes, maybe even buffered, think on exit management mutable cache::State mBlockchainState; mutable cache::LedgerAnchor mLedgerAnchorCache; @@ -236,22 +242,25 @@ namespace gradido { }; std::shared_ptr FileBased::create( + std::stop_token stopToken, const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, std::string_view folder, std::vector>&& hieroClients ) { - return std::make_shared(Private(), communityId, topicId, alias, folder, std::move(hieroClients)); + return std::make_shared(Private(), stopToken, communityId, topicId, alias, folder, std::move(hieroClients)); } std::shared_ptr FileBased::createWithoutHieroTopic( + std::stop_token stopToken, const std::string& communityId, std::string_view alias, std::string_view folder ) { return std::make_shared( Private(), + stopToken, communityId, hiero::TopicId(), alias, @@ -262,11 +271,13 @@ namespace gradido { std::shared_ptr FileBased::getptr() { + if (mStopToken.stop_requested()) return nullptr; return shared_from_this(); } std::shared_ptr FileBased::getptr() const { + if (mStopToken.stop_requested()) return nullptr; return shared_from_this(); } diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 16d2a67..8f57b19 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -93,11 +93,13 @@ namespace gradido { } bool FileBasedProvider::init( + std::stop_token masterStopView, const string& communityConfigFile, vector>&& hieroClients, uint8_t hieroClientsPerCommunity/* = 3 */ ) { lock_guard _lock(mWorkMutex); + mStopToken = masterStopView; mInitalized = true; bool resetAllCommunityIndices = false; mHieroClientsPerCommunity = hieroClientsPerCommunity; @@ -192,7 +194,7 @@ namespace gradido { auto folder = mGroupIndex->getFolder(communityIdIndex); std::shared_ptr blockchain; if (topicId.empty()) { - blockchain = FileBased::createWithoutHieroTopic(communityId, alias, folder); + blockchain = FileBased::createWithoutHieroTopic(mStopToken, communityId, alias, folder); } else { // with more hiero clients as per community needed, we make sure we not take always the first mHieroClientsPerCommunity from them vector> hieroClients = mHieroClients; // copy @@ -202,7 +204,7 @@ namespace gradido { } // with that call community will be initialized and start listening - blockchain = FileBased::create(communityId, topicId, alias, folder, std::move(hieroClients)); + blockchain = FileBased::create(mStopToken, communityId, topicId, alias, folder, std::move(hieroClients)); } g_appContext->addBlockchain(communityIdIndex, blockchain); updateListenerCommunity(communityIdIndex, alias, blockchain); diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index eb73f07..9ef8e82 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -6,8 +6,9 @@ #include "FileBased.h" #include "../cache/GroupIndex.h" -#include #include +#include +#include #define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_ID_INDEX_CACHE_SIZE_MBYTE 1 @@ -39,6 +40,7 @@ namespace gradido { std::shared_ptr findBlockchain(hiero::TopicId& topicId); //! \return true if successfully else return false bool init( + std::stop_token masterStopView, const std::string& communityConfigFile, std::vector>&& hieroClients, uint8_t hieroClientsPerCommunity = 3 @@ -74,6 +76,8 @@ namespace gradido { ); void updateListenerCommunity(uint32_t communityIdIndex, const std::string& alias, std::shared_ptr blockchain); + //! master stop source view + std::stop_token mStopToken; cache::GroupIndex* mGroupIndex; std::vector> mHieroClients; uint8_t mHieroClientsPerCommunity; diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 012238e..fd959f7 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -62,7 +62,7 @@ namespace cache { mSerializedTransactions.clear(); } - bool Block::init() + bool Block::init(std::stop_token stop /* = std::stop_token() */) { lock_guard lock(mFastMutex); // todo: add data for address index in file, until then rebuild block index on each program start @@ -78,7 +78,7 @@ namespace cache { mBlockchain->getCommunityIdIndex() ); rebuildBlockIndexTask->scheduleTask(rebuildBlockIndexTask); - mBlockFile->readBuffered(rebuildBlockIndexTask->getAlloc(), rebuildBlockIndexTask.get()); + mBlockFile->readBuffered(rebuildBlockIndexTask->getAlloc(), rebuildBlockIndexTask.get(), stop); int sumWaited = 0; while (!rebuildBlockIndexTask->isTaskFinished() && sumWaited < GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS) { diff --git a/src/cache/Block.h b/src/cache/Block.h index dcfacac..d9f6a68 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -13,6 +13,7 @@ #include #include +#include namespace model { namespace files { @@ -56,7 +57,7 @@ namespace cache { ~Block(); //! \return false if block not exist - bool init(); + bool init(std::stop_token stop = std::stop_token()); void exit(); //! \brief put new transaction to cache and file system diff --git a/src/cache/State.h b/src/cache/State.h index 95fe19e..10ec325 100644 --- a/src/cache/State.h +++ b/src/cache/State.h @@ -9,7 +9,6 @@ #include namespace cache { - class State { public: diff --git a/src/model/files/Block.cpp b/src/model/files/Block.cpp index 4c30d6c..43415d7 100644 --- a/src/model/files/Block.cpp +++ b/src/model/files/Block.cpp @@ -141,8 +141,11 @@ namespace model { auto transactionSize = readLine(startReading, &result); return result; } - bool Block::readBuffered(grdu_memory* alloc, IBlockBufferRead* callback) + bool Block::readBuffered(grdu_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken/* = std::stop_token()*/) { + if (stopToken.stop_requested()) { + return false; + } assert(alloc && callback); auto fileStream = getOpenFile(); @@ -177,7 +180,7 @@ namespace model { unsigned char hash[crypto_generichash_KEYBYTES]; memset(hash, 0, sizeof hash); - while (fileStream->good() && readed < mCurrentFileSize) { + while (fileStream->good() && readed < mCurrentFileSize && !stopToken.stop_requested()) { auto fileCursor = readed; fileStream->read((char*)&transactionSize, sizeof(uint16_t)); readed += sizeof(uint16_t); @@ -201,19 +204,21 @@ namespace model { callback->finishedLine(memStart, transactionSize, fileCursor); } callback->flush(); - unsigned char hash2[crypto_generichash_KEYBYTES]; - fileStream->read((char*)hash2, crypto_generichash_KEYBYTES); - int filePointer = fileStream->tellg(); - if (0 != sodium_memcmp(hash, hash2, crypto_generichash_KEYBYTES)) { - throw HashMismatchException( - "block hash mismatch", - memory::Block(sizeof hash, hash), - memory::Block(sizeof hash2, hash2) - ); + if (!stopToken.stop_requested()) { + unsigned char hash2[crypto_generichash_KEYBYTES]; + fileStream->read((char*)hash2, crypto_generichash_KEYBYTES); + int filePointer = fileStream->tellg(); + if (0 != sodium_memcmp(hash, hash2, crypto_generichash_KEYBYTES)) { + throw HashMismatchException( + "block hash mismatch", + memory::Block(sizeof hash, hash), + memory::Block(sizeof hash2, hash2) + ); + } } fl->unlock(mBlockPath); - return transactionSize; + return true; } diff --git a/src/model/files/Block.h b/src/model/files/Block.h index c00a6c9..84d1455 100644 --- a/src/model/files/Block.h +++ b/src/model/files/Block.h @@ -13,6 +13,7 @@ #include #include +#include //! MAGIC NUMBER: use to check if a file is big enough to could contain a transaction #define MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE 25 @@ -72,7 +73,7 @@ namespace model { uint16_t readLine(uint32_t startReading, memory::BlockPtr* buffer); std::shared_ptr readLine(uint32_t startReading); // read whole file, validate hash - bool readBuffered(grdu_memory* alloc, IBlockBufferRead* callback); + bool readBuffered(grdu_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken = std::stop_token()); //! \brief call appendLines //! \return file cursor pos at start from this line in file (0 at start of file) From 58bc65d9d7fff42a53fef3bae8fc5b8ac3b6a1d7 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 6 Mar 2026 09:29:30 +0100 Subject: [PATCH 24/65] add fast access vector to cache::State for default keys --- src/blockchain/FileBased.cpp | 16 ++++++++-------- src/cache/DefaultStateKeys.h | 4 +++- src/cache/State.cpp | 19 ++++++++++++++++++- src/cache/State.h | 34 ++++++++++++++++++++-------------- src/model/files/BlockIndex.h | 8 +++++--- 5 files changed, 54 insertions(+), 27 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 3ea0383..f040466 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -104,12 +104,12 @@ namespace gradido { loadStateFromBlockCache(); } // read basic states into memory - mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, 0); + /*mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, 0); mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_TRANSACTION_ID, 0); mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0); - mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); - + mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum()); + */ return true; } @@ -146,11 +146,11 @@ namespace gradido { std::shared_ptr FileBased::initOnline() { - auto hieroTopicIdString = mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); - if (hieroTopicIdString.size()) { + auto hieroTopicIdNum = mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum()); + if (hieroTopicIdNum) { return std::make_shared( mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), - hieroTopicIdString, + hiero::TopicId(0, 0, hieroTopicIdNum), getptr() ); } @@ -163,7 +163,7 @@ namespace gradido { if (mHieroMessageListener) { LOG_F(WARNING, "called again, while listener where already existing"); } - auto hieroTopicId = hiero::TopicId(mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString())); + auto hieroTopicId = hiero::TopicId(0, 0, mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum())); if (hieroTopicId.empty()) { LOG_F(WARNING, "startListening called without valid hiero topic id"); return; @@ -178,7 +178,7 @@ namespace gradido { hiero::ConsensusTopicQuery( hieroTopicId, listenFrom, endTime ) ); ServerGlobals::g_HieroMirrorNode->subscribeTopic(mHieroMessageListener.get()); - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, hieroTopicId.toString()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, hieroTopicId.getTopicNum()); } void FileBased::exit() diff --git a/src/cache/DefaultStateKeys.h b/src/cache/DefaultStateKeys.h index fa136b3..703d68b 100644 --- a/src/cache/DefaultStateKeys.h +++ b/src/cache/DefaultStateKeys.h @@ -1,8 +1,10 @@ #ifndef __GRADIDO_NODE_CACHE_DEFAULT_STATE_KEYS_H #define __GRADIDO_NODE_CACHE_DEFAULT_STATE_KEYS_H +#include "gradido_blockchain/types.h" + namespace cache { - enum class DefaultStateKeys { + enum class DefaultStateKeys: uint8_t { LAST_ADDRESS_INDEX, LAST_BLOCK_NR, LAST_TRANSACTION_ID, diff --git a/src/cache/State.cpp b/src/cache/State.cpp index 33fada7..e537891 100644 --- a/src/cache/State.cpp +++ b/src/cache/State.cpp @@ -3,13 +3,19 @@ #include "../lib/LevelDBExceptions.h" #include "loguru/loguru.hpp" +#include "magic_enum/magic_enum.hpp" + +#include + +using namespace magic_enum; +using std::string_view; namespace cache { State::State(std::string_view folder) : mInitalized(false), mStateFile(folder) { - + mFastAccessDefaultStates.resize(static_cast(DefaultStateKeys::MAX), 0); } State::~State() @@ -25,6 +31,17 @@ namespace cache { if (!mStateFile.init(cacheInBytes)) { return false; } + // fill fast access default states vector + mStateFile.iterate( + [&](leveldb::Slice key, leveldb::Slice value) + { + auto state = enum_cast(string_view(key.data(), key.size())); + if (state.has_value()) { + assert(state.value() < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(state.value())] = strtoll(value.data(), nullptr, 10); + } + } + ); mInitalized = true; return true; } diff --git a/src/cache/State.h b/src/cache/State.h index 95fe19e..9682b2d 100644 --- a/src/cache/State.h +++ b/src/cache/State.h @@ -6,6 +6,7 @@ #include "magic_enum/magic_enum.hpp" +#include #include namespace cache { @@ -23,7 +24,6 @@ namespace cache { //! remove state level db folder, clear maps void reset(); - inline void updateState(DefaultStateKeys key, std::string_view value); inline void updateState(DefaultStateKeys key, int32_t value); inline void updateState(DefaultStateKeys key, uint32_t value); inline void updateState(DefaultStateKeys key, int64_t value); @@ -35,7 +35,6 @@ namespace cache { void updateState(const char* key, uint64_t value); void removeState(const char* key); - inline std::string readState(DefaultStateKeys key, const std::string& defaultValue); std::string readState(const char* key, const std::string& defaultValue); inline int32_t readInt32State(DefaultStateKeys key, int32_t defaultValue); int32_t readInt32State(const char* key, int32_t defaultValue); @@ -48,46 +47,53 @@ namespace cache { protected: bool mInitalized; model::files::LevelDBWrapper mStateFile; + std::vector mFastAccessDefaultStates; }; // simple functions for inline declarations - void State::updateState(DefaultStateKeys key, std::string_view value) - { - return updateState(magic_enum::enum_name(key).data(), value); - } - void State::updateState(DefaultStateKeys key, int32_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } void State::updateState(DefaultStateKeys key, uint32_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } void State::updateState(DefaultStateKeys key, int64_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } void State::updateState(DefaultStateKeys key, uint64_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } - std::string State::readState(DefaultStateKeys key, const std::string& defaultValue) - { - return readState(magic_enum::enum_name(key).data(), defaultValue); - } - int32_t State::readInt32State(DefaultStateKeys key, int32_t defaultValue) { - return readInt32State(magic_enum::enum_name(key).data(), defaultValue); + assert(key < DefaultStateKeys::MAX); + auto value = mFastAccessDefaultStates[static_cast(key)]; + if (value) return value; + else return defaultValue; + // return readInt32State(magic_enum::enum_name(key).data(), defaultValue); } int64_t State::readInt64State(DefaultStateKeys key, int64_t defaultValue) { - return readInt64State(magic_enum::enum_name(key).data(), defaultValue); + assert(key < DefaultStateKeys::MAX); + auto value = mFastAccessDefaultStates[static_cast(key)]; + if (value) return value; + else return defaultValue; + // return readInt64State(magic_enum::enum_name(key).data(), defaultValue); } } diff --git a/src/model/files/BlockIndex.h b/src/model/files/BlockIndex.h index f29d763..e32fe23 100644 --- a/src/model/files/BlockIndex.h +++ b/src/model/files/BlockIndex.h @@ -190,9 +190,11 @@ namespace model { addressIndices(nullptr), addressIndicesCount(_addressIndices.size()) { - addressIndices = (uint32_t*)malloc(addressIndicesCount * sizeof(uint32_t)); - assert(addressIndices); - memcpy(addressIndices, _addressIndices.data(), addressIndicesCount * sizeof(uint32_t)); + if (addressIndicesCount) { + addressIndices = (uint32_t*)malloc(addressIndicesCount * sizeof(uint32_t)); + assert(addressIndices); + memcpy(addressIndices, _addressIndices.data(), addressIndicesCount * sizeof(uint32_t)); + } } DataBlock() : From fad5de60ab1b38c595b31c8bee5f330a6ef40568 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 6 Mar 2026 11:24:52 +0100 Subject: [PATCH 25/65] use Runtime Dictionary for now, fixed bug while loading from state --- src/MainServer.cpp | 2 +- src/blockchain/FileBased.cpp | 21 ++++++++++++++------- src/blockchain/FileBased.h | 2 +- src/cache/Block.cpp | 2 +- src/cache/BlockIndex.cpp | 3 ++- src/lib/PersistentDictionary.h | 27 ++++++++++++++++++++++++--- 6 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/MainServer.cpp b/src/MainServer.cpp index 9a6546f..4a5c77a 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -77,7 +77,7 @@ bool MainServer::init() auto communityDictionary = make_unique>(ServerGlobals::g_FilesPath + "/communityIdsCache"); communityDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); - auto nameHashDictionary = make_unique>(ServerGlobals::g_FilesPath + "/nameHashCache"); + auto nameHashDictionary = make_unique>(ServerGlobals::g_FilesPath + "/nameHashCache"); nameHashDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); g_appContext = make_unique(std::move(communityDictionary), std::move(nameHashDictionary)); g_appContext->syncCommunityContextsWithCommunityIds(); diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 55ffc60..c9185be 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -92,13 +92,16 @@ namespace gradido { return false; } lock_guard _lock(mWorkMutex); - if (!mPublicKeysIndex.init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { + if (mPublicKeysIndex.init(0)) { + LOG_F(WARNING, "dictionary is again persistent, please update here"); + } + /*if (!mPublicKeysIndex.init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { // remove index files for regenration LOG_F(WARNING, "reset the public key index file"); // mCachedBlocks.clear(); mPublicKeysIndex.reset(); resetBlockIndices = true; - } + }*/ if (resetBlockIndices) { model::files::BlockIndex::removeAllBlockIndexFiles(mFolderPath); @@ -121,8 +124,14 @@ namespace gradido { bool FileBased::startValidationTransactions() { if (mStopToken.stop_requested()) return false; + lock_guard _lock(mWorkMutex); auto lastBlockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); + + // trigger block index creation, only needed here for non-persistent public key dictionary + iterateBlocks(SearchDirection::ASC, [](const cache::Block& block) -> bool { return true; }); + loadStateFromBlockCache(); + if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { mLedgerAnchorCache.reset(); if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { @@ -597,7 +606,7 @@ namespace gradido { ) const { lock_guard _lock(mWorkMutex); - auto transactionNr = mLedgerAnchorCache.has(ledgerAnchor); + auto transactionNr = mLedgerAnchorCache.getTransactionNrForLedgerAnchor(ledgerAnchor); if (transactionNr) { return getTransactionForId(transactionNr); } @@ -616,13 +625,11 @@ namespace gradido { void FileBased::loadStateFromBlockCache() { - Profiler timeUsed; mBlockchainState.updateState(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); auto lastBlockNr = model::files::Block::findLastBlockFileInFolder(mFolderPath); mBlockchainState.updateState(cache::DefaultStateKeys::LAST_BLOCK_NR, lastBlockNr); auto& block = getBlock(lastBlockNr); mBlockchainState.updateState(cache::DefaultStateKeys::LAST_TRANSACTION_ID, block.getBlockIndex().getMaxTransactionNr()); - LOG_F(INFO, "timeUsed: %s", timeUsed.string().data()); } // TODO: look for a way of reusing logic from interaction::confirmTransaction, with nearly the same code in the roles @@ -729,8 +736,8 @@ namespace gradido { Filter f; f.searchDirection = SearchDirection::ASC; int countTarget = countToValidate; - if (countToValidate) { - f.minTransactionNr = lastTransaction->getTransactionNr() - countToValidate; + if (countToValidate && lastTransaction->getTransactionNr() > countToValidate) { + f.minTransactionNr = lastTransaction->getTransactionNr() - countToValidate + 1; } else { countTarget = lastTransaction->getTransactionNr(); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 2bf0636..f42705f 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -222,7 +222,7 @@ namespace gradido { std::shared_ptr mHieroMessageListener; //! contain indices for every public key address, used overall for optimisation - mutable PersistentDictionary mPublicKeysIndex; + mutable PersistentDictionary mPublicKeysIndex; // level db to store state values like last transaction // TODO: speedup with atcual struct, write out into leveldb/lmdb only on changes, maybe even buffered, think on exit management mutable cache::State mBlockchainState; diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index fd959f7..48dd97c 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -89,7 +89,7 @@ namespace cache { LOG_F(FATAL, "rebuildBlockIndex Task isn't finished after waiting a whole minute"); Application::terminate(); } - LOG_F(INFO, "time for rebuilding block index for block %s: %s", mBlockFile->getBlockPath().data(), timeUsed.string().data()); + LOG_F(INFO, "rebuilding block index for block %s in %s", mBlockFile->getBlockPath().data(), timeUsed.string().data()); mBlockIndex->writeIntoFile(); } else { diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index 5e1803f..00ddb48 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -51,7 +51,8 @@ namespace cache { clearIndexEntries(); mTransactionNrsFileCursors.clear(); model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); - LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().c_str()); + // only needed if public key dictionary is again persistend + // LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().c_str()); blockIndexFile.reset(); mMaxTransactionNr = 0; mMinTransactionNr = 0; diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h index eed6e0a..bd4f6b7 100644 --- a/src/lib/PersistentDictionary.h +++ b/src/lib/PersistentDictionary.h @@ -3,21 +3,42 @@ #include "../model/files/LevelDBWrapper.h" #include "../serialization/String.h" +#include "gradido_blockchain/lib/Dictionary.h" #include "gradido_blockchain/lib/DictionaryInterface.h" #include "gradido_blockchain/lib/DictionaryExceptions.h" #include "loguru/loguru.hpp" +#include +#include #include #include -#include -#include + +// use simple in memory threadsafe dictionary for now, until updated transaction and address index are stored into files +template< + typename DataType, + typename Hash = std::hash, + typename Equal = std::equal_to +> +class PersistentDictionary : public ThreadsafeRuntimeDictionary +{ +public: + explicit PersistentDictionary(const std::string& directory): ThreadsafeRuntimeDictionary(directory) {} + ~PersistentDictionary() {} + + inline bool init(size_t cacheInBytes) { return false; }; + inline void exit() { ThreadsafeRuntimeDictionary::reset(); }; + inline size_t getLastIndex() { + return RuntimeDictionary::mIndexDataLookup.size(); + } +}; // TODO: check usage of LMDB // it is magnitude faster but especially it is designed for prevent data loss on system failure, data can only be corrupted through hardware failure! // - https://de.wikipedia.org/wiki/Lightning_Memory-Mapped_Database // - https://github.com/LMDB/lmdb/tree/mdb.master/libraries/liblmdb // TODO: remove bi-directionality, update whole code for not using getDataForIndex at all! +/* template requires serialization::HasString class PersistentDictionary: public IMutableDictionary @@ -153,5 +174,5 @@ bool PersistentDictionary::hasIndex(size_t index) const auto it = mIndexDataReverseLookup.find(index); return it != mIndexDataReverseLookup.end(); } - +*/ #endif //__GRADIDO_NODE_PERSISTENT_DICTIONARY_H \ No newline at end of file From 2b8adb06a64687751951898210599eeaf74ef6d6 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 6 Mar 2026 12:03:19 +0100 Subject: [PATCH 26/65] fix for linux build --- README.md | 6 ++++++ src/server/json-rpc/ApiHandler.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index d5c2602..ed9ee02 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,9 @@ install rust compiler ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` + +ON debian 12 64Bit +cmake .. -DUSE_INSTALLED_SSL=On -DCMAKE_BUILD_TYPE=Release -DTARGET=x86_64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake + +On Debian 12 ARM64 Server +cmake .. -DOPENSSL_ROOT_DIR=/usr/lib/aarch64-linux-gnu -DCMAKE_BUILD_TYPE=Release -DUSE_INSTALLED_SSL=On -DTARGET=aarch64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake \ No newline at end of file diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index 8b3faf5..f177ec8 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -5,6 +5,8 @@ #include "gradido_blockchain/types.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include + namespace gradido { namespace blockchain { class Abstract; From 1762c1427f73122752d21584d0d95cd3c632c940 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 15:09:06 +0100 Subject: [PATCH 27/65] update docker for cross compiling --- CMakeLists.txt | 80 +++++++++++------- Dockerfile | 89 ++++++++++++++++++-- Dockerfile.grpc-deps | 140 ++++++++++++++++++++++++++++++++ README.md | 15 +++- dependencies/gradido_blockchain | 2 +- 5 files changed, 287 insertions(+), 39 deletions(-) create mode 100644 Dockerfile.grpc-deps diff --git a/CMakeLists.txt b/CMakeLists.txt index 301a360..afe142a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,18 +1,24 @@ cmake_minimum_required(VERSION 3.18.2) set(CMAKE_POLICY_VERSION_MINIMUM 3.5) set(CMAKE_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain.cmake") -execute_process( - COMMAND git describe --tags --dirty --always - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - OUTPUT_VARIABLE GRADIDO_NODE_GIT_VER - OUTPUT_STRIP_TRAILING_WHITESPACE -) +if(DEFINED ENV{GRADIDO_VERSION} AND NOT "$ENV{GRADIDO_VERSION}" STREQUAL "") + set(GRADIDO_NODE_GIT_VER $ENV{GRADIDO_VERSION}) +else() + execute_process( + COMMAND git describe --tags --dirty --always + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GRADIDO_NODE_GIT_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() + string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" GRADIDO_NODE_CLEAN_VERSION "${GRADIDO_NODE_GIT_VER}") project(GradidoNode VERSION ${GRADIDO_NODE_CLEAN_VERSION} LANGUAGES C CXX) SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) message("runtime output directory: ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") option(BUILD_SHARED_LIBS "Build using shared libraries" OFF) +option(GRPC_FETCH_CONTENT "Download and build gRPC via FetchContent" ON) IF(WIN32) set(CMAKE_CXX_FLAGS "/MP /EHsc /std:c++20") set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") @@ -131,38 +137,56 @@ set(ENABLE_HTTPS ON) # set(VCPKG_TARGET_TRIPLET "x64-windows" CACHE STRING "vcpkg target triplet") #endif() add_subdirectory("dependencies/gradido_blockchain") + find_package(OpenSSL REQUIRED) # reuse cacert.pem from gradido_blockchain if(WIN32) add_compile_definitions(CACERT_PEM_PATH="${CACERT_PEM}") endif() -find_package(OpenSSL REQUIRED) + add_subdirectory("dependencies/jsonrpcpp") option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" OFF) add_subdirectory("dependencies/leveldb") #add_subdirectory("dependencies/googletest") -# Fetch Content Module -include(FetchContent) - -# gRPC -set(gRPC_SSL_PROVIDER "package" CACHE STRING "gRPC SSL provider" FORCE) -set(gRPC_INSTALL ON CACHE BOOL "" FORCE) -set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE) - -message(STATUS "Make gRPC Available") -FetchContent_Declare( - grpc - GIT_REPOSITORY https://github.com/grpc/grpc.git - GIT_TAG v1.74.1 -) -FetchContent_MakeAvailable(grpc) - -if(NOT WIN32) - include_directories(${grpc_SOURCE_DIR}/include) - include_directories(${grpc_SOURCE_DIR}/third_party/abseil-cpp) - message(${grpc_SOURCE_DIR}) +#find_package(gRPC CONFIG QUIET) +#find_package(Protobuf QUIET) + +if(GRPC_FETCH_CONTENT) + # Fetch Content Module + include(FetchContent) + + # gRPC + set(FETCHCONTENT_QUIET OFF) + set(FETCHCONTENT_UPDATES_DISCONNECTED TRUE) + set(gRPC_SSL_PROVIDER "package" CACHE STRING "gRPC SSL provider" FORCE) + set(gRPC_INSTALL ON CACHE BOOL "" FORCE) + set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(gRPC_BUILD_CSHARP_EXT OFF CACHE BOOL "" FORCE) + + message(STATUS "Make gRPC Available") + FetchContent_Declare( + grpc + GIT_REPOSITORY https://github.com/grpc/grpc.git + GIT_TAG v1.74.1 + GIT_SHALLOW TRUE + ) + FetchContent_MakeAvailable(grpc) + + if(NOT WIN32) + include_directories(${grpc_SOURCE_DIR}/include) + include_directories(${grpc_SOURCE_DIR}/third_party/abseil-cpp) + message(${grpc_SOURCE_DIR}) + endif() + set(_GRPC_GRPCPP grpc++) +else() + message(STATUS "Using pre-built gRPC from docker") + list(APPEND CMAKE_PREFIX_PATH "/usr/local") + include_directories(/usr/local/include) + find_package(Protobuf REQUIRED) + find_package(gRPC CONFIG REQUIRED) + set(_GRPC_GRPCPP gRPC::grpc++) endif() # Since FetchContent uses add_subdirectory under the hood, we can use @@ -170,7 +194,7 @@ endif() # set(_PROTOBUF_LIBPROTOBUF libprotobuf) #set(_REFLECTION grpc++_reflection) #set(_PROTOBUF_PROTOC $) -set(_GRPC_GRPCPP grpc++) + #message("grpc++: ${_GRPC_GRPCPP}") SET(GradidoBlockchain_BINARY_DIR ${GradidoNode_BINARY_DIR}) diff --git a/Dockerfile b/Dockerfile index 40cf497..6584d9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ + + ##### Run environment for tests ##### FROM alpine:3.22.2 AS test_run @@ -5,11 +7,32 @@ FROM alpine:3.22.2 AS test_run ##### BUILD-ENV ##### FROM denisgolius/zig:0.15.2 AS zig_build +ARG ZIG_TARGET=x86_64-linux-musl +#aarch64-linux-musl +ARG TARGET_ARCH=x86_64 +# aarch64 +ENV ZIG_TARGET=${ZIG_TARGET} +ENV TARGET_ARCH=${TARGET_ARCH} + RUN apk add --no-cache cmake make git binutils openssl-dev openssl-libs-static + ##### BUILD debug ####### FROM zig_build AS debug_build +ARG ZIG_TARGET=x86_64-linux-musl +#aarch64-linux-musl +ARG TARGET_ARCH=x86_64 + +ARG GRADIDO_VERSION=0.0.0-dev +# aarch64 +ENV ZIG_TARGET=${ZIG_TARGET} +ENV TARGET_ARCH=${TARGET_ARCH} +ENV GRADIDO_VERSION=${GRADIDO_VERSION} + +# copy grpc-deps artifacts +COPY --from=grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local + ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} @@ -17,25 +40,73 @@ COPY . . RUN mkdir build WORKDIR ${DOCKER_WORKDIR}/build -RUN cmake .. -DENABLE_TEST=On -DENABLE_HTTPS=On -DUSE_INSTALLED_SSL=On -DCMAKE_EXE_LINKER_FLAGS="-static" -DBUILD_SHARED_LIBS=Off +RUN cmake .. \ + -DENABLE_TEST=On \ + -DENABLE_HTTPS=On \ + -DUSE_INSTALLED_SSL=On \ + -DCMAKE_EXE_LINKER_FLAGS="-static" \ + -DBUILD_SHARED_LIBS=Off \ + -DTARGET=${ZIG_TARGET} \ + -DGRPC_FETCH_CONTENT=Off + # OpenSSL paths for CMake + #-DOPENSSL_ROOT_DIR=/opt/${TARGET_ARCH}/usr \ + #-DOPENSSL_INCLUDE_DIR=/opt/${TARGET_ARCH}/usr/include \ + #-DOPENSSL_CRYPTO_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libcrypto.a \ + #-DOPENSSL_SSL_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libssl.a RUN make -j$(nproc) GradidoNode ENTRYPOINT [] -##### run tests ####### -FROM test_run AS test_build_run +##### BUILD release ####### +FROM zig_build AS release_build -COPY --from=debug_build /code/build/test/GradidoBlockchainTest ./ +ARG ZIG_TARGET=x86_64-linux-musl +#aarch64-linux-musl +ARG TARGET_ARCH=x86_64 -ENTRYPOINT [] +ARG GRADIDO_VERSION=0.0.0-dev +# aarch64 +ENV ZIG_TARGET=${ZIG_TARGET} +ENV TARGET_ARCH=${TARGET_ARCH} +ENV GRADIDO_VERSION=${GRADIDO_VERSION} + +# copy grpc-deps artifacts +COPY --from=grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local -##### BUILD release ####### -FROM zig_build AS release_build ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} COPY . . + RUN mkdir build WORKDIR ${DOCKER_WORKDIR}/build -RUN cmake -DCMAKE_BUILD_TYPE=Release .. -RUN make -j$(nproc) GradidoBlockchain +RUN cmake .. \ + -DENABLE_TEST=Off \ + -DENABLE_HTTPS=On \ + -DUSE_INSTALLED_SSL=On \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_C_FLAGS="-g0" \ + -DCMAKE_CXX_FLAGS="-g0" \ + -DCMAKE_EXE_LINKER_FLAGS="-static" \ + -DBUILD_SHARED_LIBS=Off \ + -DTARGET=${ZIG_TARGET} \ + -DGRPC_FETCH_CONTENT=Off + # OpenSSL paths for CMake + #-DOPENSSL_ROOT_DIR=/opt/${TARGET_ARCH}/usr \ + #-DOPENSSL_INCLUDE_DIR=/opt/${TARGET_ARCH}/usr/include \ + #-DOPENSSL_CRYPTO_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libcrypto.a \ + #-DOPENSSL_SSL_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libssl.a +RUN make -j$(nproc) GradidoNode + +ENTRYPOINT ["/bin/GradidoNode"] + + +##### run tests ####### +FROM test_run AS test_build_run + +COPY --from=debug_build /code/build/test/GradidoBlockchainTest ./ + +ENTRYPOINT [] + diff --git a/Dockerfile.grpc-deps b/Dockerfile.grpc-deps new file mode 100644 index 0000000..c55a8c6 --- /dev/null +++ b/Dockerfile.grpc-deps @@ -0,0 +1,140 @@ +# ============================================================================= +# grpc-deps-builder +# Builds static OpenSSL + gRPC for x86_64-linux-musl and aarch64-linux-musl +# Final image contains only the compiled artifacts (headers + .a files) +# Written by Claude Code, 2026-03-06 +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Stage 1: Common build tools +# ----------------------------------------------------------------------------- +FROM denisgolius/zig:0.15.2 AS base + +RUN apk add --no-cache \ + cmake make git binutils perl wget \ + # needed by gRPC cmake scripts + autoconf automake libtool + +# Zig wrapper helper script +# Usage: make-zig-wrappers +RUN cat > /usr/local/bin/make-zig-wrappers <<'EOF' +#!/bin/sh +TARGET="$1" +printf '#!/bin/sh\nexec zig cc -target %s "$@"\n' "$TARGET" > /usr/local/bin/zigcc +printf '#!/bin/sh\nexec zig c++ -target %s "$@"\n' "$TARGET" > /usr/local/bin/zigcxx +# ar und ranlib brauchen kein target-flag +printf '#!/bin/sh\nexec zig ar "$@"\n' > /usr/local/bin/zigar +printf '#!/bin/sh\nexec zig ranlib "$@"\n' > /usr/local/bin/zigranlib +chmod +x /usr/local/bin/zigcc /usr/local/bin/zigcxx \ + /usr/local/bin/zigar /usr/local/bin/zigranlib +EOF +RUN chmod +x /usr/local/bin/make-zig-wrappers + + +FROM base AS libraries_src + +RUN wget -q https://www.openssl.org/source/openssl-3.3.2.tar.gz && \ + tar xf openssl-3.3.2.tar.gz + +RUN git clone --depth 1 --branch v1.74.1 \ + --recurse-submodules \ + --shallow-submodules \ + https://github.com/grpc/grpc.git /tmp/grpc + +# ----------------------------------------------------------------------------- +# Stage 2a: OpenSSL for x86_64-linux-musl +# ----------------------------------------------------------------------------- +FROM libraries_src AS openssl_x86_64 + +RUN make-zig-wrappers x86_64-linux-musl + +RUN cd openssl-3.3.2 && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + ./Configure linux-x86_64 \ + no-shared no-tests no-docs \ + #--prefix=/opt/x86_64/usr \ + && \ + make -j$(nproc) && \ + make install_sw && \ + cd .. && rm -rf openssl-3.3.2* + +# ----------------------------------------------------------------------------- +# Stage 2b: OpenSSL for aarch64-linux-musl +# ----------------------------------------------------------------------------- +FROM libraries_src AS openssl_aarch64 + +RUN make-zig-wrappers aarch64-linux-musl + +RUN cd openssl-3.3.2 && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + ./Configure linux-aarch64 \ + no-shared no-tests no-docs \ + #--prefix=/opt/aarch64/usr \ + && \ + make -j$(nproc) && \ + make install_sw && \ + cd .. && rm -rf openssl-3.3.2* + +FROM openssl_x86_64 AS grpc_x86_64 + +RUN make-zig-wrappers x86_64-linux-musl + +RUN mkdir -p /tmp/grpc/cmake/build && \ + cd /tmp/grpc/cmake/build && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + cmake -DCMAKE_CXX_STANDARD=20 ../.. \ + #-DCMAKE_INSTALL_PREFIX=/opt/x86_64/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_C_FLAGS="-g0" \ + -DCMAKE_CXX_FLAGS="-g0" \ + -DZLIB_BUILD_EXAMPLES=OFF \ + -DZLIB_SHARED=OFF \ + -DZLIB_STATIC=ON \ + -DgRPC_SSL_PROVIDER=package && \ + make -j$(nproc) && make install + +FROM openssl_aarch64 AS grpc_aarch64 + +#COPY --from=grpc-deps:v1.74.1 /opt/x86_64 /opt/x86_64 + +RUN make-zig-wrappers aarch64-linux-musl + +RUN mkdir -p /tmp/grpc/cmake/build && \ + cd /tmp/grpc/cmake/build && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + cmake -DCMAKE_CXX_STANDARD=20 ../.. \ + #-DCMAKE_INSTALL_PREFIX=/opt/aarch64/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_C_FLAGS="-g0" \ + -DCMAKE_CXX_FLAGS="-g0" \ + -DZLIB_BUILD_EXAMPLES=OFF \ + -DZLIB_SHARED=OFF \ + -DZLIB_STATIC=ON \ + #-DgRPC_PROTOBUF_PROVIDER=package \ + #-DgRPC_PROTOBUF_PACKAGE_TYPE=MODULE \ + #-DProtobuf_PROTOC_EXECUTABLE=/opt/x86_64/usr/bin/protoc \ + #-DgRPC_CPP_PLUGIN_EXECUTABLE=/opt/x86_64/usr/bin/grpc_cpp_plugin \ + #-DgRPC_BUILD_GRPC_CPP_PLUGIN=OFF \ + -DgRPC_BUILD_CODEGEN=OFF \ + -DgRPC_SSL_PROVIDER=package && \ + make -j$(nproc) && make install +RUN find /usr/local/lib -name "*.so*" -delete + +# ----------------------------------------------------------------------------- +# Stage 4: Final image — only the compiled artifacts, no build tools +# ----------------------------------------------------------------------------- +FROM busybox:musl AS final + +COPY --from=grpc_x86_64 /usr/local /opt/x86_64/local +COPY --from=grpc_aarch64 /usr/local /opt/aarch64/local + +# Layout: +# /x86_64/usr/lib/ -> libgrpc.a, libssl.a, libcrypto.a, ... +# /x86_64/usr/include/ -> grpc/, openssl/, google/protobuf/, ... +# /x86_64/usr/bin/ -> grpc_cpp_plugin (host tool, x86_64 only useful) +# /aarch64/usr/lib/ -> same for aarch64 +# /aarch64/usr/include/ -> same for aarch64 \ No newline at end of file diff --git a/README.md b/README.md index ed9ee02..de7d388 100644 --- a/README.md +++ b/README.md @@ -45,4 +45,17 @@ ON debian 12 64Bit cmake .. -DUSE_INSTALLED_SSL=On -DCMAKE_BUILD_TYPE=Release -DTARGET=x86_64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake On Debian 12 ARM64 Server -cmake .. -DOPENSSL_ROOT_DIR=/usr/lib/aarch64-linux-gnu -DCMAKE_BUILD_TYPE=Release -DUSE_INSTALLED_SSL=On -DTARGET=aarch64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake \ No newline at end of file +cmake .. -DOPENSSL_ROOT_DIR=/usr/lib/aarch64-linux-gnu -DCMAKE_BUILD_TYPE=Release -DUSE_INSTALLED_SSL=On -DTARGET=aarch64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake + + +## Docker +docker build \ + --build-arg VERSION=$(git describe --tags --dirty --always) \ + -t gradido-node . +Build for aarch64 +docker build \ + --build-arg VERSION=$(git describe --tags --dirty --always) \ + --build-arg ZIG_TARGET=aarch64-linux-musl \ + --build-arg TARGET_ARCH=aarch64 \ + --target debug_build \ + -t gradido-node-aarch64 . diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 1bf5f15..c10843b 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 1bf5f15e47c5585da3b3e2075ec9091ddd7e2621 +Subproject commit c10843b405bff0a0c9e826ae02f3db26b5889501 From 6de65ca46a3b3730636270330c0baa7e2ec93f98 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 15:25:14 +0100 Subject: [PATCH 28/65] use gRPC dependencies uploaded to docker hub --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6584d9e..8389e50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ ENV TARGET_ARCH=${TARGET_ARCH} ENV GRADIDO_VERSION=${GRADIDO_VERSION} # copy grpc-deps artifacts -COPY --from=grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local +COPY --from=gradido/grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} @@ -71,7 +71,7 @@ ENV TARGET_ARCH=${TARGET_ARCH} ENV GRADIDO_VERSION=${GRADIDO_VERSION} # copy grpc-deps artifacts -COPY --from=grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local +COPY --from=gradido/grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} From 484d2d74d5228e0ddc4268bed55c1294768cd121 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 16:08:19 +0100 Subject: [PATCH 29/65] update release workflow, triggered by new tag --- .github/workflows/release.yml | 195 ++++++++++++++++++++++++---------- 1 file changed, 136 insertions(+), 59 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 555bad5..b4b7ab2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,69 +2,146 @@ name: Build and Release on: push: - branches: - - 'release/*' - + tags: + - 'v*' jobs: - build: - name: Build Binaries + # ─── Linux Builds via Docker ─────────────────────── + linux-x64-build: runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Configure and build - run: | - mkdir build - cd build - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF .. - make -j$(nproc) - cmake .. - make -j$(nproc) GradidoNode - - - name: Archive binaries - uses: actions/upload-artifact@v2 - with: - name: binaries - path: build/bin/* - - release: - name: Create Release + - uses: actions/checkout@v4 + - name: Build x86_64 + run: | + docker build \ + --build-arg VERSION=${{ github.ref_name }} \ + --target release_build \ + -t gradido-node-x86_64 . + docker create --name extract-x64 gradido-node-x86_64 + docker cp extract-x64:/code/build/bin/GradidoNode ./GradidoNode-linux-x86_64 + docker rm extract-x64 + + - name: compress + run: tar -czf GradidoNode-linux-x86_64.tar.gz GradidoNode-linux-x86_64 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: linux-binaries + path: GradidoNode-linux-*.tar.gz + + linux-aarch64-build: runs-on: ubuntu-latest - needs: build - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + - name: Build aarch64 + run: | + docker build \ + --build-arg VERSION=${{ github.ref_name }} \ + --build-arg ZIG_TARGET=aarch64-linux-musl \ + --build-arg TARGET_ARCH=aarch64 \ + --target release_build \ + -t gradido-node-aarch64 . + docker create --name extract-arm gradido-node-aarch64 + docker cp extract-arm:/code/build/bin/GradidoNode ./GradidoNode-linux-aarch64 + docker rm extract-arm + + - name: compress + run: tar -czf GradidoNode-linux-aarch64.tar.gz GradidoNode-linux-aarch64 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: linux-binaries + path: GradidoNode-linux-*.tar.gz + + # ─── Windows Native Build ────────────────────────── + windows-build: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup MSVC + uses: microsoft/setup-msbuild@v2 + - name: Configure + run: | + mkdir build + cd build + cmake .. ` + -DENABLE_TEST=Off ` + -DENABLE_HTTPS=On ` + -DCMAKE_BUILD_TYPE=Release + + - name: Build + run: | + cd build + cmake --build . --target GradidoNode --config Release + + - name: Collect all and compress + run: | + $files = Get-ChildItem -Path build/Release -Include *.exe,*.dll -Recurse + Compress-Archive -Path $files -DestinationPath GradidoNode-windows-x86_64.zip + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: windows-binary + path: GradidoNode-windows-x86_64.zip + + # ─── Release erstellen ───────────────────────────── + create-release: + needs: [linux-x64-build, linux-aarch64-build, windows-build] + runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Download binaries - uses: actions/download-artifact@v2 - with: - name: binaries - path: build/bin/ - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: | - Describe your release here. - draft: false - prerelease: false - - - name: Upload binaries - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: build/bin/* - asset_name: ${{ github.ref }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # für PR-History~ + + - name: Download alle Artifacts + uses: actions/download-artifact@v4 + + - name: generate Release Notes from PRs + id: notes + uses: actions/github-script@v7 + with: + script: | + const tags = await github.rest.repos.listTags({ + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 2 + }); + const previousTag = tags.data[1]?.name; + + // Collect PRs since last tag~ + const comparison = await github.rest.repos.compareCommits({ + owner: context.repo.owner, + repo: context.repo.repo, + base: previousTag || 'HEAD~10', + head: context.ref_name + }); + + const prs = comparison.data.commits + .map(c => c.commit.message) + .filter(m => m.includes('Merge pull request')) + .join('\n'); + + return prs || "No PRs found~ Direct pushes detected~ 😈"; + + - name: create GitHub Release + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.ref_name }} + body: | + ## Changes~ + ${{ steps.notes.outputs.result }} + ## Downloads~ + | Platform | Architecture | File | + |---|---|---| + | Linux | x86_64 | GradidoNode-linux-x86_64.tar.gz | + | Linux | aarch64 | GradidoNode-linux-aarch64.tar.gz | + | Windows | x86_64 | GradidoNode-windows-x86_64.zip | + files: | + linux-binaries/GradidoNode-linux-x86_64.tar.gz + linux-binaries/GradidoNode-linux-aarch64.tar.gz + windows-binary/GradidoNode-windows-x86_64.zip From 9f4f78c58dec24ab4c9e836cd5bc46fcdcc48ada Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 16:12:58 +0100 Subject: [PATCH 30/65] use recursive checkout in release --- .github/workflows/release.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4b7ab2..c1f6d80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 - name: Build x86_64 run: | docker build \ @@ -33,6 +36,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 - name: Build aarch64 run: | docker build \ @@ -59,7 +65,9 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v4 - + with: + submodules: recursive + fetch-depth: 0 - name: Setup MSVC uses: microsoft/setup-msbuild@v2 @@ -125,7 +133,7 @@ jobs: .filter(m => m.includes('Merge pull request')) .join('\n'); - return prs || "No PRs found~ Direct pushes detected~ 😈"; + return prs || "No PRs found~ Direct pushes detected~"; - name: create GitHub Release uses: softprops/action-gh-release@v1 From 41b73a5cc81fad37f498207f6d147d33728dad05 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 16:35:33 +0100 Subject: [PATCH 31/65] add zig setup to windows build step --- .github/workflows/release.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c1f6d80..2fad609 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: linux-binaries - path: GradidoNode-linux-*.tar.gz + path: GradidoNode-linux-x86_64.tar.gz linux-aarch64-build: runs-on: ubuntu-latest @@ -58,16 +58,20 @@ jobs: uses: actions/upload-artifact@v4 with: name: linux-binaries - path: GradidoNode-linux-*.tar.gz + path: GradidoNode-linux-aarch64.tar.gz # ─── Windows Native Build ────────────────────────── windows-build: - runs-on: windows-latest + runs-on: windows-2022 steps: - uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 + - name: install zig + uses: https://codeberg.org/mlugg/setup-zig@v2 + with: + version: 0.15.2 - name: Setup MSVC uses: microsoft/setup-msbuild@v2 From 043197411e39c9d37393170a13fad340d1f73783 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 16:37:10 +0100 Subject: [PATCH 32/65] fix zig setup action name --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2fad609..91d1b6c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: submodules: recursive fetch-depth: 0 - name: install zig - uses: https://codeberg.org/mlugg/setup-zig@v2 + uses: mlugg/setup-zig@v2 with: version: 0.15.2 - name: Setup MSVC From 223585750d872e5ffe9b7336d230b2882cc19dc6 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 17:10:06 +0100 Subject: [PATCH 33/65] fix double artifact names --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91d1b6c..7211f71 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: linux-binaries + name: linux-x86_64 path: GradidoNode-linux-x86_64.tar.gz linux-aarch64-build: @@ -57,7 +57,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: linux-binaries + name: linux-aarch64 path: GradidoNode-linux-aarch64.tar.gz # ─── Windows Native Build ────────────────────────── @@ -97,7 +97,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: windows-binary + name: windows-x86_64 path: GradidoNode-windows-x86_64.zip # ─── Release erstellen ───────────────────────────── @@ -154,6 +154,6 @@ jobs: | Linux | aarch64 | GradidoNode-linux-aarch64.tar.gz | | Windows | x86_64 | GradidoNode-windows-x86_64.zip | files: | - linux-binaries/GradidoNode-linux-x86_64.tar.gz - linux-binaries/GradidoNode-linux-aarch64.tar.gz - windows-binary/GradidoNode-windows-x86_64.zip + linux-x86_64/GradidoNode-linux-x86_64.tar.gz + linux-aarch64/GradidoNode-linux-aarch64.tar.gz + windows-x86_64/GradidoNode-windows-x86_64.zip From 7b65ec4cbd7706536e93cdf0ddc6c361cd73380e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 7 Mar 2026 17:38:03 +0100 Subject: [PATCH 34/65] fix windows release build path --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7211f71..f8b8499 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ jobs: - name: Collect all and compress run: | - $files = Get-ChildItem -Path build/Release -Include *.exe,*.dll -Recurse + $files = Get-ChildItem -Path build/bin/Release -Include *.exe,*.dll -Recurse Compress-Archive -Path $files -DestinationPath GradidoNode-windows-x86_64.zip - name: Upload artifact From ee0fb861b914143b1212dcedd9d74c763bc86b07 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 9 Mar 2026 12:44:02 +0100 Subject: [PATCH 35/65] use action for collect prs as changelog --- .github/workflows/release.yml | 37 +++++++++-------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c1f6d80..568dff6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,8 +66,9 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: recursive - fetch-depth: 0 + submodules: recursive + fetch-depth: 0 + - name: Setup MSVC uses: microsoft/setup-msbuild@v2 @@ -108,32 +109,12 @@ jobs: - name: Download alle Artifacts uses: actions/download-artifact@v4 - - name: generate Release Notes from PRs - id: notes - uses: actions/github-script@v7 + - name: Update CHANGELOG + id: changelog + uses: requarks/changelog-action@v1.10.3 with: - script: | - const tags = await github.rest.repos.listTags({ - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 2 - }); - const previousTag = tags.data[1]?.name; - - // Collect PRs since last tag~ - const comparison = await github.rest.repos.compareCommits({ - owner: context.repo.owner, - repo: context.repo.repo, - base: previousTag || 'HEAD~10', - head: context.ref_name - }); - - const prs = comparison.data.commits - .map(c => c.commit.message) - .filter(m => m.includes('Merge pull request')) - .join('\n'); - - return prs || "No PRs found~ Direct pushes detected~"; + token: ${{ github.token }} + tag: ${{ github.ref_name }} - name: create GitHub Release uses: softprops/action-gh-release@v1 @@ -141,7 +122,7 @@ jobs: name: Release ${{ github.ref_name }} body: | ## Changes~ - ${{ steps.notes.outputs.result }} + ${{ steps.changelog.outputs.result }} ## Downloads~ | Platform | Architecture | File | From 7c2b326898b61d3ed35fc75ef0a9424aed0e6d3f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 9 Mar 2026 13:48:35 +0100 Subject: [PATCH 36/65] add wrong formatted commits in changelog for now --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d91bc81..c61d33a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,6 +119,7 @@ jobs: with: token: ${{ github.token }} tag: ${{ github.ref_name }} + includeInvalidCommits: true - name: create GitHub Release uses: softprops/action-gh-release@v1 From 499e986c2bbc1ea7e621cd084620ceef8472ece9 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 9 Mar 2026 17:14:14 +0100 Subject: [PATCH 37/65] =?UTF-8?q?add=20missing=20dlls=20f=C3=BCr=20windows?= =?UTF-8?q?=20build,=20strip=20linux=20builds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 46 +++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c61d33a..fab0eb0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,17 +20,19 @@ jobs: --target release_build \ -t gradido-node-x86_64 . docker create --name extract-x64 gradido-node-x86_64 - docker cp extract-x64:/code/build/bin/GradidoNode ./GradidoNode-linux-x86_64 + docker cp extract-x64:/code/build/bin/GradidoNode ./GradidoNode docker rm extract-x64 - - name: compress - run: tar -czf GradidoNode-linux-x86_64.tar.gz GradidoNode-linux-x86_64 + - name: Strip binary + run: strip ./GradidoNode + - name: compress + run: tar -czf gradido_node-${{ github.ref_name }}-linux-x64.tar.gz GradidoNode - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: linux-x86_64 - path: GradidoNode-linux-x86_64.tar.gz + path: gradido_node-${{ github.ref_name }}-linux-x64.tar.gz linux-aarch64-build: runs-on: ubuntu-latest @@ -48,11 +50,16 @@ jobs: --target release_build \ -t gradido-node-aarch64 . docker create --name extract-arm gradido-node-aarch64 - docker cp extract-arm:/code/build/bin/GradidoNode ./GradidoNode-linux-aarch64 + docker cp extract-arm:/code/build/bin/GradidoNode ./GradidoNode docker rm extract-arm + - name: Strip binary + run: | + sudo apt-get install -y binutils-aarch64-linux-gnu + aarch64-linux-gnu-strip ./GradidoNode + - name: compress - run: tar -czf GradidoNode-linux-aarch64.tar.gz GradidoNode-linux-aarch64 + run: tar -czf gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz GradidoNode - name: Upload artifacts uses: actions/upload-artifact@v4 @@ -91,15 +98,23 @@ jobs: cmake --build . --target GradidoNode --config Release - name: Collect all and compress + shell: powershell run: | - $files = Get-ChildItem -Path build/bin/Release -Include *.exe,*.dll -Recurse - Compress-Archive -Path $files -DestinationPath GradidoNode-windows-x86_64.zip + $exeAndDll = Get-ChildItem -Path build/bin/Release -Include *.exe,*.dll -Recurse | Select-Object -ExpandProperty FullName + + $opensslDlls = @( + "dependencies/gradido_blockchain/dependencies/vcpkg/packages/openssl_x64-windows/bin/libssl-3-x64.dll", + "dependencies/gradido_blockchain/dependencies/vcpkg/packages/openssl_x64-windows/bin/libcrypto-3-x64.dll" + ) + $files = $exeAndDll + $opensslDlls + $files = $files | Where-Object { Test-Path $_ } + Compress-Archive -Path $files -DestinationPath "gradido_node-${{ github.ref_name }}-win32-x64.zip" -Force - name: Upload artifact uses: actions/upload-artifact@v4 with: name: windows-x86_64 - path: GradidoNode-windows-x86_64.zip + path: gradido_node-${{ github.ref_name }}-win32-x64.zip # ─── Release erstellen ───────────────────────────── create-release: @@ -128,14 +143,15 @@ jobs: body: | ## Changes~ ${{ steps.changelog.outputs.result }} + *Auto-generated release notes from Pull-Requests* ## Downloads~ | Platform | Architecture | File | |---|---|---| - | Linux | x86_64 | GradidoNode-linux-x86_64.tar.gz | - | Linux | aarch64 | GradidoNode-linux-aarch64.tar.gz | - | Windows | x86_64 | GradidoNode-windows-x86_64.zip | + | Linux | x86_64 | gradido_node-${{ github.ref_name }}-linux-x64.tar.gz | + | Linux | aarch64 | gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz | + | Windows | x86_64 | gradido_node-${{ github.ref_name }}-win32-x64.zip | files: | - linux-x86_64/GradidoNode-linux-x86_64.tar.gz - linux-aarch64/GradidoNode-linux-aarch64.tar.gz - windows-x86_64/GradidoNode-windows-x86_64.zip + linux-x86_64/gradido_node-${{ github.ref_name }}-linux-x64.tar.gz + linux-aarch64/gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz + windows-x86_64/gradido_node-${{ github.ref_name }}-win32-x64.zip \ No newline at end of file From 4eda19d5da33534246ca8b86ce7e567a59131a2f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 9 Mar 2026 17:29:35 +0100 Subject: [PATCH 38/65] fix path --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fab0eb0..d702caa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: linux-aarch64 - path: GradidoNode-linux-aarch64.tar.gz + path: gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz # ─── Windows Native Build ────────────────────────── windows-build: From af8b422617e38a45d35224af753f2256b57fbfbb Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 10 Mar 2026 09:46:53 +0100 Subject: [PATCH 39/65] fix version handling from ci - to binary --- .github/workflows/release.yml | 12 ++++++++++-- CMakeLists.txt | 4 ++-- Dockerfile | 6 ++++-- dependencies/gradido_blockchain | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d702caa..b390cd3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,10 +13,14 @@ jobs: with: submodules: recursive fetch-depth: 0 + - name: Get blockchain version + id: blockchain_version + run: echo "version=$(git -C dependencies/gradido_blockchain describe --tags --dirty --always)" >> $GITHUB_OUTPUT - name: Build x86_64 run: | docker build \ - --build-arg VERSION=${{ github.ref_name }} \ + --build-arg GRADIDO_NODE_VERSION=${{ github.ref_name }} \ + --build-arg GRADIDO_BLOCKCHAIN_VERSION=${{ steps.blockchain_version.outputs.version }} \ --target release_build \ -t gradido-node-x86_64 . docker create --name extract-x64 gradido-node-x86_64 @@ -41,10 +45,14 @@ jobs: with: submodules: recursive fetch-depth: 0 + - name: Get blockchain version + id: blockchain_version + run: echo "version=$(git -C dependencies/gradido_blockchain describe --tags --dirty --always)" >> $GITHUB_OUTPUT - name: Build aarch64 run: | docker build \ - --build-arg VERSION=${{ github.ref_name }} \ + --build-arg GRADIDO_NODE_VERSION=${{ github.ref_name }} \ + --build-arg GRADIDO_BLOCKCHAIN_VERSION=${{ steps.blockchain_version.outputs.version }} \ --build-arg ZIG_TARGET=aarch64-linux-musl \ --build-arg TARGET_ARCH=aarch64 \ --target release_build \ diff --git a/CMakeLists.txt b/CMakeLists.txt index afe142a..b086aa8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.18.2) set(CMAKE_POLICY_VERSION_MINIMUM 3.5) set(CMAKE_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain.cmake") -if(DEFINED ENV{GRADIDO_VERSION} AND NOT "$ENV{GRADIDO_VERSION}" STREQUAL "") - set(GRADIDO_NODE_GIT_VER $ENV{GRADIDO_VERSION}) +if(DEFINED ENV{GRADIDO_NODE_VERSION} AND NOT "$ENV{GRADIDO_NODE_VERSION}" STREQUAL "") + set(GRADIDO_NODE_GIT_VER $ENV{GRADIDO_NODE_VERSION}) else() execute_process( COMMAND git describe --tags --dirty --always diff --git a/Dockerfile b/Dockerfile index 8389e50..c37255e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,11 +64,13 @@ ARG ZIG_TARGET=x86_64-linux-musl #aarch64-linux-musl ARG TARGET_ARCH=x86_64 -ARG GRADIDO_VERSION=0.0.0-dev +ARG GRADIDO_NODE_VERSION=0.0.0-dev +ARG GRADIDO_BLOCKCHAIN_VERSION=0.0.0-dev # aarch64 ENV ZIG_TARGET=${ZIG_TARGET} ENV TARGET_ARCH=${TARGET_ARCH} -ENV GRADIDO_VERSION=${GRADIDO_VERSION} +ENV GRADIDO_NODE_VERSION=${GRADIDO_NODE_VERSION} +ENV GRADIDO_BLOCKCHAIN_VERSION=${GRADIDO_BLOCKCHAIN_VERSION} # copy grpc-deps artifacts COPY --from=gradido/grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index c10843b..0b5c856 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit c10843b405bff0a0c9e826ae02f3db26b5889501 +Subproject commit 0b5c856417415a9fcbfb7b9d9ae68023828f236e From 44ffa904535306e7a2e6eee13eab5d9d5989c657 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 11 Mar 2026 14:49:32 +0100 Subject: [PATCH 40/65] bundle cacert with gradido-blockchain lib --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 0b5c856..5a4acac 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 0b5c856417415a9fcbfb7b9d9ae68023828f236e +Subproject commit 5a4acac4f8cf996cf1b6211d2361e549905ecf24 From 35fbc50c893672b76a1a6b8c3472ad402362fbc4 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 11 Mar 2026 15:18:08 +0100 Subject: [PATCH 41/65] bug fix --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 5a4acac..2131a5a 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 5a4acac4f8cf996cf1b6211d2361e549905ecf24 +Subproject commit 2131a5a791e7f2656d244d8e0ca5af1e704f172e From 46bfef083e5a2c17e619567bc8fea0de1873a92c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 11 Mar 2026 16:29:25 +0100 Subject: [PATCH 42/65] use plain cmake instead of a custom c program --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 2131a5a..c47c6e6 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 2131a5a791e7f2656d244d8e0ca5af1e704f172e +Subproject commit c47c6e6f9adecd4d2df004ee1f62d25212eadea4 From c53015ed53a7d6526b984ff23785134fcddfd304 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 12 Mar 2026 16:49:18 +0100 Subject: [PATCH 43/65] use SyncTopic also between reconnects, reconnect topic after 30 minutes no messages because of hiero/hederda/gRPC bug, prevent endless reconnect on exit --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 4 +- src/blockchain/FileBased.cpp | 9 +- src/blockchain/FileBased.h | 8 +- src/blockchain/FileBasedProvider.cpp | 4 +- src/client/hiero/MirrorClient.cpp | 20 +- src/client/hiero/TopicMessageQuery.cpp | 380 ++++++++++-------- src/client/hiero/TopicMessageQuery.h | 14 +- src/client/hiero/const.h | 4 + src/controller/SimpleOrderingManager.cpp | 12 +- src/controller/SimpleOrderingManager.h | 5 +- src/hiero/MessageListenerQuery.cpp | 23 +- src/hiero/MessageListenerQuery.h | 2 +- src/model/files/LevelDBWrapper.cpp | 4 +- .../{SyncTopicOnStartup.cpp => SyncTopic.cpp} | 16 +- .../{SyncTopicOnStartup.h => SyncTopic.h} | 6 +- 16 files changed, 298 insertions(+), 215 deletions(-) rename src/task/{SyncTopicOnStartup.cpp => SyncTopic.cpp} (93%) rename src/task/{SyncTopicOnStartup.h => SyncTopic.h} (89%) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index c47c6e6..44c3c55 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit c47c6e6f9adecd4d2df004ee1f62d25212eadea4 +Subproject commit 44c3c559146b6d07def2730f56dbacb3021f5aa3 diff --git a/src/MainServer.cpp b/src/MainServer.cpp index 4a5c77a..b86b583 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -174,10 +174,10 @@ void MainServer::exit() // iota::MqttClientWrapper::getInstance()->exit(); CacheManager::getInstance()->getFuzzyTimer()->stop(); - ServerGlobals::g_CPUScheduler->stop(); - ServerGlobals::g_WriteFileCPUScheduler->stop(); // ServerGlobals::g_IotaRequestCPUScheduler->stop(); FileBasedProvider::getInstance()->exit(); + ServerGlobals::g_CPUScheduler->stop(); + ServerGlobals::g_WriteFileCPUScheduler->stop(); } bool MainServer::configExists(const string& fileName) { diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index c9185be..5e660d9 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -10,7 +10,7 @@ #include "../ServerGlobals.h" #include "../SystemExceptions.h" #include "../task/NotifyClient.h" -#include "../task/SyncTopicOnStartup.h" +#include "../task/SyncTopic.h" #include "../client/hiero/MirrorClient.h" #include "gradido_blockchain/const.h" @@ -160,12 +160,12 @@ namespace gradido { } } - std::shared_ptr FileBased::initOnline() + std::shared_ptr FileBased::getTopicSyncTask() { if (mStopToken.stop_requested()) return nullptr; auto hieroTopicIdNum = mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum()); if (hieroTopicIdNum) { - return std::make_shared( + return std::make_shared( mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), hiero::TopicId(0, 0, hieroTopicIdNum), getptr() @@ -178,9 +178,6 @@ namespace gradido { void FileBased::startListening(data::Timestamp lastTransactionConfirmedAt) { if (mStopToken.stop_requested()) return; - if (mHieroMessageListener) { - LOG_F(WARNING, "called again, while listener where already existing"); - } auto hieroTopicId = hiero::TopicId(0, 0, mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum())); if (hieroTopicId.empty()) { LOG_F(WARNING, "startListening called without valid hiero topic id"); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index f42705f..84238b6 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -45,7 +45,7 @@ namespace controller { } namespace task { - class SyncTopicOnStartup; + class SyncTopic; } namespace gradido { @@ -108,11 +108,11 @@ namespace gradido { //! init 3 //! prepare task for syncronize with hiero topic - //! all SyncTopicOnStartup for all communities should be started/scheduled at the same time because there could need each other for new cross group transactions - std::shared_ptr initOnline(); + //! all SyncTopic for all communities should be started/scheduled at the same time because there could need each other for new cross group transactions + std::shared_ptr getTopicSyncTask(); //! init 4 - //! start listening to topic, will be called from SyncTopicOnStartup at the end, will update last known TopicId + //! start listening to topic, will be called from SyncTopic at the end, will update last known TopicId void startListening(data::Timestamp lastTransactionConfirmedAt); // clean up group, stopp all running processes diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 8f57b19..3a7db76 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -4,7 +4,7 @@ #include "../client/hiero/ConsensusClient.h" #include "../client/GraphQL.h" #include "../ServerGlobals.h" -#include "../task/SyncTopicOnStartup.h" +#include "../task/SyncTopic.h" #include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/data/hiero/TopicId.h" @@ -143,7 +143,7 @@ namespace gradido { if (pair.second->getHieroTopicId().empty()) { continue; } - auto task = pair.second->initOnline(); + auto task = pair.second->getTopicSyncTask(); auto hieroClient = pair.second->pickHieroClient(); hieroClient->getTopicInfo(pair.second->getHieroTopicId(), task); task->scheduleTask(task); diff --git a/src/client/hiero/MirrorClient.cpp b/src/client/hiero/MirrorClient.cpp index 8162ee6..788ad1b 100644 --- a/src/client/hiero/MirrorClient.cpp +++ b/src/client/hiero/MirrorClient.cpp @@ -102,18 +102,26 @@ namespace client { return ::hiero::ConsensusTopicResponse(resultJson); } - void MirrorClient::subscribeTopic(TopicMessageQuery* responseListener) { + void MirrorClient::subscribeTopic(TopicMessageQuery* responseListener) + { if (!responseListener) { throw GradidoNullPointerException("missing response listener", "TopicMessageQuery", __FUNCTION__); } grpc::StubOptions options; grpc::TemplatedGenericStub mirrorStub(mChannel); - auto readerWriter = std::move(mirrorStub.PrepareCall( - responseListener->getClientContextPtr(), - "/com.hedera.mirror.api.proto.ConsensusService/subscribeTopic", - responseListener->getCompletionQueuePtr() - )); + auto readerWriter = mirrorStub.PrepareCall( + responseListener->getClientContextPtr(), + "/com.hedera.mirror.api.proto.ConsensusService/subscribeTopic", + responseListener->getCompletionQueuePtr() + ); + if (!readerWriter) { + throw GradidoNullPointerException( + "PrepareCall failed, couldn't subscribe to topic", + "std::unique_ptr>", + __FUNCTION__ + ); + } readerWriter->StartCall(responseListener->getCallStatusPtr()); responseListener->setResponseReader(readerWriter); } diff --git a/src/client/hiero/TopicMessageQuery.cpp b/src/client/hiero/TopicMessageQuery.cpp index 586a3d2..83a39d7 100644 --- a/src/client/hiero/TopicMessageQuery.cpp +++ b/src/client/hiero/TopicMessageQuery.cpp @@ -6,6 +6,7 @@ #include "../../lib/protopuf.h" #include "../../ServerGlobals.h" +#include "gradido_blockchain/Application.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/lib/DataTypeConverter.h" @@ -15,8 +16,11 @@ #include +using std::chrono::duration_cast, std::chrono::milliseconds, std::chrono::seconds, std::chrono::system_clock; +using DataTypeConverter::timespanToString; using namespace magic_enum; using std::make_unique; +using std::this_thread::sleep_for; namespace client { namespace hiero { @@ -34,7 +38,7 @@ namespace client { TopicMessageQuery::~TopicMessageQuery() { - LOG_F(INFO, "TopicMessageQuery::~TopicMessageQuery"); + LOG_F(2, "TopicMessageQuery::~TopicMessageQuery"); mExitCalled = true; mThread.join(); } @@ -46,17 +50,20 @@ namespace client { int TopicMessageQuery::ThreadFunction() { + try { loguru::set_thread_name(mThreadName.data()); // copied most of the code from hiero cpp sdk from startSubscription from TopicMessageQuery.cc // ::hiero::ConsensusTopicQuery query; // Declare needed variables. ::grpc::ByteBuffer grpcByteBuffer; ::hiero::ConsensusTopicResponse response; - std::chrono::system_clock::duration backoff = ::hiero::DEFAULT_MIN_BACKOFF; - std::chrono::system_clock::duration maxBackoff = ::hiero::DEFAULT_MAX_BACKOFF; + system_clock::duration backoff = ::hiero::DEFAULT_MIN_BACKOFF; + system_clock::duration maxBackoff = ::hiero::DEFAULT_MAX_BACKOFF; + system_clock::time_point lastTransactionTimepoint = system_clock::now(); ::grpc::Status grpcStatus; uint64_t attempt = 0ULL; bool complete = false; + auto completeReason = ConnectionClosedReason::Subscription_Ended; bool ok = false; void* tag = nullptr; @@ -65,201 +72,232 @@ namespace client { grpcByteBuffer = memoryBuffer.createGrpcBuffer(); while (!mExitCalled) { - // Process based on the completion queue status. - auto backOffFromNow = std::chrono::system_clock::now() + std::chrono::duration_cast(backoff); - auto nextStatus = mCompletionQueues.back()->AsyncNext(&tag, &ok, backOffFromNow); - LOG_F(2, "next status: %s", enum_name(nextStatus).data()); - switch (nextStatus) + // Process based on the completion queue status. + auto backOffFromNow = system_clock::now() + duration_cast(backoff); + auto nextStatus = mCompletionQueues.back()->AsyncNext(&tag, &ok, backOffFromNow); + LOG_F(2, "next status: %s", enum_name(nextStatus).data()); + switch (nextStatus) + { + case ::grpc::CompletionQueue::TIMEOUT: + { + // Backoff if the completion queue timed out. + backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; + LOG_F(2, "new timeout: %s", timespanToString(backoff).data()); + if (system_clock::now() - lastTransactionTimepoint > ::hiero::DEFAULT_SUBSCRIPTION_INACTIVE_TIMEOUT) { + completeReason = ConnectionClosedReason::Timeout; + complete = true; + mCompletionQueues.back()->Shutdown(); + LOG_F(2, "shutdown connection after DEFAULT_SUBSCRIPTION_INACTIVE_TIMEOUT"); + } + break; + } + case ::grpc::CompletionQueue::GOT_EVENT: + { + // Decrease the backoff time. + backoff = (backoff / 2 < ::hiero::DEFAULT_MIN_BACKOFF) ? ::hiero::DEFAULT_MIN_BACKOFF : backoff / 2; + + // Process based on the call status. + switch (static_cast(*reinterpret_cast(tag))) { - case ::grpc::CompletionQueue::TIMEOUT: + case CallStatus::STATUS_CREATE: { - // Backoff if the completion queue timed out. - backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; - LOG_F(2, "new timeout: %s", DataTypeConverter::timespanToString(backoff).data()); - break; + if (ok) + { + ::grpc::WriteOptions writeOptions; + mCallStatus = CallStatus::STATUS_WRITE; + mResponseReader->WriteLast(grpcByteBuffer, writeOptions, &mCallStatus); + } + break; + } + case CallStatus::STATUS_WRITE: + { + if (ok) + { + mCallStatus = CallStatus::STATUS_PROCESSING; + mResponseReader->Read(&grpcByteBuffer, &mCallStatus); + } + break; } - case ::grpc::CompletionQueue::GOT_EVENT: + case CallStatus::STATUS_PROCESSING: { - // Decrease the backoff time. - backoff = (backoff / 2 < ::hiero::DEFAULT_MIN_BACKOFF) ? ::hiero::DEFAULT_MIN_BACKOFF : backoff / 2; + // If the response should be processed, process it. + if (ok) + { + lastTransactionTimepoint = system_clock::now(); + // Read the response. + auto block = MemoryBlock(grpcByteBuffer); + LOG_F(INFO, "response: echo \"%s\" | xxd -r -p | protoscope", block.get()->convertToHex().data()); + response = protopuf::deserialize<::hiero::ConsensusTopicResponse, ::hiero::ConsensusTopicResponseMessage>(*block.get()); - // Process based on the call status. - switch (static_cast(*reinterpret_cast(tag))) + // Adjust the query timestamp and limit, in case a retry is triggered. + auto consensusTimestamp = response.getConsensusTimestamp(); + if (!consensusTimestamp.empty()) { - case CallStatus::STATUS_CREATE: + // Add one of the smallest denomination of time + mStartQuery.setConsensusStartTime({ + consensusTimestamp.getSeconds(), consensusTimestamp.getNanos() + 1 + }); + } + + if (mStartQuery.getLimit() > 0ULL) { - if (ok) - { - ::grpc::WriteOptions writeOptions; - mCallStatus = CallStatus::STATUS_WRITE; - mResponseReader->WriteLast(grpcByteBuffer, writeOptions, &mCallStatus); - } - break; + mStartQuery.setLimit(mStartQuery.getLimit() - 1ULL); } - case CallStatus::STATUS_WRITE: + + // Process the received message. + const auto& chunkInfo = response.getChunkInfo(); + if (chunkInfo.empty() || chunkInfo.getTotal() == 1) { - if (ok) - { - mCallStatus = CallStatus::STATUS_PROCESSING; - mResponseReader->Read(&grpcByteBuffer, &mCallStatus); - } - break; + try { + onMessageArrived(std::move(response)); + } + catch (GradidoBlockchainException& ex) { + LOG_F(ERROR, "error calling onMessageArrived: %s", ex.getFullString().c_str()); + } + catch (std::exception& ex) { + LOG_F(ERROR, "std error calling onMessageArrived: %s", ex.what()); + } + catch (...) { + LOG_F(ERROR, "unknown error calling onMessageArrived"); + } } - case CallStatus::STATUS_PROCESSING: + else { - // If the response should be processed, process it. - if (ok) - { - // Read the response. - auto block = MemoryBlock(grpcByteBuffer); - LOG_F(INFO, "response: echo \"%s\" | xxd -r -p | protoscope", block.get()->convertToHex().data()); - response = protopuf::deserialize<::hiero::ConsensusTopicResponse, ::hiero::ConsensusTopicResponseMessage>(*block.get()); - - // Adjust the query timestamp and limit, in case a retry is triggered. - auto consensusTimestamp = response.getConsensusTimestamp(); - if (!consensusTimestamp.empty()) - { - // Add one of the smallest denomination of time - mStartQuery.setConsensusStartTime({ - consensusTimestamp.getSeconds(), consensusTimestamp.getNanos() + 1 - }); - } + LOG_F(FATAL, "Chunked Messages not implemented yet"); + /* + const TransactionId transactionId = + TransactionId::fromProtobuf(response.chunkinfo().initialtransactionid()); + pendingMessages[transactionId].push_back(response); - if (mStartQuery.getLimit() > 0ULL) - { - mStartQuery.setLimit(mStartQuery.getLimit() - 1ULL); - } - - // Process the received message. - const auto& chunkInfo = response.getChunkInfo(); - if (chunkInfo.empty() || chunkInfo.getTotal() == 1) - { - try { - onMessageArrived(std::move(response)); - } - catch (GradidoBlockchainException& ex) { - LOG_F(ERROR, "error calling onMessageArrived: %s", ex.getFullString().c_str()); - } - catch (std::exception& ex) { - LOG_F(ERROR, "std error calling onMessageArrived: %s", ex.what()); - } - catch (...) { - LOG_F(ERROR, "unknown error calling onMessageArrived"); - } - } - else - { - LOG_F(FATAL, "Chunked Messages not implemented yet"); - /* - const TransactionId transactionId = - TransactionId::fromProtobuf(response.chunkinfo().initialtransactionid()); - pendingMessages[transactionId].push_back(response); - - if (pendingMessages[transactionId].size() == response.chunkinfo().total()) - { - onNext(TopicMessage::ofMany(pendingMessages[transactionId])); - } - */ - } - mResponseReader->Read(&grpcByteBuffer, &mCallStatus); - } - - // If the response shouldn't be processed (due to completion or error), finish the RPC. - else - { - mCallStatus = CallStatus::STATUS_FINISH; - mResponseReader->Finish(&grpcStatus, &mCallStatus); - } - - break; + if (pendingMessages[transactionId].size() == response.chunkinfo().total()) + { + onNext(TopicMessage::ofMany(pendingMessages[transactionId])); + } + */ } - case CallStatus::STATUS_FINISH: - { - if (grpcStatus.ok()) - { - // RPC completed successfully. - // completionHandler(); - LOG_F(INFO, "RPC subscription complete!"); + mResponseReader->Read(&grpcByteBuffer, &mCallStatus); + } - // Shutdown the completion queue. - mCompletionQueues.back()->Shutdown(); + // If the response shouldn't be processed (due to completion or error), finish the RPC. + else + { + mCallStatus = CallStatus::STATUS_FINISH; + mResponseReader->Finish(&grpcStatus, &mCallStatus); + } - // Mark the RPC as complete. - complete = true; - break; - } - else - { - // An error occurred. Whether retrying or not, cancel the call and close the queue. - mClientContexts.back()->TryCancel(); - mCompletionQueues.back()->Shutdown(); + break; + } + case CallStatus::STATUS_FINISH: + { + if (grpcStatus.ok()) + { + // RPC completed successfully. + // completionHandler(); + LOG_F(INFO, "RPC subscription complete!"); - if (attempt >= ::hiero::DEFAULT_MAX_ATTEMPTS || !shouldRetry(grpcStatus)) - { - // This RPC call shouldn't be retried, handle the error and mark as complete to exit. - //errorHandler(grpcStatus); - LOG_F(ERROR, "Subscription error: %s", grpcStatus.error_message().data()); - complete = true; - } - } - - break; - } - default: - { - // Unrecognized call status, do nothing for now (not sure if this is correct). - break; - } - } + // Shutdown the completion queue. + mCompletionQueues.back()->Shutdown(); + // Mark the RPC as complete. + complete = true; + completeReason = ConnectionClosedReason::Subscription_Ended; break; - } - case ::grpc::CompletionQueue::SHUTDOWN: - { - // Getting here means the RPC is reached completion or encountered an un-retriable error, and the completion - // queue has been shut down. End the subscription. - if (complete) + } + else + { + // An error occurred. Whether retrying or not, cancel the call and close the queue. + mClientContexts.back()->TryCancel(); + mCompletionQueues.back()->Shutdown(); + + if (attempt >= ::hiero::DEFAULT_MAX_ATTEMPTS || !shouldRetry(grpcStatus)) { - // Give a second for the queue to finish its processing. - std::this_thread::sleep_for(std::chrono::seconds(1)); - LOG_F(INFO, "RPC Subscription for topic %s ended.", mStartQuery.getTopicId().toString().data()); - return 0; + // This RPC call shouldn't be retried, handle the error and mark as complete to exit. + //errorHandler(grpcStatus); + LOG_F(ERROR, "Subscription error: %s", grpcStatus.error_message().data()); + complete = true; + completeReason = ConnectionClosedReason::Error; } + } - // If the completion queue has been shut down and the RPC hasn't completed, that means the RPC needs to be - // retried. Increase the backoff for the retry. - backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; - std::this_thread::sleep_for(backoff); - ++attempt; - - // Resend the query to a different node with a different completion queue and client context. - // contexts.push_back(std::make_unique()); - // queues.push_back(std::make_unique()); - - // Reset the call status and send the query. - /* - *callStatus = CallStatus::STATUS_CREATE; - reader = getConnectedMirrorNode(network)->getConsensusServiceStub()->AsyncsubscribeTopic( - contexts.back().get(), query, queues.back().get(), callStatus.get()); - */ - mCallStatus = CallStatus::STATUS_CREATE; - mCompletionQueues.push_back(make_unique()); - mClientContexts.push_back(make_unique()); - MemoryBlock memoryBuffer(protopuf::serialize<::hiero::ConsensusTopicQuery, ::hiero::ConsensusTopicQueryMessage>(mStartQuery)); - grpcByteBuffer = memoryBuffer.createGrpcBuffer(); - ServerGlobals::g_HieroMirrorNode->subscribeTopic(this); - onConnectionClosed(); - break; + break; } default: { - // Not sure what to do here, just fail out. - std::cout << "Unknown gRPC completion queue event, failing.." << std::endl; - return -1; + // Unrecognized call status, do nothing for now (not sure if this is correct). + break; } } + + break; + } + case ::grpc::CompletionQueue::SHUTDOWN: + { + // Getting here means the RPC is reached completion or encountered an un-retriable error, and the completion + // queue has been shut down. End the subscription. + if (complete) + { + // Give a second for the queue to finish its processing. + sleep_for(seconds(1)); + LOG_F(INFO, "RPC Subscription for topic %s ended.", mStartQuery.getTopicId().toString().data()); + onConnectionClosed(completeReason); + return 0; + } + + // If the completion queue has been shut down and the RPC hasn't completed, that means the RPC needs to be + // retried. Increase the backoff for the retry. + backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; + sleep_for(backoff); + ++attempt; + + // Resend the query to a different node with a different completion queue and client context. + // contexts.push_back(std::make_unique()); + // queues.push_back(std::make_unique()); + + // Reset the call status and send the query. + /* + *callStatus = CallStatus::STATUS_CREATE; + reader = getConnectedMirrorNode(network)->getConsensusServiceStub()->AsyncsubscribeTopic( + contexts.back().get(), query, queues.back().get(), callStatus.get()); + */ + if (!Application::getStopToken().stop_requested()) { + mCallStatus = CallStatus::STATUS_CREATE; + mCompletionQueues.push_back(make_unique()); + mClientContexts.push_back(make_unique()); + MemoryBlock memoryBuffer(protopuf::serialize<::hiero::ConsensusTopicQuery, ::hiero::ConsensusTopicQueryMessage>(mStartQuery)); + grpcByteBuffer = memoryBuffer.createGrpcBuffer(); + ServerGlobals::g_HieroMirrorNode->subscribeTopic(this); + onConnectionClosed(ConnectionClosedReason::Reconnect); + } + break; + } + default: + { + // Not sure what to do here, just fail out. + std::cout << "Unknown gRPC completion queue event, failing.." << std::endl; + onConnectionClosed(ConnectionClosedReason::Error); + return -1; + } + } } - return 0; + } + catch (GradidoBlockchainException& ex) { + LOG_F(ERROR, "Thread has uncaught gradido blockchain exception: %s", ex.getFullString().c_str()); + onConnectionClosed(ConnectionClosedReason::Exception); + return -3; + } + catch (std::exception& ex) { + LOG_F(ERROR, "Thread has uncaught exception: %s", ex.what()); + onConnectionClosed(ConnectionClosedReason::Exception); + return -3; + } + catch (...) { + LOG_F(ERROR, "Thread has uncaught unknown exception"); + onConnectionClosed(ConnectionClosedReason::Exception); + return -3; + } + + onConnectionClosed(ConnectionClosedReason::Deconstruct); + return 0; } bool TopicMessageQuery::shouldRetry(::grpc::Status status) diff --git a/src/client/hiero/TopicMessageQuery.h b/src/client/hiero/TopicMessageQuery.h index 3b6b404..7968562 100644 --- a/src/client/hiero/TopicMessageQuery.h +++ b/src/client/hiero/TopicMessageQuery.h @@ -17,7 +17,7 @@ namespace client { namespace hiero { // Enum to track the status of the gRPC call. - enum class CallStatus + enum class CallStatus : long { STATUS_CREATE = 0, STATUS_WRITE = 1, @@ -25,6 +25,16 @@ namespace client { STATUS_FINISH = 3 }; + enum class ConnectionClosedReason + { + Subscription_Ended, + Error, + Exception, + Reconnect, + Deconstruct, + Timeout // it seems that grpc/hiero topic subscribe is unstable around ~ 30 minutes without messages so better close connection than and create new one + }; + class TopicMessageQuery { public: @@ -48,7 +58,7 @@ namespace client { // will be called from grpc client if connection was closed // so no more messageArrived calls - virtual void onConnectionClosed() = 0; + virtual void onConnectionClosed(ConnectionClosedReason reason) noexcept = 0; // called inside loop if connection was closed, default implementation copied from hiero cpp sdk virtual bool shouldRetry(grpc::Status status); diff --git a/src/client/hiero/const.h b/src/client/hiero/const.h index df1a37b..4e18e63 100644 --- a/src/client/hiero/const.h +++ b/src/client/hiero/const.h @@ -18,6 +18,10 @@ namespace hiero { * The default maximum number of times a request will attempt to be submitted before considering the execution failed. */ constexpr auto DEFAULT_MAX_ATTEMPTS = 10U; + /* + * Header/Hiero gRPC seems becoming unstable around 30 minutes without messages + */ + constexpr auto DEFAULT_SUBSCRIPTION_INACTIVE_TIMEOUT = std::chrono::minutes(30); } #endif // __GRADIDO_NODE_CLIENT_HIERO_CONST_H_ \ No newline at end of file diff --git a/src/controller/SimpleOrderingManager.cpp b/src/controller/SimpleOrderingManager.cpp index a699d8c..4bf8c4c 100644 --- a/src/controller/SimpleOrderingManager.cpp +++ b/src/controller/SimpleOrderingManager.cpp @@ -19,7 +19,7 @@ using gradido::data::LedgerAnchor; namespace controller { SimpleOrderingManager::SimpleOrderingManager(std::string_view communityId) - : task::Thread("SimpleOrderingManager"), + : task::Thread("SimpleOrderingManager"), mInitalized(false), mLastTransactions(MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION * 2), mCommunityId(communityId), mLastSequenceNumber(0) @@ -30,10 +30,16 @@ namespace controller { { } - void SimpleOrderingManager::init(uint64_t lastKnownSequenceNumber) + void SimpleOrderingManager::reinitialize(uint64_t lastKnownSequenceNumber) { - mLastSequenceNumber = lastKnownSequenceNumber; + std::unique_lock _lock(mTransactionsMutex); + mLastSequenceNumber = lastKnownSequenceNumber; + mTransactions.clear(); + mLastTransactions.clear(); + if (!mInitalized) { + mInitalized = true; Thread::init(); + } } int SimpleOrderingManager::ThreadFunction() diff --git a/src/controller/SimpleOrderingManager.h b/src/controller/SimpleOrderingManager.h index 2994e86..73b532a 100644 --- a/src/controller/SimpleOrderingManager.h +++ b/src/controller/SimpleOrderingManager.h @@ -39,7 +39,8 @@ namespace controller { SimpleOrderingManager(std::string_view communityId); ~SimpleOrderingManager(); - void init(uint64_t lastKnownSequenceNumber); + // combine init and reset + void reinitialize(uint64_t lastKnownSequenceNumber); enum class PushResult { FOUND_IN_LAST_TRANSACTIONS, @@ -51,7 +52,6 @@ namespace controller { std::shared_ptr findCrossGroupTransactionPair(const gradido::data::LedgerAnchor& transactionId) const; protected: - inline uint64_t getLastSequenceNumber() const { return mLastSequenceNumber; } void updateSequenceNumber(uint64_t newSequenceNumber); @@ -93,6 +93,7 @@ namespace controller { void processTransaction(const TopicResponseDeserializer& gradidoTransactionWorkData); + bool mInitalized; // fast duplication check // use first 4 and last 4 Byte of transaction hash as key // data part is consensusTimestamp which already should be unique.. but better make sure it is really unique diff --git a/src/hiero/MessageListenerQuery.cpp b/src/hiero/MessageListenerQuery.cpp index db86395..33d3a27 100644 --- a/src/hiero/MessageListenerQuery.cpp +++ b/src/hiero/MessageListenerQuery.cpp @@ -1,13 +1,18 @@ #include "MessageListenerQuery.h" -#include "../controller/SimpleOrderingManager.h" #include "../blockchain/FileBasedProvider.h" +#include "../client/hiero/ConsensusClient.h" +#include "../controller/SimpleOrderingManager.h" +#include "../task/SyncTopic.h" #include "ConsensusTopicResponse.h" +#include "gradido_blockchain/Application.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "loguru/loguru.hpp" +using client::hiero::ConnectionClosedReason; + namespace hiero { MessageListenerQuery::MessageListenerQuery(const TopicId& topicId, std::string_view communityId, ConsensusTopicQuery startQuery) @@ -40,9 +45,21 @@ namespace hiero { } // will be called from grpc client if connection was closed - void MessageListenerQuery::onConnectionClosed() + void MessageListenerQuery::onConnectionClosed(ConnectionClosedReason reason) noexcept { //mIsClosed = true; - LOG_F(WARNING, "connection closed on topic: %s, try reconnect", mTopicId.toString().data()); + if (ConnectionClosedReason::Reconnect == reason) { + LOG_F(WARNING, "connection closed on topic: %s, try reconnect", mTopicId.toString().data()); + } + else { + if (!Application::getStopToken().stop_requested()) { + auto blockchain = gradido::blockchain::FileBasedProvider::getInstance()->findBlockchain(mCommunityId); + auto fileBasedBlockchain = static_cast(blockchain.get()); + auto task = fileBasedBlockchain->getTopicSyncTask(); + auto hieroClient = fileBasedBlockchain->pickHieroClient(); + hieroClient->getTopicInfo(fileBasedBlockchain->getHieroTopicId(), task); + task->scheduleTask(task); + } + } } } \ No newline at end of file diff --git a/src/hiero/MessageListenerQuery.h b/src/hiero/MessageListenerQuery.h index 83aa317..b84602f 100644 --- a/src/hiero/MessageListenerQuery.h +++ b/src/hiero/MessageListenerQuery.h @@ -20,7 +20,7 @@ namespace hiero { void onMessageArrived(ConsensusTopicResponse&& consensusTopicResponse) override; // will be called from grpc client if connection was closed - void onConnectionClosed() override; + void onConnectionClosed(client::hiero::ConnectionClosedReason reason) noexcept override; inline bool isClosed() const { return mIsClosed; } inline void cancelConnection() { getClientContextPtr()->TryCancel(); } diff --git a/src/model/files/LevelDBWrapper.cpp b/src/model/files/LevelDBWrapper.cpp index f6fb305..ee959ee 100644 --- a/src/model/files/LevelDBWrapper.cpp +++ b/src/model/files/LevelDBWrapper.cpp @@ -25,7 +25,9 @@ namespace model { LevelDBWrapper::~LevelDBWrapper() { - exit(); + if (mLevelDB) { + exit(); + } } bool LevelDBWrapper::init(size_t cacheInByte/* = 0*/) diff --git a/src/task/SyncTopicOnStartup.cpp b/src/task/SyncTopic.cpp similarity index 93% rename from src/task/SyncTopicOnStartup.cpp rename to src/task/SyncTopic.cpp index f90a1c3..547c36a 100644 --- a/src/task/SyncTopicOnStartup.cpp +++ b/src/task/SyncTopic.cpp @@ -1,4 +1,4 @@ -#include "SyncTopicOnStartup.h" +#include "SyncTopic.h" #include "../controller/SimpleOrderingManager.h" #include "../blockchain/FileBased.h" #include "../client/hiero/MirrorClient.h" @@ -25,7 +25,7 @@ using std::shared_ptr; using std::string, std::to_string; namespace task { - SyncTopicOnStartup::SyncTopicOnStartup( + SyncTopic::SyncTopic( uint64_t lastKnownSequenceNumber, hiero::TopicId lastKnowTopicId, shared_ptr blockchain @@ -37,12 +37,12 @@ namespace task { } - SyncTopicOnStartup::~SyncTopicOnStartup() + SyncTopic::~SyncTopic() { } - int SyncTopicOnStartup::run() + int SyncTopic::run() { // check topic info const auto& topicInfo = mObject; @@ -54,8 +54,8 @@ namespace task { // or check if maybe or block files get corrupted auto lastTransactionIdentical = checkLastTransaction(); - // start ordering manager - orderingManager->init(mLastKnownSequenceNumber); + // (re-)start ordering manager + orderingManager->reinitialize(mLastKnownSequenceNumber); // load transactions from mirror node mConfirmedAtLastReadedTransaction = data::Timestamp(); @@ -89,7 +89,7 @@ namespace task { return 0; } - SyncTopicOnStartup::LastTransactionState SyncTopicOnStartup::checkLastTransaction() + SyncTopic::LastTransactionState SyncTopic::checkLastTransaction() { const auto& mirrorNode = ServerGlobals::g_HieroMirrorNode; @@ -147,7 +147,7 @@ namespace task { return LastTransactionState::IDENTICAL; } - uint32_t SyncTopicOnStartup::loadTransactionsFromMirrorNode(hiero::TopicId topicId) + uint32_t SyncTopic::loadTransactionsFromMirrorNode(hiero::TopicId topicId) { const auto& mirrorNode = ServerGlobals::g_HieroMirrorNode; const auto& orderingManager = mBlockchain->getOrderingManager(); diff --git a/src/task/SyncTopicOnStartup.h b/src/task/SyncTopic.h similarity index 89% rename from src/task/SyncTopicOnStartup.h rename to src/task/SyncTopic.h index 32065a0..e47944c 100644 --- a/src/task/SyncTopicOnStartup.h +++ b/src/task/SyncTopic.h @@ -15,14 +15,14 @@ namespace gradido { #define MAGIC_NUMBER_MIRROR_API_GET_TOPIC_MESSAGES_BULK_SIZE 25 namespace task { - class SyncTopicOnStartup: public CPUTaskGRPCReactor { + class SyncTopic: public CPUTaskGRPCReactor { public: - SyncTopicOnStartup( + SyncTopic( uint64_t lastKnownSequenceNumber, hiero::TopicId lastKnowTopicId, std::shared_ptr blockchain ); - virtual ~SyncTopicOnStartup(); + virtual ~SyncTopic(); int run() override; const char* getResourceType() const override { return "SyncTopicOnStartup"; } From 696d2cbad85853a4e773e8f274544ecf7ed9b83e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 12 Mar 2026 17:02:23 +0100 Subject: [PATCH 44/65] api, getTransactions add searchDirection, make DESC default --- src/server/json-rpc/ApiHandler.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 2cd0c02..66cb13d 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -141,17 +141,22 @@ namespace server { std::string format; uint64_t transactionId = 0; uint32_t maxResultCount = 100; + std::string searchDirectionString; - if (!getUInt64Parameter(responseJson, params, "fromTransactionId", transactionId) || + if (!getUInt64Parameter(responseJson, params, "fromTransactionId", transactionId) || !getStringParameter(responseJson, params, "format", format)) { return; } getUIntParameter(responseJson, params, "maxResultCount", maxResultCount, true); + //printf("group: %s, id: %d\n", groupAlias.data(), transactionId); FilterBuilder builder; auto filter = builder .setMinTransactionNr(transactionId) .setPagination({ maxResultCount }) - .setSearchDirection(SearchDirection::ASC) + .setSearchDirection(SearchDirection::DESC) .build(); + if (getStringParameter(responseJson, params, "searchDirection", searchDirectionString, true) && searchDirectionString == "ASC") { + filter.searchDirection = SearchDirection::ASC; + } findAllTransactions(resultJson, filter, blockchain, format); } else if (method == "getAddressBalance") { From c549aa84dae27de92d5211247477a0808962c691 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 16 Mar 2026 11:13:20 +0100 Subject: [PATCH 45/65] add WireFilter and fromJson for this, use for getTransactions api function --- dependencies/gradido_blockchain | 2 +- src/model/Apollo/TransactionList.h | 2 +- src/server/json-rpc/ApiHandler.cpp | 48 ++--- src/server/json-rpc/ApiHandler.h | 17 +- src/server/json-rpc/WireFilter.cpp | 83 +++++++++ src/server/json-rpc/WireFilter.h | 67 +++++++ src/server/json-rpc/fromJson.cpp | 284 +++++++++++++++++++++++++++++ src/server/json-rpc/fromJson.h | 33 ++++ 8 files changed, 502 insertions(+), 34 deletions(-) create mode 100644 src/server/json-rpc/WireFilter.cpp create mode 100644 src/server/json-rpc/WireFilter.h create mode 100644 src/server/json-rpc/fromJson.cpp create mode 100644 src/server/json-rpc/fromJson.h diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 44c3c55..207fe71 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 44c3c559146b6d07def2730f56dbacb3021f5aa3 +Subproject commit 207fe71b274b76a25c33b88dfdd28e2279398001 diff --git a/src/model/Apollo/TransactionList.h b/src/model/Apollo/TransactionList.h index 9049402..e86d279 100644 --- a/src/model/Apollo/TransactionList.h +++ b/src/model/Apollo/TransactionList.h @@ -8,7 +8,7 @@ namespace gradido { namespace blockchain { class Abstract; class TransactionEntry; - class Filter; + struct Filter; } } diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 66cb13d..ae1a72e 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -1,4 +1,6 @@ #include "ApiHandler.h" +#include "fromJson.h" +#include "WireFilter.h" // need to be here, else it produce a linker error, or more precisly the member function generateList // TODO: fix the reason @@ -8,6 +10,7 @@ #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/data/adapter/byteArray.h" #include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" #include "gradido_blockchain/data/LedgerAnchor.h" @@ -21,11 +24,13 @@ #include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJson.h" +#include "gradido_protobuf_zig.h" #include "../../blockchain/FileBased.h" #include "../../blockchain/FileBasedProvider.h" #include "../../blockchain/NodeTransactionEntry.h" +#include #include "rapidjson/prettywriter.h" #include "magic_enum/magic_enum.hpp" #include "loguru/loguru.hpp" @@ -43,7 +48,7 @@ using namespace magic_enum; using std::optional, std::nullopt; using gradido::g_appContext; -using gradido::data::compact::PublicKeyIndex; +using gradido::data::compact::PublicKeyIndex, gradido::data::compact::ConfirmedTxs; namespace server { namespace json_rpc { @@ -138,26 +143,12 @@ namespace server { } // TODO: rename to listsinceblock else if (method == "getTransactions") { - std::string format; - uint64_t transactionId = 0; - uint32_t maxResultCount = 100; - std::string searchDirectionString; - - if (!getUInt64Parameter(responseJson, params, "fromTransactionId", transactionId) || - !getStringParameter(responseJson, params, "format", format)) { return; } - getUIntParameter(responseJson, params, "maxResultCount", maxResultCount, true); - - //printf("group: %s, id: %d\n", groupAlias.data(), transactionId); - FilterBuilder builder; - auto filter = builder - .setMinTransactionNr(transactionId) - .setPagination({ maxResultCount }) - .setSearchDirection(SearchDirection::DESC) - .build(); - if (getStringParameter(responseJson, params, "searchDirection", searchDirectionString, true) && searchDirectionString == "ASC") { - filter.searchDirection = SearchDirection::ASC; + WireFilter filter; + auto result = fromJson(params, filter); + if (JsonParseResultType::Ok != result.type) { + error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, result.error.c_str()); } - findAllTransactions(resultJson, filter, blockchain, format); + findAllTransactions(resultJson, filter.toCompactFilter(*g_appContext), blockchain, filter.format); } else if (method == "getAddressBalance") { std::string date_string; @@ -298,16 +289,16 @@ namespace server { void ApiHandler::findAllTransactions( rapidjson::Value& resultJson, - const Filter& filter, + const CompactFilter& filter, std::shared_ptr blockchain, - const std::string& format + WireOutputFormat format ) { Profiler timeUsed; - auto alloc = mRootJson.GetAllocator(); + auto& alloc = mRootJson.GetAllocator(); // count for pagination - Filter countFilter = filter; + CompactFilter countFilter = filter; countFilter.pagination = Pagination(); // remove pagination for count countFilter.minTransactionNr = 0; // remove minTransactionNr for count countFilter.maxTransactionNr = 0; // remove maxTransactionNr for count @@ -317,7 +308,7 @@ namespace server { auto transactions = blockchain->findAll(filter); - if (format == "json") { + if (WireOutputFormat::Json == format) { resultJson.AddMember("type", "json", alloc); } else { @@ -325,10 +316,11 @@ namespace server { } Value jsonTransactionArray(kArrayType); for (auto it = transactions.begin(); it != transactions.end(); it++) { - auto transactionSerialized = (*it)->getSerializedTransaction(); + auto legacyTx = blockchain->getTransactionForId((*it)->txNr); + auto transactionSerialized = legacyTx->getSerializedTransaction(); if (transactionSerialized->size() > 0) { - if (format == "json") { - jsonTransactionArray.PushBack(toJson(*(*it)->getConfirmedTransaction(), alloc), alloc); + if (WireOutputFormat::Json == format) { + jsonTransactionArray.PushBack(toJson(*legacyTx->getConfirmedTransaction(), alloc), alloc); } else { auto base64TransactionString = transactionSerialized->convertToBase64(); diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index f177ec8..ff01b63 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -5,15 +5,24 @@ #include "gradido_blockchain/types.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include #include +#include + namespace gradido { namespace blockchain { class Abstract; class Filter; + struct CompactFilter; } namespace data { class LedgerAnchor; + namespace compact { + class ConfirmedGradidoTx; + using ConstConfirmedTxPtr = std::shared_ptr; + using ConfirmedTxs = std::vector; + } } } @@ -25,6 +34,8 @@ namespace memory { namespace server { namespace json_rpc { + enum class WireOutputFormat; + // TODO: write api doc and help on command class ApiHandler : public RequestHandler { @@ -40,9 +51,9 @@ namespace server { */ void findAllTransactions( rapidjson::Value& resultJson, - const gradido::blockchain::Filter& filter, + const gradido::blockchain::CompactFilter& filter, std::shared_ptr blockchain, - const std::string& format + WireOutputFormat format ); /*! * TODO: implement index for iota message id if it is used much @@ -102,8 +113,6 @@ namespace server { memory::ConstBlockPtr nameHash, std::shared_ptr blockchain ); - // helper - }; } } diff --git a/src/server/json-rpc/WireFilter.cpp b/src/server/json-rpc/WireFilter.cpp new file mode 100644 index 0000000..379d7e4 --- /dev/null +++ b/src/server/json-rpc/WireFilter.cpp @@ -0,0 +1,83 @@ +#include "WireFilter.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" +#include "gradido_blockchain/blockchain/PublicKeySearchType.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/data/TransactionType.h" + +using gradido::AppContext; +using gradido::blockchain::CompactFilter, gradido::blockchain::PublicKeySearchType, gradido::blockchain::SearchDirection; +using gradido::data::compact::PublicKeyIndex; +using gradido::data::TransactionType; + +namespace server::json_rpc { + WireFilter::WireFilter() + : searchDirection(SearchDirection::DESC), transactionType(TransactionType::NONE), + publicKeySearchType(PublicKeySearchType::None), format(WireOutputFormat::Base64), + maxTransactionNr(0), minTransactionNr(0), pagination(20) + { + } + + CompactFilter WireFilter::toCompactFilter(AppContext& appContext) const + { + CompactFilter resultFilter; + resultFilter.searchDirection = searchDirection; + resultFilter.transactionType = transactionType; + resultFilter.publicKeySearchType = publicKeySearchType; + if (!coinCommunityId.empty()) { + resultFilter.coinCommunityIdIndex = appContext.getOrAddCommunityIdIndex(coinCommunityId); + } + resultFilter.maxTransactionNr = maxTransactionNr; + resultFilter.minTransactionNr = minTransactionNr; + if (!publicKey.isEmpty()) { + uint32_t communityIdIndex = 0; + if (!communityId.empty()) { + communityIdIndex = appContext.getOrAddCommunityIdIndex(communityId); + } + else if (resultFilter.coinCommunityIdIndex) { + communityIdIndex = resultFilter.coinCommunityIdIndex; + } + PublicKeyIndex publicKeyIndex{}; + if (communityIdIndex) { + auto& communityBlockchain = appContext.getCommunityContext(communityIdIndex).getBlockchain(); + if (!communityBlockchain) { + throw GradidoNullPointerException("missing blockchain for valid community id index", "shared_ptr", __FUNCTION__); + } + auto publicKeyIndexSizeT = communityBlockchain->getPublicKeyDictionary().getIndexForData(publicKey); + if (publicKeyIndexSizeT != (uint32_t)publicKeyIndexSizeT) { + throw DictionaryOverflowException("to big public key index found", "PublicKey"); + } + publicKeyIndex = { + .communityIdIndex = communityIdIndex, + .publicKeyIndex = (uint32_t)publicKeyIndexSizeT + }; + } + if (publicKeyIndex.empty()) { + // search in all communities for public key + for (uint32_t i = 1; publicKeyIndex.empty() && appContext.getCommunityIds().hasIndex(i); i++) + { + if (i == communityIdIndex) continue; + auto& communityBlockchain = appContext.getCommunityContext(i).getBlockchain(); + if (!communityBlockchain) { + throw GradidoNullPointerException("missing blockchain for valid community id index from loop", "shared_ptr", __FUNCTION__); + } + auto publicKeyIndexSizeT = communityBlockchain->getPublicKeyDictionary().getIndexForData(publicKey); + if (publicKeyIndexSizeT != (uint32_t)publicKeyIndexSizeT) { + throw DictionaryOverflowException("to big public key index found", "PublicKey"); + } + publicKeyIndex = { + .communityIdIndex = i, + .publicKeyIndex = (uint32_t)publicKeyIndexSizeT + }; + } + } + if (!publicKeyIndex.empty()) { + resultFilter.publicKeyIndex = publicKeyIndex; + } + } + resultFilter.pagination = pagination; + resultFilter.timepointInterval = timepointInterval; + return resultFilter; + } +} \ No newline at end of file diff --git a/src/server/json-rpc/WireFilter.h b/src/server/json-rpc/WireFilter.h new file mode 100644 index 0000000..d9d954d --- /dev/null +++ b/src/server/json-rpc/WireFilter.h @@ -0,0 +1,67 @@ +#ifndef GRADIDO_NODE_SERVER_JSON_RPC_WIRE_FILTER_H +#define GRADIDO_NODE_SERVER_JSON_RPC_WIRE_FILTER_H + +#include "gradido_blockchain/types.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" +#include "gradido_blockchain/blockchain/Pagination.h" +#include "gradido_blockchain/blockchain/PublicKeySearchType.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/TransactionType.h" +#include "gradido_blockchain/lib/TimepointInterval.h" + +#include +#include + +namespace gradido { + class AppContext; +} + +namespace server::json_rpc { + + enum class WireOutputFormat { + Base64, // protobuf serialized, encoded in base64 + Json // json format, encoded in string + }; + + struct WireFilter + { + WireFilter(); + //! search direction and result order, default: DESC + gradido::blockchain::SearchDirection searchDirection; + + //! transaction type + gradido::data::TransactionType transactionType; + + //! type of data publicKey contains + gradido::blockchain::PublicKeySearchType publicKeySearchType; + + //! format for result returned to caller + WireOutputFormat format; + + //! index starts with 1 + std::string communityId; + //! for colored coins, index starts with 1 + std::string coinCommunityId; + + //! transaction number to stop search, 0 means no stop + uint64_t maxTransactionNr; + //! transaction number to start from, 0 default + uint64_t minTransactionNr; + + //! return only transaction in which the public key is involved, either directly in the transaction or as signer + PublicKey publicKey; + + //! search result scope + gradido::blockchain::Pagination pagination; + //! interval between two dates with 1 month resolution + TimepointInterval timepointInterval; + + //! for the future, but not yet implemented + std::string luaFilterFunction; + + gradido::blockchain::CompactFilter toCompactFilter(gradido::AppContext& appContext) const; + }; +} + +#endif // GRADIDO_NODE_SERVER_JSON_RPC_WIRE_FILTER_H \ No newline at end of file diff --git a/src/server/json-rpc/fromJson.cpp b/src/server/json-rpc/fromJson.cpp new file mode 100644 index 0000000..c759ac3 --- /dev/null +++ b/src/server/json-rpc/fromJson.cpp @@ -0,0 +1,284 @@ +#include "fromJson.h" +#include "WireFilter.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" +#include "gradido_blockchain/blockchain/Pagination.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/CommunityContext.h" +#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/lib/DataTypeConverter.h" +#include "gradido_blockchain/lib/TimepointInterval.h" + +#include +#include +#include + +#include +#include +#include + +using namespace magic_enum; +using DataTypeConverter::dateTimeStringToTimePoint; +using gradido::AppContext; +using gradido::blockchain::CompactFilter, gradido::blockchain::Pagination, gradido::blockchain::SearchDirection; +using gradido::CommunityContext; +using gradido::data::compact::PublicKeyIndex; +using rapidjson::Value; +using std::string, std::string_view; + +namespace server::json_rpc { + + static JsonParseResult checkFieldName(const Value& json, const char* fieldName) + { + if (!json.HasMember(fieldName)) return { .type = JsonParseResultType::Missing_Field }; + if (json[fieldName].IsNull()) return { .type = JsonParseResultType::Null_Field }; + return { .type = JsonParseResultType::Ok }; + } + + static JsonParseResult isNonEmptyStringField(const Value& json, const char* fieldName) + { + auto result = checkFieldName(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + if (!json[fieldName].IsString()) { + string errorMessage = "expect string type for field: "; + errorMessage += fieldName; + return { + .type = JsonParseResultType::Type_Mismatch, + .error = errorMessage + }; + } + return { .type = JsonParseResultType::Ok }; + } + + static JsonParseResult isNonEmptyNumericalField(const Value& json, const char* fieldName) + { + auto result = checkFieldName(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + if (!json[fieldName].IsNumber()) { + string errorMessage = "expect number type for field: "; + errorMessage += fieldName; + return { + .type = JsonParseResultType::Type_Mismatch, + .error = errorMessage + }; + } + return { .type = JsonParseResultType::Ok }; + } + + static JsonParseResult isNonEmptyObject(const Value& json, const char* fieldName) + { + auto result = checkFieldName(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + if (!json[fieldName].IsObject()) { + string errorMessage = "expect object type for field: "; + errorMessage += fieldName; + return { + .type = JsonParseResultType::Type_Mismatch, + .error = errorMessage + }; + } + return { .type = JsonParseResultType::Ok }; + } + + // enum + template + requires std::is_enum_v + JsonParseResult fromJson(const Value& json, const char* fieldName, T& value) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + string_view valueString(json[fieldName].GetString(), json[fieldName].GetStringLength()); + auto optionalValue = enum_cast(valueString); + if (!optionalValue) + { + string errorMessage = fieldName; + errorMessage += ": "; + errorMessage += valueString; + errorMessage += " is invalid for enum "; + errorMessage += enum_type_name(); + return { + .type = JsonParseResultType::Invalid_Enum, + .error = errorMessage + }; + } + value = optionalValue.value(); + return { .type = JsonParseResultType::Ok }; + } + + // unsigned integer + static JsonParseResult fromJson(const Value& json, const char* fieldName, uint32_t& value) + { + auto result = isNonEmptyNumericalField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + value = json[fieldName].GetUint(); + return { .type = JsonParseResultType::Ok }; + } + + // unsigned long long + static JsonParseResult fromJson(const Value& json, const char* fieldName, uint64_t& value) + { + auto result = isNonEmptyNumericalField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + value = json[fieldName].GetUint64(); + return { .type = JsonParseResultType::Ok }; + } + + // string + static JsonParseResult fromJson(const Value& json, const char* fieldName, std::string& value) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + value = string(json[fieldName].GetString(), json[fieldName].GetStringLength()); + return { .type = JsonParseResultType::Ok }; + } + + // Timepoint + static JsonParseResult fromJson(const Value& json, const char* fieldName, Timepoint& value) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + // expect JavaScript DateTime String + // TODO: allow more formats, maybe with fmtLib + value = dateTimeStringToTimePoint(string(json[fieldName].GetString(), json[fieldName].GetStringLength()), "%FT%F"); + return { .type = JsonParseResultType::Ok }; + } + + // PublicKey + static JsonParseResult fromJson(const Value& json, const char* fieldName, PublicKey& publicKey) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + string_view valueString(json[fieldName].GetString(), json[fieldName].GetStringLength()); + if (valueString.size() != 65) { + string errorMessage = fieldName; + errorMessage += " hasn't expected size of 64 hex character + string end character"; + return { + .type = JsonParseResultType::Invalid_Field_Value, + .error = errorMessage + }; + } + + size_t resultBinSize = 0; + unsigned char buffer[32]; + if (0 != sodium_hex2bin(buffer, 32, valueString.data(), valueString.size(), nullptr, &resultBinSize, nullptr)) { + string errorMessage = fieldName; + errorMessage += " contain invalid hex"; + return { + .type = JsonParseResultType::Invalid_Hex, + .error = errorMessage + }; + } + + publicKey = PublicKey(buffer); + return { .type = JsonParseResultType::Ok }; + } + + // Pagination + static JsonParseResult fromJson(const Value& json, const char* fieldName, Pagination& pagination) + { + auto result = isNonEmptyObject(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + const auto& valuePagination = json[fieldName]; + // size + auto sizeResult = isNonEmptyNumericalField(valuePagination, "size"); + if (JsonParseResultType::Type_Mismatch == sizeResult.type) { + return sizeResult; + } + if (JsonParseResultType::Ok == sizeResult.type) { + pagination.size = valuePagination["size"].GetUint(); + } + if (pagination.size > 100) { + pagination.size = 100; + } + // page + auto pageResult = isNonEmptyNumericalField(valuePagination, "page"); + if (JsonParseResultType::Type_Mismatch == pageResult.type) { + return pageResult; + } + if (JsonParseResultType::Ok == pageResult.type) { + pagination.page = valuePagination["page"].GetUint(); + } + return { .type = JsonParseResultType::Ok }; + } + + // TimepointInterval + static JsonParseResult fromJson(const Value& json, const char* fieldName, TimepointInterval& timepointInterval) + { + auto result = isNonEmptyObject(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + const auto& valueTimepointIterval = json[fieldName]; + // start date + Timepoint tempTimepoint; + auto startDateResult = fromJson(valueTimepointIterval, "startDate", tempTimepoint); + if (JsonParseResultType::Ok == startDateResult.type) { + timepointInterval.setStartDate(tempTimepoint); + } + else if (JsonParseResultType::Type_Mismatch == startDateResult.type) { + return startDateResult; + } + // end date + auto endDateResult = fromJson(valueTimepointIterval, "endDate", tempTimepoint); + if (JsonParseResultType::Ok == endDateResult.type) { + timepointInterval.setEndDate(tempTimepoint); + } + else if (JsonParseResultType::Type_Mismatch == endDateResult.type) { + return endDateResult; + } + return { .type = JsonParseResultType::Ok }; + } + + // WireFilter + JsonParseResult fromJson(const Value& value, WireFilter& filter) + { + auto result = fromJson(value, "searchDirection", filter.searchDirection); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "transactionType", filter.transactionType); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "publicKeySearchType", filter.publicKeySearchType); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "format", filter.format); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "communityId", filter.communityId); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "coinCommunityId", filter.coinCommunityId); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "maxTransactionNr", filter.maxTransactionNr); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "minTransactionNr", filter.minTransactionNr); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "publicKey", filter.publicKey); + if ( + JsonParseResultType::Invalid_Field_Value == result.type || + JsonParseResultType::Invalid_Hex == result.type || + JsonParseResultType::Type_Mismatch == result.type + ) { + return result; + } + + result = fromJson(value, "pagination", filter.pagination); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "timepointInterval", filter.timepointInterval); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + return { .type = JsonParseResultType::Ok }; + } + +} \ No newline at end of file diff --git a/src/server/json-rpc/fromJson.h b/src/server/json-rpc/fromJson.h new file mode 100644 index 0000000..e98b51d --- /dev/null +++ b/src/server/json-rpc/fromJson.h @@ -0,0 +1,33 @@ +#ifndef GRADIDO_NODE_SERVER_JSON_RPC_FROM_JSON_H +#define GRADIDO_NODE_SERVER_JSON_RPC_FROM_JSON_H + +#include "WireFilter.h" + +#include + +#include + + +namespace server::json_rpc { + enum JsonParseResultType : uint8_t { + Ok, + Missing_Field, + Null_Field, + Invalid_Enum, + Invalid_Field_Value, + Invalid_Hex, + Type_Mismatch, + General_Error + }; + + struct JsonParseResult + { + JsonParseResultType type; + std::string error; + }; + + + JsonParseResult fromJson(const rapidjson::Value& value, WireFilter& filter); +} + +#endif // GRADIDO_NODE_SERVER_JSON_RPC_FROM_JSON_H \ No newline at end of file From 73106014357f2a9152ffaf944f058d07d39b994f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 16 Mar 2026 15:49:16 +0100 Subject: [PATCH 46/65] use new transaction index --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 256 ++++++++++------------ src/blockchain/FileBased.h | 8 +- src/blockchain/NodeTransactionEntry.cpp | 18 ++ src/blockchain/NodeTransactionEntry.h | 4 + src/cache/BlockIndex.cpp | 111 ++-------- src/cache/BlockIndex.h | 101 ++------- src/serialization/cacheToJson.cpp | 4 +- src/server/json-rpc/ApiHandler.cpp | 24 -- src/server/json-rpc/ApiHandler.h | 5 - src/task/RebuildBlockIndexTask.cpp | 9 +- src/task/WriteTransactionsToBlockTask.cpp | 2 +- src/task/WriteTransactionsToBlockTask.h | 2 +- 13 files changed, 179 insertions(+), 367 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 207fe71..59ffbbd 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 207fe71b274b76a25c33b88dfdd28e2279398001 +Subproject commit 59ffbbd2323f6d5c8635f6912d14cbce57748d3b diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 5e660d9..246467e 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -357,141 +357,142 @@ namespace gradido { TransactionEntries FileBased::findAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS */) const { - TransactionEntries result; - // if pagination is used, filterCopy contain count of still to find transactions - Filter filterCopy(filter); - bool stopped = false; - iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, mPublicKeysIndex); - for (auto transactionNr : transactionNrs) { - if (!filter.pagination.hasCapacityLeft(result.size())) { + TransactionEntries resultTxs; + CompactFilter compactFilter(filter, mPublicKeysIndex, mCommunityIdIndex); + FilterResult lastFilterResult = FilterResult::DISMISS; + + bool hasPagination = filter.pagination.size > 0; + if (hasPagination) { + resultTxs.reserve(filter.pagination.size); + } + + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool + { + do { + auto txs = block.getBlockIndex().findTransactions(compactFilter); + // nothing found? search next block + if (!txs.size()) return true; + + if (resultTxs.capacity() - resultTxs.size() < txs.size()) { + resultTxs.reserve(txs.size() + resultTxs.size()); + } + for (const auto& tx : txs) { + // cannot short cut like in blockchain::InMemory, because FileBased uses a Cache and when we don't copy the shared_ptr, + // it is possible it will be deleted while we are using it :/ + auto confirmedTx = getTransactionForId(tx); + if (!confirmedTx) { + throw GradidoBlockchainTransactionNotFoundException("cannot found confirmed tx in iterateAllImpl").setTransactionId(tx); + } + if (filter.filterFunction) { + lastFilterResult = filter.filterFunction(*confirmedTx); + } + else { + lastFilterResult = FilterResult::USE; + } + if ((FilterResult::USE & lastFilterResult) == FilterResult::USE) { + resultTxs.emplace_back(confirmedTx); + } + } + } while (hasPagination && filter.pagination.hasCapacityLeft(resultTxs.size()) && (FilterResult::STOP & lastFilterResult) != FilterResult::STOP); + // we have enough, stop + if ((FilterResult::STOP & lastFilterResult) == FilterResult::STOP || !filter.pagination.hasCapacityLeft(resultTxs.size())) { return false; } - auto transaction = block.getTransaction(transactionNr, *g_appContext); - auto filterResult = filter.matches(transaction, FilterCriteria::FILTER_FUNCTION | FilterCriteria::TIMEPOINT_INTERVAL); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - result.push_back(transaction); + return true; + } + ); + return resultTxs; + } + + ConfirmedTxs FileBased::findAll(const CompactFilter& filter) const + { + ConfirmedTxs resultTxs; + + bool hasPagination = filter.pagination.size > 0; + if (hasPagination) { + resultTxs.reserve(filter.pagination.size); + } + + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool + { + auto txs = block.getBlockIndex().findTransactions(filter); + // nothing found? search next block + if (!txs.size()) return true; + + if (!hasPagination && (resultTxs.capacity() - resultTxs.size() < txs.size())) { + resultTxs.reserve(txs.size() + resultTxs.size()); } - if ((filterResult & FilterResult::STOP) == FilterResult::STOP) { - stopped = true; - break; + for (const auto& tx : txs) { + auto confirmedTx = getConfirmedTxForId(tx); + if (!confirmedTx) { + throw GradidoBlockchainTransactionNotFoundException("cannot found confirmed tx in iterateAllImpl").setTransactionId(tx); + } + resultTxs.emplace_back(confirmedTx); } - } - if (filter.pagination.size) { - filterCopy.pagination.size = filter.pagination.size - result.size(); - // we have requested result count, let's exit here - if (filterCopy.pagination.size <= 0) { + // we have enough, stop + // without pagination, hasCapacityLeft returns always true + if (!filter.pagination.hasCapacityLeft(resultTxs.size())) { return false; } + return true; } - return !stopped; - }); - return result; + ); + return resultTxs; } - ConfirmedTxs FileBased::findAll(const CompactFilter& filter) const + data::compact::ConfirmedTxs FileBased::findAll( + const CompactFilter& filter, + std::function elementFilter + ) const { - ConfirmedTxs results; - // if pagination is used, filterCopy contain count of still to find transactions - CompactFilter filterCopy(filter); - auto skipEntries = filter.pagination.skipEntriesCount(); - int paginationCursor = 0; - iterateBlocks(filterCopy.searchDirection, + ConfirmedTxs resultTxs; + FilterResult lastFilterResult = FilterResult::DISMISS; + + bool hasPagination = filter.pagination.size > 0; + if (hasPagination) { + resultTxs.reserve(filter.pagination.size); + } + + iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - const auto& transactionIndex = block.getBlockIndex(); - if (PublicKeySearchType::BalanceChangingPublicKey == filterCopy.publicKeySearchType && filterCopy.publicKeyIndex.communityIdIndex == mCommunityIdIndex) - { - filterCopy.pagination.page = 1; - do { - auto balanceChangingTxsInRange = transactionIndex.findTransactionsBalanceChangingForPublicKey(filterCopy); - if (balanceChangingTxsInRange.empty()) { - break; - } - for (const auto& tx : balanceChangingTxsInRange) { - auto transaction = getConfirmedTxForId(tx); - if (!transaction) { - throw GradidoBlockchainTransactionNotFoundException("confirmed tx not found").setTransactionId(tx); - } - auto filterResult = filterCopy.matches(*transaction, FilterCriteria::TIMEPOINT_INTERVAL); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - if (paginationCursor >= skipEntries) { - results.push_back(transaction); - if (!filterCopy.pagination.hasCapacityLeft(results.size())) { - return false; - } - } - paginationCursor++; - } - if ((filterResult & FilterResult::STOP) == FilterResult::STOP) { - return false; - } - } - if (filterCopy.pagination.empty() || filter.pagination.size > balanceChangingTxsInRange.size()) { - break; - } - filterCopy.pagination.page++; - } while (filter.pagination.hasCapacityLeft(results.size())); - return true; - } + do { + auto txs = block.getBlockIndex().findTransactions(filter); + // nothing found? search next block + if (!txs.size()) return true; - transactionIndex.lock(); - try { - auto startIt = transactionIndex.begin(filter); - auto endIt = transactionIndex.end(filter); - auto it = startIt; - for (; it != endIt; ++it) - { - auto transaction = block.getCompactTransaction(*it, *g_appContext); - if (!transaction) { - throw GradidoBlockchainTransactionNotFoundException("confirmed tx not found").setTransactionId(*it); - } - auto filterResult = filter.matches(*transaction, FilterCriteria::TIMEPOINT_INTERVAL); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - if (paginationCursor >= skipEntries) { - results.push_back(transaction); - if (!filter.pagination.hasCapacityLeft(results.size())) { - transactionIndex.unlock(); - return false; - } - } - paginationCursor++; + if (resultTxs.capacity() - resultTxs.size() < txs.size()) { + resultTxs.reserve(txs.size() + resultTxs.size()); + } + for (const auto& tx : txs) { + // cannot short cut like in blockchain::InMemory, because FileBased uses a Cache and when we don't copy the shared_ptr, + // it is possible it will be deleted while we are using it :/ + auto confirmedTx = getConfirmedTxForId(tx); + if (!confirmedTx) { + throw GradidoBlockchainTransactionNotFoundException("cannot found confirmed tx in iterateAllImpl").setTransactionId(tx); } - if ((filterResult & FilterResult::STOP) == FilterResult::STOP) { - transactionIndex.unlock(); - return false; + lastFilterResult = elementFilter(*confirmedTx); + if ((FilterResult::USE & lastFilterResult) == FilterResult::USE) { + resultTxs.emplace_back(getConfirmedTxForId(tx)); } } - transactionIndex.unlock(); - return true; - } - catch (...) { - transactionIndex.unlock(); - throw; + } while (hasPagination && filter.pagination.hasCapacityLeft(resultTxs.size()) && (FilterResult::STOP & lastFilterResult) != FilterResult::STOP); + // we have enough, stop + if ((FilterResult::STOP & lastFilterResult) == FilterResult::STOP || !filter.pagination.hasCapacityLeft(resultTxs.size())) { + return false; } - }); - return results; + return true; + } + ); + return resultTxs; } size_t FileBased::countAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS*/) const { - size_t count = 0; - // check if filter has fields which aren't checked by index - if (!gradido::blockchain::TransactionsIndex::canMatchWithoutDeserialize(filter)) { - LOG_F( - WARNING, - "slow count, detect fields in Filter which aren't covered by index: %s", - toJsonString(filter).c_str() - ); - return findAll(filter).size(); - } - iterateBlocks(filter.searchDirection, - [&](const cache::Block& block) -> bool { - count += block.getBlockIndex().countTransactions(filter, mPublicKeysIndex); - return true; - } - ); - return count; + CompactFilter compactFilter(filter, mPublicKeysIndex, mCommunityIdIndex); + return countAll(compactFilter); } size_t FileBased::countAll(const CompactFilter& filter) const @@ -507,33 +508,6 @@ namespace gradido { return count; } - std::vector FileBased::findAllFast(const Filter& filter) const - { - // check if filter has fields which aren't checked by index - if (!gradido::blockchain::TransactionsIndex::canMatchWithoutDeserialize(filter)) { - LOG_F( - ERROR, - "findAllFast call with invalid filter not covered by index: %s", - toJsonString(filter).c_str() - ); - return {}; - } - std::vector result; - // if pagination is used, filterCopy contain count of still to find transactions - Filter filterCopy(filter); - iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, mPublicKeysIndex); - result.insert(result.end(), transactionNrs.begin(), transactionNrs.end()); - if (filter.pagination.size) { - filterCopy.pagination.size = filter.pagination.size - result.size(); - // we have requested result count, let's exit here - if (filterCopy.pagination.size <= 0) return false; - } - return true; - }); - return result; - } - AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const { // return getAddressTypeSlow(filter); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 84238b6..6398b7f 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -150,15 +150,15 @@ namespace gradido { virtual TransactionEntries findAll(const Filter& filter) const override; virtual data::compact::ConfirmedTxs findAll(const CompactFilter& filter) const override; + virtual data::compact::ConfirmedTxs findAll( + const CompactFilter& filter, + std::function elementFilter + ) const override; // find all optimized for counting transaction nrs, better not use the filter.function for that, because this would slow down virtual size_t countAll(const Filter& filter = Filter::ALL_TRANSACTIONS) const override; virtual size_t countAll(const CompactFilter& filter) const override; - //! use only index for searching, ignore filter function - //! \return vector with transaction nrs - std::vector findAllFast(const Filter& filter) const; - virtual data::AddressType getAddressType(const Filter& filter = Filter::LAST_TRANSACTION) const override; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index c0f44f4..e0e4da0 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -2,9 +2,12 @@ #include "FileBased.h" #include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_protobuf_zig.h" namespace gradido { using data::adapter::toPublicKey; + using data::compact::ConfirmedGradidoTx; namespace blockchain { @@ -62,5 +65,20 @@ namespace gradido { { } + + ConfirmedGradidoTx NodeTransactionEntry::convertToCompactConfirmedTx() const + { + uint8_t buffer[1024]; + grdu_memory alloc; + grdu_memory_init_static(&alloc, buffer, 1024); + grdw_confirmed_transaction tx{}; + getConfirmedTransaction()->toGrdw(&alloc, &tx, mBlockchainCommunityIdIndex); + auto confirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, mBlockchainCommunityIdIndex, *g_appContext); + alloc.last_index = 0; + grdw_transaction_body txBody{}; + getTransactionBody()->toGrdw(&alloc, &txBody); + confirmedTx.fillFromGrdwTransactionBody(&txBody, *g_appContext); + return confirmedTx; + } } } diff --git a/src/blockchain/NodeTransactionEntry.h b/src/blockchain/NodeTransactionEntry.h index ea222b2..cec4ff1 100644 --- a/src/blockchain/NodeTransactionEntry.h +++ b/src/blockchain/NodeTransactionEntry.h @@ -2,10 +2,12 @@ #define __GRADIDO_NODE_BLOCKCHAIN_NODE_TRANSACTION_ENTRY_H #include "gradido_blockchain/blockchain/TransactionEntry.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include namespace gradido { + namespace blockchain { class FileBased; @@ -62,6 +64,8 @@ namespace gradido { inline const std::vector& getAddressIndices() const { std::scoped_lock lock(mFastMutex); return mPublicKeyIndices; } inline bool isAddressIndexInvolved(uint32_t addressIndex) const; + data::compact::ConfirmedGradidoTx convertToCompactConfirmedTx() const; + protected: int32_t mFileCursor; std::vector mPublicKeyIndices; diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index 00ddb48..97c3bdf 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -8,18 +8,22 @@ #include "gradido_blockchain/blockchain/RangeUtils.h" #include "gradido_blockchain/blockchain/SearchDirection.h" #include "gradido_blockchain/data/TransactionType.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/serialization/toJson.h" #include "loguru/loguru.hpp" using namespace rapidjson; -using gradido::blockchain::AbstractProvider, gradido::blockchain::Filter, gradido::blockchain::SearchDirection, gradido::blockchain::TransactionsIndex; +using gradido::blockchain::AbstractProvider; +using gradido::blockchain::Filter, gradido::blockchain::SearchDirection; +using gradido::blockchain::TransactionsIndexRoaringBitmaps; +using gradido::data::compact::ConfirmedGradidoTx; using gradido::data::TransactionType; namespace cache { BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) - : TransactionsIndex(blockchainCommunityIdIndex), mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) + : TransactionsIndexRoaringBitmaps(blockchainCommunityIdIndex), mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) { } @@ -41,131 +45,44 @@ namespace cache { std::lock_guard _lock(mRecursiveMutex); // Todo: store at runtime like Dictionary writeIntoFile(); - clearIndexEntries(); + reset(); mTransactionNrsFileCursors.clear(); } void BlockIndex::reset() { std::lock_guard _lock(mRecursiveMutex); - clearIndexEntries(); + TransactionsIndexRoaringBitmaps::reset(); mTransactionNrsFileCursors.clear(); model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); // only needed if public key dictionary is again persistend // LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().c_str()); blockIndexFile.reset(); - mMaxTransactionNr = 0; - mMinTransactionNr = 0; } bool BlockIndex::loadFromFile(const IDictionary& publicKeysDictionary) { + return false; + /* + * // need to be rewritten std::lock_guard _lock(mRecursiveMutex); assert(!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size()); model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); return blockIndexFile.readFromFile(this); - } - - std::unique_ptr BlockIndex::serialize() - { - if (!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size() && !mMaxTransactionNr && !mMinTransactionNr) { - // we haven't anything to save - return nullptr; - } - - assert(mYearMonthAddressIndexEntries.size() && mTransactionNrsFileCursors.size()); - auto blockIndexFile = std::make_unique(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); - blockIndexFile->addYearBlock(mMinYearMonth.year()); - - std::vector publicKeyIndicesTemp; - publicKeyIndicesTemp.reserve(10); - for (auto monthYearIndex = 0; monthYearIndex < mYearMonthAddressIndexEntries.size(); monthYearIndex++) - { - auto monthYear = indexToYearMonth(monthYearIndex); - if (monthYear.month() <= date::month(1) && mMinYearMonth != monthYear) { - blockIndexFile->addYearBlock(monthYear.year()); - } - blockIndexFile->addMonthBlock(monthYear.month()); - for (const auto& itEntry : mYearMonthAddressIndexEntries[monthYearIndex]) - { - auto fileCursorIt = mTransactionNrsFileCursors.find(itEntry.transactionNr); - if (fileCursorIt == mTransactionNrsFileCursors.end()) { - throw GradidoNodeInvalidDataException("missing file cursor for transaction"); - } - publicKeyIndicesTemp.clear(); - for (auto i = 0; i < itEntry.addressIndiceCount; i++) { - publicKeyIndicesTemp.push_back(itEntry.addressIndices[i]); - } - blockIndexFile->addDataBlock( - itEntry.transactionNr, - fileCursorIt->second, - itEntry.transactionType, - itEntry.coinCommunityIdIndex, - itEntry.isBalanceChanging, - publicKeyIndicesTemp - ); - } - } - // finally write down to file - return std::move(blockIndexFile); + */ } bool BlockIndex::writeIntoFile() { - auto blockIndexFile = serialize(); - if (blockIndexFile) { - blockIndexFile->writeToFile(); - return true; - } return false; } - bool BlockIndex::addIndicesForTransaction( - gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, - date::year year, - date::month month, - uint64_t transactionNr, - int32_t fileCursor, - const uint32_t* addressIndices, - uint16_t addressIndiceCount, - uint8_t isBalanceChanging - ) - { - std::lock_guard _lock(mRecursiveMutex); - mDirty = true; - - TransactionsIndex::addIndicesForTransaction( - transactionType, - coinCommunityIdIndex, - year, - month, - transactionNr, - addressIndices, - addressIndiceCount, - isBalanceChanging - ); - - addFileCursorForTransaction(transactionNr, fileCursor); - return true; - } - - bool BlockIndex::addIndicesForTransaction( - std::shared_ptr transactionEntry, - IMutableDictionary& publicKeyDictionary - ) - { - std::lock_guard _lock(mRecursiveMutex); - TransactionsIndex::addIndicesForTransaction(transactionEntry, publicKeyDictionary); - addFileCursorForTransaction(transactionEntry->getTransactionNr(), transactionEntry->getFileCursor()); - return true; - } - bool BlockIndex::addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx) + bool BlockIndex::addIndicesForTransaction(const ConfirmedGradidoTx& compactTx, const IDictionary& publicKeyDict) { std::lock_guard _lock(mRecursiveMutex); - TransactionsIndex::addIndicesForTransaction(compactTx); + TransactionsIndexRoaringBitmaps::addTransactionIndices(compactTx, publicKeyDict); return true; } diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index 18f87d9..4c19f82 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -3,7 +3,7 @@ #include "gradido_blockchain/blockchain/CompactFilter.h" #include "gradido_blockchain/blockchain/Filter.h" -#include "gradido_blockchain/blockchain/TransactionsIndex.h" +#include "gradido_blockchain/blockchain/TransactionsIndexRoaringBitmaps.h" #include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/lib/DictionaryInterface.h" @@ -40,7 +40,7 @@ namespace cache { TODO: Auto-Recover if missing, and maybe check with saved block on startup */ - class BlockIndex : public gradido::blockchain::TransactionsIndex, public model::files::IBlockIndexReceiver + class BlockIndex : public gradido::blockchain::TransactionsIndexRoaringBitmaps { // friend model::files::BlockIndex; public: @@ -56,29 +56,12 @@ namespace cache { //! \brief write block index into files std::unique_ptr serialize(); - inline rapidjson::Value serializeToJson(rapidjson::Document::AllocatorType& alloc) const; + //! \brief //! \return true if there was something to write into file, after writing it to file bool writeIntoFile(); - bool addIndicesForTransaction( - std::shared_ptr transactionEntry, - IMutableDictionary& publicKeyDictionary - ); - bool addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx); - - //! implement from model::files::IBlockIndexReceiver, called by loading block index from file - bool addIndicesForTransaction( - gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, - date::year year, - date::month month, - uint64_t transactionNr, - int32_t fileCursor, - const uint32_t* addressIndices, - uint16_t addressIndiceCount, - uint8_t isBalanceChanging - ); + bool addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx, const IDictionary& publicKeyDict); //! \brief add transactionNr - fileCursor pair to map if not already exist //! \return false if transactionNr exist, else return true @@ -86,19 +69,12 @@ namespace cache { //! \brief search transaction nrs for search criteria in filter, ignore filter function //! \return transaction nrs - inline std::vector findTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; inline std::vector findTransactions(const gradido::blockchain::CompactFilter& filter) const; - inline std::vector findTransactionsBalanceChangingForPublicKey(const gradido::blockchain::CompactFilter& filter) const; //! count all, ignore pagination - inline size_t countTransactions(const gradido::blockchain::Filter& filter, const IDictionary& publicKeysDictionary) const; inline size_t countTransactions(const gradido::blockchain::CompactFilter& filter) const; - //! \brief find transaction nrs from specific month and year - //! \return {0, 0} if nothing found - inline std::pair findTransactionsForMonthYear(date::year year, date::month month) const; - //! \param fileCursor reference to be filled with fileCursor //! \return true if transaction nr was found and fileCursor was set, else return false bool getFileCursorForTransactionNr(uint64_t transactionNr, int32_t& fileCursor) const; @@ -129,45 +105,25 @@ namespace cache { }; - rapidjson::Value BlockIndex::serializeToJson(rapidjson::Document::AllocatorType& alloc) const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::serializeToJson(alloc); - } - - std::vector BlockIndex::findTransactions( - const gradido::blockchain::Filter& filter, - const IDictionary& publicKeysDictionary - ) const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::findTransactions(filter, publicKeysDictionary, mBlockchainCommunityIdIndex); - } - std::vector BlockIndex::findTransactions(const gradido::blockchain::CompactFilter& filter) const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::findTransactions(filter); - } - std::vector BlockIndex::findTransactionsBalanceChangingForPublicKey(const gradido::blockchain::CompactFilter& filter) const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::findTransactionsBalanceChangingForPublicKey(filter); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::findTransactions(filter); } - size_t BlockIndex::countTransactions( - const gradido::blockchain::Filter& filter, - const IDictionary& publicKeysDictionary - ) const + size_t BlockIndex::countTransactions(const gradido::blockchain::CompactFilter& filter) const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::countTransactions(gradido::blockchain::CompactFilter(filter, publicKeysDictionary, mBlockchainCommunityIdIndex)); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::countTransactions(filter); } - size_t BlockIndex::countTransactions(const gradido::blockchain::CompactFilter& filter) const + size_t BlockIndex::getTransactionsCount() const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::countTransactions(filter); + if (!mMaxTransactionNr && !mMinTransactionNr) { + return 0; + } + return mMaxTransactionNr - mMinTransactionNr + 1; } bool BlockIndex::hasTransactionNr(uint64_t transactionNr) const @@ -180,41 +136,12 @@ namespace cache { uint64_t BlockIndex::getMaxTransactionNr() const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::getMaxTransactionNr(); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::getMaxTransactionNr(); } uint64_t BlockIndex::getMinTransactionNr() const { std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::getMinTransactionNr(); - } - - uint64_t BlockIndex::getTransactionsCount() const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::getTransactionsCount(); - } - - date::year_month BlockIndex::getOldestYearMonth() const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::getOldestYearMonth(); - } - date::year_month BlockIndex::getNewestYearMonth() const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::getNewestYearMonth(); - } - - TimepointInterval BlockIndex::filteredTimepointInterval(const gradido::blockchain::CompactFilter& filter) const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::filteredTimepointInterval(filter); - } - - std::pair BlockIndex::findTransactionsForMonthYear(date::year year, date::month month) const - { - std::lock_guard _lock(mRecursiveMutex); - return gradido::blockchain::TransactionsIndex::findTransactionsForMonthYear({ year, month }); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::getMinTransactionNr(); } } diff --git a/src/serialization/cacheToJson.cpp b/src/serialization/cacheToJson.cpp index 9c145d8..83c80e0 100644 --- a/src/serialization/cacheToJson.cpp +++ b/src/serialization/cacheToJson.cpp @@ -5,9 +5,9 @@ using namespace rapidjson; namespace serialization { - template<> + /*template<> Value toJson(const cache::BlockIndex& value, Document::AllocatorType& alloc) { return value.serializeToJson(alloc); - } + }*/ } \ No newline at end of file diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index ae1a72e..b448ecc 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -169,9 +169,6 @@ namespace server { else if (method == "getAddressType") { getAddressType(resultJson, pubkey, blockchain); } - else if (method == "getAddressTxids") { - getAddressTxids(resultJson, pubkey, blockchain); - } else if (method == "getTransaction") { std::string format; uint64_t transactionId = 0; @@ -442,27 +439,6 @@ namespace server { resultJson.AddMember("addressType", Value(typeString.data(), typeString.size(), alloc), alloc); } - void ApiHandler::getAddressTxids(Value& resultJson, memory::ConstBlockPtr pubkey, std::shared_ptr blockchain) - { - assert(blockchain); - assert(pubkey); - - auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); - assert(fileBasedBlockchain); - - auto transactionNrs = fileBasedBlockchain->findAllFast({ 0, 0, pubkey }); - - auto alloc = mRootJson.GetAllocator(); - Value transactionNrsJson(kArrayType); - for (auto& transactionNr : transactionNrs) { - transactionNrsJson.PushBack(transactionNr, alloc); - } - - resultJson.AddMember("transactionNrs", transactionNrsJson, alloc); - } - - - void ApiHandler::listTransactions( Value& resultJson, std::shared_ptr blockchain, diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index ff01b63..febfb54 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -90,11 +90,6 @@ namespace server { memory::ConstBlockPtr pubkey, std::shared_ptr blockchain ); - void getAddressTxids( - rapidjson::Value& resultJson, - memory::ConstBlockPtr pubkey, - std::shared_ptr blockchain - ); void listTransactions( rapidjson::Value& resultJson, std::shared_ptr blockchain, diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp index 3b8093e..78622e1 100644 --- a/src/task/RebuildBlockIndexTask.cpp +++ b/src/task/RebuildBlockIndexTask.cpp @@ -14,10 +14,11 @@ #include -using namespace magic_enum; -using memory::Block; +using gradido::g_appContext; using gradido::blockchain::FileBased; using gradido::data::compact::ConfirmedGradidoTx; +using namespace magic_enum; +using memory::Block; using std::shared_ptr, std::make_shared, std::unique_lock; using ServerGlobals::g_CPUScheduler; @@ -57,8 +58,8 @@ namespace task { throw GradidoNodeInvalidDataException("error deserialize transaction body"); } - auto compactConfirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, &body, mCommunityIdIndex, *gradido::g_appContext); - mBlockIndex->addIndicesForTransaction(compactConfirmedTx); + auto compactConfirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, &body, mCommunityIdIndex, *g_appContext); + mBlockIndex->addIndicesForTransaction(compactConfirmedTx, g_appContext->getCommunityContext(mCommunityIdIndex).getBlockchain()->getPublicKeyDictionary()); mBlockIndex->addFileCursorForTransaction(compactConfirmedTx.txNr, fileCursor); grdu_memory_init_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); diff --git a/src/task/WriteTransactionsToBlockTask.cpp b/src/task/WriteTransactionsToBlockTask.cpp index 0ecb658..64fd838 100644 --- a/src/task/WriteTransactionsToBlockTask.cpp +++ b/src/task/WriteTransactionsToBlockTask.cpp @@ -71,7 +71,7 @@ namespace task { assert(!isTaskFinished()); std::lock_guard lock(mFastMutex); mTransactions.insert({transaction->getTransactionNr(), transaction}); - mBlockIndex->addIndicesForTransaction(transaction, publicKeyDictionary); + mBlockIndex->addTransactionIndices(transaction->convertToCompactConfirmedTx(), publicKeyDictionary); } std::shared_ptr WriteTransactionsToBlockTask::getTransaction(uint64_t nr) diff --git a/src/task/WriteTransactionsToBlockTask.h b/src/task/WriteTransactionsToBlockTask.h index 43a0b8b..3713318 100644 --- a/src/task/WriteTransactionsToBlockTask.h +++ b/src/task/WriteTransactionsToBlockTask.h @@ -60,7 +60,7 @@ namespace task { std::shared_ptr transaction, IMutableDictionary& publicKeyDictionary ); - + //! return transaction by nr std::shared_ptr getTransaction(uint64_t nr); From 9cc4c9e5d9863cf57cfe860e65b89e3ba87dd253 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 17 Mar 2026 09:25:24 +0100 Subject: [PATCH 47/65] fix build error --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 59ffbbd..3c97d72 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 59ffbbd2323f6d5c8635f6912d14cbce57748d3b +Subproject commit 3c97d72e69022f09ec6a92d6b9b3731eb4b9a7da From ce84def9c827a7d243e02124df366b0ec9bd9472 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 17 Mar 2026 13:29:31 +0100 Subject: [PATCH 48/65] fix problem with findAll and returning entries double tofullfill pagination --- dependencies/gradido_blockchain | 2 +- src/blockchain/FileBased.cpp | 29 +++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 59ffbbd..6510963 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 59ffbbd2323f6d5c8635f6912d14cbce57748d3b +Subproject commit 6510963e06e2ecea9c4c76a6b86a8d2c870ee7c3 diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 246467e..7fe77a2 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -360,6 +360,7 @@ namespace gradido { TransactionEntries resultTxs; CompactFilter compactFilter(filter, mPublicKeysIndex, mCommunityIdIndex); FilterResult lastFilterResult = FilterResult::DISMISS; + size_t lastFindTransactionResultCount = 0; bool hasPagination = filter.pagination.size > 0; if (hasPagination) { @@ -370,14 +371,18 @@ namespace gradido { [&](const cache::Block& block) -> bool { do { + compactFilter.pagination = filter.pagination; auto txs = block.getBlockIndex().findTransactions(compactFilter); + lastFindTransactionResultCount = txs.size(); + // nothing found? search next block if (!txs.size()) return true; if (resultTxs.capacity() - resultTxs.size() < txs.size()) { resultTxs.reserve(txs.size() + resultTxs.size()); } - for (const auto& tx : txs) { + for (const auto& tx : txs) + { // cannot short cut like in blockchain::InMemory, because FileBased uses a Cache and when we don't copy the shared_ptr, // it is possible it will be deleted while we are using it :/ auto confirmedTx = getTransactionForId(tx); @@ -394,7 +399,13 @@ namespace gradido { resultTxs.emplace_back(confirmedTx); } } - } while (hasPagination && filter.pagination.hasCapacityLeft(resultTxs.size()) && (FilterResult::STOP & lastFilterResult) != FilterResult::STOP); + ++compactFilter.pagination.page; + } while ( + hasPagination && + filter.pagination.hasCapacityLeft(resultTxs.size()) && + (FilterResult::STOP & lastFilterResult) != FilterResult::STOP && + lastFindTransactionResultCount == filter.pagination.size + ); // we have enough, stop if ((FilterResult::STOP & lastFilterResult) == FilterResult::STOP || !filter.pagination.hasCapacityLeft(resultTxs.size())) { return false; @@ -448,7 +459,9 @@ namespace gradido { ) const { ConfirmedTxs resultTxs; + CompactFilter filterCopy(filter); FilterResult lastFilterResult = FilterResult::DISMISS; + size_t lastFindTransactionResultCount = 0; bool hasPagination = filter.pagination.size > 0; if (hasPagination) { @@ -459,7 +472,9 @@ namespace gradido { [&](const cache::Block& block) -> bool { do { - auto txs = block.getBlockIndex().findTransactions(filter); + filterCopy.pagination = filter.pagination; + auto txs = block.getBlockIndex().findTransactions(filterCopy); + lastFindTransactionResultCount = txs.size(); // nothing found? search next block if (!txs.size()) return true; @@ -478,7 +493,13 @@ namespace gradido { resultTxs.emplace_back(getConfirmedTxForId(tx)); } } - } while (hasPagination && filter.pagination.hasCapacityLeft(resultTxs.size()) && (FilterResult::STOP & lastFilterResult) != FilterResult::STOP); + ++filterCopy.pagination.page; + } while ( + hasPagination && + filter.pagination.hasCapacityLeft(resultTxs.size()) && + (FilterResult::STOP & lastFilterResult) != FilterResult::STOP && + lastFindTransactionResultCount == filter.pagination.size + ); // we have enough, stop if ((FilterResult::STOP & lastFilterResult) == FilterResult::STOP || !filter.pagination.hasCapacityLeft(resultTxs.size())) { return false; From eed8f0398a9b1f249bc9ecce2c2dd5d5b7579859 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 18 Mar 2026 10:03:56 +0100 Subject: [PATCH 49/65] fix problem with validation last X transaction not starting by 0 --- src/blockchain/FileBased.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 7fe77a2..510a989 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -734,6 +734,12 @@ namespace gradido { else { countTarget = lastTransaction->getTransactionNr(); } + if (f.minTransactionNr) { + auto previousTxEntry = getTransactionForId(f.minTransactionNr - 1); + if (previousTxEntry) { + previousConfirmedTransaction = previousTxEntry->getConfirmedTransaction(); + } + } f.filterFunction = [&](const TransactionEntry& transactionEntry) -> FilterResult From 0ce774e51d8707f725502fdc9da56ef3d4c61b3b Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 5 Apr 2026 19:23:51 +0200 Subject: [PATCH 50/65] hot fix in gradido blockchain --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 6510963..e581ae5 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 6510963e06e2ecea9c4c76a6b86a8d2c870ee7c3 +Subproject commit e581ae55b490bf4cd74cc9d74bd613d632c1e494 From 60a7d2d22ca19c48d53201855d9b88ac06828243 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 12 Apr 2026 15:21:16 +0200 Subject: [PATCH 51/65] speedup json rpc calls, use static buffer for rapid json --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 23 +++++- src/server/json-rpc/ApiHandlerFactory.h | 28 ------- src/server/json-rpc/RequestHandler.cpp | 102 ++++++++++++------------ src/server/json-rpc/RequestHandler.h | 10 ++- 5 files changed, 81 insertions(+), 84 deletions(-) delete mode 100644 src/server/json-rpc/ApiHandlerFactory.h diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index e581ae5..58d52e3 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit e581ae55b490bf4cd74cc9d74bd613d632c1e494 +Subproject commit 58d52e354e07cc060e752553c6e9f38a8922cb6a diff --git a/src/MainServer.cpp b/src/MainServer.cpp index b86b583..487ae4b 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -3,8 +3,8 @@ #include "ServerGlobals.h" #include "blockchain/FileBasedProvider.h" +#include "server/json-rpc/ApiHandler.h" // #include "iota/MqttClientWrapper.h" -#include "server/json-rpc/ApiHandlerFactory.h" #include "SingletonManager/CacheManager.h" #include "hiero/Addressbook.h" @@ -24,10 +24,12 @@ #include #include +#include "cpp-httplib/httplib.h" #include "loguru.hpp" using gradido::blockchain::FileBasedProvider; using gradido::g_appContext, gradido::AppContext; +using server::json_rpc::ApiHandler; using std::filesystem::create_directories, std::filesystem::exists, std::filesystem::is_regular_file, std::filesystem::path; using std::shared_ptr, std::make_unique; using std::string; @@ -149,7 +151,24 @@ bool MainServer::init() // start jsonrpc 2.0 server mHttpServer = new Server("0.0.0.0", jsonrpc_port, "http-server"); mHttpServer->init(); - mHttpServer->registerResponseHandler("/api", new server::json_rpc::ApiHandlerFactory()); + // mHttpServer->registerResponseHandler("/api", new server::json_rpc::ApiHandlerFactory()); + mHttpServer->registerCallbackHandler("/api", MethodType::OPTIONS, [](const httplib::Request& req, httplib::Response& res) { + ApiHandler::cors(res); + } + ); + mHttpServer->registerCallbackHandler("/api", MethodType::GET, [](const httplib::Request& req, httplib::Response& res) { + res.set_content("REST API", "text/plain"); + } + ); + mHttpServer->registerCallbackHandler("/api", MethodType::DEL, [](const httplib::Request& req, httplib::Response& res) { + res.set_content("---", "text/plain"); + } + ); + mHttpServer->registerCallbackHandler("/api", MethodType::POST, [](const httplib::Request& req, httplib::Response& res) { + ApiHandler handler; + handler.handlePostPut(req, res); + } + ); mHttpServer->run(); LOG_F(INFO, "started in %s, json rpc port: %d", usedTime.string().c_str(), jsonrpc_port); } diff --git a/src/server/json-rpc/ApiHandlerFactory.h b/src/server/json-rpc/ApiHandlerFactory.h deleted file mode 100644 index 6d1e177..0000000 --- a/src/server/json-rpc/ApiHandlerFactory.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef __GRADIDO_NODE_SERVER_JSON_RPC_API_HANDLER_FACTORY_H -#define __GRADIDO_NODE_SERVER_JSON_RPC_API_HANDLER_FACTORY_H - -#include "gradido_blockchain/http/AbstractResponseHandlerFactory.h" -#include "ApiHandler.h" - -namespace server { - namespace json_rpc { - class ApiHandlerFactory : public AbstractResponseHandlerFactory - { - public: - virtual std::unique_ptr getResponseHandler(MethodType method) { - return std::make_unique(); - }; - //! \return check if factory has a response handler for a specific method, else server didn't need to listen for requests - virtual bool has(MethodType method) { - switch (method) { - case MethodType::GET: - case MethodType::POST: - case MethodType::OPTIONS: return true; - default: return false; - } - }; - }; - } -} - -#endif //__GRADIDO_NODE_SERVER_JSON_RPC_API_HANDLER_FACTORY_H \ No newline at end of file diff --git a/src/server/json-rpc/RequestHandler.cpp b/src/server/json-rpc/RequestHandler.cpp index 135a75b..fb626f2 100644 --- a/src/server/json-rpc/RequestHandler.cpp +++ b/src/server/json-rpc/RequestHandler.cpp @@ -1,6 +1,7 @@ #include "RequestHandler.h" #include "gradido_blockchain/lib/DataTypeConverter.h" +#include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/http/ServerConfig.h" @@ -24,18 +25,22 @@ using namespace rapidjson; namespace server { namespace json_rpc { + static thread_local void* gtl_valueBuffer = malloc(RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY * 4); + RequestHandler::RequestHandler() - : mRootJson(kObjectType) + : mRootJson(kObjectType, new MemoryPoolAllocator<>(gtl_valueBuffer, RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY * 4)) { - + // MemoryPoolAllocator<> valueAllocator(valueBuffer, valueBuffer->size()); + //MemoryPoolAllocator<> parseAllocator(parseBuffer, parseBuffer->size()); +// mRootJson = Document() } Value RequestHandler::handleOneRpcCall(const rapidjson::Value& jsonRpcRequest) { - LOG_F(2, "handleOneRpcCall"); + // LOG_F(2, "handleOneRpcCall"); Value responseJson(kObjectType); std::string method; - auto alloc = mRootJson.GetAllocator(); + auto& alloc = mRootJson.GetAllocator(); if (checkObjectOrArrayParameter(responseJson, jsonRpcRequest, "params") && getStringParameter(responseJson, jsonRpcRequest, "method", method)) { try { @@ -61,72 +66,71 @@ namespace server { return responseJson; } - void RequestHandler::handleRequest(const httplib::Request& request, httplib::Response& response, MethodType method) + void RequestHandler::cors(httplib::Response& response) { - loguru::set_thread_name("json-rpc"); if (ServerConfig::g_AllowUnsecureFlags & ServerConfig::UNSECURE_CORS_ALL) { response.set_header("Access-Control-Allow-Origin", "*"); response.set_header("Access-Control-Allow-Headers", "Authorization, Access-Control-Allow-Headers, Origin,Accept, X-Requested-With, Content-Type, Access-Control-Request-Method, Access-Control-Request-Headers"); } + } - auto alloc = mRootJson.GetAllocator(); + void RequestHandler::handlePostPut(const httplib::Request& request, httplib::Response& response) + { + loguru::set_thread_name("json-rpc"); + cors(response); + + auto& alloc = mRootJson.GetAllocator(); Value responseJson(kObjectType); responseJson.AddMember("jsonrpc", "2.0", alloc); responseJson.AddMember("id", Value(kNullType), alloc); - // TODO: put group name in request url to keep function calls as similar as possible to bitcoin and co - Document rapidjson_params; - if (MethodType::POST == method || MethodType::PUT == method) - { - rapidjson_params.Parse(request.body.data()); - if (rapidjson_params.HasParseError()) - { - Value data(kObjectType); - data.AddMember("parseError", Value(GetParseError_En(rapidjson_params.GetParseError()), alloc), alloc); - data.AddMember("errorOffset", rapidjson_params.GetErrorOffset(), alloc); - error(responseJson, JSON_RPC_ERROR_PARSE_ERROR, "error parsing request to json", &data); - } - else - { - if (rapidjson_params.IsObject()) { - responseJson = handleOneRpcCall(rapidjson_params); - } - else if (rapidjson_params.IsArray()) { - responseJson = Value(kArrayType); - for (auto& v : rapidjson_params.GetArray()) { - responseJson.PushBack(handleOneRpcCall(v), alloc); - } - } - else { - error(responseJson, JSON_RPC_ERROR_INVALID_REQUEST, "empty body"); - } - } - } - else if (MethodType::GET == method) - { - parseQueryParametersToRapidjson(request.params, rapidjson_params); - - //rapid_json_result = handle(rapidjson_params); - LOG_F(ERROR, "[%s:%d] must be implemented\n", __FUNCTION__, __LINE__); - } - else if (MethodType::OPTIONS == method) + Document rapidjson_params; + + rapidjson_params.Parse(request.body.data()); + if (rapidjson_params.HasParseError()) { - return; + Value data(kObjectType); + data.AddMember("parseError", Value(GetParseError_En(rapidjson_params.GetParseError()), alloc), alloc); + data.AddMember("errorOffset", rapidjson_params.GetErrorOffset(), alloc); + error(responseJson, JSON_RPC_ERROR_PARSE_ERROR, "error parsing request to json", &data); } else { - error(responseJson, JSON_RPC_ERROR_INVALID_REQUEST, "HTTP method unknown"); - } + if (rapidjson_params.IsObject()) { + responseJson = handleOneRpcCall(rapidjson_params); + } + else if (rapidjson_params.IsArray()) { + responseJson = Value(kArrayType); + for (auto& v : rapidjson_params.GetArray()) { + responseJson.PushBack(handleOneRpcCall(v), alloc); + } + } + else { + error(responseJson, JSON_RPC_ERROR_INVALID_REQUEST, "empty body"); + } + } // 3. Stringify the DOM - StringBuffer buffer; - Writer writer(buffer); + static thread_local void* writeMemoryBuffer = malloc(RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY); + MemoryPoolAllocator<> writeBufferAllocator(writeMemoryBuffer, RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY); + GenericStringBuffer, MemoryPoolAllocator<>> buffer(&writeBufferAllocator, RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY); + + // StringBuffer buffer(); + Writer writer(buffer); responseJson.Accept(writer); - // printf("response: %s\n", buffer.GetString()); response.set_content(buffer.GetString(), "application/json"); } + void RequestHandler::handleOptions(const httplib::Request& request, httplib::Response& response) + { + cors(response); + } + + void RequestHandler::handleGet(const httplib::Request& request, httplib::Response& response) + { + throw GradidoNotImplementedException("REST Api not implemented yet"); + } bool RequestHandler::parseQueryParametersToRapidjson(const std::multimap& params, Document& rapidParams) { diff --git a/src/server/json-rpc/RequestHandler.h b/src/server/json-rpc/RequestHandler.h index 4be09d8..9a96262 100644 --- a/src/server/json-rpc/RequestHandler.h +++ b/src/server/json-rpc/RequestHandler.h @@ -1,8 +1,6 @@ #ifndef __GRADIDO_NODE_SERVER_JSON_RPC_REQUEST_HANDLER_H #define __GRADIDO_NODE_SERVER_JSON_RPC_REQUEST_HANDLER_H -#include "cpp-httplib/httplib.h" - #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/http/JsonRPCRequest.h" // need JsonRPCErrorCodes from that #include "gradido_blockchain/http/AbstractResponseHandler.h" @@ -11,12 +9,15 @@ namespace server { namespace json_rpc { - class RequestHandler : public AbstractResponseHandler + class RequestHandler { public: RequestHandler(); - virtual void handleRequest(const httplib::Request& request, httplib::Response& response, MethodType method); + void handlePostPut(const httplib::Request& request, httplib::Response& response); + void handleOptions(const httplib::Request& request, httplib::Response& response); + void handleGet(const httplib::Request& request, httplib::Response& response); + static void cors(httplib::Response& response); rapidjson::Value handleOneRpcCall(const rapidjson::Value& jsonRpcRequest); //virtual Poco::JSON::Object* handle(Poco::Dynamic::Var params) = 0; @@ -48,6 +49,7 @@ namespace server { void error(rapidjson::Value& responseJson, JsonRPCErrorCodes code, GradidoBlockchainException& ex); protected: + rapidjson::Document mRootJson; }; From 506f9435c1b84a0d3e0b20a957c60ea64c446740 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 14 Apr 2026 14:36:13 +0200 Subject: [PATCH 52/65] update gradido blockchain, fix fix on linux --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 58d52e3..ce54d25 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 58d52e354e07cc060e752553c6e9f38a8922cb6a +Subproject commit ce54d25f264b4feef3b2711edd498bfe43b1a7cd From 68f8bc321a9fcddfda13ccb17c8047ed92e8f06e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 28 Apr 2026 17:35:40 +0200 Subject: [PATCH 53/65] adapt to changed GradidoBlockchain with moving logic to blockchain core, rename Profiler to MonotonicTimer --- CMakeLists.txt | 1 + src/MainServer.cpp | 4 ++-- src/ServerGlobals.cpp | 1 - src/blockchain/FileBased.cpp | 12 ++++++------ src/cache/Block.cpp | 4 ++-- src/client/hiero/RequestReactor.h | 4 ++-- src/hiero/Addressbook.cpp | 4 ++-- src/lib/protopuf.h | 4 ++-- src/main.cpp | 1 - src/model/files/Block.cpp | 6 +++--- src/server/json-rpc/ApiHandler.cpp | 18 +++++++++--------- src/task/CPUShedulerThread.cpp | 4 ++-- src/task/HieroMessageToTransactionTask.cpp | 2 -- src/task/RebuildBlockIndexTask.cpp | 1 - 14 files changed, 31 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b086aa8..f9c6d62 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,7 @@ include_directories( "dependencies/gradido_blockchain/build" "dependencies/gradido_blockchain/dependencies" "dependencies/gradido_blockchain/dependencies/date/include" + "dependencies/gradido_blockchain/dependencies/gradido-blockchain-core/include" "dependencies/gradido_blockchain/dependencies/magic_enum/include" "dependencies/gradido_blockchain/dependencies/protopuf/include" "dependencies/gradido_blockchain/dependencies/rapidjson/include" diff --git a/src/MainServer.cpp b/src/MainServer.cpp index 487ae4b..d41202c 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -12,7 +12,7 @@ #include "lib/PersistentDictionary.h" #include "gradido_blockchain/AppContext.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/http/ServerConfig.h" #include @@ -46,7 +46,7 @@ MainServer::~MainServer() bool MainServer::init() { - Profiler usedTime; + MonotonicTimer usedTime; ServerGlobals::g_FilesPath = getHomeDir() + "/.gradido"; create_directories(ServerGlobals::g_FilesPath); diff --git a/src/ServerGlobals.cpp b/src/ServerGlobals.cpp index bf4b55e..1831987 100644 --- a/src/ServerGlobals.cpp +++ b/src/ServerGlobals.cpp @@ -1,6 +1,5 @@ #include "ServerGlobals.h" -#include "gradido_blockchain/lib/Profiler.h" // #include "gradido_blockchain/http/IotaRequest.h" #include "client/hiero/MirrorClient.h" diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 510a989..41eebab 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -25,7 +25,7 @@ #include "gradido_blockchain/interaction/confirmTransaction/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" #include "gradido_blockchain/serialization/toJsonString.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "loguru/loguru.hpp" @@ -146,7 +146,7 @@ namespace gradido { } if (!mTransactionTriggerEventsCache.init(GRADIDO_NODE_MAGIC_NUMBER_TRANSACTION_TRIGGER_EVENTS_CACHE_MEGA_BTYES * 1024 * 1024)) { - Profiler timeUsed; + MonotonicTimer timeUsed; rescanForTransactionTriggerEvents(); LOG_F(INFO, "rescan blockchain for transaction trigger events, time: %s", timeUsed.string().data()); } @@ -207,7 +207,7 @@ namespace gradido { mHieroMessageListener->cancelConnection(); } - Profiler timeUsed; + MonotonicTimer timeUsed; // wait until all Task of TaskObeserver are finished, wait a second and check if number decreased, // if number no longer descrease after a second and we wait more than 10 seconds total, exit loop while (auto pendingTasksCount = mTaskObserver->getPendingTasksCount()) { @@ -715,8 +715,8 @@ namespace gradido { { // load first GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE transaction into cache and validate the transaction to check file integrity if (mStopToken.stop_requested()) return false; - Profiler timeUsed; - Profiler timeSinceLastPrint; + MonotonicTimer timeUsed; + MonotonicTimer timeSinceLastPrint; data::ConstConfirmedTransactionPtr previousConfirmedTransaction = nullptr; auto lastTransaction = findOne(Filter::LAST_TRANSACTION); if (!lastTransaction) { @@ -771,7 +771,7 @@ namespace gradido { findAll(f); // printf("\r"); f.filterFunction = nullptr; - Profiler batchVerifyTime; + MonotonicTimer batchVerifyTime; auto invalidSignatures = verifySignatures(f, mCommunityId, ThreadingPolicy::ThreeQuarter); LOG_F(INFO, "time used for loading and validating last: %d transactions: %s (%s for batch verify)", count, diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 48dd97c..b4e80b5 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -19,7 +19,7 @@ #include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_protobuf_zig.h" #include "loguru/loguru.hpp" @@ -71,7 +71,7 @@ namespace cache { // check if Block exist if (mBlockFile->getCurrentFileSize()) { - Profiler timeUsed; + MonotonicTimer timeUsed; mBlockIndex->reset(); auto rebuildBlockIndexTask = make_shared( mBlockIndex, diff --git a/src/client/hiero/RequestReactor.h b/src/client/hiero/RequestReactor.h index f847ed5..4ed9c7a 100644 --- a/src/client/hiero/RequestReactor.h +++ b/src/client/hiero/RequestReactor.h @@ -8,7 +8,7 @@ #include "../Exceptions.h" #include "MessageObserver.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include #include @@ -43,7 +43,7 @@ namespace client { ); } else { - Profiler timeUsed; + MonotonicTimer timeUsed; auto block = MemoryBlock(mBuffer); auto result = pp::message_coder::decode(block.get()->span()); if (!result.has_value()) { diff --git a/src/hiero/Addressbook.cpp b/src/hiero/Addressbook.cpp index bdd08a9..ab4e36c 100644 --- a/src/hiero/Addressbook.cpp +++ b/src/hiero/Addressbook.cpp @@ -1,6 +1,6 @@ #include "../SystemExceptions.h" #include "Addressbook.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "loguru/loguru.hpp" @@ -21,7 +21,7 @@ namespace hiero { void Addressbook::load() { - Profiler timeUsed; + MonotonicTimer timeUsed; // read addressbook from binary file std::ifstream file(mFilePath, std::ios::binary | std::ios::ate); diff --git a/src/lib/protopuf.h b/src/lib/protopuf.h index ad98ea4..73ccaa6 100644 --- a/src/lib/protopuf.h +++ b/src/lib/protopuf.h @@ -7,7 +7,7 @@ #include "protopuf/message.h" */ -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/memory/Block.h" #include "loguru/loguru.hpp" @@ -34,7 +34,7 @@ namespace protopuf { template T deserialize(const memory::Block& raw) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto result = pp::message_coder::decode(raw.span()); if (!result.has_value()) { // TODO: check if using exception is better diff --git a/src/main.cpp b/src/main.cpp index a6eee90..cf4df4f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -7,7 +7,6 @@ #include "ServerGlobals.h" #include "generated/version.h" -#include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/version.h" #include "sodium.h" diff --git a/src/model/files/Block.cpp b/src/model/files/Block.cpp index 43415d7..8d8158b 100644 --- a/src/model/files/Block.cpp +++ b/src/model/files/Block.cpp @@ -10,7 +10,7 @@ #include "gradido_blockchain/data/ConfirmedTransaction.h" #include "gradido_blockchain/interaction/deserialize/Context.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "loguru/loguru.hpp" @@ -94,7 +94,7 @@ namespace model { if (startReading > mCurrentFileSize - minimalFileSize) { throw EndReachingException("file is to small for read request", mBlockPath.data(), startReading, minimalFileSize); } - Profiler timeUsed; + MonotonicTimer timeUsed; //Poco::FastMutex::ScopedLock lock(mFastMutex); auto fl = FileLockManager::getInstance(); if (!fl->tryLockTimeout(mBlockPath, 100)) { @@ -281,7 +281,7 @@ namespace model { std::shared_ptr Block::calculateHash() { - Profiler timeUsed; + MonotonicTimer timeUsed; auto fl = FileLockManager::getInstance(); if (mCurrentFileSize == 0) { diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index b448ecc..8494253 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -21,7 +21,7 @@ #include "gradido_blockchain/interaction/serialize/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" #include "gradido_blockchain/lib/DataTypeConverter.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJson.h" #include "gradido_protobuf_zig.h" @@ -117,7 +117,7 @@ namespace server { } if (method == "getLastTransaction") { - Profiler timeUsed; + MonotonicTimer timeUsed; std::string format = "base64"; getStringParameter(responseJson, params, "format", format); auto lastTransaction = blockchain->findOne(Filter::LAST_TRANSACTION); @@ -266,7 +266,7 @@ namespace server { void ApiHandler::listCommunities(rapidjson::Value& resultJson) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); const auto& groupIndex = FileBasedProvider::getInstance()->getGroupIndex(); Value communities(kArrayType); @@ -291,7 +291,7 @@ namespace server { WireOutputFormat format ) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); // count for pagination @@ -358,7 +358,7 @@ namespace server { gradido::data::LedgerAnchor* ledgerAnchor/* = nullptr */ ) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); std::shared_ptr transactionEntry; @@ -404,7 +404,7 @@ namespace server { std::shared_ptr blockchain ) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); assert(blockchain); @@ -459,7 +459,7 @@ namespace server { "query": "query ($currentPage: Int = 1, $pageSize: Int = 25, $order: Order = DESC, $onlyCreations: Boolean = false) {\n transactionList(\n currentPage: $currentPage\n pageSize: $pageSize\n order: $order\n onlyCreations: $onlyCreations\n ) {\n gdtSum\n count\n balance\n decay\n decayDate\n transactions {\n type\n balance\n decayStart\n decayEnd\n decayDuration\n memo\n transactionId\n name\n email\n date\n decay {\n balance\n decayStart\n decayEnd\n decayDuration\n decayStartBlock\n __typename\n }\n firstTransaction\n __typename\n }\n __typename\n }\n}\n" } */ - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); model::Apollo::TransactionList transactionList(blockchain, filter.updatedBalancePublicKey); @@ -483,7 +483,7 @@ namespace server { std::shared_ptr blockchain ) { - Profiler timeUsed; + MonotonicTimer timeUsed; Filter f; f.involvedPublicKey = pubkey; f.minTransactionNr = firstTransactionNr; @@ -506,7 +506,7 @@ namespace server { std::shared_ptr blockchain ) { - Profiler timeUsed; + MonotonicTimer timeUsed; Filter f; auto nameHashId = g_appContext->getUserNameHashs().getIndexForData(adapter::toByteArray<32>(nameHash)); if (!nameHashId) { diff --git a/src/task/CPUShedulerThread.cpp b/src/task/CPUShedulerThread.cpp index 1d6159d..058e9d6 100644 --- a/src/task/CPUShedulerThread.cpp +++ b/src/task/CPUShedulerThread.cpp @@ -5,7 +5,7 @@ #include "../ServerGlobals.h" #ifdef _UNI_LIB_DEBUG -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #endif //_UNI_LIB_DEBUG #include "gradido_blockchain/Application.h" @@ -31,7 +31,7 @@ namespace task { { #ifdef _UNI_LIB_DEBUG - Profiler counter; + MonotonicTimer counter; //debug::CPUShedulerTasksLog* l = debug::CPUShedulerTasksLog::getInstance(); std::string name = mWaitingTask->getName(); //l->addTaskLogEntry((HASH)mWaitingTask.getResourcePtrHolder(), mWaitingTask->getResourceType(), mName.data(), name); diff --git a/src/task/HieroMessageToTransactionTask.cpp b/src/task/HieroMessageToTransactionTask.cpp index 42fabfe..dba2454 100644 --- a/src/task/HieroMessageToTransactionTask.cpp +++ b/src/task/HieroMessageToTransactionTask.cpp @@ -1,7 +1,5 @@ #include "HieroMessageToTransactionTask.h" -#include "gradido_blockchain/lib/Profiler.h" - #include "../blockchain/FileBasedProvider.h" #include "../controller/SimpleOrderingManager.h" #include "gradido_blockchain/blockchain/Filter.h" diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp index 78622e1..8223d83 100644 --- a/src/task/RebuildBlockIndexTask.cpp +++ b/src/task/RebuildBlockIndexTask.cpp @@ -4,7 +4,6 @@ #include "../blockchain/FileBased.h" #include "gradido_blockchain/AppContext.h" -#include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_protobuf_zig.h" From 6a51c85a420a1ded5a4aafe5813f5360ae576319 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 29 Apr 2026 08:45:51 +0200 Subject: [PATCH 54/65] update submodule --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index ce54d25..af3ae7f 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit ce54d25f264b4feef3b2711edd498bfe43b1a7cd +Subproject commit af3ae7f6b6f9347fcdc2567da2a17df43ccba473 From 4d2204ab58c039df9128ae1eb55ba6ebe23eb057 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 3 May 2026 09:06:14 +0200 Subject: [PATCH 55/65] more comments about shutdown process, removed some white spaces --- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 9 +++++- src/controller/SimpleOrderingManager.cpp | 38 +++++++++++++----------- src/controller/SimpleOrderingManager.h | 15 +++++----- src/controller/TaskObserver.cpp | 6 ++-- 5 files changed, 41 insertions(+), 29 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index af3ae7f..390d6ad 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit af3ae7f6b6f9347fcdc2567da2a17df43ccba473 +Subproject commit 390d6ad3fdd00c2e72b2f9dcff5df68311ee7e95 diff --git a/src/MainServer.cpp b/src/MainServer.cpp index d41202c..811ce3a 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -182,7 +182,7 @@ bool MainServer::init() void MainServer::exit() { LOG_F(INFO, "Running Tasks Count on shutdown: %lu", ServerGlobals::g_NumberExistingTasks.load()); - + // stop worker scheduler // TODO: make sure that pending transaction are still write out to storage if (mHttpServer) { @@ -190,13 +190,20 @@ void MainServer::exit() delete mHttpServer; mHttpServer = nullptr; } + printf("[shutdown] Stopped HTTP Server...\n"); // iota::MqttClientWrapper::getInstance()->exit(); CacheManager::getInstance()->getFuzzyTimer()->stop(); + printf("[shutdown] Stopped Fuzzy Timer...\n"); // ServerGlobals::g_IotaRequestCPUScheduler->stop(); FileBasedProvider::getInstance()->exit(); + printf("[shutdown] Stopped File Based Provider...\n"); ServerGlobals::g_CPUScheduler->stop(); + printf("[shutdown] Stopped CPU Scheduler...\n"); ServerGlobals::g_WriteFileCPUScheduler->stop(); + printf("[shutdown] Stopped IO Worker...\n"); + ServerGlobals::clearMemory(); + printf("[shutdown] Cleared Memory...\n"); } bool MainServer::configExists(const string& fileName) { diff --git a/src/controller/SimpleOrderingManager.cpp b/src/controller/SimpleOrderingManager.cpp index 4bf8c4c..5da3238 100644 --- a/src/controller/SimpleOrderingManager.cpp +++ b/src/controller/SimpleOrderingManager.cpp @@ -18,34 +18,38 @@ using gradido::data::LedgerAnchor; namespace controller { - SimpleOrderingManager::SimpleOrderingManager(std::string_view communityId) - : task::Thread("SimpleOrderingManager"), mInitalized(false), - mLastTransactions(MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION * 2), - mCommunityId(communityId), + SimpleOrderingManager::SimpleOrderingManager(std::string_view communityId, std::stop_token stopToken) + : task::Thread("SimpleOrderingManager"), mStopToken(stopToken), mInitalized(false), + mLastTransactions(MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION * 2), + mCommunityId(communityId), mLastSequenceNumber(0) { } SimpleOrderingManager::~SimpleOrderingManager() { + } void SimpleOrderingManager::reinitialize(uint64_t lastKnownSequenceNumber) { - std::unique_lock _lock(mTransactionsMutex); - mLastSequenceNumber = lastKnownSequenceNumber; - mTransactions.clear(); - mLastTransactions.clear(); - if (!mInitalized) { - mInitalized = true; - Thread::init(); - } + std::unique_lock _lock(mTransactionsMutex); + mLastSequenceNumber = lastKnownSequenceNumber; + mTransactions.clear(); + mLastTransactions.clear(); + if (!mInitalized) { + mInitalized = true; + Thread::init(); + } } int SimpleOrderingManager::ThreadFunction() { size_t transactionsCount = 0; do { + if (mStopToken.stop_requested()) { + return 0; + } std::unique_lock _lock(mTransactionsMutex); auto it = mTransactions.begin(); // if no transaction is in map or first transaction deserialize task is still running (or is waiting to be scheduled) @@ -87,7 +91,7 @@ namespace controller { Timepoint now = std::chrono::system_clock::now(); if (it->second.putIntoListTime + MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION < now) { - // timeouted + // timeouted task->notificateFailedTransaction(blockchain, "Transaction skipped (pairing not found)"); mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); @@ -124,7 +128,7 @@ namespace controller { } if (task->isSuccess()) { processTransaction(it->second); - } + } mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); transactionsCount = mTransactions.size(); @@ -156,7 +160,7 @@ namespace controller { LOG_F(INFO, "Transaction confirmed, msgId: %s, confirmedAt: %s", transactionId.toString().data(), confirmedAt.toString().data() ); - + } catch (GradidoBlockchainException& ex) { auto communityServer = fileBasedBlockchain->getListeningCommunityServer(); @@ -191,7 +195,7 @@ namespace controller { return PushResult::FOUND_IN_LAST_TRANSACTIONS; } } - + auto range = mTransactions.equal_range(consensusTimestamp); for (auto& it = range.first; it != range.second; ++it) { if (it->second.consensusTopicResponse.isMessageSame(consensusTopicResponse)) { @@ -231,7 +235,7 @@ namespace controller { void SimpleOrderingManager::updateSequenceNumber(uint64_t newSequenceNumber) { - + if (!mLastSequenceNumber) { mLastSequenceNumber = newSequenceNumber; } diff --git a/src/controller/SimpleOrderingManager.h b/src/controller/SimpleOrderingManager.h index 73b532a..71a16fb 100644 --- a/src/controller/SimpleOrderingManager.h +++ b/src/controller/SimpleOrderingManager.h @@ -36,8 +36,8 @@ namespace controller { class SimpleOrderingManager : public task::Thread { public: - SimpleOrderingManager(std::string_view communityId); - ~SimpleOrderingManager(); + SimpleOrderingManager(std::string_view communityId, std::stop_token stopToken); + virtual ~SimpleOrderingManager(); // combine init and reset void reinitialize(uint64_t lastKnownSequenceNumber); @@ -51,7 +51,7 @@ namespace controller { PushResult pushTransaction(hiero::ConsensusTopicResponse&& consensusTopicResponse); std::shared_ptr findCrossGroupTransactionPair(const gradido::data::LedgerAnchor& transactionId) const; - protected: + protected: inline uint64_t getLastSequenceNumber() const { return mLastSequenceNumber; } void updateSequenceNumber(uint64_t newSequenceNumber); @@ -63,21 +63,21 @@ namespace controller { TopicResponseDeserializer( hiero::ConsensusTopicResponse&& _consensusTopicResponse, std::shared_ptr _deserializeTask - ) : consensusTopicResponse(std::move(_consensusTopicResponse)), - deserializeTask(_deserializeTask), + ) : consensusTopicResponse(std::move(_consensusTopicResponse)), + deserializeTask(_deserializeTask), putIntoListTime(std::chrono::system_clock::now()) { } TopicResponseDeserializer(TopicResponseDeserializer&& move) noexcept - : consensusTopicResponse(std::move(move.consensusTopicResponse)), + : consensusTopicResponse(std::move(move.consensusTopicResponse)), deserializeTask(std::move(move.deserializeTask)), putIntoListTime(std::move(move.putIntoListTime)) { } TopicResponseDeserializer(const TopicResponseDeserializer& other) noexcept - : consensusTopicResponse(other.consensusTopicResponse), + : consensusTopicResponse(other.consensusTopicResponse), deserializeTask(other.deserializeTask), putIntoListTime(other.putIntoListTime) { @@ -93,6 +93,7 @@ namespace controller { void processTransaction(const TopicResponseDeserializer& gradidoTransactionWorkData); + std::stop_token mStopToken; bool mInitalized; // fast duplication check // use first 4 and last 4 Byte of transaction hash as key diff --git a/src/controller/TaskObserver.cpp b/src/controller/TaskObserver.cpp index e31195b..ccd6bce 100644 --- a/src/controller/TaskObserver.cpp +++ b/src/controller/TaskObserver.cpp @@ -29,7 +29,7 @@ bool TaskObserver::addBlockWriteTask(std::shared_ptrgetTransactionEntriesList(); mTransactionsFromPendingTasks.insert(transactions->begin(), transactions->end()); - + return true; } @@ -89,7 +89,7 @@ std::shared_ptr TaskObserver::getTransaction(uint64_t tran // *********************** Finish command ************************************** int TaskObserverFinishCommand::taskFinished(task::Task* task) -{ - mBlockchain->getTaskObserver().removeTask(task); +{ + mBlockchain->getTaskObserver().removeTask(task); return 0; } \ No newline at end of file From aa8067f452fc3a37fe6b29a948dc6803bbbad55d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 3 May 2026 09:09:27 +0200 Subject: [PATCH 56/65] fix build --- src/blockchain/FileBased.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index 41eebab..2202ed2 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -65,7 +65,7 @@ namespace gradido { mFolderPath(folder), mCommunityId(communityId), mTaskObserver(std::make_shared()), - mOrderingManager(std::make_shared(communityId)), + mOrderingManager(std::make_shared(communityId, stopToken)), // mIotaMessageListener(new iota::MessageListener(communityId, alias)), mPublicKeysIndex((string(folder).append("/pubkeysCache"))), mBlockchainState(string(folder).append("/.state")), @@ -125,13 +125,13 @@ namespace gradido { { if (mStopToken.stop_requested()) return false; - lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); auto lastBlockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); // trigger block index creation, only needed here for non-persistent public key dictionary iterateBlocks(SearchDirection::ASC, [](const cache::Block& block) -> bool { return true; }); loadStateFromBlockCache(); - + if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { mLedgerAnchorCache.reset(); if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { From 9c0dc64a5b0a2d9e0b8c9a1aa17382ab5212613f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 4 May 2026 08:57:55 +0200 Subject: [PATCH 57/65] wip exit process --- src/controller/SimpleOrderingManager.cpp | 2 +- src/task/HieroMessageToTransactionTask.cpp | 2 +- src/task/SyncTopic.cpp | 26 ++++++++++++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/controller/SimpleOrderingManager.cpp b/src/controller/SimpleOrderingManager.cpp index 5da3238..fde3127 100644 --- a/src/controller/SimpleOrderingManager.cpp +++ b/src/controller/SimpleOrderingManager.cpp @@ -72,7 +72,7 @@ namespace controller { // maybe we have an error // or hiero has used the same sequence number twice? LOG_F( - ERROR, + ERROR, "this transaction or after this was already put into blockchain, fatal error, programm code must be fixed, communityId: %s, last sequence number: %lu, current sequence number: %lu", mCommunityId.data(), lastSequenceNumber, currentSequenceNumber ); diff --git a/src/task/HieroMessageToTransactionTask.cpp b/src/task/HieroMessageToTransactionTask.cpp index dba2454..cdc6163 100644 --- a/src/task/HieroMessageToTransactionTask.cpp +++ b/src/task/HieroMessageToTransactionTask.cpp @@ -79,7 +79,7 @@ namespace task { } // check if transaction already exist - // if this transaction doesn't belong to us, we can quit here + // if this transaction doesn't belong to us, we can quit here // also if we already have this transaction auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); assert(fileBasedBlockchain); diff --git a/src/task/SyncTopic.cpp b/src/task/SyncTopic.cpp index 547c36a..4e0cd1f 100644 --- a/src/task/SyncTopic.cpp +++ b/src/task/SyncTopic.cpp @@ -11,6 +11,7 @@ #include "gradido_blockchain/serialization/toJsonString.h" #include "loguru/loguru.hpp" +#include "magic_enum/magic_enum.hpp" #include #include @@ -18,6 +19,7 @@ using namespace gradido; using namespace blockchain; using namespace interaction; +using namespace magic_enum; using namespace serialization; using gradido::blockchain::FileBased; @@ -53,10 +55,11 @@ namespace task { // for example: check previous transaction until one was found, // or check if maybe or block files get corrupted auto lastTransactionIdentical = checkLastTransaction(); + LOG_F(INFO, "last transaction state: %s, last known sequence number: %lu", enum_name(lastTransactionIdentical).data(), mLastKnownSequenceNumber); // (re-)start ordering manager orderingManager->reinitialize(mLastKnownSequenceNumber); - + // load transactions from mirror node mConfirmedAtLastReadedTransaction = data::Timestamp(); if (LastTransactionState::IDENTICAL == lastTransactionIdentical) { @@ -119,13 +122,13 @@ namespace task { deserializer.run(mBlockchain->getCommunityIdIndex()); if (!deserializer.isGradidoTransaction()) { LOG_F( - ERROR, + ERROR, "last transaction on mirrors is invalid Gradido Transaction. CommunityId: %s, lastKnownTopicId: %s, sequenceNumber: %lu, transactionNr: %lu, confirmedAt: %s", mBlockchain->getCommunityId().data(), mLastKnowTopicId.toString().data(), mLastKnownSequenceNumber, lastTransaction->getTransactionNr(), - lastTransaction->getConfirmedTransaction()->getConfirmedAt().toString().data() + confirmedAt.toString().data() ); return LastTransactionState::INVALID; } @@ -135,10 +138,10 @@ namespace task { bool prettyJson = true; LOG_F(ERROR, "own transaction: %s", toJsonString(*lastTransaction->getConfirmedTransaction(), prettyJson).data()); LOG_F( - ERROR, - "from topic: %s, sequenceNumber: %lu: %s", - mLastKnowTopicId.toString().data(), - mLastKnownSequenceNumber, + ERROR, + "from topic: %s, sequenceNumber: %lu: %s", + mLastKnowTopicId.toString().data(), + mLastKnownSequenceNumber, toJsonString(*lastTransactionMirror, prettyJson).data() ); return LastTransactionState::NOT_IDENTICAL; @@ -160,6 +163,15 @@ namespace task { auto responses = mirrorNode->listTopicMessagesById(topicId, mConfirmedAtLastReadedTransaction, limit, "asc"); if (responses.empty()) break; responsesCount = responses.size(); + if (responses.back().getConsensusTimestamp() <= mConfirmedAtLastReadedTransaction) { + LOG_F(WARNING, "unexpected response from mirror node, transaction has the same or lower consensus timestamp than last readed transaction. CommunityId: %s, topicId: %s, last readed consensus timestamp: %s, last readed sequence number: %lu, responses count: %u", + mBlockchain->getCommunityId().data(), + topicId.toString().data(), + mConfirmedAtLastReadedTransaction.toString().data(), + mLastKnownSequenceNumber, + responsesCount + ); + } mConfirmedAtLastReadedTransaction = responses.back().getConsensusTimestamp(); addedTransactionsSum += responsesCount; for (auto& response : responses) { From 89f405d83c1597a73170b1a1fc9afbe2afa9c0ab Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 13 May 2026 09:46:06 +0200 Subject: [PATCH 58/65] fix bug with missing add compact tx to cache, after removing --- dependencies/gradido_blockchain | 2 +- src/cache/Block.cpp | 15 ++++++++++++--- src/cache/Block.h | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 390d6ad..9c521cf 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 390d6ad3fdd00c2e72b2f9dcff5df68311ee7e95 +Subproject commit 9c521cf9143e4ab725b2529f0b411990d76c8c15 diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index b4e80b5..8abd187 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -149,7 +149,7 @@ namespace cache { // mBlockIndex->updateAddressIndex(transactionEntry, publicKeyDictionary); } - void Block::addCompactTransaction(shared_ptr transactionEntry, AppContext& appContext) const + std::shared_ptr Block::addCompactTransaction(shared_ptr transactionEntry, AppContext& appContext) const { // create compact version try { @@ -165,6 +165,7 @@ namespace cache { transactionEntry->getTransactionBody()->toGrdw(&alloc, &txBody); confirmedTxPtr->fillFromGrdwTransactionBody(&txBody, appContext); mConfirmedTxByNr.add(confirmedTxPtr->txNr, confirmedTxPtr); + return confirmedTxPtr; } catch (GradidoBlockchainException& ex) { LOG_F(WARNING, "%s on create compact", ex.getFullString().c_str()); @@ -249,10 +250,18 @@ namespace cache { auto confirmedTx = mConfirmedTxByNr.get(transactionNr); if (!confirmedTx) { // check write cache, else try to read from storage - getTransaction(transactionNr, appContext); + // return always a valid ptr or throw exception + auto transactionEntry = getTransaction(transactionNr, appContext); + + // we use two different access expire caches, after this call succeed it is sure, that the transaction is in Serialized Transactions, + // but it can be still missing in mConfirmedTxByNr + confirmedTx = mConfirmedTxByNr.get(transactionNr); + if (!confirmedTx) { + return addCompactTransaction(transactionEntry, appContext); + } } // should only don't work, if getTransaction failed, but this will throw an exception anyway - return mConfirmedTxByNr.get(transactionNr).value(); + return confirmedTx.value(); } bool Block::hasSpaceLeft() { diff --git a/src/cache/Block.h b/src/cache/Block.h index d9f6a68..ed80728 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -95,7 +95,7 @@ namespace cache { int32_t fileCursor, gradido::AppContext& appContext ) const; - void addCompactTransaction(std::shared_ptr transactionEntry, gradido::AppContext& appContext) const; + std::shared_ptr addCompactTransaction(std::shared_ptr transactionEntry, gradido::AppContext& appContext) const; mutable std::mutex mFastMutex; uint32_t mBlockNr; From 53c89419496e592703c640dd142db994d99440a0 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 13 May 2026 10:47:26 +0200 Subject: [PATCH 59/65] fix linux build error --- src/cache/Block.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 8abd187..cbcd350 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -66,10 +66,10 @@ namespace cache { { lock_guard lock(mFastMutex); // todo: add data for address index in file, until then rebuild block index on each program start - // if (!mBlockIndex->loadFromFile(publicKeyDictionary)) + // if (!mBlockIndex->loadFromFile(publicKeyDictionary)) { // check if Block exist - if (mBlockFile->getCurrentFileSize()) + if (mBlockFile->getCurrentFileSize()) { MonotonicTimer timeUsed; mBlockIndex->reset(); @@ -99,7 +99,7 @@ namespace cache { /*else { // hot fix: init address index // if block index was loaded from file, we load all transactions which change something in address index - // TODO: persistent storage for address index + // TODO: persistent storage for address index } */ return true; @@ -137,7 +137,7 @@ namespace cache { } void Block::addTransaction( - memory::ConstBlockPtr serializedTransaction, + memory::ConstBlockPtr serializedTransaction, int32_t fileCursor, AppContext& appContext ) const @@ -170,6 +170,7 @@ namespace cache { catch (GradidoBlockchainException& ex) { LOG_F(WARNING, "%s on create compact", ex.getFullString().c_str()); } + return nullptr; } shared_ptr Block::getTransaction(uint64_t transactionNr, AppContext& appContext) const @@ -199,7 +200,7 @@ namespace cache { int32_t fileCursor = 0; if (!mBlockIndex->getFileCursorForTransactionNr(transactionNr, fileCursor)) { LOG_F(INFO, "writeTransactionTaskExist: %d, writeTransactionTaskIsObserved: %d", - (int)writeTransactionTaskExist, + (int)writeTransactionTaskExist, (int)writeTransactionTaskIsObserved ); throw GradidoBlockchainTransactionNotFoundException("transaction not found in cache, in write task or file").setTransactionId(transactionNr); @@ -252,7 +253,7 @@ namespace cache { // check write cache, else try to read from storage // return always a valid ptr or throw exception auto transactionEntry = getTransaction(transactionNr, appContext); - + // we use two different access expire caches, after this call succeed it is sure, that the transaction is in Serialized Transactions, // but it can be still missing in mConfirmedTxByNr confirmedTx = mConfirmedTxByNr.get(transactionNr); @@ -279,7 +280,7 @@ namespace cache { if (mTransactionWriteTask) { Timepoint now = std::chrono::system_clock::now(); - if (now - mTransactionWriteTask->getCreationDate() > ServerGlobals::g_WriteToDiskTimeout) + if (now - mTransactionWriteTask->getCreationDate() > ServerGlobals::g_WriteToDiskTimeout) { auto copyTask = mTransactionWriteTask; mBlockchain->getTaskObserver().addBlockWriteTask(copyTask); From 4a942fdd55462b82c564e502aeec94855ae5bce3 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 17 May 2026 12:32:45 +0200 Subject: [PATCH 60/65] remove iota, fix for changed gradido blockchain (core) --- config/gradido.yaml | 23 +++++++-------------- dependencies/gradido_blockchain | 2 +- src/ServerGlobals.cpp | 24 ---------------------- src/ServerGlobals.h | 2 -- src/blockchain/NodeTransactionEntry.cpp | 8 +++++--- src/cache/Block.cpp | 9 ++++++--- src/model/files/Block.cpp | 9 +++++---- src/model/files/Block.h | 7 +++---- src/server/json-rpc/ApiHandler.cpp | 1 - src/task/RebuildBlockIndexTask.cpp | 27 ++++++++++++++----------- src/task/RebuildBlockIndexTask.h | 8 +++++--- 11 files changed, 47 insertions(+), 73 deletions(-) diff --git a/config/gradido.yaml b/config/gradido.yaml index 1916735..fcd601c 100644 --- a/config/gradido.yaml +++ b/config/gradido.yaml @@ -1,26 +1,17 @@ # Client configuration, accessing extern services -clients: - # IOTA Configuration - iota: - rest_api: - host: api.lb-0.h.chrysalis-devnet.iota.cafe - port: 443 # automatic using HTTPS if 443 - mqtt: - host: api.lb-0.h.chrysalis-devnet.iota.cafe - port: 1883 - +clients: hiero: networkType: testnet - + # Server Configuration, which services are served on which ports server: json_rpc: 8340 # Port for JSON RPC 2.0 Requests # Cache Settings cache: - timeout: 600 # Cache Timeout in seconds, how long to keep transactions and other data in memory cache - checks_interval: 10 # How often to check for cache timeout in seconds - write_to_disk_interval: 10 # How often block and index data will be flushed to disk in seconds + timeout: 600 # Cache Timeout in seconds, how long to keep transactions and other data in memory cache + checks_interval: 10 # How often to check for cache timeout in seconds + write_to_disk_interval: 10 # How often block and index data will be flushed to disk in seconds # Logging Settings logging: @@ -28,8 +19,8 @@ logging: # I/O Settings io: - worker_count: 1 # How many workers are allowed to access disk at the same time + worker_count: 1 # How many workers are allowed to access disk at the same time # Security Settings unsecure: - allow_cors_all: 0 # Set to 1 if js frontend should be allowed to send requests to it + allow_cors_all: 0 # Set to 1 if js frontend should be allowed to send requests to it diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 9c521cf..b9f5929 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 9c521cf9143e4ab725b2529f0b411990d76c8c15 +Subproject commit b9f5929acca75214f24e8c44cf11f622dc5e4b90 diff --git a/src/ServerGlobals.cpp b/src/ServerGlobals.cpp index 1831987..a90b2cd 100644 --- a/src/ServerGlobals.cpp +++ b/src/ServerGlobals.cpp @@ -1,6 +1,5 @@ #include "ServerGlobals.h" -// #include "gradido_blockchain/http/IotaRequest.h" #include "client/hiero/MirrorClient.h" using namespace std::chrono; @@ -15,7 +14,6 @@ namespace ServerGlobals { std::chrono::seconds g_CacheTimeout(600); std::chrono::seconds g_TimeoutCheck(60); std::chrono::seconds g_WriteToDiskTimeout(10); - IotaRequest* g_IotaRequestHandler = nullptr; std::string g_IotaMqttBrokerUri; std::atomic g_NumberExistingTasks; bool g_LogTransactions = false; @@ -40,10 +38,6 @@ namespace ServerGlobals { delete g_GroupIndex; g_GroupIndex = nullptr; } - if (g_IotaRequestHandler) { - delete g_IotaRequestHandler; - g_IotaRequestHandler = nullptr; - } if (g_HieroMirrorNode) { delete g_HieroMirrorNode; g_HieroMirrorNode = nullptr; @@ -51,24 +45,6 @@ namespace ServerGlobals { } - /*bool initIota(const MapEnvironmentToConfig& cfg) - { - // testnet - // api.lb-0.h.chrysalis-devnet.iota.cafe - // mainnet: - // chrysalis-nodes.iota.org - std::string iotaHost = cfg.getString("clients.iota.rest_api.host", "api.lb-0.h.chrysalis-devnet.iota.cafe"); - int iotaPort = cfg.getInt("clients.iota.rest_api.port", 443); - g_IotaRequestHandler = new IotaRequest(iotaHost, iotaPort, "/api/v1/"); - - std::string iotaMqttHost = cfg.getString("clients.iota.mqtt.host", "api.lb-0.h.chrysalis-devnet.iota.cafe"); - int mqttPort = cfg.getInt("clients.iota.mqtt.port", 1883); - g_IotaMqttBrokerUri = iotaHost + ":" + std::to_string(mqttPort); - - g_isOfflineMode = cfg.getBool("clients.isOfflineMode", false); - return true; - }*/ - bool initHiero(std::string_view hieroNetworkType) { g_HieroMirrorNode = new client::hiero::MirrorClient(hieroNetworkType); return true; diff --git a/src/ServerGlobals.h b/src/ServerGlobals.h index 8c346ca..6ef983e 100644 --- a/src/ServerGlobals.h +++ b/src/ServerGlobals.h @@ -1,7 +1,6 @@ #ifndef GRADIDO_NODE_SERVER_GLOBALS #define GRADIDO_NODE_SERVER_GLOBALS -#include "gradido_blockchain/http/IotaRequest.h" #include "gradido_blockchain/lib/MapEnvironmentToConfig.h" #include "cache/GroupIndex.h" #include "task/CPUSheduler.h" @@ -28,7 +27,6 @@ namespace ServerGlobals { extern std::chrono::seconds g_TimeoutCheck; //! in which timespan data will be flushed to disk, in seconds, default 10 seconds extern std::chrono::seconds g_WriteToDiskTimeout; - extern IotaRequest* g_IotaRequestHandler; extern std::string g_IotaMqttBrokerUri; extern std::atomic g_NumberExistingTasks; extern bool g_LogTransactions; diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index e0e4da0..49861e5 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -1,9 +1,11 @@ #include "NodeTransactionEntry.h" #include "FileBased.h" +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain_core/data/wire/confirmed_transaction.h" +#include "gradido_blockchain_core/data/wire/transaction_body.h" #include "gradido_blockchain/data/adapter/publicKey.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" -#include "gradido_protobuf_zig.h" namespace gradido { using data::adapter::toPublicKey; @@ -69,8 +71,8 @@ namespace gradido { ConfirmedGradidoTx NodeTransactionEntry::convertToCompactConfirmedTx() const { uint8_t buffer[1024]; - grdu_memory alloc; - grdu_memory_init_static(&alloc, buffer, 1024); + grd_memory alloc; + grd_memory_init_arena_static(&alloc, buffer, 1024); grdw_confirmed_transaction tx{}; getConfirmedTransaction()->toGrdw(&alloc, &tx, mBlockchainCommunityIdIndex); auto confirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, mBlockchainCommunityIdIndex, *g_appContext); diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 8abd187..04f3100 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -12,6 +12,9 @@ #include "../SingletonManager/CacheManager.h" +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain_core/data/wire/confirmed_transaction.h" +#include "gradido_blockchain_core/data/wire/transaction_body.h" #include "gradido_blockchain/Application.h" #include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" @@ -20,7 +23,6 @@ #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_blockchain/lib/MonotonicTimer.h" -#include "gradido_protobuf_zig.h" #include "loguru/loguru.hpp" @@ -154,8 +156,8 @@ namespace cache { // create compact version try { uint8_t buffer[1024]; - grdu_memory alloc; - grdu_memory_init_static(&alloc, buffer, 1024); + grd_memory alloc; + grd_memory_init_arena_static(&alloc, buffer, 1024); grdw_confirmed_transaction tx{}; auto communityIdIndex = mBlockchain->getCommunityIdIndex(); transactionEntry->getConfirmedTransaction()->toGrdw(&alloc, &tx, communityIdIndex); @@ -170,6 +172,7 @@ namespace cache { catch (GradidoBlockchainException& ex) { LOG_F(WARNING, "%s on create compact", ex.getFullString().c_str()); } + return nullptr; } shared_ptr Block::getTransaction(uint64_t transactionNr, AppContext& appContext) const diff --git a/src/model/files/Block.cpp b/src/model/files/Block.cpp index 8d8158b..9840a3a 100644 --- a/src/model/files/Block.cpp +++ b/src/model/files/Block.cpp @@ -141,7 +141,7 @@ namespace model { auto transactionSize = readLine(startReading, &result); return result; } - bool Block::readBuffered(grdu_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken/* = std::stop_token()*/) + bool Block::readBuffered(grd_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken/* = std::stop_token()*/) { if (stopToken.stop_requested()) { return false; @@ -193,14 +193,15 @@ namespace model { throw InvalidReadBlockSize("transactionSize is to small to contain a transaction", mBlockPath.data(), readed, transactionSize); } auto memStart = alloc->last_index; - auto buffer = grdu_memory_alloc(alloc, transactionSize); + grd_memory_block buffer; + grd_memory_block_alloc(&buffer, alloc, transactionSize); if (alloc->out_of_memory_capacity) { // TODO: own exception throw GradidoNodeInvalidDataException("memory buffer to small"); } - fileStream->read(reinterpret_cast(buffer), transactionSize); + fileStream->read(reinterpret_cast(buffer.data), buffer.size); readed += transactionSize; - calculateOneHashStep(hash, buffer, transactionSize); + calculateOneHashStep(hash, buffer.data, buffer.size); callback->finishedLine(memStart, transactionSize, fileCursor); } callback->flush(); diff --git a/src/model/files/Block.h b/src/model/files/Block.h index 84d1455..e888cee 100644 --- a/src/model/files/Block.h +++ b/src/model/files/Block.h @@ -7,8 +7,6 @@ #include "../../task/CPUTask.h" -#include "gradido_protobuf_zig.h" - #include #include @@ -18,11 +16,12 @@ //! MAGIC NUMBER: use to check if a file is big enough to could contain a transaction #define MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE 25 +struct grd_memory; + namespace cache { class BlockIndex; } - namespace controller { class AddressIndex; } @@ -73,7 +72,7 @@ namespace model { uint16_t readLine(uint32_t startReading, memory::BlockPtr* buffer); std::shared_ptr readLine(uint32_t startReading); // read whole file, validate hash - bool readBuffered(grdu_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken = std::stop_token()); + bool readBuffered(grd_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken = std::stop_token()); //! \brief call appendLines //! \return file cursor pos at start from this line in file (0 at start of file) diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 8494253..02cfb13 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -24,7 +24,6 @@ #include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJson.h" -#include "gradido_protobuf_zig.h" #include "../../blockchain/FileBased.h" #include "../../blockchain/FileBasedProvider.h" diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp index 8223d83..43ea2d1 100644 --- a/src/task/RebuildBlockIndexTask.cpp +++ b/src/task/RebuildBlockIndexTask.cpp @@ -3,10 +3,12 @@ #include "../ServerGlobals.h" #include "../blockchain/FileBased.h" +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain_core/data/wire/confirmed_transaction.h" +#include "gradido_blockchain_core/data/wire/transaction_body.h" #include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" -#include "gradido_protobuf_zig.h" #include "loguru/loguru.hpp" #include "magic_enum/magic_enum.hpp" @@ -28,7 +30,7 @@ namespace task { ): task::CPUTask(g_CPUScheduler), mBlockIndex(blockIndex), mCommunityIdIndex(communityIdIndex), mLastLineReaded(false) { - grdu_memory_init_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + grd_memory_init_arena_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); } int RebuildBlockIndexTask::run() @@ -40,20 +42,21 @@ namespace task { void RebuildBlockIndexTask::finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) { - grdu_memory decodeMemoryAlloc; - grdu_memory_init_static(&decodeMemoryAlloc, mBuffers[1], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + grd_memory decodeMemoryAlloc; + grd_memory_init_arena_static(&decodeMemoryAlloc, mBuffers[1], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); grdw_confirmed_transaction tx{}; - auto encodeResult = grdw_confirmed_transaction_decode(&decodeMemoryAlloc, &tx, mBuffers[0], size); - if (GRDW_ENCODING_ERROR_SUCCESS != encodeResult.state) { - LOG_F(ERROR, "decode error: %s", enum_name(encodeResult.state).data()); + grd_memory_block src = { .data = mBuffers[0], .size = size }; + auto result = grdw_confirmed_transaction_decode(&tx, &src, &decodeMemoryAlloc); + if (GRD_SUCCESS != result) { + LOG_F(ERROR, "decode error: %s", enum_name(result).data()); throw GradidoNodeInvalidDataException("error deserialize confirmed transaction"); } grdw_transaction_body body{}; - grdu_memory_init_static(&decodeMemoryAlloc, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); - encodeResult = grdw_transaction_body_decode(&decodeMemoryAlloc, &body, tx.transaction.body_bytes, tx.transaction.body_bytes_size); - if (GRDW_ENCODING_ERROR_SUCCESS != encodeResult.state) { - LOG_F(ERROR, "body decode error: %s", enum_name(encodeResult.state).data()); + grd_memory_init_arena_static(&decodeMemoryAlloc, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + result = grdw_transaction_body_decode(&body, &tx.transaction.body_bytes, &decodeMemoryAlloc); + if (GRD_SUCCESS != result) { + LOG_F(ERROR, "body decode error: %s", enum_name(result).data()); throw GradidoNodeInvalidDataException("error deserialize transaction body"); } @@ -61,7 +64,7 @@ namespace task { mBlockIndex->addIndicesForTransaction(compactConfirmedTx, g_appContext->getCommunityContext(mCommunityIdIndex).getBlockchain()->getPublicKeyDictionary()); mBlockIndex->addFileCursorForTransaction(compactConfirmedTx.txNr, fileCursor); - grdu_memory_init_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + grd_memory_init_arena_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); } void RebuildBlockIndexTask::flush() diff --git a/src/task/RebuildBlockIndexTask.h b/src/task/RebuildBlockIndexTask.h index 015a10c..209a448 100644 --- a/src/task/RebuildBlockIndexTask.h +++ b/src/task/RebuildBlockIndexTask.h @@ -3,6 +3,7 @@ #include "CPUTask.h" #include "../model/files/Block.h" +#include "gradido_blockchain_core/memory.h" #include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/lib/DictionaryInterface.h" @@ -13,6 +14,7 @@ #include #include + namespace gradido { namespace blockchain { class FileBased; @@ -30,7 +32,7 @@ namespace cache { } // TODO: maybe put into config -#define REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE 1024 +#define REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE 2048 namespace task { @@ -50,12 +52,12 @@ namespace task { virtual void finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) override; virtual void flush() override; - inline grdu_memory* getAlloc() { return &mReadInAllocator; } + inline grd_memory* getAlloc() { return &mReadInAllocator; } protected: uint8_t mBuffers[2][REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE]; - grdu_memory mReadInAllocator; + grd_memory mReadInAllocator; std::mutex mWorkConfirmedMutex; gradido::data::compact::ConfirmedGradidoTx mConfirmedTx; std::condition_variable mConfirmedTxReadyCondition; From 2d0ba76a00f5b5ffe49b6e33fc6b2f45b39e9c52 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 17 May 2026 15:42:21 +0200 Subject: [PATCH 61/65] fix some bugs with protobuf --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index b9f5929..a697ccc 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit b9f5929acca75214f24e8c44cf11f622dc5e4b90 +Subproject commit a697ccc4bd5812376dc3434ff137dfb1996804e8 From 12935a9e858771223fa1066871e68475e70b1bb4 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 24 May 2026 13:26:59 +0200 Subject: [PATCH 62/65] adapt to changed Uuid handling --- CMakeLists.txt | 8 +++----- dependencies/gradido_blockchain | 2 +- src/MainServer.cpp | 4 +++- src/blockchain/FileBased.h | 10 +++++----- src/blockchain/FileBasedProvider.cpp | 7 +++++-- src/cache/Block.cpp | 4 +++- src/cache/Block.h | 2 +- src/cache/BlockIndex.cpp | 4 +++- src/cache/BlockIndex.h | 8 ++++---- src/server/json-rpc/ApiHandler.cpp | 7 ++++++- src/server/json-rpc/WireFilter.h | 4 ++-- src/server/json-rpc/fromJson.cpp | 3 ++- src/task/RebuildBlockIndexTask.h | 1 - src/task/WriteTransactionsToBlockTask.cpp | 3 ++- src/task/WriteTransactionsToBlockTask.h | 4 ++-- 15 files changed, 42 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f9c6d62..2bb1c60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,7 +46,7 @@ configure_file( set(GRADIDO_NODE_VERSION "@PROJECT_VERSION@") include_directories( - "dependencies" + "dependencies" "dependencies/jsonrpcpp/include" "dependencies/gradido_blockchain" "dependencies/gradido_blockchain/include" @@ -213,7 +213,7 @@ endif() #add_subdirectory(dependencies/protobuf/cmake) #set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -# prevent problems with two libs including googletest +# prevent problems with two libs including googletest #option(protobuf_BUILD_TESTS "Build tests" OFF) #message("binary dir: ${CMAKE_BINARY_DIR}") @@ -244,7 +244,7 @@ else() endif() target_link_libraries(GradidoNode ${_GRPC_GRPCPP}) -IF(WIN32) +IF(WIN32 AND BUILD_SHARED_LIBS) add_custom_command(TARGET GradidoNode POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ $ COMMAND_EXPAND_LISTS @@ -259,5 +259,3 @@ endif(UNIX) # add test #include(CTest) #add_subdirectory(test) - - diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index a697ccc..f0545d2 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit a697ccc4bd5812376dc3434ff137dfb1996804e8 +Subproject commit f0545d2481e12f23755af8320a0cef0453a12382 diff --git a/src/MainServer.cpp b/src/MainServer.cpp index 811ce3a..9f28e76 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -29,6 +29,8 @@ using gradido::blockchain::FileBasedProvider; using gradido::g_appContext, gradido::AppContext; +using gradido::data::GenericHash, gradido::data::GenericHashHash, gradido::data::GenericHashEqual; +using gradido::data::Uuid, gradido::data::UuidHash, gradido::data::UuidEqual; using server::json_rpc::ApiHandler; using std::filesystem::create_directories, std::filesystem::exists, std::filesystem::is_regular_file, std::filesystem::path; using std::shared_ptr, std::make_unique; @@ -77,7 +79,7 @@ bool MainServer::init() unsigned short jsonrpc_port = (unsigned short)config.getInt("server.json_rpc", 8340); - auto communityDictionary = make_unique>(ServerGlobals::g_FilesPath + "/communityIdsCache"); + auto communityDictionary = make_unique>(ServerGlobals::g_FilesPath + "/communityIdsCache"); communityDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); auto nameHashDictionary = make_unique>(ServerGlobals::g_FilesPath + "/nameHashCache"); nameHashDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 6398b7f..d76dd8f 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -12,7 +12,7 @@ #include "../lib/PersistentDictionary.h" #include "gradido_blockchain/blockchain/Abstract.h" -#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/hiero/TopicId.h" #include "gradido_blockchain/lib/AccessExpireCache.h" @@ -173,12 +173,12 @@ namespace gradido { inline void setListeningCommunityServer(std::shared_ptr client); inline std::shared_ptr getListeningCommunityServer() const; - inline virtual const IDictionary& getPublicKeyDictionary() const override { return mPublicKeysIndex; } - inline virtual uint32_t getOrAddPublicKey(const PublicKey& publicKey) override { + inline virtual const IDictionary& getPublicKeyDictionary() const override { return mPublicKeysIndex; } + inline virtual uint32_t getOrAddPublicKey(const data::PublicKey& publicKey) override { return mPublicKeysIndex.getOrAddIndexForData(publicKey); } - inline uint32_t getOrAddIndexForPublicKey(const PublicKey& publicKey) const { + inline uint32_t getOrAddIndexForPublicKey(const data::PublicKey& publicKey) const { return mPublicKeysIndex.getOrAddIndexForData(publicKey); } @@ -222,7 +222,7 @@ namespace gradido { std::shared_ptr mHieroMessageListener; //! contain indices for every public key address, used overall for optimisation - mutable PersistentDictionary mPublicKeysIndex; + mutable PersistentDictionary mPublicKeysIndex; // level db to store state values like last transaction // TODO: speedup with atcual struct, write out into leveldb/lmdb only on changes, maybe even buffered, think on exit management mutable cache::State mBlockchainState; diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index 3a7db76..d41bdf5 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -7,6 +7,8 @@ #include "../task/SyncTopic.h" #include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/data/adapter/uuid.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/hiero/TopicId.h" #include "gradido_blockchain/lib/DictionaryExceptions.h" @@ -24,6 +26,7 @@ using std::string; using std::vector; namespace gradido { + using data::adapter::uuidFromString; namespace blockchain { FileBasedProvider::FileBasedProvider() @@ -64,7 +67,7 @@ namespace gradido { shared_ptr FileBasedProvider::findBlockchain(const string& communityId) { - auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(communityId); + auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(uuidFromString(communityId.c_str())); if (!communityIdIndex) { LOG_F(WARNING, "no community id index for %s", communityId.c_str()); } @@ -78,7 +81,7 @@ namespace gradido { { try { const auto& groupIndexEntry = mGroupIndex->getCommunityDetails(topicId); - auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(groupIndexEntry.communityId); + auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(uuidFromString(groupIndexEntry.communityId.c_str())); if (!communityIdIndex) { LOG_F(WARNING, "no community id index for %s", groupIndexEntry.communityId.c_str()); } diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 6ddab63..430b2f8 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -17,6 +17,7 @@ #include "gradido_blockchain_core/data/wire/transaction_body.h" #include "gradido_blockchain/Application.h" #include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/interaction/deserialize/Context.h" @@ -33,7 +34,8 @@ using gradido::AppContext; using namespace gradido::blockchain; -using gradido::data::TransactionType, gradido::data::compact::ConfirmedGradidoTx; +using gradido::data::compact::ConfirmedGradidoTx, gradido::data::TransactionType; +using gradido::data::PublicKey; using namespace gradido::interaction; using std::shared_ptr, std::make_shared, std::lock_guard; using task::RebuildBlockIndexTask; diff --git a/src/cache/Block.h b/src/cache/Block.h index ed80728..3d1a18d 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -63,7 +63,7 @@ namespace cache { //! \brief put new transaction to cache and file system bool pushTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary, + IMutableDictionary& publicKeyDictionary, gradido::AppContext& appContext ); diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index 97c3bdf..5d762d8 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -7,8 +7,9 @@ #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/RangeUtils.h" #include "gradido_blockchain/blockchain/SearchDirection.h" -#include "gradido_blockchain/data/TransactionType.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/serialization/toJson.h" #include "loguru/loguru.hpp" @@ -18,6 +19,7 @@ using gradido::blockchain::AbstractProvider; using gradido::blockchain::Filter, gradido::blockchain::SearchDirection; using gradido::blockchain::TransactionsIndexRoaringBitmaps; using gradido::data::compact::ConfirmedGradidoTx; +using gradido::data::PublicKey; using gradido::data::TransactionType; namespace cache { diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index 4c19f82..e26b2bf 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -4,7 +4,7 @@ #include "gradido_blockchain/blockchain/CompactFilter.h" #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/TransactionsIndexRoaringBitmaps.h" -#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/lib/DictionaryInterface.h" #include "../blockchain/NodeTransactionEntry.h" @@ -47,12 +47,12 @@ namespace cache { BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex); ~BlockIndex(); - bool init(const IDictionary& publicKeysDictionary); + bool init(const IDictionary& publicKeysDictionary); void exit(); void reset(); //! \brief loading block index from file (or at least try to load) - bool loadFromFile(const IDictionary& publicKeysDictionary); + bool loadFromFile(const IDictionary& publicKeysDictionary); //! \brief write block index into files std::unique_ptr serialize(); @@ -61,7 +61,7 @@ namespace cache { //! \return true if there was something to write into file, after writing it to file bool writeIntoFile(); - bool addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx, const IDictionary& publicKeyDict); + bool addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx, const IDictionary& publicKeyDict); //! \brief add transactionNr - fileCursor pair to map if not already exist //! \return false if transactionNr exist, else return true diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index 02cfb13..0b246fe 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -10,6 +10,8 @@ #include "gradido_blockchain/blockchain/FilterBuilder.h" #include "gradido_blockchain/data/adapter/byteArray.h" #include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/adapter/uuid.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" @@ -48,6 +50,7 @@ using namespace magic_enum; using std::optional, std::nullopt; using gradido::g_appContext; using gradido::data::compact::PublicKeyIndex, gradido::data::compact::ConfirmedTxs; +using gradido::data::adapter::uuidFromString; namespace server { namespace json_rpc { @@ -158,7 +161,9 @@ namespace server { auto date = DataTypeConverter::dateTimeStringToTimePoint(date_string); optional coinCommunityId = nullopt; if (params.HasMember("coinCommunityId") && params["coinCommunityId"].IsString()) { - auto coinCommunityIdIndexOptional = g_appContext->getCommunityIds().getIndexForData(params["coinCommunityId"].GetString()); + auto coinCommunityIdIndexOptional = g_appContext->getCommunityIds().getIndexForData( + uuidFromString(params["coinCommunityId"].GetString()) + ); if (coinCommunityIdIndexOptional) { coinCommunityId = static_cast(coinCommunityIdIndexOptional); } diff --git a/src/server/json-rpc/WireFilter.h b/src/server/json-rpc/WireFilter.h index d9d954d..01a4531 100644 --- a/src/server/json-rpc/WireFilter.h +++ b/src/server/json-rpc/WireFilter.h @@ -6,7 +6,7 @@ #include "gradido_blockchain/blockchain/Pagination.h" #include "gradido_blockchain/blockchain/PublicKeySearchType.h" #include "gradido_blockchain/blockchain/SearchDirection.h" -#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/lib/TimepointInterval.h" @@ -50,7 +50,7 @@ namespace server::json_rpc { uint64_t minTransactionNr; //! return only transaction in which the public key is involved, either directly in the transaction or as signer - PublicKey publicKey; + gradido::data::PublicKey publicKey; //! search result scope gradido::blockchain::Pagination pagination; diff --git a/src/server/json-rpc/fromJson.cpp b/src/server/json-rpc/fromJson.cpp index c759ac3..d28e153 100644 --- a/src/server/json-rpc/fromJson.cpp +++ b/src/server/json-rpc/fromJson.cpp @@ -5,7 +5,7 @@ #include "gradido_blockchain/blockchain/Pagination.h" #include "gradido_blockchain/blockchain/SearchDirection.h" #include "gradido_blockchain/CommunityContext.h" -#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/compact/PublicKeyIndex.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "gradido_blockchain/lib/TimepointInterval.h" @@ -24,6 +24,7 @@ using gradido::AppContext; using gradido::blockchain::CompactFilter, gradido::blockchain::Pagination, gradido::blockchain::SearchDirection; using gradido::CommunityContext; using gradido::data::compact::PublicKeyIndex; +using gradido::data::PublicKey; using rapidjson::Value; using std::string, std::string_view; diff --git a/src/task/RebuildBlockIndexTask.h b/src/task/RebuildBlockIndexTask.h index 209a448..f8e7817 100644 --- a/src/task/RebuildBlockIndexTask.h +++ b/src/task/RebuildBlockIndexTask.h @@ -4,7 +4,6 @@ #include "CPUTask.h" #include "../model/files/Block.h" #include "gradido_blockchain_core/memory.h" -#include "gradido_blockchain/crypto/ByteArray.h" #include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include "gradido_blockchain/lib/DictionaryInterface.h" diff --git a/src/task/WriteTransactionsToBlockTask.cpp b/src/task/WriteTransactionsToBlockTask.cpp index 64fd838..e42cf4d 100644 --- a/src/task/WriteTransactionsToBlockTask.cpp +++ b/src/task/WriteTransactionsToBlockTask.cpp @@ -4,6 +4,7 @@ #include "../model/files/Block.h" #include "../ServerGlobals.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/memory/Block.h" @@ -64,7 +65,7 @@ namespace task { void WriteTransactionsToBlockTask::addSerializedTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ) { assert(!isTaskSheduled()); diff --git a/src/task/WriteTransactionsToBlockTask.h b/src/task/WriteTransactionsToBlockTask.h index 3713318..fb71334 100644 --- a/src/task/WriteTransactionsToBlockTask.h +++ b/src/task/WriteTransactionsToBlockTask.h @@ -8,7 +8,7 @@ */ #include "CPUTask.h" -#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/lib/MultithreadQueue.h" #include "gradido_blockchain/lib/DictionaryInterface.h" @@ -58,7 +58,7 @@ namespace task { void addSerializedTransaction( std::shared_ptr transaction, - IMutableDictionary& publicKeyDictionary + IMutableDictionary& publicKeyDictionary ); //! return transaction by nr From 17aed77e8f49a3c18b92cc446eeecce3b27410ea Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 24 May 2026 14:12:39 +0200 Subject: [PATCH 63/65] fix build error in gradido blockchain --- dependencies/gradido_blockchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index f0545d2..d7809ef 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit f0545d2481e12f23755af8320a0cef0453a12382 +Subproject commit d7809ef9ceda2cd3a6aa646c0c6fb1f722f26728 From 1f21bdf37147504f8e39eb34a1dde28cfb771aef Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 24 May 2026 15:14:45 +0200 Subject: [PATCH 64/65] fix linux build --- src/serialization/String.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/serialization/String.cpp b/src/serialization/String.cpp index 92a3e26..6eae5fa 100644 --- a/src/serialization/String.cpp +++ b/src/serialization/String.cpp @@ -1,9 +1,10 @@ #include "String.h" -#include "gradido_blockchain/crypto/ByteArray.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/memory/Block.h" #include +using gradido::data::ByteArray; using memory::ConstBlockPtr, memory::Block; using std::string; using std::make_shared; From 6cf651b8540525614162314c0aff32c64ff17dd6 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 24 May 2026 16:53:22 +0200 Subject: [PATCH 65/65] debug github workflow --- .github/workflows/release.yml | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b390cd3..b1ac404 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Build and Release on: push: tags: - - 'v*' + - "v*" jobs: # ─── Linux Builds via Docker ─────────────────────── linux-x64-build: @@ -108,15 +108,26 @@ jobs: - name: Collect all and compress shell: powershell run: | - $exeAndDll = Get-ChildItem -Path build/bin/Release -Include *.exe,*.dll -Recurse | Select-Object -ExpandProperty FullName - - $opensslDlls = @( + Write-Host "Current directory: $(Get-Location)" + $exeAndDll = @( + "build/bin/Release/GradidoNode.exe", "dependencies/gradido_blockchain/dependencies/vcpkg/packages/openssl_x64-windows/bin/libssl-3-x64.dll", "dependencies/gradido_blockchain/dependencies/vcpkg/packages/openssl_x64-windows/bin/libcrypto-3-x64.dll" ) - $files = $exeAndDll + $opensslDlls - $files = $files | Where-Object { Test-Path $_ } - Compress-Archive -Path $files -DestinationPath "gradido_node-${{ github.ref_name }}-win32-x64.zip" -Force + Write-Host "=== Debug: gefundene Dateien ===" + Write-Host "Anzahl der gefundenen Dateien: $($exeAndDll.Count)" + if ($exeAndDll.Count -gt 0) { + Write-Host "Liste der Dateien:" + $exeAndDll | ForEach-Object { Write-Host " - $_" } + } else { + Write-Host "Keine Dateien gefunden!" + Write-Host "Existiert der Pfad? $(Test-Path build/bin/Release)" + if (Test-Path build/bin/Release) { + Write-Host "Inhalt des Verzeichnisses:" + Get-ChildItem -Path build/bin/Release -Recurse | ForEach-Object { Write-Host " - $($_.FullName)" } + } + } + Compress-Archive -Path $exeAndDll -DestinationPath "gradido_node-${{ github.ref_name }}-win32-x64.zip" -Force - name: Upload artifact uses: actions/upload-artifact@v4 @@ -131,7 +142,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # für PR-History~ + fetch-depth: 0 # für PR-History~ - name: Download alle Artifacts uses: actions/download-artifact@v4 @@ -162,4 +173,4 @@ jobs: files: | linux-x86_64/gradido_node-${{ github.ref_name }}-linux-x64.tar.gz linux-aarch64/gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz - windows-x86_64/gradido_node-${{ github.ref_name }}-win32-x64.zip \ No newline at end of file + windows-x86_64/gradido_node-${{ github.ref_name }}-win32-x64.zip