diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 555bad5a..b1ac404c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,69 +2,175 @@ name: Build and Release on: push: - branches: - - 'release/*' - + tags: + - "v*" jobs: - build: - name: Build Binaries + # ─── Linux Builds via Docker ─────────────────────── + linux-x64-build: runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Configure and build - run: | - mkdir build - cd build - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF .. - make -j$(nproc) - cmake .. - make -j$(nproc) GradidoNode - - - name: Archive binaries - uses: actions/upload-artifact@v2 - with: - name: binaries - path: build/bin/* - - release: - name: Create Release + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Get blockchain version + id: blockchain_version + run: echo "version=$(git -C dependencies/gradido_blockchain describe --tags --dirty --always)" >> $GITHUB_OUTPUT + - name: Build x86_64 + run: | + docker build \ + --build-arg GRADIDO_NODE_VERSION=${{ github.ref_name }} \ + --build-arg GRADIDO_BLOCKCHAIN_VERSION=${{ steps.blockchain_version.outputs.version }} \ + --target release_build \ + -t gradido-node-x86_64 . + docker create --name extract-x64 gradido-node-x86_64 + docker cp extract-x64:/code/build/bin/GradidoNode ./GradidoNode + docker rm extract-x64 + + - name: Strip binary + run: strip ./GradidoNode + + - name: compress + run: tar -czf gradido_node-${{ github.ref_name }}-linux-x64.tar.gz GradidoNode + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: linux-x86_64 + path: gradido_node-${{ github.ref_name }}-linux-x64.tar.gz + + linux-aarch64-build: runs-on: ubuntu-latest - needs: build - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Get blockchain version + id: blockchain_version + run: echo "version=$(git -C dependencies/gradido_blockchain describe --tags --dirty --always)" >> $GITHUB_OUTPUT + - name: Build aarch64 + run: | + docker build \ + --build-arg GRADIDO_NODE_VERSION=${{ github.ref_name }} \ + --build-arg GRADIDO_BLOCKCHAIN_VERSION=${{ steps.blockchain_version.outputs.version }} \ + --build-arg ZIG_TARGET=aarch64-linux-musl \ + --build-arg TARGET_ARCH=aarch64 \ + --target release_build \ + -t gradido-node-aarch64 . + docker create --name extract-arm gradido-node-aarch64 + docker cp extract-arm:/code/build/bin/GradidoNode ./GradidoNode + docker rm extract-arm + + - name: Strip binary + run: | + sudo apt-get install -y binutils-aarch64-linux-gnu + aarch64-linux-gnu-strip ./GradidoNode + + - name: compress + run: tar -czf gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz GradidoNode + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: linux-aarch64 + path: gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz + + # ─── Windows Native Build ────────────────────────── + windows-build: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: install zig + uses: mlugg/setup-zig@v2 + with: + version: 0.15.2 + - name: Setup MSVC + uses: microsoft/setup-msbuild@v2 + + - name: Configure + run: | + mkdir build + cd build + cmake .. ` + -DENABLE_TEST=Off ` + -DENABLE_HTTPS=On ` + -DCMAKE_BUILD_TYPE=Release + + - name: Build + run: | + cd build + cmake --build . --target GradidoNode --config Release + + - name: Collect all and compress + shell: powershell + run: | + Write-Host "Current directory: $(Get-Location)" + $exeAndDll = @( + "build/bin/Release/GradidoNode.exe", + "dependencies/gradido_blockchain/dependencies/vcpkg/packages/openssl_x64-windows/bin/libssl-3-x64.dll", + "dependencies/gradido_blockchain/dependencies/vcpkg/packages/openssl_x64-windows/bin/libcrypto-3-x64.dll" + ) + Write-Host "=== Debug: gefundene Dateien ===" + Write-Host "Anzahl der gefundenen Dateien: $($exeAndDll.Count)" + if ($exeAndDll.Count -gt 0) { + Write-Host "Liste der Dateien:" + $exeAndDll | ForEach-Object { Write-Host " - $_" } + } else { + Write-Host "Keine Dateien gefunden!" + Write-Host "Existiert der Pfad? $(Test-Path build/bin/Release)" + if (Test-Path build/bin/Release) { + Write-Host "Inhalt des Verzeichnisses:" + Get-ChildItem -Path build/bin/Release -Recurse | ForEach-Object { Write-Host " - $($_.FullName)" } + } + } + Compress-Archive -Path $exeAndDll -DestinationPath "gradido_node-${{ github.ref_name }}-win32-x64.zip" -Force + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: windows-x86_64 + path: gradido_node-${{ github.ref_name }}-win32-x64.zip + + # ─── Release erstellen ───────────────────────────── + create-release: + needs: [linux-x64-build, linux-aarch64-build, windows-build] + runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Download binaries - uses: actions/download-artifact@v2 - with: - name: binaries - path: build/bin/ - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: | - Describe your release here. - draft: false - prerelease: false - - - name: Upload binaries - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: build/bin/* - asset_name: ${{ github.ref }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # für PR-History~ + + - name: Download alle Artifacts + uses: actions/download-artifact@v4 + + - name: Update CHANGELOG + id: changelog + uses: requarks/changelog-action@v1.10.3 + with: + token: ${{ github.token }} + tag: ${{ github.ref_name }} + includeInvalidCommits: true + + - name: create GitHub Release + uses: softprops/action-gh-release@v1 + with: + name: Release ${{ github.ref_name }} + body: | + ## Changes~ + ${{ steps.changelog.outputs.result }} + *Auto-generated release notes from Pull-Requests* + ## Downloads~ + | Platform | Architecture | File | + |---|---|---| + | Linux | x86_64 | gradido_node-${{ github.ref_name }}-linux-x64.tar.gz | + | Linux | aarch64 | gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz | + | Windows | x86_64 | gradido_node-${{ github.ref_name }}-win32-x64.zip | + files: | + linux-x86_64/gradido_node-${{ github.ref_name }}-linux-x64.tar.gz + linux-aarch64/gradido_node-${{ github.ref_name }}-linux-arm64.tar.gz + windows-x86_64/gradido_node-${{ github.ref_name }}-win32-x64.zip diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b4aced2..2bb1c609 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,18 +1,24 @@ cmake_minimum_required(VERSION 3.18.2) set(CMAKE_POLICY_VERSION_MINIMUM 3.5) set(CMAKE_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain.cmake") -execute_process( - COMMAND git describe --tags --dirty --always - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - OUTPUT_VARIABLE GRADIDO_NODE_GIT_VER - OUTPUT_STRIP_TRAILING_WHITESPACE -) +if(DEFINED ENV{GRADIDO_NODE_VERSION} AND NOT "$ENV{GRADIDO_NODE_VERSION}" STREQUAL "") + set(GRADIDO_NODE_GIT_VER $ENV{GRADIDO_NODE_VERSION}) +else() + execute_process( + COMMAND git describe --tags --dirty --always + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GRADIDO_NODE_GIT_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() + string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" GRADIDO_NODE_CLEAN_VERSION "${GRADIDO_NODE_GIT_VER}") project(GradidoNode VERSION ${GRADIDO_NODE_CLEAN_VERSION} LANGUAGES C CXX) SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) message("runtime output directory: ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") option(BUILD_SHARED_LIBS "Build using shared libraries" OFF) +option(GRPC_FETCH_CONTENT "Download and build gRPC via FetchContent" ON) IF(WIN32) set(CMAKE_CXX_FLAGS "/MP /EHsc /std:c++20") set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") @@ -40,13 +46,14 @@ configure_file( set(GRADIDO_NODE_VERSION "@PROJECT_VERSION@") include_directories( - "dependencies" + "dependencies" "dependencies/jsonrpcpp/include" "dependencies/gradido_blockchain" "dependencies/gradido_blockchain/include" "dependencies/gradido_blockchain/build" "dependencies/gradido_blockchain/dependencies" "dependencies/gradido_blockchain/dependencies/date/include" + "dependencies/gradido_blockchain/dependencies/gradido-blockchain-core/include" "dependencies/gradido_blockchain/dependencies/magic_enum/include" "dependencies/gradido_blockchain/dependencies/protopuf/include" "dependencies/gradido_blockchain/dependencies/rapidjson/include" @@ -72,8 +79,8 @@ FILE(GLOB SERVER_JSON_RPC "src/server/json-rpc/*.h" "src/server/json-rpc/*.cpp") FILE(GLOB LIB_SRC "src/lib/*.h" "src/lib/*.cpp" "src/lib/*.c") FILE(GLOB MODEL "src/model/*.h" "src/model/*.cpp") FILE(GLOB MODEL_APOLLO "src/model/Apollo/*.h" "src/model/Apollo/*.cpp") -FILE(GLOB MODEL_APOLLO_CREATE_TRANSACTION - "src/model/Apollo/createTransaction/*.h" +FILE(GLOB MODEL_APOLLO_CREATE_TRANSACTION + "src/model/Apollo/createTransaction/*.h" "src/model/Apollo/createTransaction/*.cpp" ) FILE(GLOB MODEL_FILES "src/model/files/*.h" "src/model/files/*.cpp") @@ -84,7 +91,7 @@ FILE(GLOB VIEW "src/view/*.h" "src/view/*.cpp") FILE(GLOB MAIN "src/*.cpp" "src/*.c" "src/*.h") -SET(LOCAL_SRCS +SET(LOCAL_SRCS ${BLOCKCHAIN} ${CACHE_SRC} ${CONTAINER} ${CONTROLLER} ${CLIENT} ${CLIENT_HIERO} ${SERIALIZATION} @@ -131,38 +138,56 @@ set(ENABLE_HTTPS ON) # set(VCPKG_TARGET_TRIPLET "x64-windows" CACHE STRING "vcpkg target triplet") #endif() add_subdirectory("dependencies/gradido_blockchain") + find_package(OpenSSL REQUIRED) # reuse cacert.pem from gradido_blockchain if(WIN32) add_compile_definitions(CACERT_PEM_PATH="${CACERT_PEM}") endif() -find_package(OpenSSL REQUIRED) + add_subdirectory("dependencies/jsonrpcpp") option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" OFF) add_subdirectory("dependencies/leveldb") #add_subdirectory("dependencies/googletest") -# Fetch Content Module -include(FetchContent) - -# gRPC -set(gRPC_SSL_PROVIDER "package" CACHE STRING "gRPC SSL provider" FORCE) -set(gRPC_INSTALL ON CACHE BOOL "" FORCE) -set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE) - -message(STATUS "Make gRPC Available") -FetchContent_Declare( - grpc - GIT_REPOSITORY https://github.com/grpc/grpc.git - GIT_TAG v1.74.1 -) -FetchContent_MakeAvailable(grpc) - -if(NOT WIN32) - include_directories(${grpc_SOURCE_DIR}/include) - include_directories(${grpc_SOURCE_DIR}/third_party/abseil-cpp) - message(${grpc_SOURCE_DIR}) +#find_package(gRPC CONFIG QUIET) +#find_package(Protobuf QUIET) + +if(GRPC_FETCH_CONTENT) + # Fetch Content Module + include(FetchContent) + + # gRPC + set(FETCHCONTENT_QUIET OFF) + set(FETCHCONTENT_UPDATES_DISCONNECTED TRUE) + set(gRPC_SSL_PROVIDER "package" CACHE STRING "gRPC SSL provider" FORCE) + set(gRPC_INSTALL ON CACHE BOOL "" FORCE) + set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(gRPC_BUILD_CSHARP_EXT OFF CACHE BOOL "" FORCE) + + message(STATUS "Make gRPC Available") + FetchContent_Declare( + grpc + GIT_REPOSITORY https://github.com/grpc/grpc.git + GIT_TAG v1.74.1 + GIT_SHALLOW TRUE + ) + FetchContent_MakeAvailable(grpc) + + if(NOT WIN32) + include_directories(${grpc_SOURCE_DIR}/include) + include_directories(${grpc_SOURCE_DIR}/third_party/abseil-cpp) + message(${grpc_SOURCE_DIR}) + endif() + set(_GRPC_GRPCPP grpc++) +else() + message(STATUS "Using pre-built gRPC from docker") + list(APPEND CMAKE_PREFIX_PATH "/usr/local") + include_directories(/usr/local/include) + find_package(Protobuf REQUIRED) + find_package(gRPC CONFIG REQUIRED) + set(_GRPC_GRPCPP gRPC::grpc++) endif() # Since FetchContent uses add_subdirectory under the hood, we can use @@ -170,7 +195,7 @@ endif() # set(_PROTOBUF_LIBPROTOBUF libprotobuf) #set(_REFLECTION grpc++_reflection) #set(_PROTOBUF_PROTOC $) -set(_GRPC_GRPCPP grpc++) + #message("grpc++: ${_GRPC_GRPCPP}") SET(GradidoBlockchain_BINARY_DIR ${GradidoNode_BINARY_DIR}) @@ -188,7 +213,7 @@ endif() #add_subdirectory(dependencies/protobuf/cmake) #set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -# prevent problems with two libs including googletest +# prevent problems with two libs including googletest #option(protobuf_BUILD_TESTS "Build tests" OFF) #message("binary dir: ${CMAKE_BINARY_DIR}") @@ -205,7 +230,7 @@ add_executable(GradidoNode ${LOCAL_SRCS}) if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") # needed for clang/zig - target_compile_options(GradidoNode PRIVATE -Wno-error=deprecated-literal-operator) + target_compile_options(GradidoNode PRIVATE -Wno-error=deprecated-literal-operator -Wno-error=deprecated-declarations) endif() #SUBDIRS("src/test") @@ -219,7 +244,7 @@ else() endif() target_link_libraries(GradidoNode ${_GRPC_GRPCPP}) -IF(WIN32) +IF(WIN32 AND BUILD_SHARED_LIBS) add_custom_command(TARGET GradidoNode POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ $ COMMAND_EXPAND_LISTS @@ -234,5 +259,3 @@ endif(UNIX) # add test #include(CTest) #add_subdirectory(test) - - diff --git a/Dockerfile b/Dockerfile index 40cf4978..c37255ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ + + ##### Run environment for tests ##### FROM alpine:3.22.2 AS test_run @@ -5,11 +7,32 @@ FROM alpine:3.22.2 AS test_run ##### BUILD-ENV ##### FROM denisgolius/zig:0.15.2 AS zig_build +ARG ZIG_TARGET=x86_64-linux-musl +#aarch64-linux-musl +ARG TARGET_ARCH=x86_64 +# aarch64 +ENV ZIG_TARGET=${ZIG_TARGET} +ENV TARGET_ARCH=${TARGET_ARCH} + RUN apk add --no-cache cmake make git binutils openssl-dev openssl-libs-static + ##### BUILD debug ####### FROM zig_build AS debug_build +ARG ZIG_TARGET=x86_64-linux-musl +#aarch64-linux-musl +ARG TARGET_ARCH=x86_64 + +ARG GRADIDO_VERSION=0.0.0-dev +# aarch64 +ENV ZIG_TARGET=${ZIG_TARGET} +ENV TARGET_ARCH=${TARGET_ARCH} +ENV GRADIDO_VERSION=${GRADIDO_VERSION} + +# copy grpc-deps artifacts +COPY --from=gradido/grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local + ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} @@ -17,25 +40,75 @@ COPY . . RUN mkdir build WORKDIR ${DOCKER_WORKDIR}/build -RUN cmake .. -DENABLE_TEST=On -DENABLE_HTTPS=On -DUSE_INSTALLED_SSL=On -DCMAKE_EXE_LINKER_FLAGS="-static" -DBUILD_SHARED_LIBS=Off +RUN cmake .. \ + -DENABLE_TEST=On \ + -DENABLE_HTTPS=On \ + -DUSE_INSTALLED_SSL=On \ + -DCMAKE_EXE_LINKER_FLAGS="-static" \ + -DBUILD_SHARED_LIBS=Off \ + -DTARGET=${ZIG_TARGET} \ + -DGRPC_FETCH_CONTENT=Off + # OpenSSL paths for CMake + #-DOPENSSL_ROOT_DIR=/opt/${TARGET_ARCH}/usr \ + #-DOPENSSL_INCLUDE_DIR=/opt/${TARGET_ARCH}/usr/include \ + #-DOPENSSL_CRYPTO_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libcrypto.a \ + #-DOPENSSL_SSL_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libssl.a RUN make -j$(nproc) GradidoNode ENTRYPOINT [] -##### run tests ####### -FROM test_run AS test_build_run +##### BUILD release ####### +FROM zig_build AS release_build -COPY --from=debug_build /code/build/test/GradidoBlockchainTest ./ +ARG ZIG_TARGET=x86_64-linux-musl +#aarch64-linux-musl +ARG TARGET_ARCH=x86_64 -ENTRYPOINT [] +ARG GRADIDO_NODE_VERSION=0.0.0-dev +ARG GRADIDO_BLOCKCHAIN_VERSION=0.0.0-dev +# aarch64 +ENV ZIG_TARGET=${ZIG_TARGET} +ENV TARGET_ARCH=${TARGET_ARCH} +ENV GRADIDO_NODE_VERSION=${GRADIDO_NODE_VERSION} +ENV GRADIDO_BLOCKCHAIN_VERSION=${GRADIDO_BLOCKCHAIN_VERSION} + +# copy grpc-deps artifacts +COPY --from=gradido/grpc-deps:v1.74.1 /opt/${TARGET_ARCH}/local /usr/local -##### BUILD release ####### -FROM zig_build AS release_build ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} COPY . . + RUN mkdir build WORKDIR ${DOCKER_WORKDIR}/build -RUN cmake -DCMAKE_BUILD_TYPE=Release .. -RUN make -j$(nproc) GradidoBlockchain +RUN cmake .. \ + -DENABLE_TEST=Off \ + -DENABLE_HTTPS=On \ + -DUSE_INSTALLED_SSL=On \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_C_FLAGS="-g0" \ + -DCMAKE_CXX_FLAGS="-g0" \ + -DCMAKE_EXE_LINKER_FLAGS="-static" \ + -DBUILD_SHARED_LIBS=Off \ + -DTARGET=${ZIG_TARGET} \ + -DGRPC_FETCH_CONTENT=Off + # OpenSSL paths for CMake + #-DOPENSSL_ROOT_DIR=/opt/${TARGET_ARCH}/usr \ + #-DOPENSSL_INCLUDE_DIR=/opt/${TARGET_ARCH}/usr/include \ + #-DOPENSSL_CRYPTO_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libcrypto.a \ + #-DOPENSSL_SSL_LIBRARY=/opt/${TARGET_ARCH}/usr/lib/libssl.a +RUN make -j$(nproc) GradidoNode + +ENTRYPOINT ["/bin/GradidoNode"] + + +##### run tests ####### +FROM test_run AS test_build_run + +COPY --from=debug_build /code/build/test/GradidoBlockchainTest ./ + +ENTRYPOINT [] + diff --git a/Dockerfile.grpc-deps b/Dockerfile.grpc-deps new file mode 100644 index 00000000..c55a8c60 --- /dev/null +++ b/Dockerfile.grpc-deps @@ -0,0 +1,140 @@ +# ============================================================================= +# grpc-deps-builder +# Builds static OpenSSL + gRPC for x86_64-linux-musl and aarch64-linux-musl +# Final image contains only the compiled artifacts (headers + .a files) +# Written by Claude Code, 2026-03-06 +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Stage 1: Common build tools +# ----------------------------------------------------------------------------- +FROM denisgolius/zig:0.15.2 AS base + +RUN apk add --no-cache \ + cmake make git binutils perl wget \ + # needed by gRPC cmake scripts + autoconf automake libtool + +# Zig wrapper helper script +# Usage: make-zig-wrappers +RUN cat > /usr/local/bin/make-zig-wrappers <<'EOF' +#!/bin/sh +TARGET="$1" +printf '#!/bin/sh\nexec zig cc -target %s "$@"\n' "$TARGET" > /usr/local/bin/zigcc +printf '#!/bin/sh\nexec zig c++ -target %s "$@"\n' "$TARGET" > /usr/local/bin/zigcxx +# ar und ranlib brauchen kein target-flag +printf '#!/bin/sh\nexec zig ar "$@"\n' > /usr/local/bin/zigar +printf '#!/bin/sh\nexec zig ranlib "$@"\n' > /usr/local/bin/zigranlib +chmod +x /usr/local/bin/zigcc /usr/local/bin/zigcxx \ + /usr/local/bin/zigar /usr/local/bin/zigranlib +EOF +RUN chmod +x /usr/local/bin/make-zig-wrappers + + +FROM base AS libraries_src + +RUN wget -q https://www.openssl.org/source/openssl-3.3.2.tar.gz && \ + tar xf openssl-3.3.2.tar.gz + +RUN git clone --depth 1 --branch v1.74.1 \ + --recurse-submodules \ + --shallow-submodules \ + https://github.com/grpc/grpc.git /tmp/grpc + +# ----------------------------------------------------------------------------- +# Stage 2a: OpenSSL for x86_64-linux-musl +# ----------------------------------------------------------------------------- +FROM libraries_src AS openssl_x86_64 + +RUN make-zig-wrappers x86_64-linux-musl + +RUN cd openssl-3.3.2 && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + ./Configure linux-x86_64 \ + no-shared no-tests no-docs \ + #--prefix=/opt/x86_64/usr \ + && \ + make -j$(nproc) && \ + make install_sw && \ + cd .. && rm -rf openssl-3.3.2* + +# ----------------------------------------------------------------------------- +# Stage 2b: OpenSSL for aarch64-linux-musl +# ----------------------------------------------------------------------------- +FROM libraries_src AS openssl_aarch64 + +RUN make-zig-wrappers aarch64-linux-musl + +RUN cd openssl-3.3.2 && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + ./Configure linux-aarch64 \ + no-shared no-tests no-docs \ + #--prefix=/opt/aarch64/usr \ + && \ + make -j$(nproc) && \ + make install_sw && \ + cd .. && rm -rf openssl-3.3.2* + +FROM openssl_x86_64 AS grpc_x86_64 + +RUN make-zig-wrappers x86_64-linux-musl + +RUN mkdir -p /tmp/grpc/cmake/build && \ + cd /tmp/grpc/cmake/build && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + cmake -DCMAKE_CXX_STANDARD=20 ../.. \ + #-DCMAKE_INSTALL_PREFIX=/opt/x86_64/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_C_FLAGS="-g0" \ + -DCMAKE_CXX_FLAGS="-g0" \ + -DZLIB_BUILD_EXAMPLES=OFF \ + -DZLIB_SHARED=OFF \ + -DZLIB_STATIC=ON \ + -DgRPC_SSL_PROVIDER=package && \ + make -j$(nproc) && make install + +FROM openssl_aarch64 AS grpc_aarch64 + +#COPY --from=grpc-deps:v1.74.1 /opt/x86_64 /opt/x86_64 + +RUN make-zig-wrappers aarch64-linux-musl + +RUN mkdir -p /tmp/grpc/cmake/build && \ + cd /tmp/grpc/cmake/build && \ + CC=zigcc CXX=zigcxx AR="zig ar" RANLIB="zig ranlib" \ + cmake -DCMAKE_CXX_STANDARD=20 ../.. \ + #-DCMAKE_INSTALL_PREFIX=/opt/aarch64/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O2 -DNDEBUG" \ + -DCMAKE_C_FLAGS="-g0" \ + -DCMAKE_CXX_FLAGS="-g0" \ + -DZLIB_BUILD_EXAMPLES=OFF \ + -DZLIB_SHARED=OFF \ + -DZLIB_STATIC=ON \ + #-DgRPC_PROTOBUF_PROVIDER=package \ + #-DgRPC_PROTOBUF_PACKAGE_TYPE=MODULE \ + #-DProtobuf_PROTOC_EXECUTABLE=/opt/x86_64/usr/bin/protoc \ + #-DgRPC_CPP_PLUGIN_EXECUTABLE=/opt/x86_64/usr/bin/grpc_cpp_plugin \ + #-DgRPC_BUILD_GRPC_CPP_PLUGIN=OFF \ + -DgRPC_BUILD_CODEGEN=OFF \ + -DgRPC_SSL_PROVIDER=package && \ + make -j$(nproc) && make install +RUN find /usr/local/lib -name "*.so*" -delete + +# ----------------------------------------------------------------------------- +# Stage 4: Final image — only the compiled artifacts, no build tools +# ----------------------------------------------------------------------------- +FROM busybox:musl AS final + +COPY --from=grpc_x86_64 /usr/local /opt/x86_64/local +COPY --from=grpc_aarch64 /usr/local /opt/aarch64/local + +# Layout: +# /x86_64/usr/lib/ -> libgrpc.a, libssl.a, libcrypto.a, ... +# /x86_64/usr/include/ -> grpc/, openssl/, google/protobuf/, ... +# /x86_64/usr/bin/ -> grpc_cpp_plugin (host tool, x86_64 only useful) +# /aarch64/usr/lib/ -> same for aarch64 +# /aarch64/usr/include/ -> same for aarch64 \ No newline at end of file diff --git a/README.md b/README.md index d5c2602e..de7d388a 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,22 @@ install rust compiler ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` + +ON debian 12 64Bit +cmake .. -DUSE_INSTALLED_SSL=On -DCMAKE_BUILD_TYPE=Release -DTARGET=x86_64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake + +On Debian 12 ARM64 Server +cmake .. -DOPENSSL_ROOT_DIR=/usr/lib/aarch64-linux-gnu -DCMAKE_BUILD_TYPE=Release -DUSE_INSTALLED_SSL=On -DTARGET=aarch64-linux-gnu -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake + + +## Docker +docker build \ + --build-arg VERSION=$(git describe --tags --dirty --always) \ + -t gradido-node . +Build for aarch64 +docker build \ + --build-arg VERSION=$(git describe --tags --dirty --always) \ + --build-arg ZIG_TARGET=aarch64-linux-musl \ + --build-arg TARGET_ARCH=aarch64 \ + --target debug_build \ + -t gradido-node-aarch64 . diff --git a/config/gradido.yaml b/config/gradido.yaml index 19167358..fcd601cb 100644 --- a/config/gradido.yaml +++ b/config/gradido.yaml @@ -1,26 +1,17 @@ # Client configuration, accessing extern services -clients: - # IOTA Configuration - iota: - rest_api: - host: api.lb-0.h.chrysalis-devnet.iota.cafe - port: 443 # automatic using HTTPS if 443 - mqtt: - host: api.lb-0.h.chrysalis-devnet.iota.cafe - port: 1883 - +clients: hiero: networkType: testnet - + # Server Configuration, which services are served on which ports server: json_rpc: 8340 # Port for JSON RPC 2.0 Requests # Cache Settings cache: - timeout: 600 # Cache Timeout in seconds, how long to keep transactions and other data in memory cache - checks_interval: 10 # How often to check for cache timeout in seconds - write_to_disk_interval: 10 # How often block and index data will be flushed to disk in seconds + timeout: 600 # Cache Timeout in seconds, how long to keep transactions and other data in memory cache + checks_interval: 10 # How often to check for cache timeout in seconds + write_to_disk_interval: 10 # How often block and index data will be flushed to disk in seconds # Logging Settings logging: @@ -28,8 +19,8 @@ logging: # I/O Settings io: - worker_count: 1 # How many workers are allowed to access disk at the same time + worker_count: 1 # How many workers are allowed to access disk at the same time # Security Settings unsecure: - allow_cors_all: 0 # Set to 1 if js frontend should be allowed to send requests to it + allow_cors_all: 0 # Set to 1 if js frontend should be allowed to send requests to it diff --git a/dependencies/gradido_blockchain b/dependencies/gradido_blockchain index 03dcbdf4..d7809ef9 160000 --- a/dependencies/gradido_blockchain +++ b/dependencies/gradido_blockchain @@ -1 +1 @@ -Subproject commit 03dcbdf4b31deefaf79a7ef08eb05c046931f2fa +Subproject commit d7809ef9ceda2cd3a6aa646c0c6fb1f722f26728 diff --git a/src/MainServer.cpp b/src/MainServer.cpp index bf004ffe..9f28e762 100644 --- a/src/MainServer.cpp +++ b/src/MainServer.cpp @@ -3,27 +3,39 @@ #include "ServerGlobals.h" #include "blockchain/FileBasedProvider.h" +#include "server/json-rpc/ApiHandler.h" // #include "iota/MqttClientWrapper.h" -#include "server/json-rpc/ApiHandlerFactory.h" #include "SingletonManager/CacheManager.h" #include "hiero/Addressbook.h" #include "client/hiero/const.h" +#include "lib/PersistentDictionary.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/http/ServerConfig.h" #include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include "cpp-httplib/httplib.h" #include "loguru.hpp" -using namespace gradido; -using namespace blockchain; -namespace fs = std::filesystem; +using gradido::blockchain::FileBasedProvider; +using gradido::g_appContext, gradido::AppContext; +using gradido::data::GenericHash, gradido::data::GenericHashHash, gradido::data::GenericHashEqual; +using gradido::data::Uuid, gradido::data::UuidHash, gradido::data::UuidEqual; +using server::json_rpc::ApiHandler; +using std::filesystem::create_directories, std::filesystem::exists, std::filesystem::is_regular_file, std::filesystem::path; +using std::shared_ptr, std::make_unique; +using std::string; +using std::vector; MainServer::MainServer() : mHttpServer(nullptr) @@ -36,23 +48,23 @@ MainServer::~MainServer() bool MainServer::init() { - Profiler usedTime; + MonotonicTimer usedTime; ServerGlobals::g_FilesPath = getHomeDir() + "/.gradido"; - fs::create_directories(ServerGlobals::g_FilesPath); - + create_directories(ServerGlobals::g_FilesPath); + // ********** logging ************************************ - std::string logPath = ServerGlobals::g_FilesPath + "/logs"; - fs::create_directories(logPath); + string logPath = ServerGlobals::g_FilesPath + "/logs"; + create_directories(logPath); // beware, no logrotation in loguru // TODO: add config options to choose which to use // TODO: check switching to https://github.com/gabime/spdlog - std::string errorLogFile = logPath + "/errors.log"; + string errorLogFile = logPath + "/errors.log"; loguru::add_file(errorLogFile.data(), loguru::Append, loguru::Verbosity_WARNING); // info - std::string infoLogFile = logPath + "/infos.log"; + string infoLogFile = logPath + "/infos.log"; loguru::add_file(infoLogFile.data(), loguru::Append, loguru::Verbosity_INFO); // infos and above - std::string debugLogFile = logPath + "/debug.log"; + string debugLogFile = logPath + "/debug.log"; loguru::add_file(debugLogFile.data(), loguru::Truncate, loguru::Verbosity_MAX); #if defined(__linux__) || defined(__unix__) // use syslog on linux, this has logrotation build in @@ -67,6 +79,13 @@ bool MainServer::init() unsigned short jsonrpc_port = (unsigned short)config.getInt("server.json_rpc", 8340); + auto communityDictionary = make_unique>(ServerGlobals::g_FilesPath + "/communityIdsCache"); + communityDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); + auto nameHashDictionary = make_unique>(ServerGlobals::g_FilesPath + "/nameHashCache"); + nameHashDictionary->init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES); + g_appContext = make_unique(std::move(communityDictionary), std::move(nameHashDictionary)); + g_appContext->syncCommunityContextsWithCommunityIds(); + // timeouts ServerGlobals::loadTimeouts(config); ServerGlobals::g_LogTransactions = config.getBool("logging.log_transactions", ServerGlobals::g_LogTransactions); @@ -75,7 +94,7 @@ bool MainServer::init() // start cpu scheduler // std::thread::hardware_concurrency() sometime return 0 if number couldn't be determined - uint8_t worker_count = 2; // std::max(2, (int)std::thread::hardware_concurrency() * 2); + uint8_t worker_count = std::max(2, (int)std::thread::hardware_concurrency() * 2); // I think 1 or 2 by HDD is ok, more by SSD, but should be profiled on work load uint8_t io_worker_count = config.getInt("io.worker_count", 2); ServerGlobals::g_CPUScheduler = new task::CPUSheduler(worker_count, "Default Worker"); @@ -83,16 +102,16 @@ bool MainServer::init() CacheManager::getInstance()->getFuzzyTimer()->addTimer("mainCPUScheduler", ServerGlobals::g_CPUScheduler, std::chrono::milliseconds(100)); ServerGlobals::g_WriteFileCPUScheduler = new task::CPUSheduler(io_worker_count, "IO Worker"); // ServerGlobals::g_IotaRequestCPUScheduler = new task::CPUSheduler(2, "Iota Worker"); - std::string hieroNetworkType = config.getString("clients.hiero.networkType", "testnet"); + string hieroNetworkType = config.getString("clients.hiero.networkType", "testnet"); ServerGlobals::initHiero(hieroNetworkType); uint8_t hieroNodeCount = config.getInt("clients.hiero.nodeCount", 3); uint8_t hieroNodeCountPerCommunity = config.getInt("clients.hiero.nodeCountPerCommunity", 3); - std::vector> hieroClients; + vector> hieroClients; if (!ServerGlobals::g_isOfflineMode) { //iota::MqttClientWrapper::getInstance()->init(); - std::string grpcAddressesFile = ServerGlobals::g_FilesPath + "/addressbook/" + hieroNetworkType + ".pb"; + string grpcAddressesFile = ServerGlobals::g_FilesPath + "/addressbook/" + hieroNetworkType + ".pb"; if (!hieroNodeCount || !hieroNodeCountPerCommunity) { LOG_F(ERROR, "clients.hiero.nodeCountPerCommunity and clients.hiero.nodeCount need to be both >0"); @@ -100,11 +119,12 @@ bool MainServer::init() if (hieroNodeCountPerCommunity > hieroNodeCount) { LOG_F(ERROR, "clients.hiero.nodeCountPerCommunity (%d) mustn't be greate than clients.hiero.nodeCount (%d)", hieroNodeCount, hieroNodeCountPerCommunity); } - hiero::Addressbook addressbook(grpcAddressesFile.data()); + hiero::Addressbook addressbook(grpcAddressesFile.c_str()); addressbook.load(); hieroClients.reserve(hieroNodeCount); for (int i = 0; i < hieroNodeCount; i++) { + if (mMasterStopSource.stop_requested()) break; const auto& hieroNode = addressbook.pickRandomNode(); const auto& endpoint = hieroNode.pickRandomEndpoint(); auto hieroServiceEndpointUrl = endpoint.getConnectionString(); @@ -114,29 +134,50 @@ bool MainServer::init() hieroNode.getNodeCertHash() ); if (!hieroClient) { - LOG_F(ERROR, "Error connecting with hiero network via service endpoint: %s", hieroServiceEndpointUrl.data()); + LOG_F(ERROR, "Error connecting with hiero network via service endpoint: %s", hieroServiceEndpointUrl.c_str()); return false; } LOG_F(INFO, "Hiero endpoint: %s (%s)", - hieroServiceEndpointUrl.data(), - hieroNode.getDescription().data() + hieroServiceEndpointUrl.c_str(), + hieroNode.getDescription().c_str() ); hieroClients.push_back(hieroClient); } } - if (!FileBasedProvider::getInstance()->init(ServerGlobals::g_FilesPath + "/communities.json", std::move(hieroClients), hieroNodeCountPerCommunity)) { + + if (!FileBasedProvider::getInstance()->init(getStopToken(), ServerGlobals::g_FilesPath + "/communities.json", std::move(hieroClients), hieroNodeCountPerCommunity)) { LOG_F(ERROR, "Error loading communities, please try to delete communities folders and try again!"); return false; } - - // JSON Interface Server - mHttpServer = new Server("0.0.0.0", jsonrpc_port, "http-server"); - mHttpServer->init(); - mHttpServer->registerResponseHandler("/api", new server::json_rpc::ApiHandlerFactory()); - mHttpServer->run(); - LOG_F(INFO, "started in %s, json rpc port: %d", usedTime.string().data(), jsonrpc_port); - // start the json server - // doesn't return + if (!mMasterStopSource.stop_requested()) { + // start jsonrpc 2.0 server + mHttpServer = new Server("0.0.0.0", jsonrpc_port, "http-server"); + mHttpServer->init(); + // mHttpServer->registerResponseHandler("/api", new server::json_rpc::ApiHandlerFactory()); + mHttpServer->registerCallbackHandler("/api", MethodType::OPTIONS, [](const httplib::Request& req, httplib::Response& res) { + ApiHandler::cors(res); + } + ); + mHttpServer->registerCallbackHandler("/api", MethodType::GET, [](const httplib::Request& req, httplib::Response& res) { + res.set_content("REST API", "text/plain"); + } + ); + mHttpServer->registerCallbackHandler("/api", MethodType::DEL, [](const httplib::Request& req, httplib::Response& res) { + res.set_content("---", "text/plain"); + } + ); + mHttpServer->registerCallbackHandler("/api", MethodType::POST, [](const httplib::Request& req, httplib::Response& res) { + ApiHandler handler; + handler.handlePostPut(req, res); + } + ); + mHttpServer->run(); + LOG_F(INFO, "started in %s, json rpc port: %d", usedTime.string().c_str(), jsonrpc_port); + } + else { + LOG_F(INFO, "stopped before startup was finished in: %s", usedTime.string().c_str()); + return false; + } return true; } @@ -151,24 +192,31 @@ void MainServer::exit() delete mHttpServer; mHttpServer = nullptr; } + printf("[shutdown] Stopped HTTP Server...\n"); // iota::MqttClientWrapper::getInstance()->exit(); CacheManager::getInstance()->getFuzzyTimer()->stop(); - ServerGlobals::g_CPUScheduler->stop(); - ServerGlobals::g_WriteFileCPUScheduler->stop(); + printf("[shutdown] Stopped Fuzzy Timer...\n"); // ServerGlobals::g_IotaRequestCPUScheduler->stop(); FileBasedProvider::getInstance()->exit(); + printf("[shutdown] Stopped File Based Provider...\n"); + ServerGlobals::g_CPUScheduler->stop(); + printf("[shutdown] Stopped CPU Scheduler...\n"); + ServerGlobals::g_WriteFileCPUScheduler->stop(); + printf("[shutdown] Stopped IO Worker...\n"); + ServerGlobals::clearMemory(); + printf("[shutdown] Cleared Memory...\n"); } -bool MainServer::configExists(const std::string& fileName) { - return fs::exists(fileName) && fs::is_regular_file(fileName); +bool MainServer::configExists(const string& fileName) { + return exists(fileName) && is_regular_file(fileName); } -std::string MainServer::findConfigFile() +string MainServer::findConfigFile() { // possible paths - fs::path currentPath = "gradido.yaml"; // current location - fs::path homePath = fs::path(getHomeDir()) / ".gradido" / "gradido.yaml"; + path currentPath = "gradido.yaml"; // current location + path homePath = path(getHomeDir()) / ".gradido" / "gradido.yaml"; // check paths if (configExists(currentPath.string())) { @@ -183,12 +231,12 @@ std::string MainServer::findConfigFile() return ""; } -std::string MainServer::getHomeDir() +string MainServer::getHomeDir() { #if defined(_WIN32) || defined(_WIN64) - return fs::path(getenv("USERPROFILE")).string(); // windows + return path(getenv("USERPROFILE")).string(); // windows #else - return fs::path(getenv("HOME")).string(); // linux + return path(getenv("HOME")).string(); // linux #endif } diff --git a/src/MainServer.h b/src/MainServer.h index 5ec3210e..2d0a6827 100644 --- a/src/MainServer.h +++ b/src/MainServer.h @@ -5,6 +5,7 @@ #include "gradido_blockchain/http/Server.h" #include +#include class MainServer : public Application { diff --git a/src/ServerGlobals.cpp b/src/ServerGlobals.cpp index bf4b55e7..a90b2cda 100644 --- a/src/ServerGlobals.cpp +++ b/src/ServerGlobals.cpp @@ -1,7 +1,5 @@ #include "ServerGlobals.h" -#include "gradido_blockchain/lib/Profiler.h" -// #include "gradido_blockchain/http/IotaRequest.h" #include "client/hiero/MirrorClient.h" using namespace std::chrono; @@ -16,7 +14,6 @@ namespace ServerGlobals { std::chrono::seconds g_CacheTimeout(600); std::chrono::seconds g_TimeoutCheck(60); std::chrono::seconds g_WriteToDiskTimeout(10); - IotaRequest* g_IotaRequestHandler = nullptr; std::string g_IotaMqttBrokerUri; std::atomic g_NumberExistingTasks; bool g_LogTransactions = false; @@ -41,10 +38,6 @@ namespace ServerGlobals { delete g_GroupIndex; g_GroupIndex = nullptr; } - if (g_IotaRequestHandler) { - delete g_IotaRequestHandler; - g_IotaRequestHandler = nullptr; - } if (g_HieroMirrorNode) { delete g_HieroMirrorNode; g_HieroMirrorNode = nullptr; @@ -52,24 +45,6 @@ namespace ServerGlobals { } - /*bool initIota(const MapEnvironmentToConfig& cfg) - { - // testnet - // api.lb-0.h.chrysalis-devnet.iota.cafe - // mainnet: - // chrysalis-nodes.iota.org - std::string iotaHost = cfg.getString("clients.iota.rest_api.host", "api.lb-0.h.chrysalis-devnet.iota.cafe"); - int iotaPort = cfg.getInt("clients.iota.rest_api.port", 443); - g_IotaRequestHandler = new IotaRequest(iotaHost, iotaPort, "/api/v1/"); - - std::string iotaMqttHost = cfg.getString("clients.iota.mqtt.host", "api.lb-0.h.chrysalis-devnet.iota.cafe"); - int mqttPort = cfg.getInt("clients.iota.mqtt.port", 1883); - g_IotaMqttBrokerUri = iotaHost + ":" + std::to_string(mqttPort); - - g_isOfflineMode = cfg.getBool("clients.isOfflineMode", false); - return true; - }*/ - bool initHiero(std::string_view hieroNetworkType) { g_HieroMirrorNode = new client::hiero::MirrorClient(hieroNetworkType); return true; diff --git a/src/ServerGlobals.h b/src/ServerGlobals.h index 8c346caf..6ef983ee 100644 --- a/src/ServerGlobals.h +++ b/src/ServerGlobals.h @@ -1,7 +1,6 @@ #ifndef GRADIDO_NODE_SERVER_GLOBALS #define GRADIDO_NODE_SERVER_GLOBALS -#include "gradido_blockchain/http/IotaRequest.h" #include "gradido_blockchain/lib/MapEnvironmentToConfig.h" #include "cache/GroupIndex.h" #include "task/CPUSheduler.h" @@ -28,7 +27,6 @@ namespace ServerGlobals { extern std::chrono::seconds g_TimeoutCheck; //! in which timespan data will be flushed to disk, in seconds, default 10 seconds extern std::chrono::seconds g_WriteToDiskTimeout; - extern IotaRequest* g_IotaRequestHandler; extern std::string g_IotaMqttBrokerUri; extern std::atomic g_NumberExistingTasks; extern bool g_LogTransactions; diff --git a/src/SingletonManager/FileLockManager.cpp b/src/SingletonManager/FileLockManager.cpp index 19d5c164..bc212af3 100644 --- a/src/SingletonManager/FileLockManager.cpp +++ b/src/SingletonManager/FileLockManager.cpp @@ -7,12 +7,15 @@ FileLockManager::FileLockManager() + : mInitialized(true) { } FileLockManager::~FileLockManager() { + std::scoped_lock lock(mWorkingMutex); + mInitialized = false; for (auto it = mFiles.begin(); it != mFiles.end(); it++) { // printf("%s \n", it->first.data()); delete it->second; @@ -28,6 +31,10 @@ FileLockManager* FileLockManager::getInstance() bool FileLockManager::isLock(const std::string& file) { + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager, will return false regardless of real state\n", __FUNCTION__); + return false; + } std::scoped_lock lock(mWorkingMutex); auto it = mFiles.find(file); if (it != mFiles.end()) { @@ -40,6 +47,10 @@ bool FileLockManager::isLock(const std::string& file) bool FileLockManager::tryLock(const std::string& file) { std::scoped_lock lock(mWorkingMutex); + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager, will return true regardless of real state\n", __FUNCTION__); + return true; + } auto it = mFiles.find(file); if (it != mFiles.end()) { if (!*it->second) { @@ -54,6 +65,10 @@ bool FileLockManager::tryLock(const std::string& file) bool FileLockManager::tryLockTimeout(const std::string& file, int tryCount) { + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager, will return true regardless of real state\n", __FUNCTION__); + return true; + } int timeoutRounds = tryCount; bool fileLocked = false; while (!fileLocked && timeoutRounds > 0) { @@ -73,6 +88,10 @@ bool FileLockManager::tryLockTimeout(const std::string& file, int tryCount) void FileLockManager::unlock(const std::string& file) { std::scoped_lock lock(mWorkingMutex); + if (!mInitialized) { + printf("warning: %s was called after ~FileLockManager\n", __FUNCTION__); + return; + } auto it = mFiles.find(file); assert(it != mFiles.end()); diff --git a/src/SingletonManager/FileLockManager.h b/src/SingletonManager/FileLockManager.h index b3bbb780..2004bf35 100644 --- a/src/SingletonManager/FileLockManager.h +++ b/src/SingletonManager/FileLockManager.h @@ -27,6 +27,7 @@ class FileLockManager protected: FileLockManager(); + bool mInitialized; std::mutex mWorkingMutex; std::unordered_map mFiles; diff --git a/src/blockchain/FileBased.cpp b/src/blockchain/FileBased.cpp index e4f07afa..2202ed2b 100644 --- a/src/blockchain/FileBased.cpp +++ b/src/blockchain/FileBased.cpp @@ -1,3 +1,4 @@ +#include "gradido_blockchain/AppContext.h" #include "FileBased.h" #include "../client/hiero/ConsensusClient.h" #include "FileBasedProvider.h" @@ -9,49 +10,72 @@ #include "../ServerGlobals.h" #include "../SystemExceptions.h" #include "../task/NotifyClient.h" -#include "../task/SyncTopicOnStartup.h" +#include "../task/SyncTopic.h" #include "../client/hiero/MirrorClient.h" #include "gradido_blockchain/const.h" +#include "gradido_blockchain/blockchain/batch/signaturesVerify.h" +#include "gradido_blockchain/blockchain/batch/ThreadingPolicy.h" +#include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" +#include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/data/Timestamp.h" #include "gradido_blockchain/interaction/confirmTransaction/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/serialization/toJsonString.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "loguru/loguru.hpp" #include +#include #include using namespace cache; +using std::lock_guard; +using std::string, std::string_view, std::stop_token, std::vector; +using std::shared_ptr, std::make_shared; +using client::hiero::ConsensusClient; +using controller::SimpleOrderingManager; +using serialization::toJsonString; namespace gradido { + using data::adapter::toPublicKey; + using data::AddressType, data::Timestamp, data::LedgerAnchor; + using data::compact::ConstConfirmedTxPtr, data::compact::ConfirmedTxs, data::compact::PublicKeyIndex; + using namespace interaction; namespace blockchain { + using batch::ThreadingPolicy, batch::verifySignatures; + FileBased::FileBased( Private, - std::string_view communityId, + stop_token stopToken, + const string& communityId, const hiero::TopicId& topicId, - std::string_view alias, - std::string_view folder, - std::vector>&& hieroClients) - : Abstract(communityId), - mExitCalled(false), + string_view alias, + string_view folder, + vector>&& hieroClients) + : Abstract(g_appContext->getOrAddCommunityIdIndex(communityId)), + mStopToken(stopToken), mHieroTopicId(topicId), mAlias(alias), - mFolderPath(folder), + mFolderPath(folder), + mCommunityId(communityId), mTaskObserver(std::make_shared()), - mOrderingManager(std::make_shared(communityId)), + mOrderingManager(std::make_shared(communityId, stopToken)), // mIotaMessageListener(new iota::MessageListener(communityId, alias)), - mPublicKeysIndex(std::make_shared(std::string(folder).append("/pubkeysCache"))), - mBlockchainState(std::string(folder).append("/.state")), - mMessageIdsCache(std::string(folder).append("/messageIdCache")), + mPublicKeysIndex((string(folder).append("/pubkeysCache"))), + mBlockchainState(string(folder).append("/.state")), + mLedgerAnchorCache(string(folder).append("/messageIdCache")), mTransactionTriggerEventsCache(std::string(folder).append("/transactionTriggerEventCache")), mCachedBlocks(ServerGlobals::g_CacheTimeout), mTransactionHashCache(communityId), mHieroClients(std::move(hieroClients)) { - assert(mHieroClients.size()); + assert(mHieroTopicId.empty() || mHieroClients.size()); } FileBased::~FileBased() @@ -64,16 +88,21 @@ namespace gradido { } bool FileBased::init(bool resetBlockIndices) { - assert(!mExitCalled); - std::lock_guard _lock(mWorkMutex); - if (!mPublicKeysIndex->init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { + if (mStopToken.stop_requested()) { + return false; + } + lock_guard _lock(mWorkMutex); + if (mPublicKeysIndex.init(0)) { + LOG_F(WARNING, "dictionary is again persistent, please update here"); + } + /*if (!mPublicKeysIndex.init(GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES * 1024 * 1024)) { // remove index files for regenration LOG_F(WARNING, "reset the public key index file"); // mCachedBlocks.clear(); - mPublicKeysIndex->reset(); + mPublicKeysIndex.reset(); resetBlockIndices = true; - } - + }*/ + if (resetBlockIndices) { model::files::BlockIndex::removeAllBlockIndexFiles(mFolderPath); } @@ -83,103 +112,93 @@ namespace gradido { loadStateFromBlockCache(); } // read basic states into memory - mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, 0); + /*mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, 0); mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_TRANSACTION_ID, 0); mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0); - mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); + mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum()); + */ + return true; + } - if (!mMessageIdsCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { - mMessageIdsCache.reset(); - if (!mMessageIdsCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { + bool FileBased::startValidationTransactions() + { + if (mStopToken.stop_requested()) return false; + + lock_guard _lock(mWorkMutex); + auto lastBlockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 0); + + // trigger block index creation, only needed here for non-persistent public key dictionary + iterateBlocks(SearchDirection::ASC, [](const cache::Block& block) -> bool { return true; }); + loadStateFromBlockCache(); + + if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { + mLedgerAnchorCache.reset(); + if (!mLedgerAnchorCache.init(GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES * 1024 * 1024)) { throw ClassNotInitalizedException("cannot initalize message id cache", "cache::MessageId"); } // load last 20 message ids into cache FilterBuilder filterBuilder; - auto transactions = findAll(filterBuilder.setPagination({20}).setSearchDirection(SearchDirection::DESC).build()); + auto transactions = findAll(filterBuilder.setPagination({ 20 }).setSearchDirection(SearchDirection::DESC).build()); for (auto& transaction : transactions) { - mMessageIdsCache.add(transaction->getConfirmedTransaction()->getMessageId(), transaction->getTransactionNr()); + mLedgerAnchorCache.add(transaction->getConfirmedTransaction()->getLedgerAnchor(), transaction->getTransactionNr()); } } if (!mTransactionTriggerEventsCache.init(GRADIDO_NODE_MAGIC_NUMBER_TRANSACTION_TRIGGER_EVENTS_CACHE_MEGA_BTYES * 1024 * 1024)) { - Profiler timeUsed; + MonotonicTimer timeUsed; rescanForTransactionTriggerEvents(); LOG_F(INFO, "rescan blockchain for transaction trigger events, time: %s", timeUsed.string().data()); } - // load first GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE transaction into cache and validate the transaction to check file integrity - Profiler timeUsed; - FilterBuilder builder; - int count = 0; - findAll(builder - .setSearchDirection(SearchDirection::DESC) - .setPagination({ GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE }) - .setFilterFunction([this, &count](const TransactionEntry& transactionEntry) -> FilterResult { - auto previousTransactionNr = transactionEntry.getTransactionNr() - 1; - data::ConstConfirmedTransactionPtr previousConfirmedTransaction; - auto transactionBody = transactionEntry.getTransactionBody(); - validate::Context validator(*transactionEntry.getConfirmedTransaction()); - if (previousTransactionNr >= 1) { - auto entry = getTransactionForId(previousTransactionNr); - if (entry) { - previousConfirmedTransaction = entry->getConfirmedTransaction(); - } - } - validate::Type validationLevel = validate::Type::SINGLE | validate::Type::ACCOUNT; - if (transactionBody->getType() != data::CrossGroupType::LOCAL) { - validationLevel = validationLevel | validate::Type::PAIRED; - } - if (previousConfirmedTransaction) { - validationLevel = validationLevel | validate::Type::PREVIOUS; - validator.setSenderPreviousConfirmedTransaction(previousConfirmedTransaction); - } - validator.run(validationLevel, getptr()); - mTransactionHashCache.push(*transactionEntry.getConfirmedTransaction()); - count++; - return FilterResult::DISMISS; - }) - .build() - ); - LOG_F(INFO, "time used for loading and validating last: %d/%d transactions: %s", - count, - GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE, - timeUsed.string().data() - ); - return true; + // if state was empty, we better validate full blockchain + if (!lastBlockNr) { + return validateLastTransactions(0); + } + else { + return validateLastTransactions(GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE); + } } - std::shared_ptr FileBased::initOnline() + std::shared_ptr FileBased::getTopicSyncTask() { - return std::make_shared( - mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), - hiero::TopicId(mBlockchainState.readState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString())), - getptr() - ); + if (mStopToken.stop_requested()) return nullptr; + auto hieroTopicIdNum = mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum()); + if (hieroTopicIdNum) { + return std::make_shared( + mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_SEQUENCE_NUMBER, 0), + hiero::TopicId(0, 0, hieroTopicIdNum), + getptr() + ); + } + LOG_F(WARNING, "init online called for community without hiero topic id"); + return nullptr; } void FileBased::startListening(data::Timestamp lastTransactionConfirmedAt) - { - if (mHieroMessageListener) { - LOG_F(WARNING, "called again, while listener where already existing"); + { + if (mStopToken.stop_requested()) return; + auto hieroTopicId = hiero::TopicId(0, 0, mBlockchainState.readInt64State(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.getTopicNum())); + if (hieroTopicId.empty()) { + LOG_F(WARNING, "startListening called without valid hiero topic id"); + return; } data::Timestamp listenFrom = { lastTransactionConfirmedAt.getSeconds(), lastTransactionConfirmedAt.getNanos() + 1 }; auto now = std::chrono::system_clock::now(); // TODO: restart after connection was closed because of timeout auto endTime = now + std::chrono::duration(std::chrono::years(10)); mHieroMessageListener = std::make_shared( - mHieroTopicId, - mCommunityId, - hiero::ConsensusTopicQuery( mHieroTopicId, listenFrom, endTime ) + hieroTopicId, + mCommunityId, + hiero::ConsensusTopicQuery( hieroTopicId, listenFrom, endTime ) ); - ServerGlobals::g_HieroMirrorNode->subscribeTopic(mHieroMessageListener); - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, mHieroTopicId.toString()); + ServerGlobals::g_HieroMirrorNode->subscribeTopic(mHieroMessageListener.get()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_HIERO_TOPIC_ID, hieroTopicId.getTopicNum()); } void FileBased::exit() { - std::lock_guard _lock(mWorkMutex); - mExitCalled = true; + lock_guard _lock(mWorkMutex); /*if (mIotaMessageListener) { delete mIotaMessageListener; mIotaMessageListener = nullptr; @@ -188,7 +207,7 @@ namespace gradido { mHieroMessageListener->cancelConnection(); } - Profiler timeUsed; + MonotonicTimer timeUsed; // wait until all Task of TaskObeserver are finished, wait a second and check if number decreased, // if number no longer descrease after a second and we wait more than 10 seconds total, exit loop while (auto pendingTasksCount = mTaskObserver->getPendingTasksCount()) { @@ -200,25 +219,27 @@ namespace gradido { mOrderingManager->exit(); mCachedBlocks.clear(); mBlockchainState.exit(); - mPublicKeysIndex->exit(); - mMessageIdsCache.exit(); + mPublicKeysIndex.exit(); + mLedgerAnchorCache.exit(); mTransactionTriggerEventsCache.exit(); } + bool FileBased::createAndAddConfirmedTransaction( data::ConstGradidoTransactionPtr gradidoTransaction, - memory::ConstBlockPtr messageId, + const data::LedgerAnchor& ledgerAnchor, data::Timestamp confirmedAt ) { + if (mStopToken.stop_requested()) return false; if (!gradidoTransaction) { throw GradidoNullPointerException("missing transaction", "GradidoTransactionPtr", __FUNCTION__); } - if (!messageId) { - throw GradidoNullPointerException("missing messageId", "memory::ConstBlockPtr", __FUNCTION__); + if (ledgerAnchor.empty()) { + throw GradidoNullPointerException("empty ledger anchor", "gradido::data::LedgerAnchor", __FUNCTION__); } - std::lock_guard _lock(mWorkMutex); - if (mExitCalled) { return false;} + lock_guard _lock(mWorkMutex); + confirmTransaction::Context confirmTransactionContext(getptr()); - auto role = confirmTransactionContext.createRole(gradidoTransaction, messageId, confirmedAt); + auto role = confirmTransactionContext.createRole(gradidoTransaction, ledgerAnchor, confirmedAt); auto confirmedTransaction = confirmTransactionContext.run(role); // will occure if transaction already exist if (!confirmedTransaction) { @@ -226,21 +247,75 @@ namespace gradido { } auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); auto& block = getBlock(blockNr); - auto nodeTransactionEntry = std::make_shared(confirmedTransaction, getptr()); - if (!block.pushTransaction(nodeTransactionEntry)) { - // block was already stopped, so we can stop here also + auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); + if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex, *g_appContext)) { + // block was already stopped, so we can stop here also LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); return false; } role->runPastAddToBlockchain(confirmedTransaction, getptr()); mBlockchainState.updateState(DefaultStateKeys::LAST_TRANSACTION_ID, confirmedTransaction->getId()); - mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex->getLastIndex()); + mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); mTransactionHashCache.push(*nodeTransactionEntry->getConfirmedTransaction()); - mMessageIdsCache.add(confirmedTransaction->getMessageId(), confirmedTransaction->getId()); + mLedgerAnchorCache.add(confirmedTransaction->getLedgerAnchor(), confirmedTransaction->getId()); // add public keys to index auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); for (const auto& address : involvedAddresses) { - mPublicKeysIndex->getOrAddIndexForString(address->copyAsString()); + mPublicKeysIndex.getOrAddIndexForData(toPublicKey(address)); + } + if (mCommunityServer) { + task::TaskPtr notifyClientTask = std::make_shared(mCommunityServer, confirmedTransaction); + notifyClientTask->scheduleTask(notifyClientTask); + } + return true; + } + + bool FileBased::createAndAddConfirmedTransactionExtern( + data::ConstGradidoTransactionPtr gradidoTransaction, + const data::LedgerAnchor& ledgerAnchor, + std::vector accountBalances + ) + { + if (mStopToken.stop_requested()) return false; + if (!gradidoTransaction) { + throw GradidoNullPointerException("missing transaction", "GradidoTransactionPtr", __FUNCTION__); + } + if (ledgerAnchor.empty()) { + throw GradidoNullPointerException("empty ledger anchor", "gradido::data::LedgerAnchor", __FUNCTION__); + } + lock_guard _lock(mWorkMutex); + confirmTransaction::Context confirmTransactionContext(getptr()); + auto role = confirmTransactionContext.createRole( + gradidoTransaction, + ledgerAnchor, + gradidoTransaction->getTransactionBody()->getCreatedAt() + ); + if (!role) { + throw GradidoNotImplementedException("missing role for gradido transaction"); + } + role->setAccountBalances(accountBalances); + auto confirmedTransaction = confirmTransactionContext.run(role); + // will occure if transaction already exist + if (!confirmedTransaction) { + return false; + } + auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); + auto& block = getBlock(blockNr); + auto nodeTransactionEntry = make_shared(confirmedTransaction, getptr()); + if (!block.pushTransaction(nodeTransactionEntry, mPublicKeysIndex, *g_appContext)) { + // block was already stopped, so we can stop here also + LOG_F(WARNING, "couldn't push transaction: %lu to block: %d", confirmedTransaction->getId(), blockNr); + return false; + } + role->runPastAddToBlockchain(confirmedTransaction, getptr()); + mBlockchainState.updateState(DefaultStateKeys::LAST_TRANSACTION_ID, confirmedTransaction->getId()); + mBlockchainState.updateState(DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); + mTransactionHashCache.push(*nodeTransactionEntry->getConfirmedTransaction()); + mLedgerAnchorCache.add(confirmedTransaction->getLedgerAnchor(), confirmedTransaction->getId()); + // add public keys to index + auto involvedAddresses = confirmedTransaction->getInvolvedAddresses(); + for (const auto& address : involvedAddresses) { + mPublicKeysIndex.getOrAddIndexForData(toPublicKey(address)); } if (mCommunityServer) { task::TaskPtr notifyClientTask = std::make_shared(mCommunityServer, confirmedTransaction); @@ -256,115 +331,284 @@ namespace gradido { void FileBased::addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent) { - std::lock_guard _lock(mWorkMutex); + if (mStopToken.stop_requested()) return; + + lock_guard _lock(mWorkMutex); mTransactionTriggerEventsCache.addTransactionTriggerEvent(transactionTriggerEvent); } void FileBased::removeTransactionTriggerEvent(const data::TransactionTriggerEvent& transactionTriggerEvent) { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); mTransactionTriggerEventsCache.removeTransactionTriggerEvent(transactionTriggerEvent); } - std::vector> FileBased::findTransactionTriggerEventsInRange(TimepointInterval range) + std::vector> FileBased::findTransactionTriggerEventsInRange(Timestamp startDate, Timestamp endDate) { - std::lock_guard _lock(mWorkMutex); - return mTransactionTriggerEventsCache.findTransactionTriggerEventsInRange(range); + lock_guard _lock(mWorkMutex); + return mTransactionTriggerEventsCache.findTransactionTriggerEventsInRange(startDate, endDate); } - std::shared_ptr FileBased::findNextTransactionTriggerEventInRange(TimepointInterval range) + std::shared_ptr FileBased::findNextTransactionTriggerEventInRange(Timestamp startDate, Timestamp endDate) { - std::lock_guard _lock(mWorkMutex); - return mTransactionTriggerEventsCache.findNextTransactionTriggerEventInRange(range); + lock_guard _lock(mWorkMutex); + return mTransactionTriggerEventsCache.findNextTransactionTriggerEventInRange(startDate, endDate); } TransactionEntries FileBased::findAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS */) const { - TransactionEntries result; - // if pagination is used, filterCopy contain count of still to find transactions - Filter filterCopy(filter); - bool stopped = false; - iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, *mPublicKeysIndex); - for (auto transactionNr : transactionNrs) { - if (!filter.pagination.hasCapacityLeft(result.size())) { + TransactionEntries resultTxs; + CompactFilter compactFilter(filter, mPublicKeysIndex, mCommunityIdIndex); + FilterResult lastFilterResult = FilterResult::DISMISS; + size_t lastFindTransactionResultCount = 0; + + bool hasPagination = filter.pagination.size > 0; + if (hasPagination) { + resultTxs.reserve(filter.pagination.size); + } + + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool + { + do { + compactFilter.pagination = filter.pagination; + auto txs = block.getBlockIndex().findTransactions(compactFilter); + lastFindTransactionResultCount = txs.size(); + + // nothing found? search next block + if (!txs.size()) return true; + + if (resultTxs.capacity() - resultTxs.size() < txs.size()) { + resultTxs.reserve(txs.size() + resultTxs.size()); + } + for (const auto& tx : txs) + { + // cannot short cut like in blockchain::InMemory, because FileBased uses a Cache and when we don't copy the shared_ptr, + // it is possible it will be deleted while we are using it :/ + auto confirmedTx = getTransactionForId(tx); + if (!confirmedTx) { + throw GradidoBlockchainTransactionNotFoundException("cannot found confirmed tx in iterateAllImpl").setTransactionId(tx); + } + if (filter.filterFunction) { + lastFilterResult = filter.filterFunction(*confirmedTx); + } + else { + lastFilterResult = FilterResult::USE; + } + if ((FilterResult::USE & lastFilterResult) == FilterResult::USE) { + resultTxs.emplace_back(confirmedTx); + } + } + ++compactFilter.pagination.page; + } while ( + hasPagination && + filter.pagination.hasCapacityLeft(resultTxs.size()) && + (FilterResult::STOP & lastFilterResult) != FilterResult::STOP && + lastFindTransactionResultCount == filter.pagination.size + ); + // we have enough, stop + if ((FilterResult::STOP & lastFilterResult) == FilterResult::STOP || !filter.pagination.hasCapacityLeft(resultTxs.size())) { return false; } - auto transaction = block.getTransaction(transactionNr); - auto filterResult = filter.matches(transaction, FilterCriteria::FILTER_FUNCTION); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - result.push_back(transaction); + return true; + } + ); + return resultTxs; + } + + ConfirmedTxs FileBased::findAll(const CompactFilter& filter) const + { + ConfirmedTxs resultTxs; + + bool hasPagination = filter.pagination.size > 0; + if (hasPagination) { + resultTxs.reserve(filter.pagination.size); + } + + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool + { + auto txs = block.getBlockIndex().findTransactions(filter); + // nothing found? search next block + if (!txs.size()) return true; + + if (!hasPagination && (resultTxs.capacity() - resultTxs.size() < txs.size())) { + resultTxs.reserve(txs.size() + resultTxs.size()); + } + for (const auto& tx : txs) { + auto confirmedTx = getConfirmedTxForId(tx); + if (!confirmedTx) { + throw GradidoBlockchainTransactionNotFoundException("cannot found confirmed tx in iterateAllImpl").setTransactionId(tx); + } + resultTxs.emplace_back(confirmedTx); } - if ((filterResult & FilterResult::STOP) == FilterResult::STOP) { - stopped = true; - break; + // we have enough, stop + // without pagination, hasCapacityLeft returns always true + if (!filter.pagination.hasCapacityLeft(resultTxs.size())) { + return false; } + return true; } - if (filter.pagination.size) { - filterCopy.pagination.size = filter.pagination.size - result.size(); - // we have requested result count, let's exit here - if (filterCopy.pagination.size <= 0) { + ); + return resultTxs; + } + + data::compact::ConfirmedTxs FileBased::findAll( + const CompactFilter& filter, + std::function elementFilter + ) const + { + ConfirmedTxs resultTxs; + CompactFilter filterCopy(filter); + FilterResult lastFilterResult = FilterResult::DISMISS; + size_t lastFindTransactionResultCount = 0; + + bool hasPagination = filter.pagination.size > 0; + if (hasPagination) { + resultTxs.reserve(filter.pagination.size); + } + + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool + { + do { + filterCopy.pagination = filter.pagination; + auto txs = block.getBlockIndex().findTransactions(filterCopy); + lastFindTransactionResultCount = txs.size(); + // nothing found? search next block + if (!txs.size()) return true; + + if (resultTxs.capacity() - resultTxs.size() < txs.size()) { + resultTxs.reserve(txs.size() + resultTxs.size()); + } + for (const auto& tx : txs) { + // cannot short cut like in blockchain::InMemory, because FileBased uses a Cache and when we don't copy the shared_ptr, + // it is possible it will be deleted while we are using it :/ + auto confirmedTx = getConfirmedTxForId(tx); + if (!confirmedTx) { + throw GradidoBlockchainTransactionNotFoundException("cannot found confirmed tx in iterateAllImpl").setTransactionId(tx); + } + lastFilterResult = elementFilter(*confirmedTx); + if ((FilterResult::USE & lastFilterResult) == FilterResult::USE) { + resultTxs.emplace_back(getConfirmedTxForId(tx)); + } + } + ++filterCopy.pagination.page; + } while ( + hasPagination && + filter.pagination.hasCapacityLeft(resultTxs.size()) && + (FilterResult::STOP & lastFilterResult) != FilterResult::STOP && + lastFindTransactionResultCount == filter.pagination.size + ); + // we have enough, stop + if ((FilterResult::STOP & lastFilterResult) == FilterResult::STOP || !filter.pagination.hasCapacityLeft(resultTxs.size())) { return false; } + return true; } - return !stopped; - }); - return result; + ); + return resultTxs; } - std::vector FileBased::findAllFast(const Filter& filter) const + size_t FileBased::countAll(const Filter& filter/* = Filter::ALL_TRANSACTIONS*/) const { - std::vector result; - // if pagination is used, filterCopy contain count of still to find transactions - Filter filterCopy(filter); - iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - auto transactionNrs = block.getBlockIndex().findTransactions(filterCopy, *mPublicKeysIndex); - result.insert(result.end(), transactionNrs.begin(), transactionNrs.end()); - if (filter.pagination.size) { - filterCopy.pagination.size = filter.pagination.size - result.size(); - // we have requested result count, let's exit here - if (filterCopy.pagination.size <= 0) return false; - } - return true; - }); - return result; + CompactFilter compactFilter(filter, mPublicKeysIndex, mCommunityIdIndex); + return countAll(compactFilter); } - size_t FileBased::findAllResultCount(const Filter& filter) const + size_t FileBased::countAll(const CompactFilter& filter) const { size_t count = 0; - iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { - count += block.getBlockIndex().countTransactions(filter, *mPublicKeysIndex); - return true; - }); + // check if filter has fields which aren't checked by index + iterateBlocks(filter.searchDirection, + [&](const cache::Block& block) -> bool { + count += block.getBlockIndex().countTransactions(filter); + return true; + } + ); return count; } + AddressType FileBased::getAddressType(const Filter& filter/* = Filter::LAST_TRANSACTION*/) const + { + // return getAddressTypeSlow(filter); + if (!filter.involvedPublicKey || filter.involvedPublicKey->isEmpty()) { + throw GradidoNodeInvalidDataException("missing public key, please use filter with involvedPublicKey set"); + } + auto publicKeyIndexOptional = mPublicKeysIndex.getIndexForData(toPublicKey(filter.involvedPublicKey)); + if (!publicKeyIndexOptional) { + return AddressType::NONE; + } + uint32_t publicKeyUint32 = (uint32_t)publicKeyIndexOptional; + if (publicKeyUint32 != publicKeyIndexOptional) { + throw GradidoNodeInvalidDataException("public key index overflow"); + } + PublicKeyIndex publicKeyIndex = { .communityIdIndex = mCommunityIdIndex, .publicKeyIndex = publicKeyUint32 }; + data::AddressType result = data::AddressType::NONE; + iterateBlocks(filter.searchDirection, [&](const cache::Block& block) -> bool { + auto addressTypeStateChange = block.getBlockIndex().getAddressType(publicKeyIndex); + result = addressTypeStateChange.getValue(); + if (addressTypeStateChange.getTxId()) { + auto tx = getTransactionForId(addressTypeStateChange.getTxId()); + if (FilterResult::USE != (filter.matches(tx, FilterCriteria::MAX) & FilterResult::USE)) { + result = data::AddressType::NONE; + } + return false; //break iterateBlocks + } + // result + if (data::AddressType::NONE == result) { + return true; + } + return false; + }); + return result; + } std::shared_ptr FileBased::getTransactionForId(uint64_t transactionId) const { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); do { auto& block = getBlock(blockNr); if (block.getBlockIndex().hasTransactionNr(transactionId)) { - return block.getTransaction(transactionId); + return block.getTransaction(transactionId, *g_appContext); } blockNr--; } while (blockNr > 0); return nullptr; } - std::shared_ptr FileBased::findByMessageId( - memory::ConstBlockPtr messageId, - const Filter& filter/* = Filter::ALL_TRANSACTIONS */ + + ConstConfirmedTxPtr FileBased::getConfirmedTxForId(uint64_t transactionId) const + { + lock_guard _lock(mWorkMutex); + auto blockNr = mBlockchainState.readInt32State(cache::DefaultStateKeys::LAST_BLOCK_NR, 1); + do { + auto& block = getBlock(blockNr); + if (block.getBlockIndex().hasTransactionNr(transactionId)) { + return block.getCompactTransaction(transactionId, *g_appContext); + } + blockNr--; + } while (blockNr > 0); + return nullptr; + } + + std::shared_ptr FileBased::findByLedgerAnchor( + const data::LedgerAnchor& ledgerAnchor, + const Filter& filter/* = Filter::ALL_TRANSACTIONS*/ ) const { - auto transactionNr = mMessageIdsCache.has(messageId); + lock_guard _lock(mWorkMutex); + auto transactionNr = mLedgerAnchorCache.getTransactionNrForLedgerAnchor(ledgerAnchor); if (transactionNr) { return getTransactionForId(transactionNr); } - return Abstract::findByMessageId(messageId, filter); + auto result = Abstract::findByLedgerAnchor(ledgerAnchor, filter); + if (result) { + mLedgerAnchorCache.add(ledgerAnchor, result->getTransactionNr()); + } + return result; } + AbstractProvider* FileBased::getProvider() const { return FileBasedProvider::getInstance(); @@ -373,13 +617,11 @@ namespace gradido { void FileBased::loadStateFromBlockCache() { - Profiler timeUsed; - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex->getLastIndex()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_ADDRESS_INDEX, mPublicKeysIndex.getLastIndex()); auto lastBlockNr = model::files::Block::findLastBlockFileInFolder(mFolderPath); mBlockchainState.updateState(cache::DefaultStateKeys::LAST_BLOCK_NR, lastBlockNr); auto& block = getBlock(lastBlockNr); - mBlockchainState.updateState(cache::DefaultStateKeys::LAST_TRANSACTION_ID, block.getBlockIndex().getMaxTransactionNr()); - LOG_F(INFO, "timeUsed: %s", timeUsed.string().data()); + mBlockchainState.updateState(cache::DefaultStateKeys::LAST_TRANSACTION_ID, block.getBlockIndex().getMaxTransactionNr()); } // TODO: look for a way of reusing logic from interaction::confirmTransaction, with nearly the same code in the roles @@ -389,6 +631,7 @@ namespace gradido { Filter f = Filter::ALL_TRANSACTIONS; f.searchDirection = SearchDirection::ASC; f.filterFunction = [this](const TransactionEntry& entry) -> FilterResult { + if (mStopToken.stop_requested()) return FilterResult::STOP; if (entry.getTransactionType() == data::TransactionType::DEFERRED_TRANSFER) { auto confirmedTransaction = entry.getConfirmedTransaction(); auto body = entry.getTransactionBody(); @@ -399,7 +642,7 @@ namespace gradido { confirmedTransaction->getId(), targetDate, data::TransactionTriggerEventType::DEFERRED_TIMEOUT_REVERSAL - )); + )); } else if (entry.getTransactionType() == data::TransactionType::REDEEM_DEFERRED_TRANSFER) { // remove timeout transaction trigger event @@ -427,7 +670,7 @@ namespace gradido { }; findAll(f); } - + void FileBased::iterateBlocks(const SearchDirection& searchDir, std::function func) const { bool orderDesc = searchDir == SearchDirection::DESC; @@ -457,7 +700,7 @@ namespace gradido { if (!block) { auto block = std::make_shared(blockNr, getptr()); // return false if block not exist and will be created - if (!block->init()) { + if (!block->init(mStopToken)) { if (blockNr > mBlockchainState.readInt32State(DefaultStateKeys::LAST_BLOCK_NR, 1)) { mBlockchainState.updateState(DefaultStateKeys::LAST_BLOCK_NR, blockNr); } @@ -467,5 +710,78 @@ namespace gradido { } return *block.value(); } + + bool FileBased::validateLastTransactions(uint64_t countToValidate) + { + // load first GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE transaction into cache and validate the transaction to check file integrity + if (mStopToken.stop_requested()) return false; + MonotonicTimer timeUsed; + MonotonicTimer timeSinceLastPrint; + data::ConstConfirmedTransactionPtr previousConfirmedTransaction = nullptr; + auto lastTransaction = findOne(Filter::LAST_TRANSACTION); + if (!lastTransaction) { + // seems we have nothing todo here + LOG_F(WARNING, "startValidationTransactions called on empty blockchain"); + return true; + } + int count = 0; + Filter f; + f.searchDirection = SearchDirection::ASC; + int countTarget = countToValidate; + if (countToValidate && lastTransaction->getTransactionNr() > countToValidate) { + f.minTransactionNr = lastTransaction->getTransactionNr() - countToValidate + 1; + } + else { + countTarget = lastTransaction->getTransactionNr(); + } + if (f.minTransactionNr) { + auto previousTxEntry = getTransactionForId(f.minTransactionNr - 1); + if (previousTxEntry) { + previousConfirmedTransaction = previousTxEntry->getConfirmedTransaction(); + } + } + + f.filterFunction = + [&](const TransactionEntry& transactionEntry) -> FilterResult + { + if (mStopToken.stop_requested()) return FilterResult::STOP; + auto transactionBody = transactionEntry.getTransactionBody(); + validate::Context validator(*transactionEntry.getConfirmedTransaction()); + validate::Type validationLevel = validate::Type::SINGLE | validate::Type::ACCOUNT; + if (transactionBody->getType() != data::CrossGroupType::LOCAL) { + validationLevel = validationLevel | validate::Type::PAIRED; + } + if (previousConfirmedTransaction) { + validationLevel = validationLevel | validate::Type::PREVIOUS; + validator.setSenderPreviousConfirmedTransaction(previousConfirmedTransaction); + } + validator.disableVerify(); + validator.run(validationLevel, getptr()); + if (transactionEntry.getTransactionNr() > (lastTransaction->getTransactionNr() - GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE)) { + mTransactionHashCache.push(*transactionEntry.getConfirmedTransaction()); + } + previousConfirmedTransaction = transactionEntry.getConfirmedTransaction(); + count++; + /*if (timeSinceLastPrint.millis() > 150) { + printf("\r%.2f%%", ((double)count / (double)countTarget) * 100.0); + timeSinceLastPrint.reset(); + }*/ + return FilterResult::DISMISS; + }; + findAll(f); + // printf("\r"); + f.filterFunction = nullptr; + MonotonicTimer batchVerifyTime; + auto invalidSignatures = verifySignatures(f, mCommunityId, ThreadingPolicy::ThreeQuarter); + LOG_F(INFO, "time used for loading and validating last: %d transactions: %s (%s for batch verify)", + count, + timeUsed.string().c_str(), + batchVerifyTime.string().c_str() + ); + if (!invalidSignatures.empty()) { + throw GradidoNodeInvalidDataException("verify from at least one transaction failed"); + } + return true; + } } } \ No newline at end of file diff --git a/src/blockchain/FileBased.h b/src/blockchain/FileBased.h index 22c34432..d76dd8f1 100644 --- a/src/blockchain/FileBased.h +++ b/src/blockchain/FileBased.h @@ -2,16 +2,17 @@ #define __GRADIDO_NODE_BLOCKCHAIN_FILE_BASED_H #include "../cache/Block.h" -#include "../cache/Dictionary.h" -#include "../cache/HieroTransactionId.h" +#include "../cache/LedgerAnchor.h" #include "../cache/State.h" #include "../cache/TransactionHash.h" #include "../cache/TransactionTriggerEvent.h" #include "../client/Base.h" #include "../controller/TaskObserver.h" #include "../controller/SimpleOrderingManager.h" +#include "../lib/PersistentDictionary.h" #include "gradido_blockchain/blockchain/Abstract.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/hiero/TopicId.h" #include "gradido_blockchain/lib/AccessExpireCache.h" @@ -22,10 +23,12 @@ #define GRADIDO_NODE_MAGIC_NUMBER_BLOCKCHAIN_STATE_CACHE_SIZE_BYTES 192 //! TODO: Test and Profile different values, or create dynamic algorithmus #define GRADIDO_NODE_MAGIC_NUMBER_IOTA_MESSAGE_ID_CACHE_MEGA_BYTES 10 -#define GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES 1 +#define GRADIDO_NODE_MAGIC_NUMBER_PUBLIC_KEYS_INDEX_CACHE_MEGA_BTYES 10 +#define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_INDEX_CACHE_BYTES 400 #define GRADIDO_NODE_MAGIC_NUMBER_TRANSACTION_TRIGGER_EVENTS_CACHE_MEGA_BTYES 1 #include +#include namespace client { namespace hiero { @@ -42,7 +45,7 @@ namespace controller { } namespace task { - class SyncTopicOnStartup; + class SyncTopic; } namespace gradido { @@ -61,21 +64,30 @@ namespace gradido { public: // Constructor is only usable by this class FileBased( - Private, - std::string_view communityId, + Private, + std::stop_token stopToken, + const std::string& communityId, const hiero::TopicId& topicId, - std::string_view alias, + std::string_view alias, std::string_view folder, std::vector>&& hieroClients ); // make sure that all shared_ptr from FileBased Blockchain know each other static inline std::shared_ptr create( - std::string_view communityId, + std::stop_token stopToken, + const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, std::string_view folder, std::vector>&& hieroClients ); + // construct without valid HieroTopic Id. Make Blockchain txs available, but don't listen for new ones from hiero/hedera + static inline std::shared_ptr createWithoutHieroTopic( + std::stop_token stopToken, + const std::string& communityId, + std::string_view alias, + std::string_view folder + ); inline std::shared_ptr getptr(); inline std::shared_ptr getptr() const; @@ -90,12 +102,17 @@ namespace gradido { bool init(bool resetBlockIndices); //! init 2 - //! prepare task for syncronize with hiero topic - //! all SyncTopicOnStartup for all communities should be started/scheduled at the same time because there could need each other for new cross group transactions - std::shared_ptr initOnline(); + //! check last GRADIDO_NODE_MAGIC_NUMBER_STARTUP_TRANSACTIONS_CACHE_SIZE if they are valid + //! should be called, after all communities where initalized, because it would need other communities for cross group transaction validation + bool startValidationTransactions(); //! init 3 - //! start listening to topic, will be called from SyncTopicOnStartup at the end, will update last known TopicId + //! prepare task for syncronize with hiero topic + //! all SyncTopic for all communities should be started/scheduled at the same time because there could need each other for new cross group transactions + std::shared_ptr getTopicSyncTask(); + + //! init 4 + //! start listening to topic, will be called from SyncTopic at the end, will update last known TopicId void startListening(data::Timestamp lastTransactionConfirmedAt); // clean up group, stopp all running processes @@ -107,34 +124,48 @@ namespace gradido { //! \return false if transaction already exist virtual bool createAndAddConfirmedTransaction( data::ConstGradidoTransactionPtr gradidoTransaction, - memory::ConstBlockPtr messageId, + const data::LedgerAnchor& ledgerAnchor, data::Timestamp confirmedAt - ) override; + ) override; + + virtual bool createAndAddConfirmedTransactionExtern( + data::ConstGradidoTransactionPtr gradidoTransaction, + const data::LedgerAnchor& ledgerAnchor, + std::vector accountBalances + ) override; + void updateLastKnownSequenceNumber(uint64_t newSequenceNumber); virtual void addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent) override; virtual void removeTransactionTriggerEvent(const data::TransactionTriggerEvent& transactionTriggerEvent) override; - virtual bool isTransactionExist(data::ConstGradidoTransactionPtr gradidoTransaction) const override { + virtual bool isTransactionExist(data::ConstGradidoTransactionPtr gradidoTransaction, data::Timestamp confirmedAt) const override { return mTransactionHashCache.has(*gradidoTransaction); } //! return events in asc order of targetDate - virtual std::vector> findTransactionTriggerEventsInRange(TimepointInterval range) override; - virtual std::shared_ptr findNextTransactionTriggerEventInRange(TimepointInterval range) override; + virtual std::vector> findTransactionTriggerEventsInRange(data::Timestamp startDate, data::Timestamp endDate) override; + virtual std::shared_ptr findNextTransactionTriggerEventInRange(data::Timestamp startDate, data::Timestamp endDate) override; //! main search function, do all the work, reference from other functions virtual TransactionEntries findAll(const Filter& filter) const override; - //! use only index for searching, ignore filter function - //! \return vector with transaction nrs - std::vector findAllFast(const Filter& filter) const; + virtual data::compact::ConfirmedTxs findAll(const CompactFilter& filter) const override; + virtual data::compact::ConfirmedTxs findAll( + const CompactFilter& filter, + std::function elementFilter + ) const override; + + // find all optimized for counting transaction nrs, better not use the filter.function for that, because this would slow down + virtual size_t countAll(const Filter& filter = Filter::ALL_TRANSACTIONS) const override; + virtual size_t countAll(const CompactFilter& filter) const override; - //! count results for a specific filter, using only the index, ignore filter function - size_t findAllResultCount(const Filter& filter) const; + virtual data::AddressType getAddressType(const Filter& filter = Filter::LAST_TRANSACTION) const override; virtual std::shared_ptr getTransactionForId(uint64_t transactionId) const override; - virtual std::shared_ptr findByMessageId( - memory::ConstBlockPtr messageId, + virtual data::compact::ConstConfirmedTxPtr getConfirmedTxForId(uint64_t transactionId) const override; + //! \param filter use to speed up search if infos exist to narrow down search transactions range + virtual ConstTransactionEntryPtr findByLedgerAnchor( + const data::LedgerAnchor& ledgerAnchor, const Filter& filter = Filter::ALL_TRANSACTIONS ) const override; virtual AbstractProvider* getProvider() const override; @@ -142,15 +173,23 @@ namespace gradido { inline void setListeningCommunityServer(std::shared_ptr client); inline std::shared_ptr getListeningCommunityServer() const; - inline uint32_t getOrAddIndexForPublicKey(memory::ConstBlockPtr publicKey) const { - return mPublicKeysIndex->getOrAddIndexForString(publicKey->copyAsString()); + inline virtual const IDictionary& getPublicKeyDictionary() const override { return mPublicKeysIndex; } + inline virtual uint32_t getOrAddPublicKey(const data::PublicKey& publicKey) override { + return mPublicKeysIndex.getOrAddIndexForData(publicKey); } + + inline uint32_t getOrAddIndexForPublicKey(const data::PublicKey& publicKey) const { + return mPublicKeysIndex.getOrAddIndexForData(publicKey); + } + inline const hiero::TopicId& getHieroTopicId() const { return mHieroTopicId; } - inline const std::string& getFolderPath() const { return mFolderPath; } inline const std::string& getAlias() const { return mAlias; } + inline const std::string& getFolderPath() const { return mFolderPath; } + inline const std::string& getCommunityId() const { return mCommunityId; } inline TaskObserver& getTaskObserver() const { return *mTaskObserver; } - inline std::shared_ptr pickHieroClient() const { return mHieroClients[std::rand() % mHieroClients.size()]; } + inline std::shared_ptr pickHieroClient() const { return mHieroClients.size() ? mHieroClients[std::rand() % mHieroClients.size()] : nullptr; } std::shared_ptr getOrderingManager() { return mOrderingManager; } + std::stop_token getStopToken() const { return mStopToken; } protected: //! if state leveldb was invalid, recover values from block cache @@ -160,15 +199,20 @@ namespace gradido { void rescanForTransactionTriggerEvents(); //! \param func if function return false, stop iteration + //! TODO: make a template function without using std::function void iterateBlocks(const SearchDirection& searchDir, std::function func) const; cache::Block& getBlock(uint32_t blockNr) const; + //! \param countToValidate lastTransaction.nr - countToValidate = minTransaction.nr, 0 for all + bool validateLastTransactions(uint64_t countToValidate); + mutable std::recursive_mutex mWorkMutex; - bool mExitCalled; + std::stop_token mStopToken; hiero::TopicId mHieroTopicId; std::string mAlias; std::string mFolderPath; + std::string mCommunityId; //! observe write to file tasks from block, mayber later more mutable std::shared_ptr mTaskObserver; @@ -178,11 +222,12 @@ namespace gradido { std::shared_ptr mHieroMessageListener; //! contain indices for every public key address, used overall for optimisation - mutable std::shared_ptr mPublicKeysIndex; + mutable PersistentDictionary mPublicKeysIndex; // level db to store state values like last transaction + // TODO: speedup with atcual struct, write out into leveldb/lmdb only on changes, maybe even buffered, think on exit management mutable cache::State mBlockchainState; - mutable cache::HieroTransactionId mMessageIdsCache; + mutable cache::LedgerAnchor mLedgerAnchorCache; cache::TransactionTriggerEvent mTransactionTriggerEventsCache; @@ -197,22 +242,42 @@ namespace gradido { }; std::shared_ptr FileBased::create( - std::string_view communityId, + std::stop_token stopToken, + const std::string& communityId, const hiero::TopicId& topicId, std::string_view alias, std::string_view folder, std::vector>&& hieroClients ) { - return std::make_shared(Private(), communityId, topicId, alias, folder, std::move(hieroClients)); + return std::make_shared(Private(), stopToken, communityId, topicId, alias, folder, std::move(hieroClients)); + } + + std::shared_ptr FileBased::createWithoutHieroTopic( + std::stop_token stopToken, + const std::string& communityId, + std::string_view alias, + std::string_view folder + ) { + return std::make_shared( + Private(), + stopToken, + communityId, + hiero::TopicId(), + alias, + folder, + std::vector>() + ); } std::shared_ptr FileBased::getptr() { + if (mStopToken.stop_requested()) return nullptr; return shared_from_this(); } std::shared_ptr FileBased::getptr() const { + if (mStopToken.stop_requested()) return nullptr; return shared_from_this(); } diff --git a/src/blockchain/FileBasedProvider.cpp b/src/blockchain/FileBasedProvider.cpp index b5f8466e..d41bdf5e 100644 --- a/src/blockchain/FileBasedProvider.cpp +++ b/src/blockchain/FileBasedProvider.cpp @@ -4,69 +4,105 @@ #include "../client/hiero/ConsensusClient.h" #include "../client/GraphQL.h" #include "../ServerGlobals.h" -#include "../task/SyncTopicOnStartup.h" +#include "../task/SyncTopic.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/data/adapter/uuid.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/data/hiero/TopicId.h" +#include "gradido_blockchain/lib/DictionaryExceptions.h" #include "loguru/loguru.hpp" +#include #include +#include +#include +#include + +using std::lock_guard; +using std::shared_ptr, std::make_shared; +using std::string; +using std::vector; namespace gradido { + using data::adapter::uuidFromString; namespace blockchain { + FileBasedProvider::FileBasedProvider() - :mGroupIndex(nullptr), mCommunityIdIndex(ServerGlobals::g_FilesPath + "/communityIdsCache"), mInitalized(false) + :mGroupIndex(nullptr), mInitalized(false) { } FileBasedProvider::~FileBasedProvider() { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); if (mGroupIndex) { delete mGroupIndex; mGroupIndex = nullptr; } } - - FileBasedProvider* FileBasedProvider::getInstance() { static FileBasedProvider one; return &one; } - std::shared_ptr FileBasedProvider::findBlockchain(std::string_view communityId) + shared_ptr FileBasedProvider::findBlockchain(uint32_t communityIdIndex) { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); if (!mInitalized) { throw ClassNotInitalizedException("please call init before", "blockchain::FileBasedProvider"); } - auto it = mBlockchainsPerGroup.find(communityId); + auto it = mBlockchainsPerGroup.find(communityIdIndex); if (it != mBlockchainsPerGroup.end()) { return it->second; } - // go manual + + + return nullptr; + } + + shared_ptr FileBasedProvider::findBlockchain(const string& communityId) + { + auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(uuidFromString(communityId.c_str())); + if (!communityIdIndex) { + LOG_F(WARNING, "no community id index for %s", communityId.c_str()); + } + else { + return findBlockchain(communityIdIndex); + } + return nullptr; + } + + shared_ptr FileBasedProvider::findBlockchain(hiero::TopicId& topicId) + { try { - const auto& groupIndexEntry = mGroupIndex->getCommunityDetails(hiero::TopicId(std::string(communityId))); - auto it = mBlockchainsPerGroup.find(groupIndexEntry.communityId); - if (it != mBlockchainsPerGroup.end()) { - return it->second; + const auto& groupIndexEntry = mGroupIndex->getCommunityDetails(topicId); + auto communityIdIndex = g_appContext->getCommunityIds().getIndexForData(uuidFromString(groupIndexEntry.communityId.c_str())); + if (!communityIdIndex) { + LOG_F(WARNING, "no community id index for %s", groupIndexEntry.communityId.c_str()); + } + else { + return findBlockchain(communityIdIndex); } } catch (GradidoBlockchainException& ex) { LOG_F(WARNING, "%s", ex.getFullString().data()); } - return nullptr; } + bool FileBasedProvider::init( - const std::string& communityConfigFile, - std::vector>&& hieroClients, + std::stop_token masterStopView, + const string& communityConfigFile, + vector>&& hieroClients, uint8_t hieroClientsPerCommunity/* = 3 */ ) { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); + mStopToken = masterStopView; mInitalized = true; bool resetAllCommunityIndices = false; mHieroClientsPerCommunity = hieroClientsPerCommunity; @@ -75,10 +111,6 @@ namespace gradido { LOG_F(ERROR, "more hiero clients per community as hiero clients"); return false; } - if (!mCommunityIdIndex.init(GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_ID_INDEX_CACHE_SIZE_MBYTE * 1024 * 1024)) { - mCommunityIdIndex.reset(); - resetAllCommunityIndices = true; - } mGroupIndex = new cache::GroupIndex(communityConfigFile); mGroupIndex->update(); auto communitiesIds = mGroupIndex->listCommunitiesIds(); @@ -88,14 +120,33 @@ namespace gradido { // exit if at least one blockchain from config couldn't be loaded // should only occure with invalid config const auto& details = mGroupIndex->getCommunityDetails(communityId); - if (!addCommunity(communityId, hiero::TopicId(details.topicId), details.alias, resetAllCommunityIndices)) { + hiero::TopicId topicId; + if (details.topicId.size()) { + topicId = details.topicId; + } + if (!addCommunity(communityId, topicId, details.alias)) { LOG_F(ERROR, "error adding community %s in folder: %s", details.alias.data(), details.folderName.data()); return false; } } - // step 2: check for new transactions in hiero network, all blockchains at the same time + // step 2: validate last transactions for (const auto& pair : mBlockchainsPerGroup) { - auto task = pair.second->initOnline(); + if (!pair.second->startValidationTransactions()) { + LOG_F( + ERROR, + "error validate last transactions from community: %s in folder: %s", + pair.second->getCommunityId().c_str(), + pair.second->getFolderPath().c_str() + ); + } + } + + // step 3: check for new transactions in hiero network, all blockchains at the same time + for (const auto& pair : mBlockchainsPerGroup) { + if (pair.second->getHieroTopicId().empty()) { + continue; + } + auto task = pair.second->getTopicSyncTask(); auto hieroClient = pair.second->pickHieroClient(); hieroClient->getTopicInfo(pair.second->getHieroTopicId(), task); task->scheduleTask(task); @@ -105,66 +156,69 @@ namespace gradido { } void FileBasedProvider::exit() { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); mInitalized = false; for (auto blockchain : mBlockchainsPerGroup) { blockchain.second->exit(); } mBlockchainsPerGroup.clear(); - mCommunityIdIndex.exit(); } int FileBasedProvider::reloadConfig() { - std::lock_guard _lock(mWorkMutex); + lock_guard _lock(mWorkMutex); if (!mInitalized) { throw ClassNotInitalizedException("please call init before", "blockchain::FileBasedProvider"); } mGroupIndex->update(); - auto communitiesIds = mGroupIndex->listCommunitiesIds(); int addedBlockchainsCount = 0; - for (auto& communityId : communitiesIds) { - const auto& details = mGroupIndex->getCommunityDetails(communityId); - auto it = mBlockchainsPerGroup.find(communityId); + mGroupIndex->iterate([&](const cache::CommunityIndexEntry& details) -> bool { + auto it = mBlockchainsPerGroup.find(details.communityIdIndex); if (it == mBlockchainsPerGroup.end()) { - if(addCommunity(communityId, hiero::TopicId(details.topicId), details.alias, false)) { + if (addCommunity(details.communityId, hiero::TopicId(details.topicId), details.alias)) { addedBlockchainsCount++; } } else { - updateListenerCommunity(communityId, details.alias, it->second); + updateListenerCommunity(details.communityIdIndex, details.alias, it->second); } - } + return true; + }); return addedBlockchainsCount; } - std::shared_ptr FileBasedProvider::addCommunity( - const std::string& communityId, + shared_ptr FileBasedProvider::addCommunity( + const string& communityId, const hiero::TopicId& topicId, - const std::string& alias, - bool resetIndices + const string& alias ) { try { - auto folder = mGroupIndex->getFolder(communityId); + auto communityIdIndex = g_appContext->getOrAddCommunityIdIndex(communityId); + auto folder = mGroupIndex->getFolder(communityIdIndex); + std::shared_ptr blockchain; + if (topicId.empty()) { + blockchain = FileBased::createWithoutHieroTopic(mStopToken, communityId, alias, folder); + } else { + // with more hiero clients as per community needed, we make sure we not take always the first mHieroClientsPerCommunity from them + vector> hieroClients = mHieroClients; // copy + if (hieroClients.size() > mHieroClientsPerCommunity) { + std::shuffle(hieroClients.begin(), hieroClients.end(), std::mt19937{ std::random_device{}() }); + hieroClients.resize(mHieroClientsPerCommunity); + } - // with more hiero clients as per community needed, we make sure we not take always the first mHieroClientsPerCommunity from them - std::vector> hieroClients = mHieroClients; // copy - if (hieroClients.size() > mHieroClientsPerCommunity) { - std::shuffle(hieroClients.begin(), hieroClients.end(), std::mt19937{ std::random_device{}() }); - hieroClients.resize(mHieroClientsPerCommunity); + // with that call community will be initialized and start listening + blockchain = FileBased::create(mStopToken, communityId, topicId, alias, folder, std::move(hieroClients)); } + g_appContext->addBlockchain(communityIdIndex, blockchain); + updateListenerCommunity(communityIdIndex, alias, blockchain); - // with that call community will be initialized and start listening - auto blockchain = FileBased::create(communityId, topicId, alias, folder, std::move(hieroClients)); - updateListenerCommunity(communityId, alias, blockchain); // need to have blockchain in map for init able to work - mBlockchainsPerGroup.insert({ communityId, blockchain }); + mBlockchainsPerGroup.insert({ communityIdIndex, blockchain }); if (!blockchain->init(false)) { LOG_F(ERROR, "error initalizing blockchain: %s", communityId.data()); - mBlockchainsPerGroup.erase(communityId); + mBlockchainsPerGroup.erase(communityIdIndex); return nullptr; } - mCommunityIdIndex.getOrAddIndexForString(communityId); return blockchain; } catch (GradidoBlockchainException& ex) { @@ -175,19 +229,19 @@ namespace gradido { return nullptr; } } - void FileBasedProvider::updateListenerCommunity(const std::string& communityId, const std::string& alias, std::shared_ptr blockchain) + void FileBasedProvider::updateListenerCommunity(uint32_t communityIdIndex, const string& alias, shared_ptr blockchain) { - const auto& communityConfig = mGroupIndex->getCommunityDetails(communityId); + const auto& communityConfig = mGroupIndex->getCommunityDetails(communityIdIndex); // for notification of community server by new transaction // deprecated, will be replaced with mqtt in future if (!communityConfig.newBlockUri.empty()) { - std::shared_ptr clientBase; - auto uri = std::string(communityConfig.newBlockUri); + shared_ptr clientBase; + auto uri = string(communityConfig.newBlockUri); if (communityConfig.blockUriType == "json") { - clientBase = std::make_shared(uri); + clientBase = make_shared(uri); } else if (communityConfig.blockUriType == "graphql") { - clientBase = std::make_shared(uri); + clientBase = make_shared(uri); } else { LOG_F(ERROR, "unknown new block uri type: %s", communityConfig.blockUriType.data()); diff --git a/src/blockchain/FileBasedProvider.h b/src/blockchain/FileBasedProvider.h index e14f386c..9ef8e82a 100644 --- a/src/blockchain/FileBasedProvider.h +++ b/src/blockchain/FileBasedProvider.h @@ -6,6 +6,10 @@ #include "FileBased.h" #include "../cache/GroupIndex.h" +#include +#include +#include + #define GRADIDO_NODE_MAGIC_NUMBER_COMMUNITY_ID_INDEX_CACHE_SIZE_MBYTE 1 namespace hiero { @@ -30,9 +34,13 @@ namespace gradido { public: static FileBasedProvider* getInstance(); - std::shared_ptr findBlockchain(std::string_view communityId); + std::shared_ptr findBlockchain(uint32_t communityIdIndex) override; + std::shared_ptr findBlockchain(const std::string& communityId) override; + + std::shared_ptr findBlockchain(hiero::TopicId& topicId); //! \return true if successfully else return false bool init( + std::stop_token masterStopView, const std::string& communityConfigFile, std::vector>&& hieroClients, uint8_t hieroClientsPerCommunity = 3 @@ -43,16 +51,13 @@ namespace gradido { //! \return count of added blockchain int reloadConfig(); - // access community id index - inline uint32_t getCommunityIdIndex(const std::string& communityId); - inline uint32_t getCommunityIdIndex(std::string_view communityId); - inline const std::string getCommunityIdString(uint32_t index); - //! list all known communities inline std::vector listCommunityIds() const; + inline const cache::GroupIndex* getGroupIndex() const { return mGroupIndex; } protected: - std::map, StringViewCompare> mBlockchainsPerGroup; + // check if necessary, or community context is enough + std::unordered_map> mBlockchainsPerGroup; std::recursive_mutex mWorkMutex; private: @@ -67,31 +72,18 @@ namespace gradido { std::shared_ptr addCommunity( const std::string& communityId, const hiero::TopicId& topicId, - const std::string& alias, - bool resetIndices + const std::string& alias ); - void updateListenerCommunity(const std::string& communityId, const std::string& alias, std::shared_ptr blockchain); + void updateListenerCommunity(uint32_t communityIdIndex, const std::string& alias, std::shared_ptr blockchain); + //! master stop source view + std::stop_token mStopToken; cache::GroupIndex* mGroupIndex; - cache::Dictionary mCommunityIdIndex; std::vector> mHieroClients; uint8_t mHieroClientsPerCommunity; bool mInitalized; }; - uint32_t FileBasedProvider::getCommunityIdIndex(const std::string& communityId) - { - return mCommunityIdIndex.getIndexForString(communityId); - } - uint32_t FileBasedProvider::getCommunityIdIndex(std::string_view communityId) - { - return getCommunityIdIndex(std::string(communityId)); - } - const std::string FileBasedProvider::getCommunityIdString(uint32_t index) - { - return mCommunityIdIndex.getStringForIndex(index); - } - std::vector FileBasedProvider::listCommunityIds() const { return mGroupIndex->listCommunitiesIds(); diff --git a/src/blockchain/NodeTransactionEntry.cpp b/src/blockchain/NodeTransactionEntry.cpp index 137f70a2..49861e5c 100644 --- a/src/blockchain/NodeTransactionEntry.cpp +++ b/src/blockchain/NodeTransactionEntry.cpp @@ -1,20 +1,28 @@ #include "NodeTransactionEntry.h" - #include "FileBased.h" +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain_core/data/wire/confirmed_transaction.h" +#include "gradido_blockchain_core/data/wire/transaction_body.h" +#include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" + namespace gradido { + using data::adapter::toPublicKey; + using data::compact::ConfirmedGradidoTx; + namespace blockchain { NodeTransactionEntry::NodeTransactionEntry( gradido::data::ConstConfirmedTransactionPtr transaction, std::shared_ptr blockchain, int32_t fileCursor /*= -10*/ - ) : TransactionEntry(transaction), mFileCursor(fileCursor) + ) : TransactionEntry(transaction, blockchain->getCommunityIdIndex()), mFileCursor(fileCursor) { auto involvedPublicKeys = transaction->getInvolvedAddresses(); mPublicKeyIndices.reserve(involvedPublicKeys.size()); for (auto& publicKey : involvedPublicKeys) { - mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(publicKey)); + mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(toPublicKey(publicKey))); } } @@ -24,11 +32,12 @@ namespace gradido { date::month month, date::year year, gradido::data::TransactionType transactionType, - const std::string& coinGroupId, + std::optional coinCommunityIdIndex, const uint32_t* addressIndices, uint8_t addressIndiceCount, + uint32_t blockchainCommunityIdIndex, int32_t fileCursor /*= -10*/ - ) : TransactionEntry(transactionNr, month, year, transactionType, coinGroupId), mFileCursor(fileCursor) + ) : TransactionEntry(transactionNr, month, year, transactionType, coinCommunityIdIndex, blockchainCommunityIdIndex), mFileCursor(fileCursor) { mPublicKeyIndices.reserve(addressIndiceCount); for (int i = 0; i < addressIndiceCount; i++) { @@ -40,14 +49,38 @@ namespace gradido { memory::ConstBlockPtr serializedTransaction, std::shared_ptr blockchain, int32_t fileCursor/* = -10 */ - ) : TransactionEntry(serializedTransaction), mFileCursor(fileCursor) + ) : TransactionEntry(serializedTransaction, blockchain->getCommunityIdIndex()), mFileCursor(fileCursor) { auto involvedPublicKeys = getConfirmedTransaction()->getInvolvedAddresses(); mPublicKeyIndices.reserve(involvedPublicKeys.size()); for (auto& publicKey : involvedPublicKeys) { - mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(publicKey)); + mPublicKeyIndices.push_back(blockchain->getOrAddIndexForPublicKey(toPublicKey(publicKey))); } } + NodeTransactionEntry::NodeTransactionEntry( + gradido::data::ConstConfirmedTransactionPtr transaction, + memory::ConstBlockPtr serializedTransaction, + std::shared_ptr blockchain, + int32_t fileCursor/* = -10 */ + ) : TransactionEntry(serializedTransaction, transaction, blockchain->getCommunityIdIndex()), mFileCursor(fileCursor) + { + + } + + ConfirmedGradidoTx NodeTransactionEntry::convertToCompactConfirmedTx() const + { + uint8_t buffer[1024]; + grd_memory alloc; + grd_memory_init_arena_static(&alloc, buffer, 1024); + grdw_confirmed_transaction tx{}; + getConfirmedTransaction()->toGrdw(&alloc, &tx, mBlockchainCommunityIdIndex); + auto confirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, mBlockchainCommunityIdIndex, *g_appContext); + alloc.last_index = 0; + grdw_transaction_body txBody{}; + getTransactionBody()->toGrdw(&alloc, &txBody); + confirmedTx.fillFromGrdwTransactionBody(&txBody, *g_appContext); + return confirmedTx; + } } } diff --git a/src/blockchain/NodeTransactionEntry.h b/src/blockchain/NodeTransactionEntry.h index e5445fa7..cec4ff1e 100644 --- a/src/blockchain/NodeTransactionEntry.h +++ b/src/blockchain/NodeTransactionEntry.h @@ -2,10 +2,12 @@ #define __GRADIDO_NODE_BLOCKCHAIN_NODE_TRANSACTION_ENTRY_H #include "gradido_blockchain/blockchain/TransactionEntry.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" #include namespace gradido { + namespace blockchain { class FileBased; @@ -36,14 +38,22 @@ namespace gradido { int32_t fileCursor = -10 ); + NodeTransactionEntry( + gradido::data::ConstConfirmedTransactionPtr transaction, + memory::ConstBlockPtr serializedTransaction, + std::shared_ptr blockchain, + int32_t fileCursor = -10 + ); + //! \brief init entry object from details e.g. by loading from file NodeTransactionEntry( uint64_t transactionNr, date::month month, date::year year, gradido::data::TransactionType transactionType, - const std::string& coinGroupId, + std::optional coinCommunityIdIndex, const uint32_t* addressIndices, uint8_t addressIndiceCount, + uint32_t blockchainCommunityIdIndex, int32_t fileCursor = -10 ); @@ -54,6 +64,8 @@ namespace gradido { inline const std::vector& getAddressIndices() const { std::scoped_lock lock(mFastMutex); return mPublicKeyIndices; } inline bool isAddressIndexInvolved(uint32_t addressIndex) const; + data::compact::ConfirmedGradidoTx convertToCompactConfirmedTx() const; + protected: int32_t mFileCursor; std::vector mPublicKeyIndices; diff --git a/src/cache/Block.cpp b/src/cache/Block.cpp index 21a58902..430b2f8e 100644 --- a/src/cache/Block.cpp +++ b/src/cache/Block.cpp @@ -8,27 +8,45 @@ #include "../controller/TaskObserver.h" #include "../model/files/Block.h" #include "../model/files/FileExceptions.h" +#include "../task/RebuildBlockIndexTask.h" #include "../SingletonManager/CacheManager.h" +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain_core/data/wire/confirmed_transaction.h" +#include "gradido_blockchain_core/data/wire/transaction_body.h" #include "gradido_blockchain/Application.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/serialization/toJsonString.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "loguru/loguru.hpp" #include +#include +#include #include +using gradido::AppContext; using namespace gradido::blockchain; +using gradido::data::compact::ConfirmedGradidoTx, gradido::data::TransactionType; +using gradido::data::PublicKey; using namespace gradido::interaction; +using std::shared_ptr, std::make_shared, std::lock_guard; +using task::RebuildBlockIndexTask; + namespace cache { Block::Block(uint32_t blockNr, std::shared_ptr blockchain) : mBlockNr(blockNr), mSerializedTransactions(ServerGlobals::g_CacheTimeout), - mBlockIndex(std::make_shared(blockchain->getFolderPath(), blockNr)), + mConfirmedTxByNr(ServerGlobals::g_CacheTimeout), + mBlockIndex(std::make_shared(blockchain->getFolderPath(), blockNr, blockchain->getCommunityIdIndex())), mBlockFile(std::make_shared(blockchain->getFolderPath(), blockNr)), mBlockchain(blockchain), mExitCalled(false) @@ -48,45 +66,46 @@ namespace cache { mSerializedTransactions.clear(); } - bool Block::init() + bool Block::init(std::stop_token stop /* = std::stop_token() */) { - std::lock_guard lock(mFastMutex); - if (!mBlockIndex->loadFromFile()) { + lock_guard lock(mFastMutex); + // todo: add data for address index in file, until then rebuild block index on each program start + // if (!mBlockIndex->loadFromFile(publicKeyDictionary)) + { // check if Block exist - if (mBlockFile->getCurrentFileSize()) { - Profiler timeUsed; - auto rebuildBlockIndexTask = mBlockFile->rebuildBlockIndex(mBlockchain); - if (!rebuildBlockIndexTask) { - throw GradidoNullPointerException("missing rebuild block index task", "RebuildBlockIndexTask", __FUNCTION__); - } + if (mBlockFile->getCurrentFileSize()) + { + MonotonicTimer timeUsed; + mBlockIndex->reset(); + auto rebuildBlockIndexTask = make_shared( + mBlockIndex, + mBlockchain->getCommunityIdIndex() + ); rebuildBlockIndexTask->scheduleTask(rebuildBlockIndexTask); + mBlockFile->readBuffered(rebuildBlockIndexTask->getAlloc(), rebuildBlockIndexTask.get(), stop); + int sumWaited = 0; - while (!rebuildBlockIndexTask->isPendingQueueEmpty() && sumWaited < 1000) { + while (!rebuildBlockIndexTask->isTaskFinished() && sumWaited < GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); sumWaited += 100; } - if (!rebuildBlockIndexTask->isPendingQueueEmpty()) { - LOG_F(FATAL, "rebuildBlockIndex Task isn't finished after waiting a whole second"); + if (!rebuildBlockIndexTask->isTaskFinished()) { + LOG_F(FATAL, "rebuildBlockIndex Task isn't finished after waiting a whole minute"); Application::terminate(); } - auto transactionEntries = rebuildBlockIndexTask->getTransactionEntries(); - mBlockIndex->reset(); - uint64_t prevId = 0; - std::for_each(transactionEntries.begin(), transactionEntries.end(), - [&](const std::shared_ptr& transactionEntry) { - if (transactionEntry->getTransactionNr() <= prevId) { - throw BlockchainOrderException("transaction nrs aren't in ascending order"); - } - prevId = transactionEntry->getTransactionNr(); - mBlockIndex->addIndicesForTransaction(transactionEntry); - } - ); - LOG_F(INFO, "time for rebuilding block index for block %s: %s", mBlockFile->getBlockPath().data(), timeUsed.string().data()); + LOG_F(INFO, "rebuilding block index for block %s in %s", mBlockFile->getBlockPath().data(), timeUsed.string().data()); + mBlockIndex->writeIntoFile(); } else { return false; } } + /*else { + // hot fix: init address index + // if block index was loaded from file, we load all transactions which change something in address index + // TODO: persistent storage for address index + } + */ return true; } @@ -101,32 +120,67 @@ namespace cache { } //bool Block::pushTransaction(const std::string& serializedTransaction, uint64_t transactionNr) - bool Block::pushTransaction(std::shared_ptr transaction) + bool Block::pushTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary, + AppContext& appContext + ) { - std::lock_guard lock(mFastMutex); + lock_guard lock(mFastMutex); if (mExitCalled) return false; if (!mTransactionWriteTask) { // std::shared_ptr blockFile, std::shared_ptr blockIndex mTransactionWriteTask = std::make_shared(mBlockFile, mBlockIndex); } - mTransactionWriteTask->addSerializedTransaction(transaction); + mTransactionWriteTask->addSerializedTransaction(transaction, publicKeyDictionary); mSerializedTransactions.add(transaction->getTransactionNr(), transaction); + addCompactTransaction(transaction, appContext); return true; } - void Block::addTransaction(memory::ConstBlockPtr serializedTransaction, int32_t fileCursor) const + void Block::addTransaction( + memory::ConstBlockPtr serializedTransaction, + int32_t fileCursor, + AppContext& appContext + ) const { - auto transactionEntry = std::make_shared(serializedTransaction, mBlockchain, fileCursor); + auto transactionEntry = make_shared(serializedTransaction, mBlockchain, fileCursor); if (mExitCalled) return; mSerializedTransactions.add(transactionEntry->getTransactionNr(), transactionEntry); + addCompactTransaction(transactionEntry, appContext); + // mBlockIndex->updateAddressIndex(transactionEntry, publicKeyDictionary); } - std::shared_ptr Block::getTransaction(uint64_t transactionNr) const + std::shared_ptr Block::addCompactTransaction(shared_ptr transactionEntry, AppContext& appContext) const + { + // create compact version + try { + uint8_t buffer[1024]; + grd_memory alloc; + grd_memory_init_arena_static(&alloc, buffer, 1024); + grdw_confirmed_transaction tx{}; + auto communityIdIndex = mBlockchain->getCommunityIdIndex(); + transactionEntry->getConfirmedTransaction()->toGrdw(&alloc, &tx, communityIdIndex); + auto confirmedTxPtr = make_shared(ConfirmedGradidoTx::fromGrdw(&tx, communityIdIndex, appContext)); + alloc.last_index = 0; + grdw_transaction_body txBody{}; + transactionEntry->getTransactionBody()->toGrdw(&alloc, &txBody); + confirmedTxPtr->fillFromGrdwTransactionBody(&txBody, appContext); + mConfirmedTxByNr.add(confirmedTxPtr->txNr, confirmedTxPtr); + return confirmedTxPtr; + } + catch (GradidoBlockchainException& ex) { + LOG_F(WARNING, "%s on create compact", ex.getFullString().c_str()); + } + return nullptr; + } + + shared_ptr Block::getTransaction(uint64_t transactionNr, AppContext& appContext) const { assert(transactionNr); - std::lock_guard lock(mFastMutex); + lock_guard lock(mFastMutex); auto transactionEntry = mSerializedTransactions.get(transactionNr); if (!transactionEntry) { @@ -150,7 +204,7 @@ namespace cache { int32_t fileCursor = 0; if (!mBlockIndex->getFileCursorForTransactionNr(transactionNr, fileCursor)) { LOG_F(INFO, "writeTransactionTaskExist: %d, writeTransactionTaskIsObserved: %d", - (int)writeTransactionTaskExist, + (int)writeTransactionTaskExist, (int)writeTransactionTaskIsObserved ); throw GradidoBlockchainTransactionNotFoundException("transaction not found in cache, in write task or file").setTransactionId(transactionNr); @@ -163,7 +217,7 @@ namespace cache { } try { auto blockLine = mBlockFile->readLine(fileCursor); - addTransaction(blockLine, fileCursor); + addTransaction(blockLine, fileCursor, appContext); } catch (model::files::EndReachingException& ex) { LOG_F(ERROR, "%s", ex.getFullString().data()); @@ -175,7 +229,7 @@ namespace cache { LOG_F(ERROR, "fileCursor: %d", fileCursor); auto blockLine = mBlockFile->readLine(fileCursor); deserialize::Context deserializer(blockLine, deserialize::Type::CONFIRMED_TRANSACTION); - deserializer.run(); + deserializer.run(mBlockchain->getCommunityIdIndex()); if (deserializer.isConfirmedTransaction()) { LOG_F(ERROR, "block: %s", serialization::toJsonString(*deserializer.getConfirmedTransaction(), true).data()); } @@ -193,6 +247,28 @@ namespace cache { return transactionEntry.value(); } + std::shared_ptr Block::getCompactTransaction( + uint64_t transactionNr, + gradido::AppContext& appContext + ) const + { + auto confirmedTx = mConfirmedTxByNr.get(transactionNr); + if (!confirmedTx) { + // check write cache, else try to read from storage + // return always a valid ptr or throw exception + auto transactionEntry = getTransaction(transactionNr, appContext); + + // we use two different access expire caches, after this call succeed it is sure, that the transaction is in Serialized Transactions, + // but it can be still missing in mConfirmedTxByNr + confirmedTx = mConfirmedTxByNr.get(transactionNr); + if (!confirmedTx) { + return addCompactTransaction(transactionEntry, appContext); + } + } + // should only don't work, if getTransaction failed, but this will throw an exception anyway + return confirmedTx.value(); + } + bool Block::hasSpaceLeft() { return mBlockFile->getCurrentFileSize() + 32 < GRADIDO_NODE_CACHE_BLOCK_MAX_FILE_SIZE_BYTE; } @@ -208,7 +284,7 @@ namespace cache { if (mTransactionWriteTask) { Timepoint now = std::chrono::system_clock::now(); - if (now - mTransactionWriteTask->getCreationDate() > ServerGlobals::g_WriteToDiskTimeout) + if (now - mTransactionWriteTask->getCreationDate() > ServerGlobals::g_WriteToDiskTimeout) { auto copyTask = mTransactionWriteTask; mBlockchain->getTaskObserver().addBlockWriteTask(copyTask); diff --git a/src/cache/Block.h b/src/cache/Block.h index 5668b758..3d1a18d9 100644 --- a/src/cache/Block.h +++ b/src/cache/Block.h @@ -8,9 +8,12 @@ #include "../task/WriteTransactionsToBlockTask.h" #include "gradido_blockchain/lib/AccessExpireCache.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" #include "gradido_blockchain/blockchain/TransactionEntry.h" #include +#include +#include namespace model { namespace files { @@ -19,14 +22,24 @@ namespace model { } namespace gradido { + class AppContext; namespace blockchain { class FileBased; class NodeTransactionEntry; } + namespace data::compact { + struct ConfirmedGradidoTx; + } +} + +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr; } // TODO: move into config #define GRADIDO_NODE_CACHE_BLOCK_MAX_FILE_SIZE_BYTE 128 * 1024 * 1024 +#define GRADIDO_NODE_CACHE_BLOCK_MAX_WAIT_TIME_FOR_BLOCK_INDEX_REBUILD_MILLISECONDS 1000 * 60 namespace cache { class Group; @@ -44,14 +57,26 @@ namespace cache { ~Block(); //! \return false if block not exist - bool init(); + bool init(std::stop_token stop = std::stop_token()); void exit(); //! \brief put new transaction to cache and file system - bool pushTransaction(std::shared_ptr transaction); + bool pushTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary, + gradido::AppContext& appContext + ); //! \brief load transaction from cache or file system - std::shared_ptr getTransaction(uint64_t transactionNr) const; + std::shared_ptr getTransaction( + uint64_t transactionNr, + gradido::AppContext& appContext + ) const; + + std::shared_ptr getCompactTransaction( + uint64_t transactionNr, + gradido::AppContext& appContext + ) const; inline BlockIndex& getBlockIndex() { return *mBlockIndex; } inline const BlockIndex& getBlockIndex() const { return *mBlockIndex; } @@ -65,12 +90,18 @@ namespace cache { protected: //! \brief add transaction from Block File, called by Block File, adding to cache and index //! not locking mutex! - void addTransaction(memory::ConstBlockPtr serializedTransaction, int32_t fileCursor) const; + void addTransaction( + memory::ConstBlockPtr serializedTransaction, + int32_t fileCursor, + gradido::AppContext& appContext + ) const; + std::shared_ptr addCompactTransaction(std::shared_ptr transactionEntry, gradido::AppContext& appContext) const; mutable std::mutex mFastMutex; uint32_t mBlockNr; mutable AccessExpireCache> mSerializedTransactions; + mutable AccessExpireCache> mConfirmedTxByNr; std::shared_ptr mBlockIndex; std::shared_ptr mBlockFile; diff --git a/src/cache/BlockIndex.cpp b/src/cache/BlockIndex.cpp index e03fdb5c..5d762d8c 100644 --- a/src/cache/BlockIndex.cpp +++ b/src/cache/BlockIndex.cpp @@ -4,23 +4,29 @@ #include "../task/HddWriteBufferTask.h" #include "../ServerGlobals.h" #include "../blockchain/FileBasedProvider.h" +#include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/blockchain/RangeUtils.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/data/TransactionType.h" #include "gradido_blockchain/serialization/toJson.h" #include "loguru/loguru.hpp" -using namespace gradido::blockchain; -using namespace gradido::data; using namespace rapidjson; +using gradido::blockchain::AbstractProvider; +using gradido::blockchain::Filter, gradido::blockchain::SearchDirection; +using gradido::blockchain::TransactionsIndexRoaringBitmaps; +using gradido::data::compact::ConfirmedGradidoTx; +using gradido::data::PublicKey; +using gradido::data::TransactionType; namespace cache { - - BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr) - : mFolderPath(groupFolderPath), mBlockNr(blockNr), mMaxTransactionNr(0), mMinTransactionNr(0), - mDirty(false) + BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) + : TransactionsIndexRoaringBitmaps(blockchainCommunityIdIndex), mFolderPath(groupFolderPath), mBlockNr(blockNr), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex), mDirty(false) { - } BlockIndex::~BlockIndex() @@ -28,10 +34,9 @@ namespace cache { exit(); } - bool BlockIndex::init() + bool BlockIndex::init(const IDictionary& publicKeysDictionary) { - std::lock_guard _lock(mRecursiveMutex); - if (loadFromFile()) { + if (loadFromFile(publicKeysDictionary)) { return true; } return false; @@ -42,215 +47,47 @@ namespace cache { std::lock_guard _lock(mRecursiveMutex); // Todo: store at runtime like Dictionary writeIntoFile(); - clearIndexEntries(); + reset(); mTransactionNrsFileCursors.clear(); - mMaxTransactionNr = 0; - mMinTransactionNr = 0; } void BlockIndex::reset() { std::lock_guard _lock(mRecursiveMutex); - clearIndexEntries(); + TransactionsIndexRoaringBitmaps::reset(); mTransactionNrsFileCursors.clear(); - model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr); - LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().data()); + model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); + // only needed if public key dictionary is again persistend + // LOG_F(WARNING, "BlockIndex: %s was corrupted and must be rebuild", blockIndexFile.getFileName().c_str()); blockIndexFile.reset(); - mMaxTransactionNr = 0; - mMinTransactionNr = 0; } - bool BlockIndex::loadFromFile() + bool BlockIndex::loadFromFile(const IDictionary& publicKeysDictionary) { + return false; + /* + * // need to be rewritten std::lock_guard _lock(mRecursiveMutex); assert(!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size()); - model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr); + model::files::BlockIndex blockIndexFile(mFolderPath, mBlockNr, mBlockchainCommunityIdIndex); return blockIndexFile.readFromFile(this); - } - - std::unique_ptr BlockIndex::serialize() - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size() && !mMaxTransactionNr && !mMinTransactionNr) { - // we haven't anything to save - return nullptr; - } - - assert(mYearMonthAddressIndexEntries.size() && mTransactionNrsFileCursors.size()); - auto blockIndexFile = std::make_unique(mFolderPath, mBlockNr); - - std::vector publicKeyIndicesTemp; - publicKeyIndicesTemp.reserve(10); - for (auto itYear = mYearMonthAddressIndexEntries.begin(); itYear != mYearMonthAddressIndexEntries.end(); itYear++) - { - blockIndexFile->addYearBlock(itYear->first); - for (auto itMonth = itYear->second.begin(); itMonth != itYear->second.end(); itMonth++) - { - blockIndexFile->addMonthBlock(itMonth->first); - for (const auto& blockIndexEntry : itMonth->second) { - auto fileCursorIt = mTransactionNrsFileCursors.find(blockIndexEntry.transactionNr); - if (fileCursorIt == mTransactionNrsFileCursors.end()) { - throw GradidoNodeInvalidDataException("missing file cursor for transaction"); - } - publicKeyIndicesTemp.clear(); - for (auto i = 0; i < blockIndexEntry.addressIndiceCount; i++) { - publicKeyIndicesTemp.push_back(blockIndexEntry.addressIndices[i]); - } - blockIndexFile->addDataBlock( - blockIndexEntry.transactionNr, - fileCursorIt->second, - blockIndexEntry.transactionType, - blockIndexEntry.coinCommunityIdIndex, - publicKeyIndicesTemp - ); - } - } - } - // finally write down to file - return std::move(blockIndexFile); - } - - Value BlockIndex::serializeToJson(Document::AllocatorType& alloc) const - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size() && !mTransactionNrsFileCursors.size() && !mMaxTransactionNr && !mMinTransactionNr) { - // we haven't anything to show - return Value(kNullType); - } - Value rootJson(kObjectType); - for (auto itYear = mYearMonthAddressIndexEntries.begin(); itYear != mYearMonthAddressIndexEntries.end(); itYear++) - { - Value yearEntry(kObjectType); - for (auto itMonth = itYear->second.begin(); itMonth != itYear->second.end(); itMonth++) - { - Value monthEntry(kArrayType); - for (const auto& blockIndexEntry : itMonth->second) - { - Value entry(kObjectType); - entry.AddMember("transactionNr", blockIndexEntry.transactionNr, alloc); - entry.AddMember("transactionType", serialization::toJson(blockIndexEntry.transactionType, alloc), alloc); - if (blockIndexEntry.coinCommunityIdIndex) { - entry.AddMember("coinCommunityIdIndex", blockIndexEntry.coinCommunityIdIndex, alloc); - } - if (blockIndexEntry.addressIndiceCount) { - Value addressIndices(kArrayType); - for (int i = 0; i < blockIndexEntry.addressIndiceCount; i++) { - addressIndices.PushBack(blockIndexEntry.addressIndices[i], alloc); - } - entry.AddMember("addressIndices", addressIndices, alloc); - } - auto fileCursorIt = mTransactionNrsFileCursors.find(blockIndexEntry.transactionNr); - if (fileCursorIt != mTransactionNrsFileCursors.end()) { - entry.AddMember("fileCursor", fileCursorIt->second, alloc); - } - monthEntry.PushBack(entry, alloc); - } - yearEntry.AddMember(serialization::toJson(itMonth->first, alloc), monthEntry, alloc); - } - rootJson.AddMember(serialization::toJson(itYear->first, alloc), yearEntry, alloc); - } - return rootJson; + */ } bool BlockIndex::writeIntoFile() { - auto blockIndexFile = serialize(); - if (blockIndexFile) { - blockIndexFile->writeToFile(); - return true; - } return false; } - bool BlockIndex::addIndicesForTransaction( - gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, - date::year year, - date::month month, - uint64_t transactionNr, - int32_t fileCursor, - const uint32_t* addressIndices, - uint16_t addressIndiceCount - ) + + bool BlockIndex::addIndicesForTransaction(const ConfirmedGradidoTx& compactTx, const IDictionary& publicKeyDict) { std::lock_guard _lock(mRecursiveMutex); - mDirty = true; - if (transactionNr > mMaxTransactionNr) { - mMaxTransactionNr = transactionNr; - } - if (!mMinTransactionNr || transactionNr < mMinTransactionNr) { - mMinTransactionNr = transactionNr; - } - - // year - auto yearIt = mYearMonthAddressIndexEntries.find(year); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - auto result = mYearMonthAddressIndexEntries.insert({ year, {} }); - yearIt = result.first; - } - // month - auto monthIt = yearIt->second.find(month); - if (monthIt == yearIt->second.end()) { - auto result = yearIt->second.insert({ month, std::list() }); - monthIt = result.first; - } - BlockIndexEntry entry; - entry.transactionNr = transactionNr; - entry.coinCommunityIdIndex = coinCommunityIdIndex; - entry.transactionType = transactionType; - entry.addressIndiceCount = addressIndiceCount; - if (addressIndiceCount > 256) { - throw GradidoNodeInvalidDataException("addressIndiceCount is bigger than 256, that cannot be"); - } - entry.addressIndices = new uint32_t[addressIndiceCount]; - memcpy(entry.addressIndices, addressIndices, sizeof(uint32_t) * addressIndiceCount); - - if (monthIt->second.size() && monthIt->second.back().transactionNr >= entry.transactionNr) { - throw BlockIndexException("try to add new transaction to block index with same or lesser transaction nr!"); - } - monthIt->second.push_back(entry); - - addFileCursorForTransaction(transactionNr, fileCursor); + TransactionsIndexRoaringBitmaps::addTransactionIndices(compactTx, publicKeyDict); return true; } - bool BlockIndex::addIndicesForTransaction(std::shared_ptr transactionEntry) - { - auto fileCursor = transactionEntry->getFileCursor(); - auto transactionNr = transactionEntry->getTransactionNr(); - - if (transactionNr > mMaxTransactionNr) { - mMaxTransactionNr = transactionNr; - } - if (!mMinTransactionNr || transactionNr < mMinTransactionNr) { - mMinTransactionNr = transactionNr; - } - - // transaction nr - file cursor map - if (fileCursor >= 0) { - addFileCursorForTransaction(transactionNr, fileCursor); - } - auto fbp = gradido::blockchain::FileBasedProvider::getInstance(); - - uint32_t coinCommunityIndex = 0; - if (!transactionEntry->getCoinCommunityId().empty()) { - coinCommunityIndex = fbp->getCommunityIdIndex(transactionEntry->getCoinCommunityId()); - } - auto& publicKeyIndices = transactionEntry->getAddressIndices(); - return addIndicesForTransaction( - transactionEntry->getTransactionType(), - coinCommunityIndex, - transactionEntry->getYear(), - transactionEntry->getMonth(), - transactionNr, - transactionEntry->getFileCursor(), - publicKeyIndices.data(), - static_cast(publicKeyIndices.size()) - ); - - } - bool BlockIndex::addFileCursorForTransaction(uint64_t transactionNr, int32_t fileCursor) { if (fileCursor < 0) return false; @@ -285,134 +122,6 @@ namespace cache { return false; } - std::vector BlockIndex::findTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const - { - std::lock_guard _lock(mRecursiveMutex); - // prefilter - if ((filter.minTransactionNr && filter.minTransactionNr > mMaxTransactionNr) || - (filter.maxTransactionNr && filter.maxTransactionNr < mMinTransactionNr)) { - return {}; - } - - uint32_t publicKeyIndex = 0; - if (filter.involvedPublicKey && !filter.involvedPublicKey->isEmpty()) { - auto involvedPublicKeyCopy = filter.involvedPublicKey->copyAsString(); - if (publicKeysDictionary.hasString(involvedPublicKeyCopy)) { - publicKeyIndex = publicKeysDictionary.getIndexForString(involvedPublicKeyCopy); - } else { - // if public key not exist, no transaction can match - return {}; - } - } - - std::vector result; - if (filter.pagination.size) { - result.reserve(filter.pagination.size); - } - - auto interval = filteredTimepointInterval(filter); - int paginationCursor = 0; - iterateRangeInOrder(interval.begin(), interval.end(), filter.searchDirection, - [&](const date::year_month& timepoint) -> bool - { - // if for a year/month combination no entries exist, return true, so continue the loop - auto yearIt = mYearMonthAddressIndexEntries.find(timepoint.year()); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - return true; - } - auto monthIt = yearIt->second.find(timepoint.month()); - if (monthIt == yearIt->second.end()) { - return true; - } - iterateRangeInOrder( - monthIt->second.begin(), - monthIt->second.end(), - filter.searchDirection, - [&filter, publicKeyIndex, &result, &paginationCursor](const BlockIndexEntry& entry) - { - auto filterResult = entry.isMatchingFilter(filter, publicKeyIndex); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - if (paginationCursor >= filter.pagination.skipEntriesCount()) { - result.push_back(entry.transactionNr); - } - paginationCursor++; - } - if (!filter.pagination.hasCapacityLeft(result.size()) || (filterResult & FilterResult::STOP) == FilterResult::STOP) { - return false; - } - return true; - } - ); - return true; - } - ); - return result; - } - - size_t BlockIndex::countTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const - { - std::lock_guard _lock(mRecursiveMutex); - // prefilter, early exit - if ((filter.minTransactionNr && filter.minTransactionNr > mMaxTransactionNr) || - (filter.maxTransactionNr && filter.maxTransactionNr < mMinTransactionNr)) { - return 0; - } - - uint32_t publicKeyIndex = 0; - if (filter.involvedPublicKey && !filter.involvedPublicKey->isEmpty()) { - auto involvedPublicKeyCopy = filter.involvedPublicKey->copyAsString(); - if (publicKeysDictionary.hasString(involvedPublicKeyCopy)) { - publicKeyIndex = publicKeysDictionary.getIndexForString(involvedPublicKeyCopy); - } else { - // if public key not exist, no transaction can match - return 0; - } - } - - auto interval = filteredTimepointInterval(filter); - size_t result = 0; - iterateRangeInOrder(interval.begin(), interval.end(), filter.searchDirection, - [&](const date::year_month& intervalIt) -> bool - { - auto yearIt = mYearMonthAddressIndexEntries.find(intervalIt.year()); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - return true; - } - auto monthIt = yearIt->second.find(intervalIt.month()); - if (monthIt == yearIt->second.end()) { - return true; - } - // iterate over entries in the month - iterateRangeInOrder(monthIt->second.begin(), monthIt->second.end(), SearchDirection::ASC, - [&](const BlockIndexEntry& entry) -> bool - { - auto filterResult = entry.isMatchingFilter(filter, publicKeyIndex); - if ((filterResult & FilterResult::USE) == FilterResult::USE) { - ++result; - } - return true; // keep going - } - ); - return true; - } - ); - return result; - } - - std::pair BlockIndex::findTransactionsForMonthYear(date::year year, date::month month) const - { - std::lock_guard _lock(mRecursiveMutex); - auto yearIt = mYearMonthAddressIndexEntries.find(year); - if (yearIt == mYearMonthAddressIndexEntries.end()) { - return { 0, 0 }; - } - auto monthIt = yearIt->second.find(month); - if (monthIt == yearIt->second.end()) { - return { 0, 0 }; - } - return { monthIt->second.front().transactionNr, monthIt->second.back().transactionNr }; - } - bool BlockIndex::getFileCursorForTransactionNr(uint64_t transactionNr, int32_t& fileCursor) const { std::lock_guard _lock(mRecursiveMutex); @@ -450,80 +159,5 @@ namespace cache { return false; } - date::year_month BlockIndex::getOldestYearMonth() const - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size()) { - return { date::year(0), date::month(0) }; - } - auto firstEntry = mYearMonthAddressIndexEntries.begin(); - assert(firstEntry->second.size()); - return { firstEntry->first, firstEntry->second.begin()->first }; - } - date::year_month BlockIndex::getNewestYearMonth() const - { - std::lock_guard _lock(mRecursiveMutex); - if (!mYearMonthAddressIndexEntries.size()) { - return { date::year(0), date::month(0) }; - } - auto lastEntry = std::prev(mYearMonthAddressIndexEntries.end()); - assert(lastEntry->second.size()); - auto lastMonthEntry = std::prev(lastEntry->second.end()); - return { lastEntry->first, lastMonthEntry->first }; - } - - void BlockIndex::clearIndexEntries() - { - std::lock_guard _lock(mRecursiveMutex); - for (auto& yearBlock : mYearMonthAddressIndexEntries) { - for (auto& monthBlock : yearBlock.second) { - for (auto& blockIndexEntry : monthBlock.second) { - delete blockIndexEntry.addressIndices; - } - } - } - mYearMonthAddressIndexEntries.clear(); - } - - FilterResult BlockIndex::BlockIndexEntry::isMatchingFilter(const gradido::blockchain::Filter& filter, const uint32_t publicKeyIndex) const - { - if (filter.transactionType != TransactionType::NONE - && filter.transactionType != transactionType) { - return FilterResult::DISMISS; - } - uint32_t coinCommunityKeyIndex = 0; - if (!filter.coinCommunityId.empty()) { - coinCommunityKeyIndex = FileBasedProvider::getInstance()->getCommunityIdIndex(filter.coinCommunityId); - } - if (coinCommunityKeyIndex && coinCommunityKeyIndex != coinCommunityIdIndex) { - return FilterResult::DISMISS; - } - if (filter.minTransactionNr && filter.minTransactionNr > transactionNr) { - return FilterResult::DISMISS; - } - if (filter.maxTransactionNr && filter.maxTransactionNr < transactionNr) { - return FilterResult::DISMISS; - } - /*uint32_t publicKeyIndex = 0; - if (filter.involvedPublicKey && !filter.involvedPublicKey->isEmpty()) { - auto involvedPublicKeyCopy = filter.involvedPublicKey->copyAsString(); - if (publicKeysDictionary.hasString(involvedPublicKeyCopy)) { - publicKeyIndex = publicKeysDictionary.getIndexForString(involvedPublicKeyCopy); - } - }*/ - if (publicKeyIndex) { - bool found = false; - for (int iPublicKeyIndices = 0; iPublicKeyIndices < addressIndiceCount; iPublicKeyIndices++) { - if (publicKeyIndex == addressIndices[iPublicKeyIndices]) { - found = true; - break; - } - } - if (!found) { - return FilterResult::DISMISS; - } - } - return FilterResult::USE; - } - + } \ No newline at end of file diff --git a/src/cache/BlockIndex.h b/src/cache/BlockIndex.h index e6ad1c55..e26b2bf9 100644 --- a/src/cache/BlockIndex.h +++ b/src/cache/BlockIndex.h @@ -1,18 +1,31 @@ #ifndef __GRADIDO_NODE_CONTROLLER_BLOCK_INDEX_H #define __GRADIDO_NODE_CONTROLLER_BLOCK_INDEX_H +#include "gradido_blockchain/blockchain/CompactFilter.h" #include "gradido_blockchain/blockchain/Filter.h" +#include "gradido_blockchain/blockchain/TransactionsIndexRoaringBitmaps.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" #include "../blockchain/NodeTransactionEntry.h" #include "../model/files/BlockIndex.h" #include "../task/CPUTask.h" -#include "Dictionary.h" - #include "rapidjson/document.h" #include #include +#include + +namespace gradido { + namespace data::compact { + class ConfirmedGradidoTx; + } + namespace blockchain { + class AbstractProvider; + class CompactFilter; + } +} namespace cache { @@ -27,40 +40,28 @@ namespace cache { TODO: Auto-Recover if missing, and maybe check with saved block on startup */ - class BlockIndex : public model::files::IBlockIndexReceiver + class BlockIndex : public gradido::blockchain::TransactionsIndexRoaringBitmaps { - //friend model::files::BlockIndex; + // friend model::files::BlockIndex; public: - BlockIndex(std::string_view groupFolderPath, uint32_t blockNr); + BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex); ~BlockIndex(); - bool init(); + bool init(const IDictionary& publicKeysDictionary); void exit(); void reset(); //! \brief loading block index from file (or at least try to load) - bool loadFromFile(); + bool loadFromFile(const IDictionary& publicKeysDictionary); //! \brief write block index into files std::unique_ptr serialize(); - rapidjson::Value serializeToJson(rapidjson::Document::AllocatorType& alloc) const; + //! \brief //! \return true if there was something to write into file, after writing it to file bool writeIntoFile(); - bool addIndicesForTransaction(std::shared_ptr transactionEntry); - - //! implement from model::files::IBlockIndexReceiver, called by loading block index from file - bool addIndicesForTransaction( - gradido::data::TransactionType transactionType, - uint32_t coinCommunityIdIndex, - date::year year, - date::month month, - uint64_t transactionNr, - int32_t fileCursor, - const uint32_t* addressIndices, - uint16_t addressIndiceCount - ); + bool addIndicesForTransaction(const gradido::data::compact::ConfirmedGradidoTx& compactTx, const IDictionary& publicKeyDict); //! \brief add transactionNr - fileCursor pair to map if not already exist //! \return false if transactionNr exist, else return true @@ -68,57 +69,63 @@ namespace cache { //! \brief search transaction nrs for search criteria in filter, ignore filter function //! \return transaction nrs - std::vector findTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const; - //! count all, ignore pagination - size_t countTransactions(const gradido::blockchain::Filter& filter, const Dictionary& publicKeysDictionary) const; + inline std::vector findTransactions(const gradido::blockchain::CompactFilter& filter) const; - //! \brief find transaction nrs from specific month and year - //! \return {0, 0} if nothing found - std::pair findTransactionsForMonthYear(date::year year, date::month month) const; + //! count all, ignore pagination + inline size_t countTransactions(const gradido::blockchain::CompactFilter& filter) const; //! \param fileCursor reference to be filled with fileCursor //! \return true if transaction nr was found and fileCursor was set, else return false bool getFileCursorForTransactionNr(uint64_t transactionNr, int32_t& fileCursor) const; inline bool hasTransactionNr(uint64_t transactionNr) const; - inline uint64_t getMaxTransactionNr() const { std::lock_guard _lock(mRecursiveMutex); return mMaxTransactionNr; } - inline uint64_t getMinTransactionNr() const { std::lock_guard _lock(mRecursiveMutex); return mMinTransactionNr; } + inline uint64_t getMaxTransactionNr() const; + inline uint64_t getMinTransactionNr() const; inline uint64_t getTransactionsCount() const; - date::year_month getOldestYearMonth() const; - date::year_month getNewestYearMonth() const; - inline TimepointInterval filteredTimepointInterval(const gradido::blockchain::Filter& filter) const; + inline date::year_month getOldestYearMonth() const; + inline date::year_month getNewestYearMonth() const; + inline TimepointInterval filteredTimepointInterval(const gradido::blockchain::CompactFilter& filter) const; + inline void lock() const { mRecursiveMutex.lock(); } + inline void unlock() const { mRecursiveMutex.unlock(); } protected: - void clearIndexEntries(); - //! \brief called from model::files::BlockIndex while reading file std::string mFolderPath; - uint32_t mBlockNr; - uint64_t mMaxTransactionNr; - uint64_t mMinTransactionNr; - + uint32_t mBlockNr; + uint32_t mBlockchainCommunityIdIndex; + std::map mTransactionNrsFileCursors; typedef std::pair TransactionNrsFileCursorsPair; - struct BlockIndexEntry - { - uint64_t transactionNr; - uint32_t* addressIndices; - uint32_t coinCommunityIdIndex; - gradido::data::TransactionType transactionType; - uint8_t addressIndiceCount; - gradido::blockchain::FilterResult isMatchingFilter(const gradido::blockchain::Filter& filter, const uint32_t publicKeyIndex) const; - }; - - std::map>> mYearMonthAddressIndexEntries; - mutable std::recursive_mutex mRecursiveMutex; bool mDirty; + }; + std::vector BlockIndex::findTransactions(const gradido::blockchain::CompactFilter& filter) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::findTransactions(filter); + } + + size_t BlockIndex::countTransactions(const gradido::blockchain::CompactFilter& filter) const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::countTransactions(filter); + } + + size_t BlockIndex::getTransactionsCount() const + { + std::lock_guard _lock(mRecursiveMutex); + if (!mMaxTransactionNr && !mMinTransactionNr) { + return 0; + } + return mMaxTransactionNr - mMinTransactionNr + 1; + } + bool BlockIndex::hasTransactionNr(uint64_t transactionNr) const { std::lock_guard _lock(mRecursiveMutex); @@ -126,25 +133,15 @@ namespace cache { && transactionNr <= mMaxTransactionNr; } - uint64_t BlockIndex::getTransactionsCount() const + uint64_t BlockIndex::getMaxTransactionNr() const { - std::lock_guard _lock(mRecursiveMutex); - if (!mMaxTransactionNr && !mMinTransactionNr) return 0; - return mMaxTransactionNr - mMinTransactionNr + 1; + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::getMaxTransactionNr(); } - - TimepointInterval BlockIndex::filteredTimepointInterval(const gradido::blockchain::Filter& filter) const - { - TimepointInterval interval(getOldestYearMonth(), getNewestYearMonth()); - if (!filter.timepointInterval.isEmpty()) { - if (interval.getStartDate() < filter.timepointInterval.getStartDate()) { - interval.setStartDate(filter.timepointInterval.getStartDate()); - } - if (interval.getEndDate() > filter.timepointInterval.getEndDate()) { - interval.setEndDate(std::max(interval.getStartDate(), filter.timepointInterval.getEndDate())); - } - } - return interval; + uint64_t BlockIndex::getMinTransactionNr() const + { + std::lock_guard _lock(mRecursiveMutex); + return gradido::blockchain::TransactionsIndexRoaringBitmaps::getMinTransactionNr(); } } diff --git a/src/cache/DefaultStateKeys.h b/src/cache/DefaultStateKeys.h index fa136b3f..703d68bd 100644 --- a/src/cache/DefaultStateKeys.h +++ b/src/cache/DefaultStateKeys.h @@ -1,8 +1,10 @@ #ifndef __GRADIDO_NODE_CACHE_DEFAULT_STATE_KEYS_H #define __GRADIDO_NODE_CACHE_DEFAULT_STATE_KEYS_H +#include "gradido_blockchain/types.h" + namespace cache { - enum class DefaultStateKeys { + enum class DefaultStateKeys: uint8_t { LAST_ADDRESS_INDEX, LAST_BLOCK_NR, LAST_TRANSACTION_ID, diff --git a/src/cache/Dictionary.cpp b/src/cache/Dictionary.cpp deleted file mode 100644 index ece73dc4..00000000 --- a/src/cache/Dictionary.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "Dictionary.h" -#include "Exceptions.h" -#include "../ServerGlobals.h" -#include "../SystemExceptions.h" -#include "../model/files/FileExceptions.h" - -#include "loguru/loguru.hpp" - -namespace cache { - - Dictionary::Dictionary(std::string_view fileName) - : mLastIndex(0), - mDictionaryFile(fileName) - { - } - - Dictionary::~Dictionary() - { - } - - bool Dictionary::init(size_t cacheInBytes) - { - std::lock_guard _lock(mFastMutex); - if (!mDictionaryFile.init(cacheInBytes)) { - return false; - } - mDictionaryFile.iterate([&](leveldb::Slice key, leveldb::Slice value) -> void { - char* end = nullptr; - mValueKeyReverseLookup.insert({ SignatureOctet((const uint8_t*)value.data(), value.size()), strtoul(key.data(), &end, 10) }); - }); - mLastIndex = mValueKeyReverseLookup.size(); - return true; - } - - void Dictionary::exit() - { - mDictionaryFile.exit(); - } - - void Dictionary::reset() - { - mDictionaryFile.reset(); - mLastIndex = 0; - } - - bool Dictionary::addStringIndex(const std::string& string, uint32_t index) - { - if (hasStringIndexPair(string, index)) { - return false; - } - std::lock_guard _lock(mFastMutex); - if (mLastIndex + 1 != index) { - throw DictionaryInvalidNewKeyException( - "addValueKeyPair called with invalid new key", - mDictionaryFile.getFolderName().data(), - index, - mLastIndex - ); - } - mLastIndex = index; - mDictionaryFile.setKeyValue(std::to_string(index).c_str(), string); - mValueKeyReverseLookup.insert({ string, index }); - - return true; - } - - uint32_t Dictionary::getIndexForString(const std::string& string) const - { - auto result = findIndexForString(string); - if (!result) { - auto hex = DataTypeConverter::binToHex(string); - throw DictionaryNotFoundException("string not found in dictionary", mDictionaryFile.getFolderName().data(), hex.data()); - } - return result; - } - - std::string Dictionary::getStringForIndex(uint32_t index) const - { - std::string value; - if(!mDictionaryFile.getValueForKey(std::to_string(index).c_str(), &value)) { - throw DictionaryNotFoundException( - "index not found in dictionary", - mDictionaryFile.getFolderName().data(), - std::to_string(index).c_str() - ); - } - return value; - } - bool Dictionary::hasIndex(uint32_t index) const - { - std::string value; - return mDictionaryFile.getValueForKey(std::to_string(index).c_str(), &value); - } - - uint32_t Dictionary::getOrAddIndexForString(const std::string& string) - { - auto index = findIndexForString(string); - if (!index) { - std::lock_guard _lock(mFastMutex); - index = ++mLastIndex; - mDictionaryFile.setKeyValue(std::to_string(index).c_str(), string); - mValueKeyReverseLookup.insert({ string, index }); - } - return index; - } - - bool Dictionary::hasStringIndexPair(const std::string& string, uint32_t index) const - { - std::lock_guard _lock(mFastMutex); - auto itRange = mValueKeyReverseLookup.equal_range(string); - if (itRange.first == mValueKeyReverseLookup.end() && itRange.second == mValueKeyReverseLookup.end()) { - return false; - } - for (auto& it = itRange.first; it != itRange.second; ++it) { - return it->second == index; - } - return false; - } - - uint32_t Dictionary::findIndexForString(const std::string& string) const - { - if (string.empty()) { - throw GradidoNodeInvalidDataException("string is empty"); - } - std::lock_guard _lock(mFastMutex); - auto itRange = mValueKeyReverseLookup.equal_range(string); - for (auto& it = itRange.first; it != itRange.second; ++it) { - std::string key = std::to_string(it->second); - std::string value; - if (mDictionaryFile.getValueForKey(key.c_str(), &value)) { - if (string == value) { - return it->second; - } - } - } - return 0; - } -} diff --git a/src/cache/Dictionary.h b/src/cache/Dictionary.h deleted file mode 100644 index 3ca3c3c0..00000000 --- a/src/cache/Dictionary.h +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef __GRADIDO_NODE_CACHE_DICTIONARY_H -#define __GRADIDO_NODE_CACHE_DICTIONARY_H - -#include "../model/files/LevelDBWrapper.h" -#include "gradido_blockchain/crypto/SignatureOctet.h" - -#include - -namespace gradido { - namespace blockchain { - class FileBased; - } -} - -namespace cache { - - - /*! - * @author Dario Rekowski - * @date 2020-02-06 - * - * @brief Create Indices for strings, for more efficient working with strings in memory - * So basically every one use only the uint32_t index value for comparisation and stuff and only if the real value is needed, - * they will ask the Dictionary for the real value - * - */ - class Dictionary - { - public: - Dictionary(std::string_view fileName); - virtual ~Dictionary(); - - //! \param cacheInBytes level db cache in bytes, 0 for no cache - //! \return true if address indices could be loaded - //! \return false if address index file was corrupted - bool init(size_t cacheInBytes); - void exit(); - //! delete dictionary file instance and also file on disk - void reset(); - - //! \brief Get index from cache or if not in cache, loading file, maybe I/O read. - //! \param address public key as binary string - //! \return Index or throw exception if index not exist - virtual uint32_t getIndexForString(const std::string& string) const; - virtual inline bool hasString(const std::string& string) const { - return findIndexForString(string) != 0; - } - - virtual std::string getStringForIndex(uint32_t index) const; - virtual bool hasIndex(uint32_t index) const; - - //! \brief Get or add index if not exist in cache or file, maybe I/O read. - //! \param address User public key as binary string - //! \param lastIndex Last knowing index for group. - //! \return Index for address. - virtual uint32_t getOrAddIndexForString(const std::string& string); - - //! \brief Add index - //! \param address User public key as binary string - //! \param index Index for address - //! \return False if index address already exist, else true. - virtual bool addStringIndex(const std::string& string, uint32_t index); - - inline uint32_t getLastIndex() { std::lock_guard _lock(mFastMutex); return mLastIndex; } - - protected: - bool hasStringIndexPair(const std::string& string, uint32_t index) const; - uint32_t findIndexForString(const std::string& string) const; - - mutable std::mutex mFastMutex; - uint32_t mLastIndex; - // key is index, value is string - mutable model::files::LevelDBWrapper mDictionaryFile; - std::unordered_map mValueKeyReverseLookup; - }; -} - -#endif //__GRADIDO_NODE_CACHE_DICTIONARY_H diff --git a/src/cache/GroupIndex.cpp b/src/cache/GroupIndex.cpp index ec6f7786..1f2278e9 100644 --- a/src/cache/GroupIndex.cpp +++ b/src/cache/GroupIndex.cpp @@ -2,15 +2,26 @@ #include "../controller/ControllerExceptions.h" #include "../ServerGlobals.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/lib/RapidjsonHelper.h" #include "gradido_blockchain/data/hiero/TopicId.h" #include "loguru/loguru.hpp" #include +#include +#include +#include +#include + +using gradido::g_appContext; +using std::function; +using std::string; +using std::shared_lock, std::unique_lock; +using std::vector; namespace cache { - GroupIndex::GroupIndex(const std::string& jsonConfigFileName) + GroupIndex::GroupIndex(const string& jsonConfigFileName) : mConfig(jsonConfigFileName) { @@ -28,7 +39,7 @@ namespace cache { size_t GroupIndex::update() { - std::scoped_lock _lock(mWorkMutex); + unique_lock _lock(mWorkMutex); clear(); try { @@ -41,10 +52,15 @@ namespace cache { rapidjson_helper::checkMember(communityEntry, "alias", rapidjson_helper::MemberType::STRING); rapidjson_helper::checkMember(communityEntry, "communityId", rapidjson_helper::MemberType::STRING); rapidjson_helper::checkMember(communityEntry, "folder", rapidjson_helper::MemberType::STRING); - rapidjson_helper::checkMember(communityEntry, "hieroTopicId", rapidjson_helper::MemberType::STRING); + // rapidjson_helper::checkMember(communityEntry, "hieroTopicId", rapidjson_helper::MemberType::STRING); entry.alias = communityEntry["alias"].GetString(); entry.communityId = communityEntry["communityId"].GetString(); - entry.topicId = communityEntry["hieroTopicId"].GetString(); + entry.communityIdIndex = g_appContext->getOrAddCommunityIdIndex(entry.communityId); + if (communityEntry.HasMember("hieroTopicId") && communityEntry["hieroTopicId"].IsString()) { + entry.topicId = communityEntry["hieroTopicId"].GetString(); + } else { + LOG_F(WARNING, "community entry %s doesn't have hieroTopicId, this community won't be listened for new transactions", entry.communityId.c_str()); + } entry.folderName = communityEntry["folder"].GetString(); if (communityEntry.HasMember("newBlockUri")) { entry.newBlockUri = communityEntry["newBlockUri"].GetString(); @@ -52,7 +68,7 @@ namespace cache { if (communityEntry.HasMember("blockUriType")) { entry.blockUriType = communityEntry["blockUriType"].GetString(); } - mCommunities.insert({ entry.communityId, entry }); + mCommunities.insert({ entry.communityIdIndex, entry }); } } else { @@ -67,13 +83,12 @@ namespace cache { return mCommunities.size(); } - std::string GroupIndex::getFolder(const std::string& communityId) + string GroupIndex::getFolder(uint32_t communityIdIndex) const { - std::scoped_lock _lock(mWorkMutex); - - auto it = mCommunities.find(communityId); + shared_lock _lock(mWorkMutex); + auto it = mCommunities.find(communityIdIndex); if(it != mCommunities.end()) { - std::string folder = ServerGlobals::g_FilesPath + '/'; + string folder = ServerGlobals::g_FilesPath + '/'; folder += it->second.folderName; if(!std::filesystem::exists(folder)) { std::filesystem::create_directories(folder); @@ -82,19 +97,21 @@ namespace cache { } return ""; } - const CommunityIndexEntry& GroupIndex::getCommunityDetails(const std::string& communityId) const - { - std::scoped_lock _lock(mWorkMutex); - auto it = mCommunities.find(communityId); - if (it != mCommunities.end()) { - return it->second; + const CommunityIndexEntry& GroupIndex::getCommunityDetails(const string& communityId) const + { + shared_lock _lock(mWorkMutex); + for (auto& it : mCommunities) { + if (it.second.communityId == communityId) { + return it.second; + } } throw controller::GroupNotFoundException("couldn't found config details for community", communityId); } + const CommunityIndexEntry& GroupIndex::getCommunityDetails(const hiero::TopicId& topicId) const { - std::scoped_lock _lock(mWorkMutex); + shared_lock _lock(mWorkMutex); for (auto& it : mCommunities) { if (topicId == hiero::TopicId(it.second.topicId)) { return it.second; @@ -102,18 +119,41 @@ namespace cache { } throw controller::GroupNotFoundException("couldn't found config details for community by topic id", topicId.toString()); } - bool GroupIndex::isCommunityInConfig(const std::string& communityId) const + + const CommunityIndexEntry& GroupIndex::getCommunityDetails(uint32_t communityIdIndex) const + { + shared_lock _lock(mWorkMutex); + auto it = mCommunities.find(communityIdIndex); + if (it != mCommunities.end()) { + return it->second; + } + throw controller::GroupNotFoundException("couldn't found config details for community", communityIdIndex); + } + + bool GroupIndex::isCommunityInConfig(uint32_t communityIdIndex) const + { + shared_lock _lock(mWorkMutex); + return mCommunities.find(communityIdIndex) != mCommunities.end(); + } + + vector GroupIndex::listCommunitiesIds() const { - std::scoped_lock _lock(mWorkMutex); - return mCommunities.find(communityId) != mCommunities.end(); + shared_lock _lock(mWorkMutex); + vector result; + result.reserve(mCommunities.size()); + for (auto it = mCommunities.begin(); it != mCommunities.end(); it++) { + result.emplace_back(it->second.communityId); + } + return result; } - std::vector GroupIndex::listCommunitiesIds() + vector GroupIndex::listCommunitiesIdIndices() const { - std::scoped_lock _lock(mWorkMutex); - std::vector result; + shared_lock _lock(mWorkMutex); + vector result; + result.reserve(mCommunities.size()); for (auto it = mCommunities.begin(); it != mCommunities.end(); it++) { - result.push_back(it->first); + result.emplace_back(it->second.communityIdIndex); } return result; } diff --git a/src/cache/GroupIndex.h b/src/cache/GroupIndex.h index cc624584..3d0577c4 100644 --- a/src/cache/GroupIndex.h +++ b/src/cache/GroupIndex.h @@ -4,8 +4,10 @@ #include "gradido_blockchain/lib/DRHash.h" #include "../model/files/JsonFile.h" +#include #include #include +#include #include namespace hiero { @@ -22,6 +24,7 @@ namespace cache { std::string folderName; std::string newBlockUri; std::string blockUriType; + uint32_t communityIdIndex; DHASH makeHash() { return DRMakeStringHash(alias.data(), alias.size()); @@ -30,12 +33,12 @@ namespace cache { /*! * @author Dario Rekowski - * + * * @date 2020-02-06 - * + * * @brief storing group folder name in memory in hash list for fast access * - * protected by FastMutex + * protected by FastMutex * * TODO: adding function to adding group folder pair and save changed group.index file */ @@ -55,21 +58,33 @@ namespace cache { //! \brief get full folder path for group public key //! if the hash from groupPublicKey exist multiple time, getting folder from mConfig, else from mHashList (faster) //! \return complete path to group folder or empty path if not group not found - std::string getFolder(const std::string& communityId); + std::string getFolder(uint32_t communityIdIndex) const; //! throw GroupNotFoundException Exception of community don't exist in config const CommunityIndexEntry& getCommunityDetails(const std::string& communityId) const; const CommunityIndexEntry& getCommunityDetails(const hiero::TopicId& topicId) const; - bool isCommunityInConfig(const std::string& communityId) const; + const CommunityIndexEntry& getCommunityDetails(uint32_t communityIdIndex) const; + bool isCommunityInConfig(uint32_t communityIdIndex) const; + + // callback for each community, stop if return false + template + void iterate(callbackFunc callback) const { + std::shared_lock _lock(mWorkMutex); + for (const auto& it : mCommunities) { + auto result = callback(it.second); + if (!result) break; + } + } //! \brief collect all group aliases from unordered map (not the fastest operation from unordered map) //! \return vector with group aliases registered to the node server - std::vector listCommunitiesIds(); + std::vector listCommunitiesIds() const; + std::vector listCommunitiesIdIndices() const; protected: - mutable std::mutex mWorkMutex; + mutable std::shared_mutex mWorkMutex; model::files::JsonFile mConfig; - std::unordered_map mCommunities; + std::unordered_map mCommunities; //! \brief clear hash list and doublet's vector void clear(); diff --git a/src/cache/HieroTransactionId.cpp b/src/cache/HieroTransactionId.cpp deleted file mode 100644 index 2952c5b6..00000000 --- a/src/cache/HieroTransactionId.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include "HieroTransactionId.h" -#include "../ServerGlobals.h" -#include "../SystemExceptions.h" -#include "../lib/LevelDBExceptions.h" -#include "gradido_blockchain/interaction/deserialize/Context.h" - -#include "loguru/loguru.hpp" - -namespace cache { - HieroTransactionId::HieroTransactionId(std::string_view folder) - : mInitalized(false), - mLevelDBFile(folder), - mHieroTransactionIdTransactionNrs(ServerGlobals::g_CacheTimeout) - { - - } - HieroTransactionId::~HieroTransactionId() - { - - } - - // try to open db - bool HieroTransactionId::init(size_t cacheInBytes) - { - if (mInitalized) { - throw ClassAlreadyInitalizedException("init was already called", "cache::MessageId"); - } - if (!mLevelDBFile.init(cacheInBytes)) { - return false; - } - mInitalized = true; - return true; - } - - void HieroTransactionId::exit() - { - if (!mInitalized) { - LOG_F(WARNING, "init wasn't called, cache::MessageId aren't stored in leveldb file, or exit was called more than once"); - } - mInitalized = false; - mLevelDBFile.exit(); - } - - //! remove state level db folder, clear maps - void HieroTransactionId::reset() - { - mLevelDBFile.reset(); - mHieroTransactionIdTransactionNrs.clear(); - } - - void HieroTransactionId::add(memory::ConstBlockPtr transactionId, uint64_t transactionNr) - { - auto hieroTransactionId = fromProtobuf(transactionId); - if (mHieroTransactionIdTransactionNrs.get(hieroTransactionId).has_value()) { - throw GradidoAlreadyExist("cache::HieroTransactionId already has key!"); - } - mHieroTransactionIdTransactionNrs.add(hieroTransactionId, transactionNr); - if (mInitalized) { - mLevelDBFile.setKeyValue(transactionId->convertToBase64().data(), std::to_string(transactionNr).data()); - } - } - - bool HieroTransactionId::has(memory::ConstBlockPtr transactionId) - { - auto hieroTransactionId = fromProtobuf(transactionId); - if (mHieroTransactionIdTransactionNrs.get(hieroTransactionId).has_value()) { - return true; - } - return readFromLevelDb(transactionId, hieroTransactionId) != 0; - } - - uint64_t HieroTransactionId::getTransactionNrForHieroTransactionId(memory::ConstBlockPtr transactionId) - { - auto hieroTransactionId = fromProtobuf(transactionId); - auto result = mHieroTransactionIdTransactionNrs.get(hieroTransactionId); - if (result.has_value()) { - return result.value(); - } - return readFromLevelDb(transactionId, hieroTransactionId); - } - - uint64_t HieroTransactionId::readFromLevelDb(memory::ConstBlockPtr transactionId, hiero::TransactionId transactionIdObj) - { - if (mInitalized) { - std::string value; - if (mLevelDBFile.getValueForKey(transactionId->convertToBase64().data(), &value)) { - auto transactionNr = std::stoull(value); - mHieroTransactionIdTransactionNrs.add(transactionIdObj, transactionNr); - return transactionNr; - } - } - return 0; - } - - hiero::TransactionId HieroTransactionId::fromProtobuf(memory::ConstBlockPtr transactionId) const - { - gradido::interaction::deserialize::Context deserializer(transactionId, gradido::interaction::deserialize::Type::HIERO_TRANSACTION_ID); - deserializer.run(); - if (!deserializer.isHieroTransactionId()) { - throw GradidoNodeInvalidDataException("[cache::HieroTransactionId::fromProtobuf] transactionId cannot be deserialized as hiero::TransactionId"); - } - return deserializer.getHieroTransactionId(); - } - -} \ No newline at end of file diff --git a/src/cache/LedgerAnchor.cpp b/src/cache/LedgerAnchor.cpp new file mode 100644 index 00000000..080a7d60 --- /dev/null +++ b/src/cache/LedgerAnchor.cpp @@ -0,0 +1,119 @@ +#include "LedgerAnchor.h" +#include "../ServerGlobals.h" +#include "../SystemExceptions.h" +#include "../lib/LevelDBExceptions.h" +#include "gradido_blockchain/data/LedgerAnchor.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/interaction/serialize/Context.h" +#include "gradido_blockchain/memory/Block.h" + +#include "loguru/loguru.hpp" +using memory::Block; +using std::shared_ptr, std::make_shared; + +namespace cache { + LedgerAnchor::LedgerAnchor(std::string_view folder) + : mInitalized(false), + mLevelDBFile(folder), + mLedgerAnchorTransactionNrs(ServerGlobals::g_CacheTimeout) + { + + } + LedgerAnchor::~LedgerAnchor() + { + + } + + // try to open db + bool LedgerAnchor::init(size_t cacheInBytes) + { + if (mInitalized) { + throw ClassAlreadyInitalizedException("init was already called", "cache::MessageId"); + } + if (!mLevelDBFile.init(cacheInBytes)) { + return false; + } + mInitalized = true; + return true; + } + + void LedgerAnchor::exit() + { + if (!mInitalized) { + LOG_F(WARNING, "init wasn't called, cache::MessageId aren't stored in leveldb file, or exit was called more than once"); + } + mInitalized = false; + mLevelDBFile.exit(); + } + + //! remove state level db folder, clear maps + void LedgerAnchor::reset() + { + mLevelDBFile.reset(); + mLedgerAnchorTransactionNrs.clear(); + } + + void LedgerAnchor::add(const gradido::data::LedgerAnchor& transactionId, uint64_t transactionNr) + { + auto ledgerAnchorSerialized = toProtobuf(transactionId); + + if (mLedgerAnchorTransactionNrs.get(ledgerAnchorSerialized).has_value()) { + throw GradidoAlreadyExist("cache::LedgerAnchor already has key!"); + } + mLedgerAnchorTransactionNrs.add(ledgerAnchorSerialized, transactionNr); + if (mInitalized) { + mLevelDBFile.setKeyValue(ledgerAnchorSerialized, std::to_string(transactionNr).data()); + } + } + + bool LedgerAnchor::has(const gradido::data::LedgerAnchor& transactionId) + { + auto ledgerAnchorSerialized = toProtobuf(transactionId); + if (mLedgerAnchorTransactionNrs.get(ledgerAnchorSerialized).has_value()) { + return true; + } + return readFromLevelDb(ledgerAnchorSerialized) != 0; + } + + uint64_t LedgerAnchor::getTransactionNrForLedgerAnchor(const gradido::data::LedgerAnchor& transactionId) + { + auto ledgerAnchorSerialized = toProtobuf(transactionId); + auto result = mLedgerAnchorTransactionNrs.get(ledgerAnchorSerialized); + if (result.has_value()) { + return result.value(); + } + return readFromLevelDb(ledgerAnchorSerialized); + } + + uint64_t LedgerAnchor::readFromLevelDb(const std::string& ledgerAnchorSerialized) + { + if (mInitalized) { + auto result = mLevelDBFile.getValueForKey(ledgerAnchorSerialized); + if (result.has_value()) + { + auto transactionNr = std::stoull(result.value()); + mLedgerAnchorTransactionNrs.add(ledgerAnchorSerialized, transactionNr); + return transactionNr; + } + } + return 0; + } + + gradido::data::LedgerAnchor LedgerAnchor::fromProtobuf(const std::string transactionIdString) const + { + auto binTransactionId = make_shared(transactionIdString); + gradido::interaction::deserialize::Context deserializer(binTransactionId, gradido::interaction::deserialize::Type::LEDGER_ANCHOR); + deserializer.run(); + if (!deserializer.isLedgerAnchor()) { + throw GradidoNodeInvalidDataException("[cache::LedgerAnchor::fromProtobuf] transactionId cannot be deserialized as gradido::data::LedgerAnchor"); + } + return deserializer.getLedgerAnchor(); + } + + std::string LedgerAnchor::toProtobuf(const gradido::data::LedgerAnchor& transactionId) const + { + gradido::interaction::serialize::Context serializer(transactionId); + return serializer.run()->copyAsString(); + } + +} \ No newline at end of file diff --git a/src/cache/HieroTransactionId.h b/src/cache/LedgerAnchor.h similarity index 50% rename from src/cache/HieroTransactionId.h rename to src/cache/LedgerAnchor.h index 43171795..2c62d42a 100644 --- a/src/cache/HieroTransactionId.h +++ b/src/cache/LedgerAnchor.h @@ -5,13 +5,20 @@ #include "gradido_blockchain/lib/ExpireCache.h" #include "gradido_blockchain/data/hiero/TransactionId.h" -namespace cache +namespace gradido { + namespace data { + class LedgerAnchor; + } +} + +namespace cache { - class HieroTransactionId + // TODO: optimize + class LedgerAnchor { public: - HieroTransactionId(std::string_view folder); - ~HieroTransactionId(); + LedgerAnchor(std::string_view folder); + ~LedgerAnchor(); // try to open db //! \param cacheInBytes level db cache in bytes, 0 for no cache @@ -20,21 +27,22 @@ namespace cache //! remove state level db folder, clear maps void reset(); - void add(memory::ConstBlockPtr transactionId, uint64_t transactionNr); - bool has(memory::ConstBlockPtr transactionId); + void add(const gradido::data::LedgerAnchor& transactionId, uint64_t transactionNr); + bool has(const gradido::data::LedgerAnchor& transactionId); //! \return 0 if not found, else transaction nr for message id - uint64_t getTransactionNrForHieroTransactionId(memory::ConstBlockPtr messageId); + uint64_t getTransactionNrForLedgerAnchor(const gradido::data::LedgerAnchor& transactionId); protected: //! read message id as key from level db and put into mMessageIdTransactionNrs if found //! \return 0 if not found or else transactionNr for message Id - uint64_t readFromLevelDb(memory::ConstBlockPtr transactionId, hiero::TransactionId transactionIdObj); - hiero::TransactionId fromProtobuf(memory::ConstBlockPtr transactionId) const; + uint64_t readFromLevelDb(const std::string& ledgerAnchorSerialized); + gradido::data::LedgerAnchor fromProtobuf(const std::string transactionIdString) const; + std::string toProtobuf(const gradido::data::LedgerAnchor& transactionId) const; bool mInitalized; model::files::LevelDBWrapper mLevelDBFile; - //! key is iota message id, value is transaction nr - ExpireCache mHieroTransactionIdTransactionNrs; + //! key is ledger anchor serialized with protopuf, value is transaction nr + ExpireCache mLedgerAnchorTransactionNrs; }; } diff --git a/src/cache/MessageId.cpp b/src/cache/MessageId.cpp deleted file mode 100644 index 55841756..00000000 --- a/src/cache/MessageId.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include "MessageId.h" -#include "../ServerGlobals.h" -#include "../SystemExceptions.h" -#include "../lib/LevelDBExceptions.h" - -#include "loguru/loguru.hpp" - -namespace cache { - MessageId::MessageId(std::string_view folder) - : mInitalized(false), - mLevelDBFile(folder), - mMessageIdTransactionNrs(ServerGlobals::g_CacheTimeout) - { - - } - MessageId::~MessageId() - { - - } - - // try to open db - bool MessageId::init(size_t cacheInBytes) - { - if (mInitalized) { - throw ClassAlreadyInitalizedException("init was already called", "cache::MessageId"); - } - if (!mLevelDBFile.init(cacheInBytes)) { - return false; - } - mInitalized = true; - return true; - } - - void MessageId::exit() - { - if (!mInitalized) { - LOG_F(WARNING, "init wasn't called, cache::MessageId aren't stored in leveldb file, or exit was called more than once"); - } - mInitalized = false; - mLevelDBFile.exit(); - } - - //! remove state level db folder, clear maps - void MessageId::reset() - { - mLevelDBFile.reset(); - mMessageIdTransactionNrs.clear(); - } - - void MessageId::add(memory::ConstBlockPtr messageId, uint64_t transactionNr) - { - iota::MessageId messageIdObj(*messageId); - if (mMessageIdTransactionNrs.get(messageIdObj).has_value()) { - throw GradidoAlreadyExist("cache::MessageId already has key!"); - } - mMessageIdTransactionNrs.add(messageIdObj, transactionNr); - if (mInitalized) { - mLevelDBFile.setKeyValue(messageId->convertToHex().data(), std::to_string(transactionNr).data()); - } - } - - bool MessageId::has(memory::ConstBlockPtr messageId) - { - iota::MessageId messageIdObj(*messageId); - if (mMessageIdTransactionNrs.get(messageIdObj).has_value()) { - return true; - } - return readFromLevelDb(messageIdObj) != 0; - } - - uint64_t MessageId::getTransactionNrForMessageId(memory::ConstBlockPtr messageId) - { - iota::MessageId messageIdObj(*messageId); - auto result = mMessageIdTransactionNrs.get(messageIdObj); - if (result.has_value()) { - return result.value(); - } - return readFromLevelDb(messageIdObj); - } - - uint64_t MessageId::readFromLevelDb(const iota::MessageId& iotaMessageIdObj) - { - if (mInitalized) { - std::string value; - if (mLevelDBFile.getValueForKey(iotaMessageIdObj.toHex().data(), &value)) { - auto transactionNr = std::stoull(value); - mMessageIdTransactionNrs.add(iotaMessageIdObj, transactionNr); - return transactionNr; - } - } - return 0; - } - -} \ No newline at end of file diff --git a/src/cache/MessageId.h b/src/cache/MessageId.h deleted file mode 100644 index a0ecf06c..00000000 --- a/src/cache/MessageId.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef __GRADIDO_NODE_CACHE_MESSAGE_ID_H -#define __GRADIDO_NODE_CACHE_MESSAGE_ID_H - -#include "../model/files/LevelDBWrapper.h" -#include "gradido_blockchain/lib/ExpireCache.h" -#include "gradido_blockchain/data/iota/MessageId.h" - -namespace cache -{ - class MessageId - { - public: - MessageId(std::string_view folder); - ~MessageId(); - - // try to open db - //! \param cacheInBytes level db cache in bytes, 0 for no cache - bool init(size_t cacheInBytes); - void exit(); - //! remove state level db folder, clear maps - void reset(); - - void add(memory::ConstBlockPtr messageId, uint64_t transactionNr); - bool has(memory::ConstBlockPtr messageId); - //! \return 0 if not found, else transaction nr for message id - uint64_t getTransactionNrForMessageId(memory::ConstBlockPtr messageId); - - protected: - //! read message id as key from level db and put into mMessageIdTransactionNrs if found - //! \return 0 if not found or else transactionNr for message Id - uint64_t readFromLevelDb(const iota::MessageId& iotaMessageIdObj); - - bool mInitalized; - model::files::LevelDBWrapper mLevelDBFile; - //! key is iota message id, value is transaction nr - ExpireCache mMessageIdTransactionNrs; - }; -} - -#endif //__GRADIDO_NODE_CACHE_MESSAGE_ID_H \ No newline at end of file diff --git a/src/cache/State.cpp b/src/cache/State.cpp index 7ad7e1c8..e537891b 100644 --- a/src/cache/State.cpp +++ b/src/cache/State.cpp @@ -3,13 +3,19 @@ #include "../lib/LevelDBExceptions.h" #include "loguru/loguru.hpp" +#include "magic_enum/magic_enum.hpp" + +#include + +using namespace magic_enum; +using std::string_view; namespace cache { State::State(std::string_view folder) : mInitalized(false), mStateFile(folder) { - + mFastAccessDefaultStates.resize(static_cast(DefaultStateKeys::MAX), 0); } State::~State() @@ -25,6 +31,17 @@ namespace cache { if (!mStateFile.init(cacheInBytes)) { return false; } + // fill fast access default states vector + mStateFile.iterate( + [&](leveldb::Slice key, leveldb::Slice value) + { + auto state = enum_cast(string_view(key.data(), key.size())); + if (state.has_value()) { + assert(state.value() < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(state.value())] = strtoll(value.data(), nullptr, 10); + } + } + ); mInitalized = true; return true; } @@ -101,9 +118,9 @@ namespace cache { LOG_F(WARNING, "init wasn't called, leveldb file couldn't be used"); return defaultValue; } - std::string tmp; - if (mStateFile.getValueForKey(key, &tmp)) { - return tmp; + auto result = mStateFile.getValueForKey(key); + if (result.has_value()) { + return result.value(); } return defaultValue; } @@ -114,9 +131,9 @@ namespace cache { LOG_F(WARNING, "init wasn't called, leveldb file couldn't be used"); return defaultValue; } - std::string tmp; - if (mStateFile.getValueForKey(key, &tmp)) { - return atoi(tmp.data()); + auto result = mStateFile.getValueForKey(key); + if (result.has_value()) { + return atoi(result.value().data()); } return defaultValue; } @@ -127,9 +144,9 @@ namespace cache { LOG_F(WARNING, "init wasn't called, leveldb file couldn't be used"); return defaultValue; } - std::string tmp; - if (mStateFile.getValueForKey(key, &tmp)) { - return strtoll(tmp.data(), nullptr, 10); + auto result = mStateFile.getValueForKey(key); + if (result.has_value()) { + return strtoll(result.value().data(), nullptr, 10); } return defaultValue; } diff --git a/src/cache/State.h b/src/cache/State.h index 95fe19e4..1533d0b9 100644 --- a/src/cache/State.h +++ b/src/cache/State.h @@ -6,10 +6,10 @@ #include "magic_enum/magic_enum.hpp" +#include #include namespace cache { - class State { public: @@ -23,7 +23,6 @@ namespace cache { //! remove state level db folder, clear maps void reset(); - inline void updateState(DefaultStateKeys key, std::string_view value); inline void updateState(DefaultStateKeys key, int32_t value); inline void updateState(DefaultStateKeys key, uint32_t value); inline void updateState(DefaultStateKeys key, int64_t value); @@ -35,7 +34,6 @@ namespace cache { void updateState(const char* key, uint64_t value); void removeState(const char* key); - inline std::string readState(DefaultStateKeys key, const std::string& defaultValue); std::string readState(const char* key, const std::string& defaultValue); inline int32_t readInt32State(DefaultStateKeys key, int32_t defaultValue); int32_t readInt32State(const char* key, int32_t defaultValue); @@ -48,46 +46,53 @@ namespace cache { protected: bool mInitalized; model::files::LevelDBWrapper mStateFile; + std::vector mFastAccessDefaultStates; }; // simple functions for inline declarations - void State::updateState(DefaultStateKeys key, std::string_view value) - { - return updateState(magic_enum::enum_name(key).data(), value); - } - void State::updateState(DefaultStateKeys key, int32_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } void State::updateState(DefaultStateKeys key, uint32_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } void State::updateState(DefaultStateKeys key, int64_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } void State::updateState(DefaultStateKeys key, uint64_t value) { + assert(key < DefaultStateKeys::MAX); + mFastAccessDefaultStates[static_cast(key)] = value; return updateState(magic_enum::enum_name(key).data(), value); } - std::string State::readState(DefaultStateKeys key, const std::string& defaultValue) - { - return readState(magic_enum::enum_name(key).data(), defaultValue); - } - int32_t State::readInt32State(DefaultStateKeys key, int32_t defaultValue) { - return readInt32State(magic_enum::enum_name(key).data(), defaultValue); + assert(key < DefaultStateKeys::MAX); + auto value = mFastAccessDefaultStates[static_cast(key)]; + if (value) return value; + else return defaultValue; + // return readInt32State(magic_enum::enum_name(key).data(), defaultValue); } int64_t State::readInt64State(DefaultStateKeys key, int64_t defaultValue) { - return readInt64State(magic_enum::enum_name(key).data(), defaultValue); + assert(key < DefaultStateKeys::MAX); + auto value = mFastAccessDefaultStates[static_cast(key)]; + if (value) return value; + else return defaultValue; + // return readInt64State(magic_enum::enum_name(key).data(), defaultValue); } } diff --git a/src/cache/TransactionHash.cpp b/src/cache/TransactionHash.cpp index c23b8caf..5acf3cc5 100644 --- a/src/cache/TransactionHash.cpp +++ b/src/cache/TransactionHash.cpp @@ -4,6 +4,8 @@ #include "../blockchain/FileBased.h" #include "../blockchain/NodeTransactionEntry.h" #include "gradido_blockchain/data/GradidoTransaction.h" +#include "gradido_blockchain/serialization/toJsonString.h" + #include "loguru/loguru.hpp" @@ -11,47 +13,69 @@ using namespace gradido; using namespace blockchain; using namespace data; +using serialization::toJsonString; + namespace cache { TransactionHash::TransactionHash(std::string_view communityId) - : mSignaturePartTransactionNrs(MAGIC_NUMBER_SIGNATURE_CACHE), mCommunityId(communityId) + : //mSignaturePartTransactionNrs(MAGIC_NUMBER_SIGNATURE_CACHE), + mCommunityId(communityId) { } TransactionHash::~TransactionHash() { - mSignaturePartTransactionNrs.clear(); + // mSignaturePartTransactionNrs.clear(); + mBodyBytesTransactionNrs.clear(); } void TransactionHash::push(const gradido::data::ConfirmedTransaction& confirmedTransaction) { - auto gradidoTransaction = confirmedTransaction.getGradidoTransaction(); - auto firstSignature = gradidoTransaction->getSignatureMap().getSignaturePairs().front().getSignature(); - auto pair = mSignaturePartTransactionNrs.get(*firstSignature); - if (pair.has_value()) { + SignatureOctet hash = deriveHash(*confirmedTransaction.getGradidoTransaction()); + // auto pair = mSignaturePartTransactionNrs.get(hash); + auto it = mBodyBytesTransactionNrs.find(confirmedTransaction.getGradidoTransaction()->getBodyBytes()); + //if (pair.has_value()) { + if (it != mBodyBytesTransactionNrs.end()) { + auto storedTransaction = FileBasedProvider::getInstance()->findBlockchain(mCommunityId)->getTransactionForId(it->second); + LOG_F(ERROR, "Hash collision detected, %s and %s have the same hash", + toJsonString(confirmedTransaction, true).c_str(), + toJsonString(*storedTransaction->getConfirmedTransaction(), true).c_str() + ); throw GradidoAlreadyExist("key already exist"); } - mSignaturePartTransactionNrs.add(*firstSignature, confirmedTransaction.getId()); + //mSignaturePartTransactionNrs.add(hash, confirmedTransaction.getId()); + mBodyBytesTransactionNrs.insert({ confirmedTransaction.getGradidoTransaction()->getBodyBytes(), confirmedTransaction.getId() }); } bool TransactionHash::has(const data::GradidoTransaction& transaction) const { // get first signature from transaction - auto firstSignature = transaction.getSignatureMap().getSignaturePairs().front().getSignature(); - auto pair = mSignaturePartTransactionNrs.get(SignatureOctet(*firstSignature)); - if (pair.has_value()) { + auto hash = deriveHash(transaction); + // auto pair = mSignaturePartTransactionNrs.get(hash); + auto it = mBodyBytesTransactionNrs.find(transaction.getBodyBytes()); + //if (pair.has_value()) + if (it != mBodyBytesTransactionNrs.end()) + { // hash collision check - auto transaction = FileBasedProvider::getInstance()->findBlockchain(mCommunityId)->getTransactionForId(pair.value()); - auto firstSignatureFromFoundedTransaction = transaction->getConfirmedTransaction()->getGradidoTransaction()->getSignatureMap().getSignaturePairs().front().getSignature(); - if (firstSignatureFromFoundedTransaction->isTheSame(firstSignature)) { + auto storedTransaction = FileBasedProvider::getInstance()->findBlockchain(mCommunityId)->getTransactionForId(it->second); + if (storedTransaction->getConfirmedTransaction()->getGradidoTransaction()->isTheSame(transaction)) { return true; - } - else { - LOG_F(INFO, "Hash collision detected, %s and %s have the same first 8 Bytes", - firstSignature->convertToHex().data(), firstSignatureFromFoundedTransaction->convertToHex().data() + } else { + LOG_F(WARNING, "Hash collision detected, %s and %s have the same hash", + toJsonString(transaction, true).c_str(), + toJsonString(*storedTransaction->getConfirmedTransaction(), true).c_str() ); } } return false; } + + SignatureOctet TransactionHash::deriveHash(const gradido::data::GradidoTransaction& transaction) const + { + if (transaction.getSignatureMap().getSignaturePairs().size()) { + return transaction.getSignatureMap().getSignaturePairs().front().getSignature()->calculateHash(); + } else { + return transaction.getBodyBytes()->calculateHash(); + } + } } \ No newline at end of file diff --git a/src/cache/TransactionHash.h b/src/cache/TransactionHash.h index 49f34004..6d43bc98 100644 --- a/src/cache/TransactionHash.h +++ b/src/cache/TransactionHash.h @@ -2,10 +2,12 @@ #define __GRADIDO_NODE_CACHE_TRANSACTION_HASH_H #include "gradido_blockchain/crypto/SignatureOctet.h" -#include "gradido_blockchain/lib/ExpireCache.h" +//#include "gradido_blockchain/lib/AccessExpireCache.h" #include "gradido_blockchain/const.h" +#include "gradido_blockchain/memory/Block.h" #include +#include namespace gradido { namespace data { @@ -31,9 +33,11 @@ namespace cache { void push(const gradido::data::ConfirmedTransaction& confirmedTransaction); bool has(const gradido::data::GradidoTransaction& gradidoTransaction) const; protected: + SignatureOctet deriveHash(const gradido::data::GradidoTransaction& transaction) const; //! key is first 8 Byte from Transaction Signature, the distribution on ed25519 signatures should be good enough even by using only the first 8 Bytes //! data are transaction nr - ExpireCache mSignaturePartTransactionNrs; + // AccessExpireCache mSignaturePartTransactionNrs; + std::unordered_map mBodyBytesTransactionNrs; std::string mCommunityId; }; } diff --git a/src/cache/TransactionTriggerEvent.cpp b/src/cache/TransactionTriggerEvent.cpp index 574486bd..606f72a6 100644 --- a/src/cache/TransactionTriggerEvent.cpp +++ b/src/cache/TransactionTriggerEvent.cpp @@ -78,10 +78,10 @@ namespace cache { LOG_F(WARNING, "couldn't find transactionTriggerEvent for removal for transaction: %lu", transactionTriggerEvent.getLinkedTransactionId()); } - std::vector> TransactionTriggerEvent::findTransactionTriggerEventsInRange(TimepointInterval range) + std::vector> TransactionTriggerEvent::findTransactionTriggerEventsInRange(Timestamp startDate, Timestamp endDate) { - auto startIt = mTransactionTriggerEvents.lower_bound(range.getStartDate()); - auto endIt = mTransactionTriggerEvents.upper_bound(range.getEndDate()); + auto startIt = mTransactionTriggerEvents.lower_bound(startDate); + auto endIt = mTransactionTriggerEvents.upper_bound(endDate); std::vector> result; result.reserve(std::distance(startIt, endIt)); for (auto& it = startIt; it != endIt; it++) { @@ -90,10 +90,10 @@ namespace cache { return result; } - std::shared_ptr TransactionTriggerEvent::findNextTransactionTriggerEventInRange(TimepointInterval range) + std::shared_ptr TransactionTriggerEvent::findNextTransactionTriggerEventInRange(Timestamp startDate, Timestamp endDate) { - auto startIt = mTransactionTriggerEvents.lower_bound(range.getStartDate()); - if (startIt != mTransactionTriggerEvents.end() && startIt->first.getAsTimepoint() <= range.getEndDate()) { + auto startIt = mTransactionTriggerEvents.lower_bound(startDate); + if (startIt != mTransactionTriggerEvents.end() && startIt->first.getAsTimepoint() <= endDate.getAsTimepoint()) { return startIt->second; } return nullptr; diff --git a/src/cache/TransactionTriggerEvent.h b/src/cache/TransactionTriggerEvent.h index 0417a314..ded37833 100644 --- a/src/cache/TransactionTriggerEvent.h +++ b/src/cache/TransactionTriggerEvent.h @@ -25,8 +25,12 @@ namespace cache { void addTransactionTriggerEvent(std::shared_ptr transactionTriggerEvent); void removeTransactionTriggerEvent(const gradido::data::TransactionTriggerEvent& transactionTriggerEvent); - std::vector> findTransactionTriggerEventsInRange(TimepointInterval range); - std::shared_ptr findNextTransactionTriggerEventInRange(TimepointInterval range); + std::vector> findTransactionTriggerEventsInRange( + gradido::data::Timestamp startDate, gradido::data::Timestamp endDate + ); + std::shared_ptr findNextTransactionTriggerEventInRange( + gradido::data::Timestamp startDate, gradido::data::Timestamp endDate + ); protected: // store all transaction trigger events, belonging to the same target date as key,value pair in level db diff --git a/src/client/hiero/ConsensusClient.h b/src/client/hiero/ConsensusClient.h index 9fdf637f..d6c4daed 100644 --- a/src/client/hiero/ConsensusClient.h +++ b/src/client/hiero/ConsensusClient.h @@ -12,7 +12,7 @@ namespace grpc { class ChannelCredentials; - class Client; + class Client; } namespace hiero { @@ -24,7 +24,7 @@ namespace client { namespace hiero { // Client for hiero/hedera Consensus Services - class ConsensusClient + class ConsensusClient { public: ~ConsensusClient(); @@ -47,7 +47,7 @@ namespace client { std::shared_ptr> responseListener ); - private: + private: ConsensusClient(std::shared_ptr channel); static std::shared_ptr getTlsChannelCredentials(memory::ConstBlockPtr certificateHash); diff --git a/src/client/hiero/MirrorClient.cpp b/src/client/hiero/MirrorClient.cpp index 980d31b7..788ad1b9 100644 --- a/src/client/hiero/MirrorClient.cpp +++ b/src/client/hiero/MirrorClient.cpp @@ -102,18 +102,26 @@ namespace client { return ::hiero::ConsensusTopicResponse(resultJson); } - void MirrorClient::subscribeTopic(std::shared_ptr responseListener) { + void MirrorClient::subscribeTopic(TopicMessageQuery* responseListener) + { if (!responseListener) { throw GradidoNullPointerException("missing response listener", "TopicMessageQuery", __FUNCTION__); } grpc::StubOptions options; grpc::TemplatedGenericStub mirrorStub(mChannel); - auto readerWriter = std::move(mirrorStub.PrepareCall( - responseListener->getClientContextPtr(), - "/com.hedera.mirror.api.proto.ConsensusService/subscribeTopic", - responseListener->getCompletionQueuePtr() - )); + auto readerWriter = mirrorStub.PrepareCall( + responseListener->getClientContextPtr(), + "/com.hedera.mirror.api.proto.ConsensusService/subscribeTopic", + responseListener->getCompletionQueuePtr() + ); + if (!readerWriter) { + throw GradidoNullPointerException( + "PrepareCall failed, couldn't subscribe to topic", + "std::unique_ptr>", + __FUNCTION__ + ); + } readerWriter->StartCall(responseListener->getCallStatusPtr()); responseListener->setResponseReader(readerWriter); } diff --git a/src/client/hiero/MirrorClient.h b/src/client/hiero/MirrorClient.h index 79656548..d0d26211 100644 --- a/src/client/hiero/MirrorClient.h +++ b/src/client/hiero/MirrorClient.h @@ -49,7 +49,7 @@ namespace client { ::hiero::ConsensusTopicResponse getTopicMessageByConsensusTimestamp(gradido::data::Timestamp confirmedAt); // gRPC API - void subscribeTopic(std::shared_ptr responseListener); + void subscribeTopic(TopicMessageQuery* responseListener); inline std::string getProtocolHost() const { return "https://" + std::string(getHost()); } inline const char* getHost() const { return MirrorNetworkEndpoints::getByEndpointName(mNetworkType.data()); } diff --git a/src/client/hiero/RequestReactor.h b/src/client/hiero/RequestReactor.h index f847ed56..4ed9c7ac 100644 --- a/src/client/hiero/RequestReactor.h +++ b/src/client/hiero/RequestReactor.h @@ -8,7 +8,7 @@ #include "../Exceptions.h" #include "MessageObserver.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include #include @@ -43,7 +43,7 @@ namespace client { ); } else { - Profiler timeUsed; + MonotonicTimer timeUsed; auto block = MemoryBlock(mBuffer); auto result = pp::message_coder::decode(block.get()->span()); if (!result.has_value()) { diff --git a/src/client/hiero/TopicMessageQuery.cpp b/src/client/hiero/TopicMessageQuery.cpp index d4f68f10..83a39d7c 100644 --- a/src/client/hiero/TopicMessageQuery.cpp +++ b/src/client/hiero/TopicMessageQuery.cpp @@ -1,9 +1,12 @@ #include "TopicMessageQuery.h" #include "const.h" #include "MemoryBlock.h" +#include "MirrorClient.h" #include "../../hiero/ConsensusTopicQuery.h" #include "../../lib/protopuf.h" +#include "../../ServerGlobals.h" +#include "gradido_blockchain/Application.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/lib/DataTypeConverter.h" @@ -13,7 +16,11 @@ #include +using std::chrono::duration_cast, std::chrono::milliseconds, std::chrono::seconds, std::chrono::system_clock; +using DataTypeConverter::timespanToString; using namespace magic_enum; +using std::make_unique; +using std::this_thread::sleep_for; namespace client { namespace hiero { @@ -24,12 +31,14 @@ namespace client { mStartQuery(startQuery), mCallStatus(CallStatus::STATUS_CREATE) { + mCompletionQueues.push_back(make_unique()); + mClientContexts.push_back(make_unique()); mThread = std::thread(&TopicMessageQuery::ThreadFunction, this); } TopicMessageQuery::~TopicMessageQuery() { - LOG_F(INFO, "TopicMessageQuery::~TopicMessageQuery"); + LOG_F(2, "TopicMessageQuery::~TopicMessageQuery"); mExitCalled = true; mThread.join(); } @@ -41,17 +50,20 @@ namespace client { int TopicMessageQuery::ThreadFunction() { + try { loguru::set_thread_name(mThreadName.data()); // copied most of the code from hiero cpp sdk from startSubscription from TopicMessageQuery.cc - ::hiero::ConsensusTopicQuery query; + // ::hiero::ConsensusTopicQuery query; // Declare needed variables. ::grpc::ByteBuffer grpcByteBuffer; ::hiero::ConsensusTopicResponse response; - std::chrono::system_clock::duration backoff = ::hiero::DEFAULT_MIN_BACKOFF; - std::chrono::system_clock::duration maxBackoff = ::hiero::DEFAULT_MAX_BACKOFF; + system_clock::duration backoff = ::hiero::DEFAULT_MIN_BACKOFF; + system_clock::duration maxBackoff = ::hiero::DEFAULT_MAX_BACKOFF; + system_clock::time_point lastTransactionTimepoint = system_clock::now(); ::grpc::Status grpcStatus; uint64_t attempt = 0ULL; bool complete = false; + auto completeReason = ConnectionClosedReason::Subscription_Ended; bool ok = false; void* tag = nullptr; @@ -60,183 +72,232 @@ namespace client { grpcByteBuffer = memoryBuffer.createGrpcBuffer(); while (!mExitCalled) { - // Process based on the completion queue status. - auto backOffFromNow = std::chrono::system_clock::now() + std::chrono::duration_cast(backoff); - auto nextStatus = mCompletionQueue.AsyncNext(&tag, &ok, backOffFromNow); - LOG_F(2, "next status: %s", enum_name(nextStatus).data()); - switch (nextStatus) + // Process based on the completion queue status. + auto backOffFromNow = system_clock::now() + duration_cast(backoff); + auto nextStatus = mCompletionQueues.back()->AsyncNext(&tag, &ok, backOffFromNow); + LOG_F(2, "next status: %s", enum_name(nextStatus).data()); + switch (nextStatus) + { + case ::grpc::CompletionQueue::TIMEOUT: + { + // Backoff if the completion queue timed out. + backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; + LOG_F(2, "new timeout: %s", timespanToString(backoff).data()); + if (system_clock::now() - lastTransactionTimepoint > ::hiero::DEFAULT_SUBSCRIPTION_INACTIVE_TIMEOUT) { + completeReason = ConnectionClosedReason::Timeout; + complete = true; + mCompletionQueues.back()->Shutdown(); + LOG_F(2, "shutdown connection after DEFAULT_SUBSCRIPTION_INACTIVE_TIMEOUT"); + } + break; + } + case ::grpc::CompletionQueue::GOT_EVENT: + { + // Decrease the backoff time. + backoff = (backoff / 2 < ::hiero::DEFAULT_MIN_BACKOFF) ? ::hiero::DEFAULT_MIN_BACKOFF : backoff / 2; + + // Process based on the call status. + switch (static_cast(*reinterpret_cast(tag))) { - case ::grpc::CompletionQueue::TIMEOUT: + case CallStatus::STATUS_CREATE: { - // Backoff if the completion queue timed out. - backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; - LOG_F(2, "new timeout: %s", DataTypeConverter::timespanToString(backoff).data()); - break; + if (ok) + { + ::grpc::WriteOptions writeOptions; + mCallStatus = CallStatus::STATUS_WRITE; + mResponseReader->WriteLast(grpcByteBuffer, writeOptions, &mCallStatus); + } + break; } - case ::grpc::CompletionQueue::GOT_EVENT: + case CallStatus::STATUS_WRITE: { - // Decrease the backoff time. - backoff = (backoff / 2 < ::hiero::DEFAULT_MIN_BACKOFF) ? ::hiero::DEFAULT_MIN_BACKOFF : backoff / 2; + if (ok) + { + mCallStatus = CallStatus::STATUS_PROCESSING; + mResponseReader->Read(&grpcByteBuffer, &mCallStatus); + } + break; + } + case CallStatus::STATUS_PROCESSING: + { + // If the response should be processed, process it. + if (ok) + { + lastTransactionTimepoint = system_clock::now(); + // Read the response. + auto block = MemoryBlock(grpcByteBuffer); + LOG_F(INFO, "response: echo \"%s\" | xxd -r -p | protoscope", block.get()->convertToHex().data()); + response = protopuf::deserialize<::hiero::ConsensusTopicResponse, ::hiero::ConsensusTopicResponseMessage>(*block.get()); - // Process based on the call status. - switch (static_cast(*reinterpret_cast(tag))) - { - case CallStatus::STATUS_CREATE: - { - if (ok) - { - ::grpc::WriteOptions writeOptions; - mCallStatus = CallStatus::STATUS_WRITE; - mResponseReader->WriteLast(grpcByteBuffer, writeOptions, &mCallStatus); - } - break; - } - case CallStatus::STATUS_WRITE: + // Adjust the query timestamp and limit, in case a retry is triggered. + auto consensusTimestamp = response.getConsensusTimestamp(); + if (!consensusTimestamp.empty()) { - if (ok) - { - mCallStatus = CallStatus::STATUS_PROCESSING; - mResponseReader->Read(&grpcByteBuffer, &mCallStatus); - } - break; + // Add one of the smallest denomination of time + mStartQuery.setConsensusStartTime({ + consensusTimestamp.getSeconds(), consensusTimestamp.getNanos() + 1 + }); } - case CallStatus::STATUS_PROCESSING: + + if (mStartQuery.getLimit() > 0ULL) { - // If the response should be processed, process it. - if (ok) - { - // Read the response. - auto block = MemoryBlock(grpcByteBuffer); - LOG_F(INFO, "response: echo \"%s\" | xxd -r -p | protoscope", block.get()->convertToHex().data()); - response = protopuf::deserialize<::hiero::ConsensusTopicResponse, ::hiero::ConsensusTopicResponseMessage>(*block.get()); - - // Adjust the query timestamp and limit, in case a retry is triggered. - auto consensusTimestamp = response.getConsensusTimestamp(); - if (!consensusTimestamp.empty()) - { - // Add one of the smallest denomination of time - query.setConsensusStartTime({ - consensusTimestamp.getSeconds(), consensusTimestamp.getNanos() + 1 - }); - } - - if (query.getLimit() > 0ULL) - { - query.setLimit(query.getLimit() - 1ULL); - } - - // Process the received message. - const auto& chunkInfo = response.getChunkInfo(); - if (chunkInfo.empty() || chunkInfo.getTotal() == 1) - { - onMessageArrived(std::move(response)); - } - else - { - LOG_F(FATAL, "Chunked Messages not implemented yet"); - /* - const TransactionId transactionId = - TransactionId::fromProtobuf(response.chunkinfo().initialtransactionid()); - pendingMessages[transactionId].push_back(response); - - if (pendingMessages[transactionId].size() == response.chunkinfo().total()) - { - onNext(TopicMessage::ofMany(pendingMessages[transactionId])); - } - */ - } - mResponseReader->Read(&grpcByteBuffer, &mCallStatus); - } - - // If the response shouldn't be processed (due to completion or error), finish the RPC. - else - { - mCallStatus = CallStatus::STATUS_FINISH; - mResponseReader->Finish(&grpcStatus, &mCallStatus); - } - - break; + mStartQuery.setLimit(mStartQuery.getLimit() - 1ULL); } - case CallStatus::STATUS_FINISH: + + // Process the received message. + const auto& chunkInfo = response.getChunkInfo(); + if (chunkInfo.empty() || chunkInfo.getTotal() == 1) { - if (grpcStatus.ok()) - { - // RPC completed successfully. - // completionHandler(); - LOG_F(INFO, "RPC subscription complete!"); - - // Shutdown the completion queue. - mCompletionQueue.Shutdown(); - - // Mark the RPC as complete. - complete = true; - break; - } - else - { - // An error occurred. Whether retrying or not, cancel the call and close the queue. - mClientContext.TryCancel(); - mCompletionQueue.Shutdown(); - - if (attempt >= ::hiero::DEFAULT_MAX_ATTEMPTS || !shouldRetry(grpcStatus)) - { - // This RPC call shouldn't be retried, handle the error and mark as complete to exit. - //errorHandler(grpcStatus); - LOG_F(ERROR, "Subscription error: %s", grpcStatus.error_message().data()); - complete = true; - } - } - - break; + try { + onMessageArrived(std::move(response)); + } + catch (GradidoBlockchainException& ex) { + LOG_F(ERROR, "error calling onMessageArrived: %s", ex.getFullString().c_str()); + } + catch (std::exception& ex) { + LOG_F(ERROR, "std error calling onMessageArrived: %s", ex.what()); + } + catch (...) { + LOG_F(ERROR, "unknown error calling onMessageArrived"); + } } - default: + else { - // Unrecognized call status, do nothing for now (not sure if this is correct). - break; - } + LOG_F(FATAL, "Chunked Messages not implemented yet"); + /* + const TransactionId transactionId = + TransactionId::fromProtobuf(response.chunkinfo().initialtransactionid()); + pendingMessages[transactionId].push_back(response); + + if (pendingMessages[transactionId].size() == response.chunkinfo().total()) + { + onNext(TopicMessage::ofMany(pendingMessages[transactionId])); + } + */ } + mResponseReader->Read(&grpcByteBuffer, &mCallStatus); + } - break; + // If the response shouldn't be processed (due to completion or error), finish the RPC. + else + { + mCallStatus = CallStatus::STATUS_FINISH; + mResponseReader->Finish(&grpcStatus, &mCallStatus); + } + + break; } - case ::grpc::CompletionQueue::SHUTDOWN: + case CallStatus::STATUS_FINISH: { - // Getting here means the RPC is reached completion or encountered an un-retriable error, and the completion - // queue has been shut down. End the subscription. - if (complete) + if (grpcStatus.ok()) + { + // RPC completed successfully. + // completionHandler(); + LOG_F(INFO, "RPC subscription complete!"); + + // Shutdown the completion queue. + mCompletionQueues.back()->Shutdown(); + + // Mark the RPC as complete. + complete = true; + completeReason = ConnectionClosedReason::Subscription_Ended; + break; + } + else + { + // An error occurred. Whether retrying or not, cancel the call and close the queue. + mClientContexts.back()->TryCancel(); + mCompletionQueues.back()->Shutdown(); + + if (attempt >= ::hiero::DEFAULT_MAX_ATTEMPTS || !shouldRetry(grpcStatus)) { - // Give a second for the queue to finish its processing. - std::this_thread::sleep_for(std::chrono::seconds(1)); - return 0; + // This RPC call shouldn't be retried, handle the error and mark as complete to exit. + //errorHandler(grpcStatus); + LOG_F(ERROR, "Subscription error: %s", grpcStatus.error_message().data()); + complete = true; + completeReason = ConnectionClosedReason::Error; } + } - // If the completion queue has been shut down and the RPC hasn't completed, that means the RPC needs to be - // retried. Increase the backoff for the retry. - backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; - std::this_thread::sleep_for(backoff); - ++attempt; - - // Resend the query to a different node with a different completion queue and client context. - // contexts.push_back(std::make_unique()); - // queues.push_back(std::make_unique()); - - // Reset the call status and send the query. - /* - *callStatus = CallStatus::STATUS_CREATE; - reader = getConnectedMirrorNode(network)->getConsensusServiceStub()->AsyncsubscribeTopic( - contexts.back().get(), query, queues.back().get(), callStatus.get()); - */ - onConnectionClosed(); - break; + break; } default: { - // Not sure what to do here, just fail out. - std::cout << "Unknown gRPC completion queue event, failing.." << std::endl; - return -1; + // Unrecognized call status, do nothing for now (not sure if this is correct). + break; } } + + break; + } + case ::grpc::CompletionQueue::SHUTDOWN: + { + // Getting here means the RPC is reached completion or encountered an un-retriable error, and the completion + // queue has been shut down. End the subscription. + if (complete) + { + // Give a second for the queue to finish its processing. + sleep_for(seconds(1)); + LOG_F(INFO, "RPC Subscription for topic %s ended.", mStartQuery.getTopicId().toString().data()); + onConnectionClosed(completeReason); + return 0; + } + + // If the completion queue has been shut down and the RPC hasn't completed, that means the RPC needs to be + // retried. Increase the backoff for the retry. + backoff = (backoff * 2 > maxBackoff) ? maxBackoff : backoff * 2; + sleep_for(backoff); + ++attempt; + + // Resend the query to a different node with a different completion queue and client context. + // contexts.push_back(std::make_unique()); + // queues.push_back(std::make_unique()); + + // Reset the call status and send the query. + /* + *callStatus = CallStatus::STATUS_CREATE; + reader = getConnectedMirrorNode(network)->getConsensusServiceStub()->AsyncsubscribeTopic( + contexts.back().get(), query, queues.back().get(), callStatus.get()); + */ + if (!Application::getStopToken().stop_requested()) { + mCallStatus = CallStatus::STATUS_CREATE; + mCompletionQueues.push_back(make_unique()); + mClientContexts.push_back(make_unique()); + MemoryBlock memoryBuffer(protopuf::serialize<::hiero::ConsensusTopicQuery, ::hiero::ConsensusTopicQueryMessage>(mStartQuery)); + grpcByteBuffer = memoryBuffer.createGrpcBuffer(); + ServerGlobals::g_HieroMirrorNode->subscribeTopic(this); + onConnectionClosed(ConnectionClosedReason::Reconnect); + } + break; + } + default: + { + // Not sure what to do here, just fail out. + std::cout << "Unknown gRPC completion queue event, failing.." << std::endl; + onConnectionClosed(ConnectionClosedReason::Error); + return -1; + } + } } - return 0; + } + catch (GradidoBlockchainException& ex) { + LOG_F(ERROR, "Thread has uncaught gradido blockchain exception: %s", ex.getFullString().c_str()); + onConnectionClosed(ConnectionClosedReason::Exception); + return -3; + } + catch (std::exception& ex) { + LOG_F(ERROR, "Thread has uncaught exception: %s", ex.what()); + onConnectionClosed(ConnectionClosedReason::Exception); + return -3; + } + catch (...) { + LOG_F(ERROR, "Thread has uncaught unknown exception"); + onConnectionClosed(ConnectionClosedReason::Exception); + return -3; + } + + onConnectionClosed(ConnectionClosedReason::Deconstruct); + return 0; } bool TopicMessageQuery::shouldRetry(::grpc::Status status) diff --git a/src/client/hiero/TopicMessageQuery.h b/src/client/hiero/TopicMessageQuery.h index 588883f5..7968562c 100644 --- a/src/client/hiero/TopicMessageQuery.h +++ b/src/client/hiero/TopicMessageQuery.h @@ -17,7 +17,7 @@ namespace client { namespace hiero { // Enum to track the status of the gRPC call. - enum class CallStatus + enum class CallStatus : long { STATUS_CREATE = 0, STATUS_WRITE = 1, @@ -25,6 +25,16 @@ namespace client { STATUS_FINISH = 3 }; + enum class ConnectionClosedReason + { + Subscription_Ended, + Error, + Exception, + Reconnect, + Deconstruct, + Timeout // it seems that grpc/hiero topic subscribe is unstable around ~ 30 minutes without messages so better close connection than and create new one + }; + class TopicMessageQuery { public: @@ -39,8 +49,8 @@ namespace client { TopicMessageQuery(TopicMessageQuery&&) = delete; TopicMessageQuery& operator=(TopicMessageQuery&&) = delete; - grpc::CompletionQueue* getCompletionQueuePtr() { return &mCompletionQueue; } - grpc::ClientContext* getClientContextPtr() { return &mClientContext; } + grpc::CompletionQueue* getCompletionQueuePtr() { return mCompletionQueues.back().get(); } + grpc::ClientContext* getClientContextPtr() { return mClientContexts.back().get(); } void setResponseReader(std::unique_ptr>& responseReader); @@ -48,7 +58,7 @@ namespace client { // will be called from grpc client if connection was closed // so no more messageArrived calls - virtual void onConnectionClosed() = 0; + virtual void onConnectionClosed(ConnectionClosedReason reason) noexcept = 0; // called inside loop if connection was closed, default implementation copied from hiero cpp sdk virtual bool shouldRetry(grpc::Status status); @@ -63,8 +73,8 @@ namespace client { ::hiero::ConsensusTopicQuery mStartQuery; - grpc::CompletionQueue mCompletionQueue; - grpc::ClientContext mClientContext; + std::vector> mCompletionQueues; + std::vector> mClientContexts; CallStatus mCallStatus; std::unique_ptr> mResponseReader; }; diff --git a/src/client/hiero/const.h b/src/client/hiero/const.h index df1a37bf..4e18e63c 100644 --- a/src/client/hiero/const.h +++ b/src/client/hiero/const.h @@ -18,6 +18,10 @@ namespace hiero { * The default maximum number of times a request will attempt to be submitted before considering the execution failed. */ constexpr auto DEFAULT_MAX_ATTEMPTS = 10U; + /* + * Header/Hiero gRPC seems becoming unstable around 30 minutes without messages + */ + constexpr auto DEFAULT_SUBSCRIPTION_INACTIVE_TIMEOUT = std::chrono::minutes(30); } #endif // __GRADIDO_NODE_CLIENT_HIERO_CONST_H_ \ No newline at end of file diff --git a/src/controller/ControllerExceptions.cpp b/src/controller/ControllerExceptions.cpp index 9aebc275..7bc5e810 100644 --- a/src/controller/ControllerExceptions.cpp +++ b/src/controller/ControllerExceptions.cpp @@ -4,17 +4,26 @@ #include +using std::string, std::to_string; + using namespace gradido::data; namespace controller { - GroupNotFoundException::GroupNotFoundException(const char* what, const std::string& groupAlias) noexcept + GroupNotFoundException::GroupNotFoundException(const char* what, const string& groupAlias) noexcept : GradidoBlockchainException(what), mGroupAlias(groupAlias) { } - std::string GroupNotFoundException::getFullString() const + + GroupNotFoundException::GroupNotFoundException(const char* what, uint32_t communityIdIndex) noexcept + : GradidoBlockchainException(what), mGroupAlias(to_string(communityIdIndex)) + { + + } + + string GroupNotFoundException::getFullString() const { - std::string resultString; + string resultString; size_t resultSize = strlen(what()) + mGroupAlias.size() + 2 + 15; resultString.reserve(resultSize); resultString = what(); @@ -23,16 +32,16 @@ namespace controller { } // ******************* BlockNotLoadedException ***************************** - BlockNotLoadedException::BlockNotLoadedException(const char* what, const std::string& groupAlias, int blockNr) noexcept + BlockNotLoadedException::BlockNotLoadedException(const char* what, const string& groupAlias, int blockNr) noexcept : GradidoBlockchainException(what), mGroupAlias(groupAlias), mBlockNr(blockNr) { } - std::string BlockNotLoadedException::getFullString() const + string BlockNotLoadedException::getFullString() const { - std::string resultString; - std::string blockNrString = std::to_string(mBlockNr); + string resultString; + string blockNrString = to_string(mBlockNr); size_t resultSize = strlen(what()) + 2 + 14 + 13 + mGroupAlias.size() + blockNrString.size(); resultString = what(); resultString += ", with group: " + mGroupAlias; @@ -54,10 +63,10 @@ namespace controller { } - std::string WrongTransactionTypeException::getFullString() const + string WrongTransactionTypeException::getFullString() const { - std::string resultString; - std::string transactionTypeString(magic_enum::enum_name(mType)); + string resultString; + string transactionTypeString(magic_enum::enum_name(mType)); size_t resultSize = strlen(what()) + 2 + 20 + transactionTypeString.size() + mPubkeyHex.size() + 10; resultString.reserve(resultSize); resultString = what(); diff --git a/src/controller/ControllerExceptions.h b/src/controller/ControllerExceptions.h index ee6b1457..772dcb75 100644 --- a/src/controller/ControllerExceptions.h +++ b/src/controller/ControllerExceptions.h @@ -10,6 +10,7 @@ namespace controller { { public: explicit GroupNotFoundException(const char* what, const std::string& groupAlias) noexcept; + explicit GroupNotFoundException(const char* what, uint32_t communityIdIndex) noexcept; std::string getFullString() const; protected: std::string mGroupAlias; diff --git a/src/controller/RemoteGroup.cpp b/src/controller/RemoteGroup.cpp index 0f37aed3..76b7943d 100644 --- a/src/controller/RemoteGroup.cpp +++ b/src/controller/RemoteGroup.cpp @@ -4,8 +4,8 @@ using namespace gradido::blockchain; using namespace gradido::data; namespace controller { - RemoteGroup::RemoteGroup(const std::string& groupAlias) - : Abstract(groupAlias) + RemoteGroup::RemoteGroup(uint32_t communityIdIndex) + : Abstract(communityIdIndex) { // get coin color on first connect to remote group } diff --git a/src/controller/RemoteGroup.h b/src/controller/RemoteGroup.h index e2e25854..b387fdd2 100644 --- a/src/controller/RemoteGroup.h +++ b/src/controller/RemoteGroup.h @@ -16,7 +16,7 @@ namespace controller { class RemoteGroup : public gradido::blockchain::Abstract { public: - RemoteGroup(const std::string& groupAlias); + RemoteGroup(uint32_t communityIdIndex); //! validate and generate confirmed transaction //! throw if gradido transaction isn't valid diff --git a/src/controller/SimpleOrderingManager.cpp b/src/controller/SimpleOrderingManager.cpp index db4bd816..fde31275 100644 --- a/src/controller/SimpleOrderingManager.cpp +++ b/src/controller/SimpleOrderingManager.cpp @@ -3,8 +3,7 @@ #include "../blockchain/FileBasedProvider.h" #include "../blockchain/Exceptions.h" #include "../task/HieroMessageToTransactionTask.h" -#include "gradido_blockchain/interaction/serialize/Context.h" -#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/data/LedgerAnchor.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "gradido_blockchain/const.h" @@ -14,32 +13,43 @@ using namespace gradido; using namespace blockchain; -using namespace interaction; + +using gradido::data::LedgerAnchor; namespace controller { - SimpleOrderingManager::SimpleOrderingManager(std::string_view communityId) - : task::Thread("SimpleOrderingManager"), - mLastTransactions(MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION * 2), - mCommunityId(communityId), + SimpleOrderingManager::SimpleOrderingManager(std::string_view communityId, std::stop_token stopToken) + : task::Thread("SimpleOrderingManager"), mStopToken(stopToken), mInitalized(false), + mLastTransactions(MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION * 2), + mCommunityId(communityId), mLastSequenceNumber(0) { } SimpleOrderingManager::~SimpleOrderingManager() { + } - void SimpleOrderingManager::init(uint64_t lastKnownSequenceNumber) + void SimpleOrderingManager::reinitialize(uint64_t lastKnownSequenceNumber) { + std::unique_lock _lock(mTransactionsMutex); mLastSequenceNumber = lastKnownSequenceNumber; - Thread::init(); + mTransactions.clear(); + mLastTransactions.clear(); + if (!mInitalized) { + mInitalized = true; + Thread::init(); + } } int SimpleOrderingManager::ThreadFunction() { size_t transactionsCount = 0; do { + if (mStopToken.stop_requested()) { + return 0; + } std::unique_lock _lock(mTransactionsMutex); auto it = mTransactions.begin(); // if no transaction is in map or first transaction deserialize task is still running (or is waiting to be scheduled) @@ -62,7 +72,7 @@ namespace controller { // maybe we have an error // or hiero has used the same sequence number twice? LOG_F( - ERROR, + ERROR, "this transaction or after this was already put into blockchain, fatal error, programm code must be fixed, communityId: %s, last sequence number: %lu, current sequence number: %lu", mCommunityId.data(), lastSequenceNumber, currentSequenceNumber ); @@ -81,30 +91,28 @@ namespace controller { Timepoint now = std::chrono::system_clock::now(); if (it->second.putIntoListTime + MAGIC_NUMBER_MAX_TIMESPAN_BETWEEN_CREATING_AND_RECEIVING_TRANSACTION < now) { - // timeouted + // timeouted task->notificateFailedTransaction(blockchain, "Transaction skipped (pairing not found)"); mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); continue; } - auto otherBlockchain = blockchainProvider->findBlockchain(body->getOtherGroup()); + auto otherBlockchain = blockchainProvider->findBlockchain(body->getOtherCommunityIdIndex().value()); if (!otherBlockchain) { task->notificateFailedTransaction(blockchain, "Transaction skipped (target community unknown)"); mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); continue; } - deserialize::Context topicIdDeserializer(gradidoTransaction->getParingMessageId(), deserialize::Type::HIERO_TRANSACTION_ID); - topicIdDeserializer.run(); - if (!topicIdDeserializer.isHieroTransactionId()) { - task->notificateFailedTransaction(blockchain, "Transaction skipped (pairing transactionId invalid)"); + if (gradidoTransaction->getPairingLedgerAnchor().empty()) { + task->notificateFailedTransaction(blockchain, "Transaction skipped (pairing ledger anchor empty)"); mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); continue; } auto pairTask = static_cast(otherBlockchain.get()) ->getOrderingManager() - ->findCrossGroupTransactionPair(topicIdDeserializer.getHieroTransactionId()) + ->findCrossGroupTransactionPair(gradidoTransaction->getPairingLedgerAnchor()) ; if (!pairTask || !pairTask->isTaskFinished()) { // not found? maybe it need some more time? @@ -120,7 +128,7 @@ namespace controller { } if (task->isSuccess()) { processTransaction(it->second); - } + } mTransactions.erase(it); updateSequenceNumber(currentSequenceNumber); transactionsCount = mTransactions.size(); @@ -136,24 +144,23 @@ namespace controller { throw CommunityNotFoundExceptions("couldn't find group", mCommunityId); } auto transaction = gradidoTransactionWorkData.deserializeTask->getGradidoTransaction(); - const auto& transactionId = gradidoTransactionWorkData.consensusTopicResponse.getChunkInfo().getInitialTransactionId(); + auto transactionId = LedgerAnchor(gradidoTransactionWorkData.consensusTopicResponse.getChunkInfo().getInitialTransactionId()); const auto& confirmedAt = gradidoTransactionWorkData.consensusTopicResponse.getConsensusTimestamp(); if (transactionId.empty()) { throw GradidoNodeInvalidDataException("missing transaction id in hiero response"); } auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); try { - serialize::Context serializer(transactionId); bool result = blockchain->createAndAddConfirmedTransaction( transaction, - serializer.run(), + transactionId, confirmedAt ); fileBasedBlockchain->updateLastKnownSequenceNumber(mLastSequenceNumber); LOG_F(INFO, "Transaction confirmed, msgId: %s, confirmedAt: %s", transactionId.toString().data(), confirmedAt.toString().data() ); - + } catch (GradidoBlockchainException& ex) { auto communityServer = fileBasedBlockchain->getListeningCommunityServer(); @@ -188,7 +195,7 @@ namespace controller { return PushResult::FOUND_IN_LAST_TRANSACTIONS; } } - + auto range = mTransactions.equal_range(consensusTimestamp); for (auto& it = range.first; it != range.second; ++it) { if (it->second.consensusTopicResponse.isMessageSame(consensusTopicResponse)) { @@ -207,13 +214,20 @@ namespace controller { return PushResult::ADDED; } - std::shared_ptr SimpleOrderingManager::findCrossGroupTransactionPair(const hiero::TransactionId& transactionId) const + std::shared_ptr SimpleOrderingManager::findCrossGroupTransactionPair(const LedgerAnchor& transactionId) const { if (isExitCalled()) { return nullptr; } std::lock_guard _lock(mTransactionsMutex); for (auto it = mTransactions.begin(); it != mTransactions.end(); it++) { - if (it->second.consensusTopicResponse.getChunkInfo().getInitialTransactionId() == transactionId) { - return it->second.deserializeTask; + if (transactionId.isHieroTransactionId()) { + if (it->second.consensusTopicResponse.getChunkInfo().getInitialTransactionId() == transactionId.getHieroTransactionId()) { + return it->second.deserializeTask; + } + } + else { + if (it->second.deserializeTask->isSuccess() && it->second.deserializeTask->getGradidoTransaction()->getPairingLedgerAnchor() == transactionId) { + return it->second.deserializeTask; + } } } return nullptr; @@ -221,7 +235,7 @@ namespace controller { void SimpleOrderingManager::updateSequenceNumber(uint64_t newSequenceNumber) { - + if (!mLastSequenceNumber) { mLastSequenceNumber = newSequenceNumber; } diff --git a/src/controller/SimpleOrderingManager.h b/src/controller/SimpleOrderingManager.h index c2480042..71a16fb4 100644 --- a/src/controller/SimpleOrderingManager.h +++ b/src/controller/SimpleOrderingManager.h @@ -2,7 +2,6 @@ #define __GRADIDO_NODE_SINGLETON_MANAGER_ORDERING_MANAGER #include "gradido_blockchain/data/GradidoTransaction.h" -#include "gradido_blockchain/data/hiero/TransactionId.h" #include "gradido_blockchain/crypto/SignatureOctet.h" #include "gradido_blockchain/lib/ExpireCache.h" #include "../task/Thread.h" @@ -16,6 +15,12 @@ namespace task { class HieroMessageToTransactionTask; } +namespace gradido { + namespace data { + class LedgerAnchor; + } +} + namespace controller { /*! @@ -31,10 +36,11 @@ namespace controller { class SimpleOrderingManager : public task::Thread { public: - SimpleOrderingManager(std::string_view communityId); - ~SimpleOrderingManager(); + SimpleOrderingManager(std::string_view communityId, std::stop_token stopToken); + virtual ~SimpleOrderingManager(); - void init(uint64_t lastKnownSequenceNumber); + // combine init and reset + void reinitialize(uint64_t lastKnownSequenceNumber); enum class PushResult { FOUND_IN_LAST_TRANSACTIONS, @@ -43,10 +49,9 @@ namespace controller { IN_SHUTDOWN }; PushResult pushTransaction(hiero::ConsensusTopicResponse&& consensusTopicResponse); - std::shared_ptr findCrossGroupTransactionPair(const hiero::TransactionId& transactionId) const; - - protected: + std::shared_ptr findCrossGroupTransactionPair(const gradido::data::LedgerAnchor& transactionId) const; + protected: inline uint64_t getLastSequenceNumber() const { return mLastSequenceNumber; } void updateSequenceNumber(uint64_t newSequenceNumber); @@ -58,21 +63,21 @@ namespace controller { TopicResponseDeserializer( hiero::ConsensusTopicResponse&& _consensusTopicResponse, std::shared_ptr _deserializeTask - ) : consensusTopicResponse(std::move(_consensusTopicResponse)), - deserializeTask(_deserializeTask), + ) : consensusTopicResponse(std::move(_consensusTopicResponse)), + deserializeTask(_deserializeTask), putIntoListTime(std::chrono::system_clock::now()) { } TopicResponseDeserializer(TopicResponseDeserializer&& move) noexcept - : consensusTopicResponse(std::move(move.consensusTopicResponse)), + : consensusTopicResponse(std::move(move.consensusTopicResponse)), deserializeTask(std::move(move.deserializeTask)), putIntoListTime(std::move(move.putIntoListTime)) { } TopicResponseDeserializer(const TopicResponseDeserializer& other) noexcept - : consensusTopicResponse(other.consensusTopicResponse), + : consensusTopicResponse(other.consensusTopicResponse), deserializeTask(other.deserializeTask), putIntoListTime(other.putIntoListTime) { @@ -88,6 +93,8 @@ namespace controller { void processTransaction(const TopicResponseDeserializer& gradidoTransactionWorkData); + std::stop_token mStopToken; + bool mInitalized; // fast duplication check // use first 4 and last 4 Byte of transaction hash as key // data part is consensusTimestamp which already should be unique.. but better make sure it is really unique diff --git a/src/controller/TaskObserver.cpp b/src/controller/TaskObserver.cpp index e31195b0..ccd6bced 100644 --- a/src/controller/TaskObserver.cpp +++ b/src/controller/TaskObserver.cpp @@ -29,7 +29,7 @@ bool TaskObserver::addBlockWriteTask(std::shared_ptrgetTransactionEntriesList(); mTransactionsFromPendingTasks.insert(transactions->begin(), transactions->end()); - + return true; } @@ -89,7 +89,7 @@ std::shared_ptr TaskObserver::getTransaction(uint64_t tran // *********************** Finish command ************************************** int TaskObserverFinishCommand::taskFinished(task::Task* task) -{ - mBlockchain->getTaskObserver().removeTask(task); +{ + mBlockchain->getTaskObserver().removeTask(task); return 0; } \ No newline at end of file diff --git a/src/hiero/Addressbook.cpp b/src/hiero/Addressbook.cpp index bdd08a92..ab4e36c4 100644 --- a/src/hiero/Addressbook.cpp +++ b/src/hiero/Addressbook.cpp @@ -1,6 +1,6 @@ #include "../SystemExceptions.h" #include "Addressbook.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "loguru/loguru.hpp" @@ -21,7 +21,7 @@ namespace hiero { void Addressbook::load() { - Profiler timeUsed; + MonotonicTimer timeUsed; // read addressbook from binary file std::ifstream file(mFilePath, std::ios::binary | std::ios::ate); diff --git a/src/hiero/MessageListenerQuery.cpp b/src/hiero/MessageListenerQuery.cpp index d3ee3449..33d3a278 100644 --- a/src/hiero/MessageListenerQuery.cpp +++ b/src/hiero/MessageListenerQuery.cpp @@ -1,13 +1,18 @@ #include "MessageListenerQuery.h" -#include "../controller/SimpleOrderingManager.h" #include "../blockchain/FileBasedProvider.h" +#include "../client/hiero/ConsensusClient.h" +#include "../controller/SimpleOrderingManager.h" +#include "../task/SyncTopic.h" #include "ConsensusTopicResponse.h" +#include "gradido_blockchain/Application.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "loguru/loguru.hpp" +using client::hiero::ConnectionClosedReason; + namespace hiero { MessageListenerQuery::MessageListenerQuery(const TopicId& topicId, std::string_view communityId, ConsensusTopicQuery startQuery) @@ -40,9 +45,21 @@ namespace hiero { } // will be called from grpc client if connection was closed - void MessageListenerQuery::onConnectionClosed() - { - mIsClosed = true; - LOG_F(WARNING, "connection closed on topic: %s", mTopicId.toString().data()); + void MessageListenerQuery::onConnectionClosed(ConnectionClosedReason reason) noexcept + { + //mIsClosed = true; + if (ConnectionClosedReason::Reconnect == reason) { + LOG_F(WARNING, "connection closed on topic: %s, try reconnect", mTopicId.toString().data()); + } + else { + if (!Application::getStopToken().stop_requested()) { + auto blockchain = gradido::blockchain::FileBasedProvider::getInstance()->findBlockchain(mCommunityId); + auto fileBasedBlockchain = static_cast(blockchain.get()); + auto task = fileBasedBlockchain->getTopicSyncTask(); + auto hieroClient = fileBasedBlockchain->pickHieroClient(); + hieroClient->getTopicInfo(fileBasedBlockchain->getHieroTopicId(), task); + task->scheduleTask(task); + } + } } } \ No newline at end of file diff --git a/src/hiero/MessageListenerQuery.h b/src/hiero/MessageListenerQuery.h index edbee669..b84602f7 100644 --- a/src/hiero/MessageListenerQuery.h +++ b/src/hiero/MessageListenerQuery.h @@ -20,10 +20,10 @@ namespace hiero { void onMessageArrived(ConsensusTopicResponse&& consensusTopicResponse) override; // will be called from grpc client if connection was closed - void onConnectionClosed() override; + void onConnectionClosed(client::hiero::ConnectionClosedReason reason) noexcept override; inline bool isClosed() const { return mIsClosed; } - inline void cancelConnection() { mClientContext.TryCancel(); } + inline void cancelConnection() { getClientContextPtr()->TryCancel(); } protected: TopicId mTopicId; std::string mCommunityId; diff --git a/src/lib/PersistentDictionary.h b/src/lib/PersistentDictionary.h new file mode 100644 index 00000000..bd4f6b7d --- /dev/null +++ b/src/lib/PersistentDictionary.h @@ -0,0 +1,178 @@ +#ifndef __GRADIDO_NODE_PERSISTENT_DICTIONARY_H +#define __GRADIDO_NODE_PERSISTENT_DICTIONARY_H + +#include "../model/files/LevelDBWrapper.h" +#include "../serialization/String.h" +#include "gradido_blockchain/lib/Dictionary.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" +#include "gradido_blockchain/lib/DictionaryExceptions.h" + +#include "loguru/loguru.hpp" + +#include +#include +#include +#include + +// use simple in memory threadsafe dictionary for now, until updated transaction and address index are stored into files +template< + typename DataType, + typename Hash = std::hash, + typename Equal = std::equal_to +> +class PersistentDictionary : public ThreadsafeRuntimeDictionary +{ +public: + explicit PersistentDictionary(const std::string& directory): ThreadsafeRuntimeDictionary(directory) {} + ~PersistentDictionary() {} + + inline bool init(size_t cacheInBytes) { return false; }; + inline void exit() { ThreadsafeRuntimeDictionary::reset(); }; + inline size_t getLastIndex() { + return RuntimeDictionary::mIndexDataLookup.size(); + } +}; + +// TODO: check usage of LMDB +// it is magnitude faster but especially it is designed for prevent data loss on system failure, data can only be corrupted through hardware failure! +// - https://de.wikipedia.org/wiki/Lightning_Memory-Mapped_Database +// - https://github.com/LMDB/lmdb/tree/mdb.master/libraries/liblmdb +// TODO: remove bi-directionality, update whole code for not using getDataForIndex at all! +/* +template +requires serialization::HasString +class PersistentDictionary: public IMutableDictionary +{ +public: + explicit PersistentDictionary(const std::string& directory) : mDictionaryFile(directory) {} + ~PersistentDictionary() {} + + bool init(size_t cacheInBytes); + void exit(); + void reset() override; + size_t getLastIndex(); + + virtual size_t getIndexForData(const DataType& data) const override; + virtual std::optional getDataForIndex(size_t index) const override; + virtual DataType getDataForIndexOrThrow(size_t index) const override; + virtual size_t getOrAddIndexForData(const DataType& data) override; + virtual bool hasIndex(size_t index) const override; + +private: + // LevelDB reads are logically const but mutate internal state + mutable model::files::LevelDBWrapper mDictionaryFile; + mutable std::shared_mutex mWorkingMutex; + std::unordered_map mIndexDataReverseLookup; +}; + + +template +requires serialization::HasString +bool PersistentDictionary::init(size_t cacheInBytes) +{ + std::unique_lock _lock(mWorkingMutex); + if (!mDictionaryFile.init(cacheInBytes)) { + return false; + } + + // key is DataType, value is size_t + mDictionaryFile.iterate([&](leveldb::Slice key, leveldb::Slice value) -> void { + mIndexDataReverseLookup.insert({ + serialization::fromString(value.data(), value.size()), + serialization::fromString(key.data(), key.size()) + }); + }); + return true; +} + +template +requires serialization::HasString +void PersistentDictionary::exit() +{ + std::unique_lock _lock(mWorkingMutex); + mDictionaryFile.exit(); +} + +template +requires serialization::HasString +void PersistentDictionary::reset() +{ + std::unique_lock _lock(mWorkingMutex); + mDictionaryFile.reset(); +} + +template +requires serialization::HasString +size_t PersistentDictionary::getLastIndex() +{ + std::unique_lock _lock(mWorkingMutex); + return mIndexDataReverseLookup.size(); +} + +template +requires serialization::HasString +size_t PersistentDictionary::getIndexForData(const DataType& data) const +{ + std::shared_lock _lock(mWorkingMutex); + auto result = mDictionaryFile.getValueForKey(serialization::toString(data)); + if (result.has_value()) { + const auto& value = result.value(); + return serialization::fromString(value.data(), value.size()); + } + return 0; +} + +template +requires serialization::HasString +std::optional PersistentDictionary::getDataForIndex(size_t index) const +{ + std::shared_lock _lock(mWorkingMutex); + + auto it = mIndexDataReverseLookup.find(index); + if (it == mIndexDataReverseLookup.end()) { + return std::nullopt; + } + return it->second; +} + +template + requires serialization::HasString +DataType PersistentDictionary::getDataForIndexOrThrow(size_t index) const +{ + auto data = getDataForIndex(index); + if (!data) { + throw DictionaryMissingEntryException(mDictionaryFile.getFolderName().data(), std::to_string(index)); + } + return data.value(); +} + +template +requires serialization::HasString +size_t PersistentDictionary::getOrAddIndexForData(const DataType& data) +{ + auto dataString = serialization::toString(data); + std::unique_lock _lock(mWorkingMutex); + auto result = mDictionaryFile.getValueForKey(dataString); + if (result.has_value()) { + const auto& value = result.value(); + return serialization::fromString(value.data(), value.size()); + } + if (mIndexDataReverseLookup.size() >= static_cast(std::numeric_limits::max())) { + throw DictionaryOverflowException("try to add more index data set's as size_t as index can handle", mDictionaryFile.getFolderName()); + } + size_t index = mIndexDataReverseLookup.size() + 1; + mDictionaryFile.setKeyValue(dataString, serialization::toString(index)); + mIndexDataReverseLookup.insert({ index, data }); + return index; +} + +template +requires serialization::HasString +bool PersistentDictionary::hasIndex(size_t index) const +{ + std::shared_lock _lock(mWorkingMutex); + auto it = mIndexDataReverseLookup.find(index); + return it != mIndexDataReverseLookup.end(); +} +*/ +#endif //__GRADIDO_NODE_PERSISTENT_DICTIONARY_H \ No newline at end of file diff --git a/src/lib/protopuf.h b/src/lib/protopuf.h index ad98ea43..73ccaa68 100644 --- a/src/lib/protopuf.h +++ b/src/lib/protopuf.h @@ -7,7 +7,7 @@ #include "protopuf/message.h" */ -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/memory/Block.h" #include "loguru/loguru.hpp" @@ -34,7 +34,7 @@ namespace protopuf { template T deserialize(const memory::Block& raw) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto result = pp::message_coder::decode(raw.span()); if (!result.has_value()) { // TODO: check if using exception is better diff --git a/src/main.cpp b/src/main.cpp index a6eee909..cf4df4f9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -7,7 +7,6 @@ #include "ServerGlobals.h" #include "generated/version.h" -#include "gradido_blockchain/lib/Profiler.h" #include "gradido_blockchain/version.h" #include "sodium.h" diff --git a/src/model/Apollo/Decay.cpp b/src/model/Apollo/Decay.cpp index eff27069..04648856 100644 --- a/src/model/Apollo/Decay.cpp +++ b/src/model/Apollo/Decay.cpp @@ -1,10 +1,13 @@ #include "Decay.h" #include "gradido_blockchain/lib/DataTypeConverter.h" +#include "magic_enum/magic_enum.hpp" + #include using namespace rapidjson; using namespace std::chrono; +using namespace magic_enum; namespace model { namespace Apollo { @@ -14,19 +17,26 @@ namespace model { std::string formatJsCompatible(Timepoint date) { auto dateString = DataTypeConverter::timePointToString(date, jsDateTimeFormat); - return dateString; + // add Z to say it is UTC + return dateString + "Z"; // return dateString.substr(0, dateString.find_last_of('.')); } Decay::Decay(Decay* parent) - : mDecayStart(parent->mDecayStart), mDecayEnd(parent->mDecayEnd), mDecayAmount(parent->mDecayAmount) + : mDecayStart(parent->mDecayStart), mDecayEnd(parent->mDecayEnd), mDecayAmount(parent->mDecayAmount), mDecayType(parent->getDecayType()) { } - Decay::Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) - : mDecayStart(decayStart), mDecayEnd(decayEnd) + + Decay::Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance, GradidoUnit decayAmount) + : mDecayStart(decayStart), mDecayEnd(decayEnd), mDecayAmount(decayAmount), mDecayType(decideDecayType(decayStart, decayEnd)) { - mDecayAmount = startBalance.calculateDecay(mDecayStart, mDecayEnd) - startBalance; + if (mDecayStart < DECAY_START_TIME) { + mDecayStart = DECAY_START_TIME; + } + if (mDecayEnd < mDecayStart) { + mDecayEnd = mDecayStart; + } } Decay::~Decay() @@ -37,14 +47,26 @@ namespace model { Value Decay::toJson(Document::AllocatorType& alloc) { Value decay(kObjectType); - decay.AddMember("decay", Value(mDecayAmount.toString(2).data(), alloc), alloc); + decay.AddMember("decay", Value(mDecayAmount.toString().data(), alloc), alloc); decay.AddMember("start", Value(formatJsCompatible(mDecayStart).data(), alloc), alloc); decay.AddMember("end", Value(formatJsCompatible(mDecayEnd).data(), alloc), alloc); decay.AddMember("duration", static_cast( duration_cast(mDecayEnd - mDecayStart).count() ), alloc); + decay.AddMember("type", Value(enum_name(mDecayType).data(), alloc), alloc); decay.AddMember("__typename", "Decay", alloc); return std::move(decay); } + + DecayType Decay::decideDecayType(Timepoint decayStart, Timepoint decayEnd) + { + if (decayEnd <= DECAY_START_TIME) { + return DecayType::BEFORE_START_BLOCK; + } + if (decayStart >= DECAY_START_TIME) { + return DecayType::AFTER_START_BLOCK; + } + return DecayType::START_BLOCK_INSIDE; + } } } \ No newline at end of file diff --git a/src/model/Apollo/Decay.h b/src/model/Apollo/Decay.h index 7bae5488..17d72474 100644 --- a/src/model/Apollo/Decay.h +++ b/src/model/Apollo/Decay.h @@ -10,22 +10,32 @@ namespace model { std::string formatJsCompatible(Timepoint date); + enum DecayType + { + BEFORE_START_BLOCK, + START_BLOCK_INSIDE, + AFTER_START_BLOCK + }; + class Decay { public: Decay(Decay* parent); - Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); + Decay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance, GradidoUnit decayAmount); ~Decay(); rapidjson::Value toJson(rapidjson::Document::AllocatorType& alloc); inline GradidoUnit getDecayAmount() const { return mDecayAmount; } - + inline DecayType getDecayType() const { return mDecayType; } protected: + static DecayType decideDecayType(Timepoint decayStart, Timepoint decayEnd); + Timepoint mDecayStart; Timepoint mDecayEnd; GradidoUnit mDecayAmount; + DecayType mDecayType; }; } } diff --git a/src/model/Apollo/Transaction.cpp b/src/model/Apollo/Transaction.cpp index 10130dd6..130c69fa 100644 --- a/src/model/Apollo/Transaction.cpp +++ b/src/model/Apollo/Transaction.cpp @@ -1,6 +1,7 @@ #include "Transaction.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "gradido_blockchain/blockchain/Abstract.h" +#include "gradido_blockchain/data/ConfirmedTransaction.h" #include "magic_enum/magic_enum.hpp" #include "loguru/loguru.hpp" @@ -35,18 +36,18 @@ namespace model { } mId = confirmedTransaction.getId(); mDate = confirmedTransaction.getConfirmedAt(); - mBalance = confirmedTransaction.getAccountBalance(pubkey, "").getBalance(); + mBalance = confirmedTransaction.getAccountBalance(pubkey, std::nullopt).getBalance(); } Transaction::Transaction(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) - : mType(TransactionType::DECAY), mId(-1), mDate(decayEnd), mDecay(nullptr) + : mType(TransactionType::DECAY), mId(-1), mDate(decayEnd), mDecay(nullptr), mHasChange(false) { calculateDecay(decayStart, decayEnd, startBalance); mAmount = mDecay->getDecayAmount(); mBalance = startBalance + mDecay->getDecayAmount(); mPreviousBalance = startBalance; } - + /* * TransactionType mType; mpfr_ptr mAmount; @@ -141,28 +142,38 @@ namespace model { return *this; } + void Transaction::setDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) + { + if (mDecay) { + delete mDecay; + mDecay = nullptr; + } + + mDecay = new Decay(decayStart, decayEnd, startBalance, (startBalance - mBalance + mAmount).negate()); + } void Transaction::calculateDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance) { if (mDecay) { delete mDecay; mDecay = nullptr; } - mDecay = new Decay(decayStart, decayEnd, startBalance); + mDecay = new Decay(decayStart, decayEnd, startBalance, startBalance.calculateDecay(decayStart, decayEnd) - startBalance); } - void Transaction::setBalance(GradidoUnit balance) + /* void Transaction::setBalance(GradidoUnit balance) { mBalance = balance; } + */ Value Transaction::toJson(Document::AllocatorType& alloc) { Value transaction(kObjectType); transaction.AddMember("id", mId, alloc); transaction.AddMember("typeId", Value(enum_name(mType).data(), alloc), alloc); - transaction.AddMember("amount", Value(mAmount.toString(2).data(), alloc), alloc); - transaction.AddMember("balance", Value(mBalance.toString(2).data(), alloc), alloc); - transaction.AddMember("previousBalance", Value(mPreviousBalance.toString(2).data(), alloc), alloc); + transaction.AddMember("amount", Value(mAmount.toString().data(), alloc), alloc); + transaction.AddMember("balance", Value(mBalance.toString().data(), alloc), alloc); + transaction.AddMember("previousBalance", Value(mPreviousBalance.toString().data(), alloc), alloc); transaction.AddMember("memo", Value(mMemo.data(), alloc), alloc); if (!mPubkey.empty() || !mFirstName.empty() || !mLastName.empty()) { @@ -178,7 +189,7 @@ namespace model { if(mHasChange) { Value changeObj(kObjectType); printf("Transaction::toJson adding change amount: %s, pubkey: %s\n", mChangeAmount.toString().data(), mChangePubkey.data()); - changeObj.AddMember("amount", Value(mChangeAmount.toString(2).data(), alloc), alloc); + changeObj.AddMember("amount", Value(mChangeAmount.toString().data(), alloc), alloc); changeObj.AddMember("pubkey", Value(mChangePubkey.data(), alloc), alloc); changeObj.AddMember("__typename", "Change", alloc); transaction.AddMember("change", changeObj, alloc); diff --git a/src/model/Apollo/Transaction.h b/src/model/Apollo/Transaction.h index fd13e470..013e0558 100644 --- a/src/model/Apollo/Transaction.h +++ b/src/model/Apollo/Transaction.h @@ -3,6 +3,8 @@ #include "Decay.h" +#include "gradido_blockchain/memory/Block.h" + #include namespace gradido { @@ -38,6 +40,7 @@ namespace model { const gradido::data::ConfirmedTransaction& confirmedTransaction, memory::ConstBlockPtr pubkey ); + // constructor for last decay to now transaction Transaction(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); // Move constrcutor @@ -48,8 +51,9 @@ namespace model { Transaction& operator=(const Transaction& other); // copy + void setDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); void calculateDecay(Timepoint decayStart, Timepoint decayEnd, GradidoUnit startBalance); - void setBalance(GradidoUnit balance); + // void setBalance(GradidoUnit balance); inline GradidoUnit getBalance() const {return mBalance;} inline void setPreviousBalance(GradidoUnit previousBalance) {mPreviousBalance = previousBalance;} inline void setChange(const GradidoUnit& changeAmount, memory::ConstBlockPtr changePubkey); @@ -90,10 +94,10 @@ namespace model { }; void Transaction::setChange(const GradidoUnit& changeAmount, memory::ConstBlockPtr changePubkey) { - mHasChange = true; - mChangeAmount = changeAmount; - mChangePubkey = changePubkey->convertToHex(); - } + mHasChange = true; + mChangeAmount = changeAmount; + mChangePubkey = changePubkey->convertToHex(); + } } } diff --git a/src/model/Apollo/TransactionList.cpp b/src/model/Apollo/TransactionList.cpp index acde4db5..c6e5d0b4 100644 --- a/src/model/Apollo/TransactionList.cpp +++ b/src/model/Apollo/TransactionList.cpp @@ -1,6 +1,9 @@ #include "TransactionList.h" #include "createTransaction/Context.h" #include "gradido_blockchain/blockchain/Filter.h" +#include "gradido_blockchain/data/Timestamp.h" +#include "gradido_blockchain/data/TransactionType.h" +#include "gradido_blockchain/serialization/toJsonString.h" #include "../../blockchain/FileBased.h" #include "../../blockchain/NodeTransactionEntry.h" @@ -11,6 +14,8 @@ using namespace rapidjson; using namespace gradido::interaction; using namespace gradido::blockchain; using namespace magic_enum; +using gradido::data::Timestamp; +using serialization::toJsonString; namespace model { namespace Apollo { @@ -26,6 +31,10 @@ namespace model { Value TransactionList::generateList(Timepoint now, const Filter& filter, Document& root) { auto fileBasedBlockchain = std::dynamic_pointer_cast(mBlockchain); + uint32_t coinCommunityId = mBlockchain->getCommunityIdIndex(); + if (filter.coinCommunityIdIndex.has_value()) { + coinCommunityId = filter.coinCommunityIdIndex.value(); + } assert(fileBasedBlockchain); auto& alloc = root.GetAllocator(); @@ -41,12 +50,13 @@ namespace model { auto filterOutNotForWallet = [&filter](const TransactionEntry& entry) -> FilterResult { // filter out creation transactions which this user has signed as moderator, and isn't the benefitor - if (entry.isCreation()) { + // shouldn't be needed any longer, because of Transaction Index change, using updatedBalancePublicKey instead of involvedPublicKey + /*if (entry.isCreation()) { auto creation = entry.getTransactionBody()->getCreation(); if (!creation->getRecipient().getPublicKey()->isTheSame(filter.involvedPublicKey)) { return FilterResult::DISMISS; } - } + }*/ // filter out register address transaction, because this won't show in wallet view if (entry.isRegisterAddress()) { return FilterResult::DISMISS; @@ -54,21 +64,17 @@ namespace model { return FilterResult::USE; }; - int countTransactions = 0; + size_t countTransactions = 0; Filter countFilter = filter; countFilter.pagination = Pagination(0, 0); - countFilter.filterFunction = [&filterOutNotForWallet, &countTransactions](const TransactionEntry& entry) -> FilterResult - { - auto result = filterOutNotForWallet(entry); - if ((result & FilterResult::USE) == FilterResult::USE) { - countTransactions++; - return FilterResult::DISMISS; - } - return result; - }; - fileBasedBlockchain->findAll(countFilter); + auto allTransactionsCount = mBlockchain->countAll(countFilter); + countFilter.transactionType = gradido::data::TransactionType::REGISTER_ADDRESS; + auto registerAddressTransactionsCount = mBlockchain->countAll(countFilter); + if (registerAddressTransactionsCount < allTransactionsCount) { + countTransactions = allTransactionsCount - registerAddressTransactionsCount; + } - auto addressType = mBlockchain->getAddressType(Filter(0,0,filter.involvedPublicKey)); + auto addressType = mBlockchain->getAddressType(Filter(0,0,filter.updatedBalancePublicKey)); transactionList.AddMember("addressType", Value(enum_name(addressType).data(), alloc), alloc); Filter filterCopy = filter; @@ -81,42 +87,57 @@ namespace model { return std::move(transactionList); } - // copy into vector to make reversing and loop through faster (cache-hit) - std::vector> allTransactionsVector(allTransactions.begin(), allTransactions.end()); - allTransactions.clear(); if (filter.searchDirection == SearchDirection::DESC) { - std::reverse(allTransactionsVector.begin(), allTransactionsVector.end()); + std::reverse(allTransactions.begin(), allTransactions.end()); } // all transaction is always sorted ASC, regardless of filter.searchDirection value GradidoUnit previousBalance(GradidoUnit::zero()); + // load previous balance before first transaction for decay Timepoint previousDate = mBlockchain->getStartDate(); + auto firstTransactionNr = allTransactions.front()->getTransactionNr(); + if (firstTransactionNr > 1) { + const auto& previousTransactionDate = allTransactions.front()->getConfirmedTransaction()->getConfirmedAt(); + auto beforePreviousTransactionDate = Timestamp( + previousTransactionDate.getSeconds(), + previousTransactionDate.getNanos() - 1000 + ); + + Filter previousTransactionFilter = Filter::LAST_TRANSACTION; + previousTransactionFilter.maxTransactionNr = firstTransactionNr - 1; + previousTransactionFilter.updatedBalancePublicKey = filter.updatedBalancePublicKey; + previousTransactionFilter.timepointInterval = TimepointInterval(previousDate, beforePreviousTransactionDate); + auto previousTransaction = mBlockchain->findOne(previousTransactionFilter); + + if (previousTransaction) { + auto accountBalance = previousTransaction->getConfirmedTransaction()->getAccountBalance(mPubkey, coinCommunityId); + if (accountBalance.getBalance() > GradidoUnit::zero()) { + previousBalance = accountBalance.getBalance(); + previousDate = previousTransaction->getConfirmedTransaction()->getConfirmedAt(); + } + } + } + createTransaction::Context createTransactionContext(mBlockchain, addressType); - for (auto& entry: allTransactionsVector) + for (auto& entry: allTransactions) { auto confirmedTransaction = entry->getConfirmedTransaction(); auto transactions = createTransactionContext.run(*confirmedTransaction, mPubkey); for (auto& transaction: transactions) { transactionsVector.push_back(transaction); + // TODO: choose correct coin color + auto balance = confirmedTransaction->getAccountBalance(mPubkey, coinCommunityId); transactionsVector.back().setPreviousBalance( previousBalance ); if (previousBalance > GradidoUnit::zero()) { - transactionsVector.back().calculateDecay(previousDate, confirmedTransaction->getConfirmedAt(), previousBalance); + transactionsVector.back().setDecay(previousDate, confirmedTransaction->getConfirmedAt(), previousBalance); } previousDate = confirmedTransaction->getConfirmedAt(); - auto& balances = confirmedTransaction->getAccountBalances(); - previousBalance = GradidoUnit::zero(); - for (auto& balance : balances) { - // calculate sum of all balances belonging to this user, of all coin color - // TODO: choose correct coin color - if (balance.getPublicKey()->isTheSame(mPubkey)) { - previousBalance += balance.getBalance(); - } - } + previousBalance = transactionsVector.back().getBalance(); } } - allTransactionsVector.clear(); + allTransactions.clear(); if (transactionsVector.empty()) { transactionList.AddMember("transactions", Value(kArrayType), alloc); return std::move(transactionList); diff --git a/src/model/Apollo/TransactionList.h b/src/model/Apollo/TransactionList.h index 9049402b..e86d279b 100644 --- a/src/model/Apollo/TransactionList.h +++ b/src/model/Apollo/TransactionList.h @@ -8,7 +8,7 @@ namespace gradido { namespace blockchain { class Abstract; class TransactionEntry; - class Filter; + struct Filter; } } diff --git a/src/model/Apollo/createTransaction/AbstractTransactionRole.h b/src/model/Apollo/createTransaction/AbstractTransactionRole.h index c2e6af7d..ad81235c 100644 --- a/src/model/Apollo/createTransaction/AbstractTransactionRole.h +++ b/src/model/Apollo/createTransaction/AbstractTransactionRole.h @@ -29,6 +29,7 @@ namespace model { ) = 0; protected: + std::shared_ptr mBlockchain; }; } diff --git a/src/model/Apollo/createTransaction/CreationTransactionRole.cpp b/src/model/Apollo/createTransaction/CreationTransactionRole.cpp index a616d48d..de8e15e2 100644 --- a/src/model/Apollo/createTransaction/CreationTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/CreationTransactionRole.cpp @@ -16,11 +16,11 @@ namespace model { Transaction result(confirmedTransaction, pubkey); result.setType(TransactionType::CREATE); - auto creation = transactionBody->getCreation(); - result.setAmount(creation->getRecipient().getAmount()); - result.setFirstName("Gradido"); - result.setLastName("Akademie"); - result.setPubkey(gradidoTransaction->getSignatureMap().getSignaturePairs().front().getPublicKey()); + auto creation = transactionBody->getCreation(); + result.setAmount(creation->getRecipient().getAmount()); + result.setFirstName("Gradido"); + result.setLastName("Akademie"); + result.setPubkey(gradidoTransaction->getSignatureMap().getSignaturePairs().front().getPublicKey()); return result; } } diff --git a/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp index b89c9c8c..2f02e588 100644 --- a/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/DeferredTransferTransactionRole.cpp @@ -15,22 +15,21 @@ namespace model { Transaction result(confirmedTransaction, pubkey); auto deferredTransfer = transactionBody->getDeferredTransfer(); - auto transfer = deferredTransfer->getTransfer(); + const auto& transfer = deferredTransfer->getTransfer(); auto amount = transfer.getSender().getAmount(); if (transfer.getRecipient()->isTheSame(pubkey)) { - result.setType(TransactionType::LINK_CHARGE); - result.setPubkey(transfer.getSender().getPublicKey()); - } - else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { - result.setType(TransactionType::LINK_SEND); - result.setPubkey(transfer.getRecipient()); - amount.negate(); - } else { + result.setType(TransactionType::LINK_CHARGE); + result.setPubkey(transfer.getSender().getPublicKey()); + } else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { + result.setType(TransactionType::LINK_SEND); + result.setPubkey(transfer.getRecipient()); + amount.negate(); + } else { throw GradidoNodeInvalidDataException("unhandled case in model::Apollo::createTransaction::DeferredTransferTransactionRole if pubkey is neither sender or recipient"); - } + } - result.setAmount(amount); + result.setAmount(amount); return result; } } diff --git a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp index 8b798057..3a99efab 100644 --- a/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/RedeemDeferredTransferTransactionRole.cpp @@ -19,18 +19,17 @@ namespace model { Transaction result(confirmedTransaction, pubkey); auto redeemDeferredTransfer = transactionBody->getRedeemDeferredTransfer(); - auto transfer = redeemDeferredTransfer->getTransfer(); + const auto& transfer = redeemDeferredTransfer->getTransfer(); auto amount = transfer.getSender().getAmount(); if (transfer.getRecipient()->isTheSame(pubkey)) { result.setType(TransactionType::LINK_RECEIVE); result.setPubkey(transfer.getSender().getPublicKey()); - } - else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { - result.setType(TransactionType::SEND); - result.setPubkey(transfer.getRecipient()); - amount.negate(); - } else { + } else if (transfer.getSender().getPublicKey()->isTheSame(pubkey)) { + result.setType(TransactionType::SEND); + result.setPubkey(transfer.getRecipient()); + amount.negate(); + } else { amount = calculateChange( redeemDeferredTransfer->getDeferredTransferTransactionNr(), confirmedTransaction.getConfirmedAt(), @@ -38,22 +37,16 @@ namespace model { ).getBalance(); result.setType(TransactionType::LINK_CHANGE); result.setPubkey(transfer.getSender().getPublicKey()); - } + } if (data::AddressType::DEFERRED_TRANSFER == mAddressType) { auto change = calculateChange( redeemDeferredTransfer->getDeferredTransferTransactionNr(), confirmedTransaction.getConfirmedAt(), transfer.getSender() ); - printf( - "RedeemDeferredTransferTransactionRole::createTransaction setting change amount: %s, negated: %s, pubkey: %s\n", - change.getBalance().toString().data(), - change.getBalance().negated().toString().data(), - change.getPublicKey()->convertToHex().data() - ); result.setChange(change.getBalance().negated(), change.getPublicKey()); } - result.setAmount(amount); + result.setAmount(amount); return result; } @@ -69,7 +62,7 @@ namespace model { return { decayedAccountBalance.getPublicKey(), decayedAccountBalance.getBalance() - transferAmount.getAmount(), - decayedAccountBalance.getCommunityId() + decayedAccountBalance.getCoinCommunityIdIndex() }; } @@ -90,7 +83,7 @@ namespace model { targetDate ); - return data::AccountBalance(transferAmount.getPublicKey(), decayed, transferAmount.getCommunityId()); + return data::AccountBalance(transferAmount.getPublicKey(), decayed, transferAmount.getCoinCommunityIdIndex()); } } } diff --git a/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp b/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp index 20f964dc..e40494d9 100644 --- a/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/TimeoutDeferredTransferTransactionRole.cpp @@ -1,5 +1,6 @@ #include "TimeoutDeferredTransferTransactionRole.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/blockchain/Abstract.h" using namespace gradido; @@ -9,10 +10,10 @@ namespace model { Transaction TimeoutDeferredTransferTransactionRole::createTransaction( const data::ConfirmedTransaction& confirmedTransaction, - memory::ConstBlockPtr pubkey + memory::ConstBlockPtr pubkey ) { auto gradidoTransaction = confirmedTransaction.getGradidoTransaction(); - auto transactionBody = gradidoTransaction->getTransactionBody(); + auto transactionBody = gradidoTransaction->getTransactionBody(); assert(transactionBody->isTimeoutDeferredTransfer()); Transaction result(confirmedTransaction, pubkey); @@ -21,10 +22,18 @@ namespace model { timeoutDeferredTransfer->getDeferredTransferTransactionNr(), confirmedTransaction.getConfirmedAt() ); + const auto& deferredTransferEntry = mBlockchain->getTransactionForId(timeoutDeferredTransfer->getDeferredTransferTransactionNr()); + const auto& deferredTransferBody = deferredTransferEntry->getTransactionBody(); + assert(deferredTransferBody->isDeferredTransfer()); + const auto& deferredTransfer = deferredTransferBody->getDeferredTransfer(); result.setType(TransactionType::LINK_TIMEOUT); - result.setAmount(changeAccountBalance.getBalance()); - result.setPubkey(changeAccountBalance.getPublicKey()); + auto balance = changeAccountBalance.getBalance(); + if (pubkey->isTheSame(deferredTransfer->getRecipientPublicKey())) { + balance.negate(); + } + result.setAmount(balance); + result.setPubkey(changeAccountBalance.getPublicKey()); return result; } } diff --git a/src/model/Apollo/createTransaction/TransferTransactionRole.cpp b/src/model/Apollo/createTransaction/TransferTransactionRole.cpp index b400d7d7..db03457d 100644 --- a/src/model/Apollo/createTransaction/TransferTransactionRole.cpp +++ b/src/model/Apollo/createTransaction/TransferTransactionRole.cpp @@ -14,21 +14,20 @@ namespace model { assert(transactionBody->isTransfer()); Transaction result(confirmedTransaction, pubkey); - auto transfer = transactionBody->getTransfer(); + const auto& transfer = transactionBody->getTransfer(); auto amount = transfer->getSender().getAmount(); if (transfer->getRecipient()->isTheSame(pubkey)) { - result.setType(TransactionType::RECEIVE); - result.setPubkey(transfer->getSender().getPublicKey()); - } - else if (transfer->getSender().getPublicKey()->isTheSame(pubkey)) { - result.setType(TransactionType::SEND); - result.setPubkey(transfer->getRecipient()); - amount.negate(); - } else { + result.setType(TransactionType::RECEIVE); + result.setPubkey(transfer->getSender().getPublicKey()); + } else if (transfer->getSender().getPublicKey()->isTheSame(pubkey)) { + result.setType(TransactionType::SEND); + result.setPubkey(transfer->getRecipient()); + amount.negate(); + } else { throw GradidoNodeInvalidDataException("unhandled case in model::Apollo::createTransaction::TransferTransactionRole if pubkey is neither sender or recipient"); - } - result.setAmount(amount); + } + result.setAmount(amount); return result; } } diff --git a/src/model/files/Block.cpp b/src/model/files/Block.cpp index 1952e351..9840a3a8 100644 --- a/src/model/files/Block.cpp +++ b/src/model/files/Block.cpp @@ -6,10 +6,11 @@ #include "../../ServerGlobals.h" #include "../../SingletonManager/FileLockManager.h" #include "../../SingletonManager/CacheManager.h" +#include "../../task/RebuildBlockIndexTask.h" #include "gradido_blockchain/data/ConfirmedTransaction.h" #include "gradido_blockchain/interaction/deserialize/Context.h" -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "loguru/loguru.hpp" @@ -93,7 +94,7 @@ namespace model { if (startReading > mCurrentFileSize - minimalFileSize) { throw EndReachingException("file is to small for read request", mBlockPath.data(), startReading, minimalFileSize); } - Profiler timeUsed; + MonotonicTimer timeUsed; //Poco::FastMutex::ScopedLock lock(mFastMutex); auto fl = FileLockManager::getInstance(); if (!fl->tryLockTimeout(mBlockPath, 100)) { @@ -140,6 +141,86 @@ namespace model { auto transactionSize = readLine(startReading, &result); return result; } + bool Block::readBuffered(grd_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken/* = std::stop_token()*/) + { + if (stopToken.stop_requested()) { + return false; + } + assert(alloc && callback); + + auto fileStream = getOpenFile(); + if (fileStream->fail()) { + throw std::runtime_error("[model::files::Block::readBuffered] file stream is failing!"); + } + auto minimalFileSize = sizeof(uint16_t) + MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE; + if (mCurrentFileSize <= minimalFileSize) { + throw EndReachingException("file is smaller than minimal block size", mBlockPath.data(), 0, minimalFileSize); + } + auto fl = FileLockManager::getInstance(); + if (!fl->tryLockTimeout(mBlockPath, 100)) { + throw LockException("cannot lock file for reading", mBlockPath.data()); + } + + uint16_t transactionSize = 0; + // call seek only if it is really necessary + // for example if a block file is read in complete line by line on program startup + // https://stackoverflow.com/questions/2438953/how-is-fseek-implemented-in-the-filesystem + // "One observation I have made about fseek on Solaris, is that each call to it resets the read buffer of the FILE.The next read will then always read a full block(8K by default)." + // https://bytes.com/topic/c/answers/218188-fseek-speed + // "However, a side - effect of the fseek is the flushing of the buffer.Without + // the fseek(), your output will(actually, I suppose "can" is correct in the + // general sense) be buffered, and only written when the buffer fille.With + // the fseek(), you are forcing the buffer to be written for every character." + // https://stackoverflow.com/questions/9349470/whats-the-difference-between-fseek-lseek-seekg-seekp + // "The difference between the various seek functions is just the kind of file/stream objects on which they operate. + // On Linux, seekg and fseek are probably implemented in terms of lseek." + + fileStream->seekg(0, std::ios_base::beg); + int32_t readed = 0; + unsigned char hash[crypto_generichash_KEYBYTES]; + memset(hash, 0, sizeof hash); + + while (fileStream->good() && readed < mCurrentFileSize && !stopToken.stop_requested()) { + auto fileCursor = readed; + fileStream->read((char*)&transactionSize, sizeof(uint16_t)); + readed += sizeof(uint16_t); + if (readed + transactionSize > mCurrentFileSize) { + fl->unlock(mBlockPath); + throw EndReachingException("file is to small for transaction size", mBlockPath.data(), readed, transactionSize); + } + if (transactionSize < MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE) { + fl->unlock(mBlockPath); + throw InvalidReadBlockSize("transactionSize is to small to contain a transaction", mBlockPath.data(), readed, transactionSize); + } + auto memStart = alloc->last_index; + grd_memory_block buffer; + grd_memory_block_alloc(&buffer, alloc, transactionSize); + if (alloc->out_of_memory_capacity) { + // TODO: own exception + throw GradidoNodeInvalidDataException("memory buffer to small"); + } + fileStream->read(reinterpret_cast(buffer.data), buffer.size); + readed += transactionSize; + calculateOneHashStep(hash, buffer.data, buffer.size); + callback->finishedLine(memStart, transactionSize, fileCursor); + } + callback->flush(); + if (!stopToken.stop_requested()) { + unsigned char hash2[crypto_generichash_KEYBYTES]; + fileStream->read((char*)hash2, crypto_generichash_KEYBYTES); + int filePointer = fileStream->tellg(); + if (0 != sodium_memcmp(hash, hash2, crypto_generichash_KEYBYTES)) { + throw HashMismatchException( + "block hash mismatch", + memory::Block(sizeof hash, hash), + memory::Block(sizeof hash2, hash2) + ); + } + } + + fl->unlock(mBlockPath); + return true; + } int32_t Block::appendLine(memory::ConstBlockPtr line) @@ -201,7 +282,7 @@ namespace model { std::shared_ptr Block::calculateHash() { - Profiler timeUsed; + MonotonicTimer timeUsed; auto fl = FileLockManager::getInstance(); if (mCurrentFileSize == 0) { @@ -267,50 +348,7 @@ namespace model { return result; } - - std::shared_ptr Block::rebuildBlockIndex(std::shared_ptr blockchain) - { - auto fl = FileLockManager::getInstance(); - std::shared_ptr rebuildTask = std::make_shared(blockchain); - - int32_t fileCursor = 0; - std::shared_ptr readBuffer; - unsigned char hash[crypto_generichash_KEYBYTES]; - memset(hash, 0, sizeof hash); - - // read in every line - while (fileCursor + sizeof(uint16_t) + MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE <= mCurrentFileSize) { - auto lineSize = readLine(fileCursor, &readBuffer); - rebuildTask->pushLine(fileCursor, readBuffer); - calculateOneHashStep(hash, (const unsigned char*)readBuffer->data(), readBuffer->size()); - fileCursor += lineSize + sizeof(uint16_t); - } - - unsigned char hash2[crypto_generichash_KEYBYTES]; - if (!fl->tryLockTimeout(mBlockPath, 100)) { - throw LockException("couldn't lock file in time", mBlockPath.data()); - } - auto fileStream = getOpenFile(); - fileStream->read((char*)hash2, crypto_generichash_KEYBYTES); - fl->unlock(mBlockPath); - - bool result = false; - if (0 == sodium_memcmp(hash, hash2, crypto_generichash_KEYBYTES)) { - result = true; - } - - if (result) { - return rebuildTask; - } - else { - throw HashMismatchException( - "block hash mismatch", - memory::Block(sizeof hash, hash), - memory::Block(sizeof hash2, hash2) - ); - } - } - + uint32_t Block::findLastBlockFileInFolder(std::string_view groupFolderPath) { std::filesystem::path groupFolder(groupFolderPath); @@ -358,43 +396,5 @@ namespace model { return 0; } - - - RebuildBlockIndexTask::RebuildBlockIndexTask(std::shared_ptr blockchain) - : task::CPUTask(ServerGlobals::g_CPUScheduler), mBlockchain(blockchain) - { - - } - - int RebuildBlockIndexTask::run() - { - while (!mPendingFileCursorLine.empty()) - { - std::pair> fileCursorLine; - if (!mPendingFileCursorLine.pop(fileCursorLine)) { - throw std::runtime_error("don't get next file cursor line"); - } - auto& serializedTransaction = fileCursorLine.second; - deserialize::Context deserializer(serializedTransaction, deserialize::Type::CONFIRMED_TRANSACTION); - deserializer.run(); - if (!deserializer.isConfirmedTransaction()) { - throw InvalidGradidoTransaction("invalid transaction from block file while rebuilding block index", serializedTransaction); - } - lock(); - std::shared_ptr transactionEntry = std::make_shared( - deserializer.getConfirmedTransaction(), - mBlockchain, - fileCursorLine.first - ); - mTransactionEntries.push_back(transactionEntry); - unlock(); - } - return 0; - } - - void RebuildBlockIndexTask::pushLine(int32_t fileCursor, std::shared_ptr line) - { - mPendingFileCursorLine.push({ fileCursor, line }); - } } } diff --git a/src/model/files/Block.h b/src/model/files/Block.h index f44fe3ef..e888ceec 100644 --- a/src/model/files/Block.h +++ b/src/model/files/Block.h @@ -7,11 +7,21 @@ #include "../../task/CPUTask.h" +#include + #include +#include +#include //! MAGIC NUMBER: use to check if a file is big enough to could contain a transaction #define MAGIC_NUMBER_MINIMAL_TRANSACTION_SIZE 25 +struct grd_memory; + +namespace cache { + class BlockIndex; +} + namespace controller { class AddressIndex; } @@ -23,10 +33,27 @@ namespace gradido { } } +namespace memory { + class Block; + using BlockPtr = std::shared_ptr; +} + +namespace task { + class RebuildBlockIndexTask; +} + namespace model { namespace files { class RebuildBlockIndexTask; + class IBlockBufferRead + { + public: + virtual void finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) = 0; + // will be called after last line was finished + virtual void flush() = 0; + }; + class Block : public TimerCallback { public: @@ -44,6 +71,8 @@ namespace model { //! \return size of line (without size field in file) uint16_t readLine(uint32_t startReading, memory::BlockPtr* buffer); std::shared_ptr readLine(uint32_t startReading); + // read whole file, validate hash + bool readBuffered(grd_memory* alloc, IBlockBufferRead* callback, std::stop_token stopToken = std::stop_token()); //! \brief call appendLines //! \return file cursor pos at start from this line in file (0 at start of file) @@ -58,9 +87,6 @@ namespace model { // very expensive, read in whole file and calculate hash bool validateHash(); - // read whole file, validate hash - std::shared_ptr rebuildBlockIndex(std::shared_ptr blockchain); - static uint32_t findLastBlockFileInFolder(std::string_view groupFolderPath); protected: @@ -100,27 +126,7 @@ namespace model { std::vector mCursorPositions; }; - //! TODO: update for able to start with first line, while calling function is still loading more and more lines from file - //! gives the additional option to prevent task for storing to many lines at once - //! use ability of Task Object for resheduling - class RebuildBlockIndexTask : public task::CPUTask - { - public: - RebuildBlockIndexTask(std::shared_ptr blockchain); - const char* getResourceType() const { return "RebuildBlockIndexTask"; }; - - int run(); - //! \param line will be moved - void pushLine(int32_t fileCursor, std::shared_ptr line); - const std::list>& getTransactionEntries() const { return mTransactionEntries; } - - inline bool isPendingQueueEmpty() { return mPendingFileCursorLine.empty(); } - - protected: - std::shared_ptr mBlockchain; - std::list> mTransactionEntries; - MultithreadQueue>> mPendingFileCursorLine; - }; + } } diff --git a/src/model/files/BlockIndex.cpp b/src/model/files/BlockIndex.cpp index 10474dfb..6b22cc11 100644 --- a/src/model/files/BlockIndex.cpp +++ b/src/model/files/BlockIndex.cpp @@ -31,6 +31,7 @@ namespace model { vFile->write(&fileCursor, sizeof(int32_t)); vFile->write(&transactionType, sizeof(TransactionType)); vFile->write(&coinCommunityIdIndex, sizeof(uint32_t)); + vFile->write(&isBalanceChanging, sizeof(uint8_t)); vFile->write(&addressIndicesCount, sizeof(uint8_t)); //vFile->write(this, sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint16_t)); @@ -51,9 +52,10 @@ namespace model { "gradido::data::TransactionType", std::to_string((uint8_t)transactionType).data() ); - } - if(!vFile->read(&coinCommunityIdIndex, sizeof(uint32_t))) return false; - if(!vFile->read(&addressIndicesCount, sizeof(uint8_t))) return false; + } + if (!vFile->read(&coinCommunityIdIndex, sizeof(uint32_t))) return false; + if (!vFile->read(&isBalanceChanging, sizeof(uint8_t))) return false; + if (!vFile->read(&addressIndicesCount, sizeof(uint8_t))) return false; auto addressIndexSize = sizeof(uint32_t) * addressIndicesCount; addressIndices = (uint32_t*)malloc(addressIndexSize); @@ -71,17 +73,16 @@ namespace model { crypto_generichash_update(state, (const unsigned char*)&fileCursor, sizeof(int32_t)); crypto_generichash_update(state, (const unsigned char*)&coinCommunityIdIndex, sizeof(uint32_t)); crypto_generichash_update(state, (const unsigned char*)&addressIndicesCount, sizeof(uint8_t)); + crypto_generichash_update(state, (const unsigned char*)&isBalanceChanging, sizeof(uint8_t)); // second part crypto_generichash_update(state, (const unsigned char*)addressIndices, sizeof(uint32_t) * addressIndicesCount); } - std::shared_ptr BlockIndex::DataBlock::createTransactionEntry(date::month month, date::year year) + std::shared_ptr BlockIndex::DataBlock::createTransactionEntry(date::month month, date::year year, uint32_t blockchainCommunityIdIndex) { - auto coinCommunityId = FileBasedProvider::getInstance()->getCommunityIdString(coinCommunityIdIndex); - // TransactionEntry(uint64_t transactionNr, int32_t fileCursor, uint8_t month, uint16_t year, uint32_t* addressIndices, uint8_t addressIndiceCount); auto transactionEntry = std::make_shared( - transactionNr, month, year, transactionType, coinCommunityId, addressIndices, addressIndicesCount + transactionNr, month, year, transactionType, coinCommunityIdIndex, addressIndices, addressIndicesCount, blockchainCommunityIdIndex ); transactionEntry->setFileCursor(fileCursor); return transactionEntry; @@ -89,8 +90,8 @@ namespace model { // ************************************************************************** - BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr) - : mDataBlockSumSize(0), mFileName(groupFolderPath) + BlockIndex::BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex) + : mDataBlockSumSize(0), mFileName(groupFolderPath), mBlockchainCommunityIdIndex(blockchainCommunityIdIndex) { std::stringstream fileNameStream; fileNameStream << "/blk" << std::setw(8) << std::setfill('0') << blockNr << ".index"; @@ -195,8 +196,8 @@ namespace model { return false; } - unsigned char hashFromFile[crypto_generichash_BYTES]; - unsigned char hashCalculated[crypto_generichash_BYTES]; + unsigned char hashFromFile[crypto_generichash_BYTES]; memset(hashFromFile, 0, crypto_generichash_BYTES); + unsigned char hashCalculated[crypto_generichash_BYTES]; memset(hashCalculated, 0, crypto_generichash_BYTES); crypto_generichash_state state; crypto_generichash_init(&state, nullptr, 0, crypto_generichash_BYTES); @@ -249,7 +250,8 @@ namespace model { dataBlock->coinCommunityIdIndex, yearCursor, monthCursor, dataBlock->transactionNr, dataBlock->fileCursor, - dataBlock->addressIndices, dataBlock->addressIndicesCount + dataBlock->addressIndices, dataBlock->addressIndicesCount, + dataBlock->isBalanceChanging ); } diff --git a/src/model/files/BlockIndex.h b/src/model/files/BlockIndex.h index af5f6811..e32fe232 100644 --- a/src/model/files/BlockIndex.h +++ b/src/model/files/BlockIndex.h @@ -6,6 +6,7 @@ #include "../../lib/VirtualFile.h" #include "date/date.h" +#include #include #include @@ -34,7 +35,8 @@ namespace model { uint64_t transactionNr, int32_t fileCursor, const uint32_t* addressIndices, - uint16_t addressIndiceCount + uint16_t addressIndiceCount, + uint8_t isBalanceChanging ) = 0; }; @@ -53,7 +55,7 @@ namespace model { { public: //! create filename from path and blocknr - BlockIndex(std::string_view groupFolderPath, uint32_t blockNr); + BlockIndex(std::string_view groupFolderPath, uint32_t blockNr, uint32_t blockchainCommunityIdIndex); //! use full filename which includes also the block nr BlockIndex(std::string_view filename); ~BlockIndex(); @@ -71,9 +73,10 @@ namespace model { int32_t fileCursor, gradido::data::TransactionType transactionType, uint32_t coinCommunityIdIndex, + uint8_t isBalanceChanging, const std::vector& addressIndices ) { - mDataBlocks.push(new DataBlock(transactionNr, fileCursor, transactionType, coinCommunityIdIndex, addressIndices)); + mDataBlocks.push(new DataBlock(transactionNr, fileCursor, transactionType, coinCommunityIdIndex, isBalanceChanging, addressIndices)); mDataBlockSumSize += mDataBlocks.back()->size(); } @@ -99,9 +102,6 @@ namespace model { protected: //! \brief replace Index File with new one, clear blocks after writing into file - - - enum BlockTypes { YEAR_BLOCK = 0xad, MONTH_BLOCK = 0x50, @@ -178,19 +178,23 @@ namespace model { int32_t _fileCursor, gradido::data::TransactionType _transactionType, uint32_t _coinCommunityIdIndex, - const std::vector& _addressIndices + uint8_t _isBalanceChanging, + const std::vector& _addressIndices ) : Block(DATA_BLOCK), transactionNr(_transactionNr), fileCursor(_fileCursor), transactionType(_transactionType), coinCommunityIdIndex(_coinCommunityIdIndex), + isBalanceChanging(_isBalanceChanging), addressIndices(nullptr), - addressIndicesCount(_addressIndices.size()) + addressIndicesCount(_addressIndices.size()) { - addressIndices = (uint32_t*)malloc(addressIndicesCount * sizeof(uint32_t)); - assert(addressIndices); - memcpy(addressIndices, _addressIndices.data(), addressIndicesCount * sizeof(uint32_t)); + if (addressIndicesCount) { + addressIndices = (uint32_t*)malloc(addressIndicesCount * sizeof(uint32_t)); + assert(addressIndices); + memcpy(addressIndices, _addressIndices.data(), addressIndicesCount * sizeof(uint32_t)); + } } DataBlock() : @@ -199,8 +203,9 @@ namespace model { fileCursor(-10), transactionType(gradido::data::TransactionType::NONE), coinCommunityIdIndex(0), + isBalanceChanging(0), addressIndices(nullptr), - addressIndicesCount(0) + addressIndicesCount(0) { } @@ -211,13 +216,15 @@ namespace model { addressIndices = nullptr; addressIndicesCount = 0; fileCursor = 0; + isBalanceChanging = 0; } uint64_t transactionNr; int32_t fileCursor; gradido::data::TransactionType transactionType; uint32_t coinCommunityIdIndex; + uint8_t isBalanceChanging; uint8_t addressIndicesCount; - uint32_t* addressIndices; + uint32_t* addressIndices; size_t size() { return sizeof(uint8_t) // Block Type @@ -225,17 +232,20 @@ namespace model { + sizeof(int32_t) // fileCursor + sizeof(gradido::data::TransactionType) // transaction type + sizeof(uint32_t) // coin community id index size - + sizeof(uint8_t) + sizeof(uint32_t) * addressIndicesCount; // address index count, address indices array + + sizeof(uint8_t) // isBalanceChanging + + sizeof(uint8_t) + sizeof(uint32_t) * addressIndicesCount // address index count, address indices array + ; } virtual void writeIntoFile(VirtualFile* vFile); virtual bool readFromFile(VirtualFile* vFile); virtual void updateHash(crypto_generichash_state* state); - std::shared_ptr createTransactionEntry(date::month month, date::year year); + std::shared_ptr createTransactionEntry(date::month month, date::year year, uint32_t blockchainCommunityIdIndex); }; std::string mFileName; + uint32_t mBlockchainCommunityIdIndex; std::queue mDataBlocks; size_t mDataBlockSumSize; }; diff --git a/src/model/files/LMDBWrapper.cpp b/src/model/files/LMDBWrapper.cpp new file mode 100644 index 00000000..e69de29b diff --git a/src/model/files/LMDBWrapper.h b/src/model/files/LMDBWrapper.h new file mode 100644 index 00000000..e69de29b diff --git a/src/model/files/LevelDBWrapper.cpp b/src/model/files/LevelDBWrapper.cpp index dea8030e..ee959ee0 100644 --- a/src/model/files/LevelDBWrapper.cpp +++ b/src/model/files/LevelDBWrapper.cpp @@ -1,5 +1,6 @@ #include "LevelDBWrapper.h" #include "../../lib/LevelDBExceptions.h" +#include "../../SingletonManager/FileLockManager.h" #include "loguru/loguru.hpp" #include "leveldb/cache.h" @@ -9,48 +10,62 @@ #include #include +using std::chrono::milliseconds, std::this_thread::sleep_for; +using std::filesystem::remove_all; +using std::optional, std::nullopt, std::string, std::string_view, std::function; +using leveldb::Status, leveldb::DB, leveldb::Slice, leveldb::ReadOptions, leveldb::WriteOptions, leveldb::NewLRUCache; + namespace model { namespace files { - - // use this global mutex to prevent an error with level db if one group is deconstructed while the same group is created new at the same time - std::mutex g_StateMutex; - LevelDBWrapper::LevelDBWrapper(std::string_view folderName) + LevelDBWrapper::LevelDBWrapper(string_view folderName) : mFolderName(folderName), mLevelDB(nullptr) { } LevelDBWrapper::~LevelDBWrapper() { - exit(); + if (mLevelDB) { + exit(); + } } bool LevelDBWrapper::init(size_t cacheInByte/* = 0*/) { - std::lock_guard _lock(g_StateMutex); + auto fm = FileLockManager::getInstance(); + if (!fm->tryLockTimeout(mFolderName, 100)) { + LOG_F(ERROR, "path: %s couldn't locked, another process still use this folder?", mFolderName.c_str()); + return false; + } mOptions.create_if_missing = true; mOptions.paranoid_checks = true; if (cacheInByte) { - mOptions.block_cache = leveldb::NewLRUCache(cacheInByte); + mOptions.block_cache = NewLRUCache(cacheInByte); } - leveldb::Status status = leveldb::DB::Open(mOptions, mFolderName, &mLevelDB); + Status status = DB::Open(mOptions, mFolderName, &mLevelDB); // if blockchain::FileBased is removed from cache and created new at the same time, the lock file from other level db instance is maybe still there // and trigger an io error, so give it same time an try it again, maximal 100 times. // TODO: Maybe use the FileLockManager for this int maxTry = 100; while (status.IsIOError() && maxTry > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - status = leveldb::DB::Open(mOptions, mFolderName, &mLevelDB); + sleep_for(milliseconds(100)); + status = DB::Open(mOptions, mFolderName, &mLevelDB); maxTry--; } + fm->unlock(mFolderName); if (!status.ok()) { LOG_F(ERROR, "path: %s, state: %s, ioError: %d", mFolderName.data(), status.ToString().data(), status.IsIOError()); } return status.ok(); } + void LevelDBWrapper::exit() { - std::lock_guard _lock(g_StateMutex); + auto fm = FileLockManager::getInstance(); + if (!fm->tryLockTimeout(mFolderName, 1000)) { + LOG_F(FATAL, "on exit: path: %s couldn't locked, another process still use this folder, data maybe corrupted, please refresh!", mFolderName.c_str()); + return; + } if (mLevelDB) { delete mLevelDB; mLevelDB = nullptr; @@ -59,38 +74,43 @@ namespace model { mOptions.block_cache = nullptr; } } + fm->unlock(mFolderName); } void LevelDBWrapper::reset() { exit(); - std::filesystem::remove_all(mFolderName); + remove_all(mFolderName); init(); } - bool LevelDBWrapper::getValueForKey(const char* key, std::string* value) + optional LevelDBWrapper::getValueForKey(const std::string& key) { - leveldb::Status s = mLevelDB->Get(leveldb::ReadOptions(), key, value); - return s.ok(); + string value; + Status s = mLevelDB->Get(ReadOptions(), key, &value); + if (!s.ok()) { return nullopt; } + return value; } - void LevelDBWrapper::setKeyValue(const char* key, const std::string& value) + void LevelDBWrapper::setKeyValue(const std::string& key, const std::string& value) { - leveldb::Status s = mLevelDB->Put(leveldb::WriteOptions(), key, value); + WriteOptions writeOptions; + writeOptions.sync = false; + Status s = mLevelDB->Put(writeOptions, key, value); if (!s.ok()) { throw LevelDBStatusException("cannot put to level db", s); } } - void LevelDBWrapper::removeKey(const char* key) + void LevelDBWrapper::removeKey(const std::string& key) { - mLevelDB->Delete(leveldb::WriteOptions(), key); + mLevelDB->Delete(WriteOptions(), key); } - void LevelDBWrapper::iterate(std::function callback) + void LevelDBWrapper::iterate(function callback) { //! \brief get iterator for looping over every entry - leveldb::ReadOptions options; + ReadOptions options; options.fill_cache = false; options.verify_checksums = true; auto it = mLevelDB->NewIterator(options); diff --git a/src/model/files/LevelDBWrapper.h b/src/model/files/LevelDBWrapper.h index 10071b4e..45ef9d16 100644 --- a/src/model/files/LevelDBWrapper.h +++ b/src/model/files/LevelDBWrapper.h @@ -5,6 +5,7 @@ #include #include +#include /*! * @author Dario Rekowski @@ -33,13 +34,13 @@ namespace model { //! read value for key from leveldb //! \param value pointer to string in which result will be write //! \return true if value could be found - bool getValueForKey(const char* key, std::string* value); + std::optional getValueForKey(const std::string& key); //! add new value key pair or update value if key exist //! \return true, throw exception on error - void setKeyValue(const char* key, const std::string& value); + void setKeyValue(const std::string& key, const std::string& value); - void removeKey(const char* key); + void removeKey(const std::string& key); //! go through all entries and call callback for each with key, value //! don't fill level db cache, verify checksum diff --git a/src/serialization/String.cpp b/src/serialization/String.cpp new file mode 100644 index 00000000..6eae5fa7 --- /dev/null +++ b/src/serialization/String.cpp @@ -0,0 +1,39 @@ +#include "String.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/memory/Block.h" + +#include + +using gradido::data::ByteArray; +using memory::ConstBlockPtr, memory::Block; +using std::string; +using std::make_shared; + +namespace serialization { + template<> + string toString(const ConstBlockPtr& ptr) + { + return ptr->copyAsString(); + } + + template<> + ConstBlockPtr fromString(const char* data, size_t size) + { + return make_shared(size, reinterpret_cast(data)); + } + + template<> + string toString>(const ByteArray<32>& bytes) + { + return { (char*)bytes.data(), 32 }; + } + + template<> + ByteArray<32> fromString>(const char* data, size_t size) + { + if (size != 32) { + throw InvalidSizeException("fromString for ByteArray<32> called with non-32 sized string", 32, size); + } + return ByteArray<32>(reinterpret_cast(data)); + } +} \ No newline at end of file diff --git a/src/serialization/String.h b/src/serialization/String.h new file mode 100644 index 00000000..c1f5343a --- /dev/null +++ b/src/serialization/String.h @@ -0,0 +1,53 @@ +#ifndef __GRADIDO_NODE_SERIALIZATION_STRING_H +#define __GRADIDO_NODE_SERIALIZATION_STRING_H + +#include +#include + +namespace serialization { + template + std::string toString(const T& value); + + template + T fromString(const char* data, size_t size); + + template + concept HasString = + requires(const T& t, const char* c, size_t size) { + { toString(t) } -> std::same_as; + { fromString(c, size) } -> std::same_as; + }; + + // for string, it's only need's to copy + template<> + inline std::string toString(const std::string& s) { + return s; + } + + template<> + inline std::string fromString(const char* data, size_t size) { + return std::string(data, size); + } + + template<> + inline std::string toString(const uint32_t& v) { + return std::to_string(v); + } + + template<> + inline uint32_t fromString(const char* data, size_t size) { + return static_cast(strtoul(data, nullptr, 0)); + } + + template<> + inline std::string toString(const size_t& v) { + return std::to_string(v); + } + + template<> + inline size_t fromString(const char* data, size_t size) { + return strtoull(data, nullptr, 0); + } +} + +#endif //__GRADIDO_NODE_SERIALIZATION_STRING_H \ No newline at end of file diff --git a/src/serialization/cacheToJson.cpp b/src/serialization/cacheToJson.cpp index 9c145d8e..83c80e03 100644 --- a/src/serialization/cacheToJson.cpp +++ b/src/serialization/cacheToJson.cpp @@ -5,9 +5,9 @@ using namespace rapidjson; namespace serialization { - template<> + /*template<> Value toJson(const cache::BlockIndex& value, Document::AllocatorType& alloc) { return value.serializeToJson(alloc); - } + }*/ } \ No newline at end of file diff --git a/src/server/json-rpc/ApiHandler.cpp b/src/server/json-rpc/ApiHandler.cpp index a62715e9..0b246fed 100644 --- a/src/server/json-rpc/ApiHandler.cpp +++ b/src/server/json-rpc/ApiHandler.cpp @@ -1,27 +1,42 @@ #include "ApiHandler.h" +#include "fromJson.h" +#include "WireFilter.h" // need to be here, else it produce a linker error, or more precisly the member function generateList // TODO: fix the reason #include "../../model/Apollo/TransactionList.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" #include "gradido_blockchain/blockchain/FilterBuilder.h" +#include "gradido_blockchain/data/adapter/byteArray.h" +#include "gradido_blockchain/data/adapter/publicKey.h" +#include "gradido_blockchain/data/adapter/uuid.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/data/LedgerAnchor.h" +#include "gradido_blockchain/data/hiero/TransactionId.h" #include "gradido_blockchain/interaction/calculateAccountBalance/Context.h" #include "gradido_blockchain/interaction/calculateCreationSum/Context.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/interaction/serialize/Context.h" #include "gradido_blockchain/interaction/validate/Context.h" -#include "gradido_blockchain/serialization/toJson.h" #include "gradido_blockchain/lib/DataTypeConverter.h" -#include "gradido_blockchain/lib/Profiler.h" -#include "gradido_blockchain/data/ConfirmedTransaction.h" -#include "gradido_blockchain/data/hiero/TransactionId.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" +#include "gradido_blockchain/memory/Block.h" +#include "gradido_blockchain/serialization/toJson.h" #include "../../blockchain/FileBased.h" #include "../../blockchain/FileBasedProvider.h" #include "../../blockchain/NodeTransactionEntry.h" +#include #include "rapidjson/prettywriter.h" #include "magic_enum/magic_enum.hpp" #include "loguru/loguru.hpp" +#include #include using namespace rapidjson; @@ -32,6 +47,11 @@ using namespace serialization; using namespace data; using namespace magic_enum; +using std::optional, std::nullopt; +using gradido::g_appContext; +using gradido::data::compact::PublicKeyIndex, gradido::data::compact::ConfirmedTxs; +using gradido::data::adapter::uuidFromString; + namespace server { namespace json_rpc { @@ -84,6 +104,8 @@ namespace server { // load public key for nearly all requests memory::BlockPtr pubkey; + PublicKeyIndex publickKeyIndex; + std::string pubkeyHex; std::set noNeedForPubkey = { "getLastTransaction", "getTransactions","getTransaction", "findUserByNameHash" @@ -93,10 +115,11 @@ namespace server { return; } pubkey = std::make_shared(memory::Block::fromHex(pubkeyHex)); + publickKeyIndex = adapter::toPublicKeyIndex(pubkey, blockchain->getCommunityIdIndex()); } if (method == "getLastTransaction") { - Profiler timeUsed; + MonotonicTimer timeUsed; std::string format = "base64"; getStringParameter(responseJson, params, "format", format); auto lastTransaction = blockchain->findOne(Filter::LAST_TRANSACTION); @@ -122,21 +145,12 @@ namespace server { } // TODO: rename to listsinceblock else if (method == "getTransactions") { - std::string format; - uint64_t transactionId = 0; - uint32_t maxResultCount = 100; - - if (!getUInt64Parameter(responseJson, params, "fromTransactionId", transactionId) || - !getStringParameter(responseJson, params, "format", format)) { return; } - getUIntParameter(responseJson, params, "maxResultCount", maxResultCount, true); - //printf("group: %s, id: %d\n", groupAlias.data(), transactionId); - FilterBuilder builder; - auto filter = builder - .setMinTransactionNr(transactionId) - .setPagination({ maxResultCount }) - .setSearchDirection(SearchDirection::ASC) - .build(); - findAllTransactions(resultJson, filter, blockchain, format); + WireFilter filter; + auto result = fromJson(params, filter); + if (JsonParseResultType::Ok != result.type) { + error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, result.error.c_str()); + } + findAllTransactions(resultJson, filter.toCompactFilter(*g_appContext), blockchain, filter.format); } else if (method == "getAddressBalance") { std::string date_string; @@ -145,24 +159,25 @@ namespace server { } auto date = DataTypeConverter::dateTimeStringToTimePoint(date_string); - std::string coinCommunityId = ""; + optional coinCommunityId = nullopt; if (params.HasMember("coinCommunityId") && params["coinCommunityId"].IsString()) { - coinCommunityId = params["coinCommunityId"].GetString(); + auto coinCommunityIdIndexOptional = g_appContext->getCommunityIds().getIndexForData( + uuidFromString(params["coinCommunityId"].GetString()) + ); + if (coinCommunityIdIndexOptional) { + coinCommunityId = static_cast(coinCommunityIdIndexOptional); + } } getAddressBalance(resultJson, pubkey, date, blockchain, coinCommunityId); - } else if (method == "getAddressType") { getAddressType(resultJson, pubkey, blockchain); } - else if (method == "getAddressTxids") { - getAddressTxids(resultJson, pubkey, blockchain); - } else if (method == "getTransaction") { std::string format; uint64_t transactionId = 0; std::string hieroTransactionIdString; - hiero::TransactionId hieroTransactionId; + LedgerAnchor ledgerAnchor; std::shared_ptr iotaMessageId; if (!getStringParameter(responseJson, params, "format", format)) { @@ -171,17 +186,18 @@ namespace server { getUInt64Parameter(responseJson, params, "transactionId", transactionId, true); getStringParameter(responseJson, params, "hieroTransactionId", hieroTransactionIdString, true); getBinaryFromHexStringParameter(responseJson, params, "iotaMessageId", iotaMessageId, true); - if (!iotaMessageId && !hieroTransactionIdString.empty()) { - hieroTransactionId = hiero::TransactionId(hieroTransactionIdString); - serialize::Context serializeContext(hieroTransactionId); - iotaMessageId = serializeContext.run(); + if (iotaMessageId) { + error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, "iotaMessageId is not longer supported"); } - if (!transactionId && !iotaMessageId && hieroTransactionId.empty()) { - error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, "transactionId, hieroTransactionId or iotaMessageId needed"); + if (!hieroTransactionIdString.empty()) { + ledgerAnchor = LedgerAnchor(hiero::TransactionId(hieroTransactionIdString)); + } + if (!transactionId && !iotaMessageId && ledgerAnchor.empty()) { + error(responseJson, JSON_RPC_ERROR_INVALID_PARAMS, "transactionId or hieroTransactionId needed"); return; } - getTransaction(resultJson, responseJson, blockchain, format, transactionId, iotaMessageId); + getTransaction(resultJson, responseJson, blockchain, format, transactionId, &ledgerAnchor); } else if (method == "getCreationSumForMonth") { int month, year; @@ -198,13 +214,13 @@ namespace server { } auto date = DataTypeConverter::dateTimeStringToTimePoint(date_string); - getCreationSumForMonth(resultJson, pubkey, targetDate, date, blockchain); + getCreationSumForMonth(resultJson, publickKeyIndex, targetDate, date, blockchain); } // TODO: think about better name, explain that this is extra formatted for the gradido frontend, to mimic current graphql backend response else if (method == "listTransactions") { Filter f; f.pagination = Pagination(25, 1); - f.involvedPublicKey = pubkey; + f.updatedBalancePublicKey = pubkey; if (params.HasMember("currentPage") && params["currentPage"].IsInt()) { f.pagination.page = params["currentPage"].GetInt(); } @@ -254,39 +270,46 @@ namespace server { void ApiHandler::listCommunities(rapidjson::Value& resultJson) { - Profiler timeUsed; - auto alloc = mRootJson.GetAllocator(); - auto groups = FileBasedProvider::getInstance()->listCommunityIds(); - resultJson.AddMember("communities", toJson(groups, alloc), alloc); + MonotonicTimer timeUsed; + auto& alloc = mRootJson.GetAllocator(); + const auto& groupIndex = FileBasedProvider::getInstance()->getGroupIndex(); + Value communities(kArrayType); + groupIndex->iterate( + [&communities, &alloc](const cache::CommunityIndexEntry& comInfos) -> bool + { + Value community(kObjectType); + community.AddMember("communityId", toJson(comInfos.communityId, alloc), alloc); + community.AddMember("alias", toJson(comInfos.alias, alloc), alloc); + communities.PushBack(community, alloc); + return true; + } + ); + resultJson.AddMember("communities", communities, alloc); resultJson.AddMember("timeUsed", Value(timeUsed.string().data(), alloc).Move(), alloc); } void ApiHandler::findAllTransactions( rapidjson::Value& resultJson, - const Filter& filter, + const CompactFilter& filter, std::shared_ptr blockchain, - const std::string& format + WireOutputFormat format ) { - Profiler timeUsed; - auto alloc = mRootJson.GetAllocator(); + MonotonicTimer timeUsed; + auto& alloc = mRootJson.GetAllocator(); // count for pagination - uint64_t totalCount = 0; - Filter countFilter = filter; + CompactFilter countFilter = filter; countFilter.pagination = Pagination(); // remove pagination for count countFilter.minTransactionNr = 0; // remove minTransactionNr for count countFilter.maxTransactionNr = 0; // remove maxTransactionNr for count - countFilter.filterFunction = [&totalCount](const TransactionEntry& transactionEntry) { - totalCount++; - return FilterResult::DISMISS; - }; - blockchain->findAll(countFilter); + auto totalCount = blockchain->countAll(countFilter); + resultJson.AddMember("totalCount", totalCount, alloc); auto transactions = blockchain->findAll(filter); - if (format == "json") { + if (WireOutputFormat::Json == format) { resultJson.AddMember("type", "json", alloc); } else { @@ -294,10 +317,11 @@ namespace server { } Value jsonTransactionArray(kArrayType); for (auto it = transactions.begin(); it != transactions.end(); it++) { - auto transactionSerialized = (*it)->getSerializedTransaction(); + auto legacyTx = blockchain->getTransactionForId((*it)->txNr); + auto transactionSerialized = legacyTx->getSerializedTransaction(); if (transactionSerialized->size() > 0) { - if (format == "json") { - jsonTransactionArray.PushBack(toJson(*(*it)->getConfirmedTransaction(), alloc), alloc); + if (WireOutputFormat::Json == format) { + jsonTransactionArray.PushBack(toJson(*legacyTx->getConfirmedTransaction(), alloc), alloc); } else { auto base64TransactionString = transactionSerialized->convertToBase64(); @@ -309,15 +333,15 @@ namespace server { // read gmw and auf balance Timepoint now = std::chrono::system_clock::now(); calculateAccountBalance::Context calculateAddressBalance(blockchain); - auto communityRootEntry = blockchain->findOne(Filter::FIRST_TRANSACTION); + CompactFilter communityRootFindFilter; + communityRootFindFilter.searchDirection = SearchDirection::ASC; + communityRootFindFilter.pagination.size = 1; + auto communityRootEntry = blockchain->findOne(communityRootFindFilter); if (communityRootEntry) { - auto communityRootBody = communityRootEntry->getTransactionBody(); - assert(communityRootBody->isCommunityRoot()); - auto communityRoot = communityRootBody->getCommunityRoot(); - auto gmwAddress = communityRoot->getGmwPubkey(); - auto aufAddress = communityRoot->getAufPubkey(); - auto gmwBalance = calculateAddressBalance.fromEnd(gmwAddress, now, ""); - auto aufBalance = calculateAddressBalance.fromEnd(aufAddress, now, ""); + auto& tx = *communityRootEntry; + assert(tx.isCommunityRoot()); + auto gmwBalance = calculateAddressBalance.fromEnd(tx.getGmw(), now, blockchain->getCommunityIdIndex()); + auto aufBalance = calculateAddressBalance.fromEnd(tx.getAuf(), now, blockchain->getCommunityIdIndex()); resultJson.AddMember("gmwBalance", Value(gmwBalance.toString().data(), alloc), alloc); resultJson.AddMember("aufBalance", Value(aufBalance.toString().data(), alloc), alloc); } else { @@ -335,10 +359,10 @@ namespace server { std::shared_ptr blockchain, const std::string& format, uint64_t transactionId/* = 0*/, - std::shared_ptr iotaMessageId /* = nullptr */ + gradido::data::LedgerAnchor* ledgerAnchor/* = nullptr */ ) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); std::shared_ptr transactionEntry; @@ -346,9 +370,12 @@ namespace server { transactionEntry = blockchain->getTransactionForId(transactionId); } else { - transactionEntry = blockchain->findByMessageId(iotaMessageId); + if (ledgerAnchor && !ledgerAnchor->empty()) { + transactionEntry = blockchain->findByLedgerAnchor(*ledgerAnchor); + } } if (!transactionEntry) { + printf("not found after: %s\n", timeUsed.string().c_str()); error(responseJson, JSON_RPC_ERROR_TRANSACTION_NOT_FOUND, "transaction not found"); return; } @@ -375,17 +402,17 @@ namespace server { void ApiHandler::getCreationSumForMonth( rapidjson::Value& resultJson, - memory::ConstBlockPtr pubkey, + gradido::data::compact::PublicKeyIndex publicKeyIndex, Timepoint targetDate, Timepoint transactionCreationDate, std::shared_ptr blockchain ) { - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); assert(blockchain); - calculateCreationSum::Context calculateCreationSum(transactionCreationDate, targetDate, pubkey); + calculateCreationSum::Context calculateCreationSum(transactionCreationDate, targetDate, publicKeyIndex); auto sumString = calculateCreationSum.run(*blockchain).toString(); resultJson.AddMember("sum", Value(sumString.data(), sumString.size(), alloc), alloc); resultJson.AddMember("timeUsed", Value(timeUsed.string().data(), alloc).Move(), alloc); @@ -396,14 +423,13 @@ namespace server { memory::ConstBlockPtr pubkey, Timepoint date, std::shared_ptr blockchain, - const std::string& coinCommunityId /* = "" */ + optional coinCommunityIdIndex /* = nullopt */ ) { assert(blockchain); auto& alloc = mRootJson.GetAllocator(); calculateAccountBalance::Context calculateAccountBalance(blockchain); - // TODO: add coinCommunity�d Filter to calculateAccountBalance Context - auto balanceString = calculateAccountBalance.fromEnd(pubkey, date, coinCommunityId, 0).toString(); + auto balanceString = calculateAccountBalance.fromEnd(pubkey, date, coinCommunityIdIndex).toString(); resultJson.AddMember("balance", Value(balanceString.data(), balanceString.size(), alloc), alloc); } @@ -417,27 +443,6 @@ namespace server { resultJson.AddMember("addressType", Value(typeString.data(), typeString.size(), alloc), alloc); } - void ApiHandler::getAddressTxids(Value& resultJson, memory::ConstBlockPtr pubkey, std::shared_ptr blockchain) - { - assert(blockchain); - assert(pubkey); - - auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); - assert(fileBasedBlockchain); - - auto transactionNrs = fileBasedBlockchain->findAllFast({ 0, 0, pubkey }); - - auto alloc = mRootJson.GetAllocator(); - Value transactionNrsJson(kArrayType); - for (auto& transactionNr : transactionNrs) { - transactionNrsJson.PushBack(transactionNr, alloc); - } - - resultJson.AddMember("transactionNrs", transactionNrsJson, alloc); - } - - - void ApiHandler::listTransactions( Value& resultJson, std::shared_ptr blockchain, @@ -458,17 +463,16 @@ namespace server { "query": "query ($currentPage: Int = 1, $pageSize: Int = 25, $order: Order = DESC, $onlyCreations: Boolean = false) {\n transactionList(\n currentPage: $currentPage\n pageSize: $pageSize\n order: $order\n onlyCreations: $onlyCreations\n ) {\n gdtSum\n count\n balance\n decay\n decayDate\n transactions {\n type\n balance\n decayStart\n decayEnd\n decayDuration\n memo\n transactionId\n name\n email\n date\n decay {\n balance\n decayStart\n decayEnd\n decayDuration\n decayStartBlock\n __typename\n }\n firstTransaction\n __typename\n }\n __typename\n }\n}\n" } */ - Profiler timeUsed; + MonotonicTimer timeUsed; auto& alloc = mRootJson.GetAllocator(); - model::Apollo::TransactionList transactionList(blockchain, filter.involvedPublicKey); + model::Apollo::TransactionList transactionList(blockchain, filter.updatedBalancePublicKey); Timepoint now = std::chrono::system_clock::now(); auto transactionListValue = transactionList.generateList(now, filter, mRootJson); calculateAccountBalance::Context calculateAddressBalance(blockchain); - // TODO: add balances from another communities - auto balance = calculateAddressBalance.fromEnd(filter.involvedPublicKey, now, ""); + auto balance = calculateAddressBalance.fromEnd(filter.updatedBalancePublicKey, now, filter.coinCommunityIdIndex); std::string balanceString = balance.toString(); transactionListValue.AddMember("balance", Value(balanceString.data(), balanceString.size(), alloc), alloc); resultJson.AddMember("transactionList", transactionListValue, alloc); @@ -483,7 +487,7 @@ namespace server { std::shared_ptr blockchain ) { - Profiler timeUsed; + MonotonicTimer timeUsed; Filter f; f.involvedPublicKey = pubkey; f.minTransactionNr = firstTransactionNr; @@ -506,15 +510,20 @@ namespace server { std::shared_ptr blockchain ) { - Profiler timeUsed; + MonotonicTimer timeUsed; Filter f; + auto nameHashId = g_appContext->getUserNameHashs().getIndexForData(adapter::toByteArray<32>(nameHash)); + if (!nameHashId) { + error(responseJson, JSON_RPC_ERROR_ADDRESS_NOT_FOUND, "user not found"); + return; + } f.transactionType = data::TransactionType::REGISTER_ADDRESS; // std::function filterFunction; - f.filterFunction = [nameHash](const TransactionEntry& entry) { + f.filterFunction = [nameHashId](const TransactionEntry& entry) { auto body = entry.getTransactionBody(); assert(body->isRegisterAddress()); auto registerAddress = body->getRegisterAddress(); - if (registerAddress->getNameHash()->isTheSame(nameHash)) { + if (nameHashId == static_cast(registerAddress->nameHashIndex)) { return FilterResult::USE | FilterResult::STOP; } return FilterResult::DISMISS; @@ -528,9 +537,9 @@ namespace server { assert(body); auto registerAddress = body->getRegisterAddress(); assert(registerAddress); - auto accountPubkey = registerAddress->getAccountPublicKey(); - assert(accountPubkey); - resultJson.AddMember("pubkey", Value(accountPubkey->convertToHex().data(), alloc), alloc); + const auto& dict = blockchain->getPublicKeyDictionary(); + + resultJson.AddMember("pubkey", toJson(dict.getDataForIndexOrThrow(registerAddress->accountPublicKeyIndex).convertToHex(), alloc), alloc); } else { error(responseJson, JSON_RPC_ERROR_ADDRESS_NOT_FOUND, "user not found"); diff --git a/src/server/json-rpc/ApiHandler.h b/src/server/json-rpc/ApiHandler.h index d599daa4..febfb54f 100644 --- a/src/server/json-rpc/ApiHandler.h +++ b/src/server/json-rpc/ApiHandler.h @@ -3,17 +3,39 @@ #include "RequestHandler.h" #include "gradido_blockchain/types.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" + +#include +#include +#include + namespace gradido { namespace blockchain { class Abstract; class Filter; + struct CompactFilter; + } + namespace data { + class LedgerAnchor; + namespace compact { + class ConfirmedGradidoTx; + using ConstConfirmedTxPtr = std::shared_ptr; + using ConfirmedTxs = std::vector; + } } } +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr ; +} + namespace server { namespace json_rpc { + enum class WireOutputFormat; + // TODO: write api doc and help on command class ApiHandler : public RequestHandler { @@ -29,16 +51,16 @@ namespace server { */ void findAllTransactions( rapidjson::Value& resultJson, - const gradido::blockchain::Filter& filter, + const gradido::blockchain::CompactFilter& filter, std::shared_ptr blockchain, - const std::string& format + WireOutputFormat format ); /*! * TODO: implement index for iota message id if it is used much * @param resultJson: for success result * @param responseJson: for the overall response, used for example for errors * @param transactionId: this parameter or - * @param iotaMessageId: this parameter for finding transaction + * @param ledgerAnchor: this parameter for finding transaction */ void getTransaction( rapidjson::Value& resultJson, @@ -46,12 +68,12 @@ namespace server { std::shared_ptr blockchain, const std::string& format, uint64_t transactionId = 0, - std::shared_ptr iotaMessageId = nullptr + gradido::data::LedgerAnchor* ledgerAnchor = nullptr ); //! \param searchStartDate start date for reverse search for creation transactions range -2 month from there void getCreationSumForMonth( rapidjson::Value& resultJson, - memory::ConstBlockPtr pubkey, + gradido::data::compact::PublicKeyIndex publicKeyIndex, Timepoint targetDate, Timepoint transactionCreationDate, std::shared_ptr blockchain @@ -61,18 +83,13 @@ namespace server { memory::ConstBlockPtr pubkey, Timepoint date, std::shared_ptr blockchain, - const std::string& coinCommunityId = "" + std::optional coinCommunityIdIndex = std::nullopt ); void getAddressType( rapidjson::Value& resultJson, memory::ConstBlockPtr pubkey, std::shared_ptr blockchain ); - void getAddressTxids( - rapidjson::Value& resultJson, - memory::ConstBlockPtr pubkey, - std::shared_ptr blockchain - ); void listTransactions( rapidjson::Value& resultJson, std::shared_ptr blockchain, @@ -91,8 +108,6 @@ namespace server { memory::ConstBlockPtr nameHash, std::shared_ptr blockchain ); - // helper - }; } } diff --git a/src/server/json-rpc/ApiHandlerFactory.h b/src/server/json-rpc/ApiHandlerFactory.h deleted file mode 100644 index 6d1e177e..00000000 --- a/src/server/json-rpc/ApiHandlerFactory.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef __GRADIDO_NODE_SERVER_JSON_RPC_API_HANDLER_FACTORY_H -#define __GRADIDO_NODE_SERVER_JSON_RPC_API_HANDLER_FACTORY_H - -#include "gradido_blockchain/http/AbstractResponseHandlerFactory.h" -#include "ApiHandler.h" - -namespace server { - namespace json_rpc { - class ApiHandlerFactory : public AbstractResponseHandlerFactory - { - public: - virtual std::unique_ptr getResponseHandler(MethodType method) { - return std::make_unique(); - }; - //! \return check if factory has a response handler for a specific method, else server didn't need to listen for requests - virtual bool has(MethodType method) { - switch (method) { - case MethodType::GET: - case MethodType::POST: - case MethodType::OPTIONS: return true; - default: return false; - } - }; - }; - } -} - -#endif //__GRADIDO_NODE_SERVER_JSON_RPC_API_HANDLER_FACTORY_H \ No newline at end of file diff --git a/src/server/json-rpc/RequestHandler.cpp b/src/server/json-rpc/RequestHandler.cpp index 135a75b5..fb626f23 100644 --- a/src/server/json-rpc/RequestHandler.cpp +++ b/src/server/json-rpc/RequestHandler.cpp @@ -1,6 +1,7 @@ #include "RequestHandler.h" #include "gradido_blockchain/lib/DataTypeConverter.h" +#include "gradido_blockchain/memory/Block.h" #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/http/ServerConfig.h" @@ -24,18 +25,22 @@ using namespace rapidjson; namespace server { namespace json_rpc { + static thread_local void* gtl_valueBuffer = malloc(RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY * 4); + RequestHandler::RequestHandler() - : mRootJson(kObjectType) + : mRootJson(kObjectType, new MemoryPoolAllocator<>(gtl_valueBuffer, RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY * 4)) { - + // MemoryPoolAllocator<> valueAllocator(valueBuffer, valueBuffer->size()); + //MemoryPoolAllocator<> parseAllocator(parseBuffer, parseBuffer->size()); +// mRootJson = Document() } Value RequestHandler::handleOneRpcCall(const rapidjson::Value& jsonRpcRequest) { - LOG_F(2, "handleOneRpcCall"); + // LOG_F(2, "handleOneRpcCall"); Value responseJson(kObjectType); std::string method; - auto alloc = mRootJson.GetAllocator(); + auto& alloc = mRootJson.GetAllocator(); if (checkObjectOrArrayParameter(responseJson, jsonRpcRequest, "params") && getStringParameter(responseJson, jsonRpcRequest, "method", method)) { try { @@ -61,72 +66,71 @@ namespace server { return responseJson; } - void RequestHandler::handleRequest(const httplib::Request& request, httplib::Response& response, MethodType method) + void RequestHandler::cors(httplib::Response& response) { - loguru::set_thread_name("json-rpc"); if (ServerConfig::g_AllowUnsecureFlags & ServerConfig::UNSECURE_CORS_ALL) { response.set_header("Access-Control-Allow-Origin", "*"); response.set_header("Access-Control-Allow-Headers", "Authorization, Access-Control-Allow-Headers, Origin,Accept, X-Requested-With, Content-Type, Access-Control-Request-Method, Access-Control-Request-Headers"); } + } - auto alloc = mRootJson.GetAllocator(); + void RequestHandler::handlePostPut(const httplib::Request& request, httplib::Response& response) + { + loguru::set_thread_name("json-rpc"); + cors(response); + + auto& alloc = mRootJson.GetAllocator(); Value responseJson(kObjectType); responseJson.AddMember("jsonrpc", "2.0", alloc); responseJson.AddMember("id", Value(kNullType), alloc); - // TODO: put group name in request url to keep function calls as similar as possible to bitcoin and co - Document rapidjson_params; - if (MethodType::POST == method || MethodType::PUT == method) - { - rapidjson_params.Parse(request.body.data()); - if (rapidjson_params.HasParseError()) - { - Value data(kObjectType); - data.AddMember("parseError", Value(GetParseError_En(rapidjson_params.GetParseError()), alloc), alloc); - data.AddMember("errorOffset", rapidjson_params.GetErrorOffset(), alloc); - error(responseJson, JSON_RPC_ERROR_PARSE_ERROR, "error parsing request to json", &data); - } - else - { - if (rapidjson_params.IsObject()) { - responseJson = handleOneRpcCall(rapidjson_params); - } - else if (rapidjson_params.IsArray()) { - responseJson = Value(kArrayType); - for (auto& v : rapidjson_params.GetArray()) { - responseJson.PushBack(handleOneRpcCall(v), alloc); - } - } - else { - error(responseJson, JSON_RPC_ERROR_INVALID_REQUEST, "empty body"); - } - } - } - else if (MethodType::GET == method) - { - parseQueryParametersToRapidjson(request.params, rapidjson_params); - - //rapid_json_result = handle(rapidjson_params); - LOG_F(ERROR, "[%s:%d] must be implemented\n", __FUNCTION__, __LINE__); - } - else if (MethodType::OPTIONS == method) + Document rapidjson_params; + + rapidjson_params.Parse(request.body.data()); + if (rapidjson_params.HasParseError()) { - return; + Value data(kObjectType); + data.AddMember("parseError", Value(GetParseError_En(rapidjson_params.GetParseError()), alloc), alloc); + data.AddMember("errorOffset", rapidjson_params.GetErrorOffset(), alloc); + error(responseJson, JSON_RPC_ERROR_PARSE_ERROR, "error parsing request to json", &data); } else { - error(responseJson, JSON_RPC_ERROR_INVALID_REQUEST, "HTTP method unknown"); - } + if (rapidjson_params.IsObject()) { + responseJson = handleOneRpcCall(rapidjson_params); + } + else if (rapidjson_params.IsArray()) { + responseJson = Value(kArrayType); + for (auto& v : rapidjson_params.GetArray()) { + responseJson.PushBack(handleOneRpcCall(v), alloc); + } + } + else { + error(responseJson, JSON_RPC_ERROR_INVALID_REQUEST, "empty body"); + } + } // 3. Stringify the DOM - StringBuffer buffer; - Writer writer(buffer); + static thread_local void* writeMemoryBuffer = malloc(RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY); + MemoryPoolAllocator<> writeBufferAllocator(writeMemoryBuffer, RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY); + GenericStringBuffer, MemoryPoolAllocator<>> buffer(&writeBufferAllocator, RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY); + + // StringBuffer buffer(); + Writer writer(buffer); responseJson.Accept(writer); - // printf("response: %s\n", buffer.GetString()); response.set_content(buffer.GetString(), "application/json"); } + void RequestHandler::handleOptions(const httplib::Request& request, httplib::Response& response) + { + cors(response); + } + + void RequestHandler::handleGet(const httplib::Request& request, httplib::Response& response) + { + throw GradidoNotImplementedException("REST Api not implemented yet"); + } bool RequestHandler::parseQueryParametersToRapidjson(const std::multimap& params, Document& rapidParams) { diff --git a/src/server/json-rpc/RequestHandler.h b/src/server/json-rpc/RequestHandler.h index 4be09d82..9a96262a 100644 --- a/src/server/json-rpc/RequestHandler.h +++ b/src/server/json-rpc/RequestHandler.h @@ -1,8 +1,6 @@ #ifndef __GRADIDO_NODE_SERVER_JSON_RPC_REQUEST_HANDLER_H #define __GRADIDO_NODE_SERVER_JSON_RPC_REQUEST_HANDLER_H -#include "cpp-httplib/httplib.h" - #include "gradido_blockchain/GradidoBlockchainException.h" #include "gradido_blockchain/http/JsonRPCRequest.h" // need JsonRPCErrorCodes from that #include "gradido_blockchain/http/AbstractResponseHandler.h" @@ -11,12 +9,15 @@ namespace server { namespace json_rpc { - class RequestHandler : public AbstractResponseHandler + class RequestHandler { public: RequestHandler(); - virtual void handleRequest(const httplib::Request& request, httplib::Response& response, MethodType method); + void handlePostPut(const httplib::Request& request, httplib::Response& response); + void handleOptions(const httplib::Request& request, httplib::Response& response); + void handleGet(const httplib::Request& request, httplib::Response& response); + static void cors(httplib::Response& response); rapidjson::Value handleOneRpcCall(const rapidjson::Value& jsonRpcRequest); //virtual Poco::JSON::Object* handle(Poco::Dynamic::Var params) = 0; @@ -48,6 +49,7 @@ namespace server { void error(rapidjson::Value& responseJson, JsonRPCErrorCodes code, GradidoBlockchainException& ex); protected: + rapidjson::Document mRootJson; }; diff --git a/src/server/json-rpc/WireFilter.cpp b/src/server/json-rpc/WireFilter.cpp new file mode 100644 index 00000000..379d7e47 --- /dev/null +++ b/src/server/json-rpc/WireFilter.cpp @@ -0,0 +1,83 @@ +#include "WireFilter.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" +#include "gradido_blockchain/blockchain/PublicKeySearchType.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/data/TransactionType.h" + +using gradido::AppContext; +using gradido::blockchain::CompactFilter, gradido::blockchain::PublicKeySearchType, gradido::blockchain::SearchDirection; +using gradido::data::compact::PublicKeyIndex; +using gradido::data::TransactionType; + +namespace server::json_rpc { + WireFilter::WireFilter() + : searchDirection(SearchDirection::DESC), transactionType(TransactionType::NONE), + publicKeySearchType(PublicKeySearchType::None), format(WireOutputFormat::Base64), + maxTransactionNr(0), minTransactionNr(0), pagination(20) + { + } + + CompactFilter WireFilter::toCompactFilter(AppContext& appContext) const + { + CompactFilter resultFilter; + resultFilter.searchDirection = searchDirection; + resultFilter.transactionType = transactionType; + resultFilter.publicKeySearchType = publicKeySearchType; + if (!coinCommunityId.empty()) { + resultFilter.coinCommunityIdIndex = appContext.getOrAddCommunityIdIndex(coinCommunityId); + } + resultFilter.maxTransactionNr = maxTransactionNr; + resultFilter.minTransactionNr = minTransactionNr; + if (!publicKey.isEmpty()) { + uint32_t communityIdIndex = 0; + if (!communityId.empty()) { + communityIdIndex = appContext.getOrAddCommunityIdIndex(communityId); + } + else if (resultFilter.coinCommunityIdIndex) { + communityIdIndex = resultFilter.coinCommunityIdIndex; + } + PublicKeyIndex publicKeyIndex{}; + if (communityIdIndex) { + auto& communityBlockchain = appContext.getCommunityContext(communityIdIndex).getBlockchain(); + if (!communityBlockchain) { + throw GradidoNullPointerException("missing blockchain for valid community id index", "shared_ptr", __FUNCTION__); + } + auto publicKeyIndexSizeT = communityBlockchain->getPublicKeyDictionary().getIndexForData(publicKey); + if (publicKeyIndexSizeT != (uint32_t)publicKeyIndexSizeT) { + throw DictionaryOverflowException("to big public key index found", "PublicKey"); + } + publicKeyIndex = { + .communityIdIndex = communityIdIndex, + .publicKeyIndex = (uint32_t)publicKeyIndexSizeT + }; + } + if (publicKeyIndex.empty()) { + // search in all communities for public key + for (uint32_t i = 1; publicKeyIndex.empty() && appContext.getCommunityIds().hasIndex(i); i++) + { + if (i == communityIdIndex) continue; + auto& communityBlockchain = appContext.getCommunityContext(i).getBlockchain(); + if (!communityBlockchain) { + throw GradidoNullPointerException("missing blockchain for valid community id index from loop", "shared_ptr", __FUNCTION__); + } + auto publicKeyIndexSizeT = communityBlockchain->getPublicKeyDictionary().getIndexForData(publicKey); + if (publicKeyIndexSizeT != (uint32_t)publicKeyIndexSizeT) { + throw DictionaryOverflowException("to big public key index found", "PublicKey"); + } + publicKeyIndex = { + .communityIdIndex = i, + .publicKeyIndex = (uint32_t)publicKeyIndexSizeT + }; + } + } + if (!publicKeyIndex.empty()) { + resultFilter.publicKeyIndex = publicKeyIndex; + } + } + resultFilter.pagination = pagination; + resultFilter.timepointInterval = timepointInterval; + return resultFilter; + } +} \ No newline at end of file diff --git a/src/server/json-rpc/WireFilter.h b/src/server/json-rpc/WireFilter.h new file mode 100644 index 00000000..01a45314 --- /dev/null +++ b/src/server/json-rpc/WireFilter.h @@ -0,0 +1,67 @@ +#ifndef GRADIDO_NODE_SERVER_JSON_RPC_WIRE_FILTER_H +#define GRADIDO_NODE_SERVER_JSON_RPC_WIRE_FILTER_H + +#include "gradido_blockchain/types.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" +#include "gradido_blockchain/blockchain/Pagination.h" +#include "gradido_blockchain/blockchain/PublicKeySearchType.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/data/TransactionType.h" +#include "gradido_blockchain/lib/TimepointInterval.h" + +#include +#include + +namespace gradido { + class AppContext; +} + +namespace server::json_rpc { + + enum class WireOutputFormat { + Base64, // protobuf serialized, encoded in base64 + Json // json format, encoded in string + }; + + struct WireFilter + { + WireFilter(); + //! search direction and result order, default: DESC + gradido::blockchain::SearchDirection searchDirection; + + //! transaction type + gradido::data::TransactionType transactionType; + + //! type of data publicKey contains + gradido::blockchain::PublicKeySearchType publicKeySearchType; + + //! format for result returned to caller + WireOutputFormat format; + + //! index starts with 1 + std::string communityId; + //! for colored coins, index starts with 1 + std::string coinCommunityId; + + //! transaction number to stop search, 0 means no stop + uint64_t maxTransactionNr; + //! transaction number to start from, 0 default + uint64_t minTransactionNr; + + //! return only transaction in which the public key is involved, either directly in the transaction or as signer + gradido::data::PublicKey publicKey; + + //! search result scope + gradido::blockchain::Pagination pagination; + //! interval between two dates with 1 month resolution + TimepointInterval timepointInterval; + + //! for the future, but not yet implemented + std::string luaFilterFunction; + + gradido::blockchain::CompactFilter toCompactFilter(gradido::AppContext& appContext) const; + }; +} + +#endif // GRADIDO_NODE_SERVER_JSON_RPC_WIRE_FILTER_H \ No newline at end of file diff --git a/src/server/json-rpc/fromJson.cpp b/src/server/json-rpc/fromJson.cpp new file mode 100644 index 00000000..d28e1534 --- /dev/null +++ b/src/server/json-rpc/fromJson.cpp @@ -0,0 +1,285 @@ +#include "fromJson.h" +#include "WireFilter.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/blockchain/CompactFilter.h" +#include "gradido_blockchain/blockchain/Pagination.h" +#include "gradido_blockchain/blockchain/SearchDirection.h" +#include "gradido_blockchain/CommunityContext.h" +#include "gradido_blockchain/data/ByteArray.h" +#include "gradido_blockchain/data/compact/PublicKeyIndex.h" +#include "gradido_blockchain/lib/DataTypeConverter.h" +#include "gradido_blockchain/lib/TimepointInterval.h" + +#include +#include +#include + +#include +#include +#include + +using namespace magic_enum; +using DataTypeConverter::dateTimeStringToTimePoint; +using gradido::AppContext; +using gradido::blockchain::CompactFilter, gradido::blockchain::Pagination, gradido::blockchain::SearchDirection; +using gradido::CommunityContext; +using gradido::data::compact::PublicKeyIndex; +using gradido::data::PublicKey; +using rapidjson::Value; +using std::string, std::string_view; + +namespace server::json_rpc { + + static JsonParseResult checkFieldName(const Value& json, const char* fieldName) + { + if (!json.HasMember(fieldName)) return { .type = JsonParseResultType::Missing_Field }; + if (json[fieldName].IsNull()) return { .type = JsonParseResultType::Null_Field }; + return { .type = JsonParseResultType::Ok }; + } + + static JsonParseResult isNonEmptyStringField(const Value& json, const char* fieldName) + { + auto result = checkFieldName(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + if (!json[fieldName].IsString()) { + string errorMessage = "expect string type for field: "; + errorMessage += fieldName; + return { + .type = JsonParseResultType::Type_Mismatch, + .error = errorMessage + }; + } + return { .type = JsonParseResultType::Ok }; + } + + static JsonParseResult isNonEmptyNumericalField(const Value& json, const char* fieldName) + { + auto result = checkFieldName(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + if (!json[fieldName].IsNumber()) { + string errorMessage = "expect number type for field: "; + errorMessage += fieldName; + return { + .type = JsonParseResultType::Type_Mismatch, + .error = errorMessage + }; + } + return { .type = JsonParseResultType::Ok }; + } + + static JsonParseResult isNonEmptyObject(const Value& json, const char* fieldName) + { + auto result = checkFieldName(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + if (!json[fieldName].IsObject()) { + string errorMessage = "expect object type for field: "; + errorMessage += fieldName; + return { + .type = JsonParseResultType::Type_Mismatch, + .error = errorMessage + }; + } + return { .type = JsonParseResultType::Ok }; + } + + // enum + template + requires std::is_enum_v + JsonParseResult fromJson(const Value& json, const char* fieldName, T& value) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + string_view valueString(json[fieldName].GetString(), json[fieldName].GetStringLength()); + auto optionalValue = enum_cast(valueString); + if (!optionalValue) + { + string errorMessage = fieldName; + errorMessage += ": "; + errorMessage += valueString; + errorMessage += " is invalid for enum "; + errorMessage += enum_type_name(); + return { + .type = JsonParseResultType::Invalid_Enum, + .error = errorMessage + }; + } + value = optionalValue.value(); + return { .type = JsonParseResultType::Ok }; + } + + // unsigned integer + static JsonParseResult fromJson(const Value& json, const char* fieldName, uint32_t& value) + { + auto result = isNonEmptyNumericalField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + value = json[fieldName].GetUint(); + return { .type = JsonParseResultType::Ok }; + } + + // unsigned long long + static JsonParseResult fromJson(const Value& json, const char* fieldName, uint64_t& value) + { + auto result = isNonEmptyNumericalField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + value = json[fieldName].GetUint64(); + return { .type = JsonParseResultType::Ok }; + } + + // string + static JsonParseResult fromJson(const Value& json, const char* fieldName, std::string& value) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + value = string(json[fieldName].GetString(), json[fieldName].GetStringLength()); + return { .type = JsonParseResultType::Ok }; + } + + // Timepoint + static JsonParseResult fromJson(const Value& json, const char* fieldName, Timepoint& value) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + // expect JavaScript DateTime String + // TODO: allow more formats, maybe with fmtLib + value = dateTimeStringToTimePoint(string(json[fieldName].GetString(), json[fieldName].GetStringLength()), "%FT%F"); + return { .type = JsonParseResultType::Ok }; + } + + // PublicKey + static JsonParseResult fromJson(const Value& json, const char* fieldName, PublicKey& publicKey) + { + auto result = isNonEmptyStringField(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + string_view valueString(json[fieldName].GetString(), json[fieldName].GetStringLength()); + if (valueString.size() != 65) { + string errorMessage = fieldName; + errorMessage += " hasn't expected size of 64 hex character + string end character"; + return { + .type = JsonParseResultType::Invalid_Field_Value, + .error = errorMessage + }; + } + + size_t resultBinSize = 0; + unsigned char buffer[32]; + if (0 != sodium_hex2bin(buffer, 32, valueString.data(), valueString.size(), nullptr, &resultBinSize, nullptr)) { + string errorMessage = fieldName; + errorMessage += " contain invalid hex"; + return { + .type = JsonParseResultType::Invalid_Hex, + .error = errorMessage + }; + } + + publicKey = PublicKey(buffer); + return { .type = JsonParseResultType::Ok }; + } + + // Pagination + static JsonParseResult fromJson(const Value& json, const char* fieldName, Pagination& pagination) + { + auto result = isNonEmptyObject(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + const auto& valuePagination = json[fieldName]; + // size + auto sizeResult = isNonEmptyNumericalField(valuePagination, "size"); + if (JsonParseResultType::Type_Mismatch == sizeResult.type) { + return sizeResult; + } + if (JsonParseResultType::Ok == sizeResult.type) { + pagination.size = valuePagination["size"].GetUint(); + } + if (pagination.size > 100) { + pagination.size = 100; + } + // page + auto pageResult = isNonEmptyNumericalField(valuePagination, "page"); + if (JsonParseResultType::Type_Mismatch == pageResult.type) { + return pageResult; + } + if (JsonParseResultType::Ok == pageResult.type) { + pagination.page = valuePagination["page"].GetUint(); + } + return { .type = JsonParseResultType::Ok }; + } + + // TimepointInterval + static JsonParseResult fromJson(const Value& json, const char* fieldName, TimepointInterval& timepointInterval) + { + auto result = isNonEmptyObject(json, fieldName); + if (result.type != JsonParseResultType::Ok) return result; + + const auto& valueTimepointIterval = json[fieldName]; + // start date + Timepoint tempTimepoint; + auto startDateResult = fromJson(valueTimepointIterval, "startDate", tempTimepoint); + if (JsonParseResultType::Ok == startDateResult.type) { + timepointInterval.setStartDate(tempTimepoint); + } + else if (JsonParseResultType::Type_Mismatch == startDateResult.type) { + return startDateResult; + } + // end date + auto endDateResult = fromJson(valueTimepointIterval, "endDate", tempTimepoint); + if (JsonParseResultType::Ok == endDateResult.type) { + timepointInterval.setEndDate(tempTimepoint); + } + else if (JsonParseResultType::Type_Mismatch == endDateResult.type) { + return endDateResult; + } + return { .type = JsonParseResultType::Ok }; + } + + // WireFilter + JsonParseResult fromJson(const Value& value, WireFilter& filter) + { + auto result = fromJson(value, "searchDirection", filter.searchDirection); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "transactionType", filter.transactionType); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "publicKeySearchType", filter.publicKeySearchType); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "format", filter.format); + if (JsonParseResultType::Invalid_Enum == result.type || JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "communityId", filter.communityId); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "coinCommunityId", filter.coinCommunityId); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "maxTransactionNr", filter.maxTransactionNr); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "minTransactionNr", filter.minTransactionNr); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "publicKey", filter.publicKey); + if ( + JsonParseResultType::Invalid_Field_Value == result.type || + JsonParseResultType::Invalid_Hex == result.type || + JsonParseResultType::Type_Mismatch == result.type + ) { + return result; + } + + result = fromJson(value, "pagination", filter.pagination); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + result = fromJson(value, "timepointInterval", filter.timepointInterval); + if (JsonParseResultType::Type_Mismatch == result.type) return result; + + return { .type = JsonParseResultType::Ok }; + } + +} \ No newline at end of file diff --git a/src/server/json-rpc/fromJson.h b/src/server/json-rpc/fromJson.h new file mode 100644 index 00000000..e98b51d0 --- /dev/null +++ b/src/server/json-rpc/fromJson.h @@ -0,0 +1,33 @@ +#ifndef GRADIDO_NODE_SERVER_JSON_RPC_FROM_JSON_H +#define GRADIDO_NODE_SERVER_JSON_RPC_FROM_JSON_H + +#include "WireFilter.h" + +#include + +#include + + +namespace server::json_rpc { + enum JsonParseResultType : uint8_t { + Ok, + Missing_Field, + Null_Field, + Invalid_Enum, + Invalid_Field_Value, + Invalid_Hex, + Type_Mismatch, + General_Error + }; + + struct JsonParseResult + { + JsonParseResultType type; + std::string error; + }; + + + JsonParseResult fromJson(const rapidjson::Value& value, WireFilter& filter); +} + +#endif // GRADIDO_NODE_SERVER_JSON_RPC_FROM_JSON_H \ No newline at end of file diff --git a/src/task/BatchDeserializeConfirmedTransactionTask.cpp b/src/task/BatchDeserializeConfirmedTransactionTask.cpp new file mode 100644 index 00000000..fdddda9c --- /dev/null +++ b/src/task/BatchDeserializeConfirmedTransactionTask.cpp @@ -0,0 +1,48 @@ +#include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/interaction/deserialize/Type.h" +#include "gradido_blockchain/GradidoBlockchainException.h" + +#include "BatchDeserializeConfirmedTransactionTask.h" +#include "../ServerGlobals.h" + +#include "loguru/loguru.hpp" + +using gradido::data::ConfirmedTransaction; +using std::shared_ptr; +using Deserializer = gradido::interaction::deserialize::Context; +using DeserializerType = gradido::interaction::deserialize::Type; + +namespace task { + BatchDeserializeConfirmedTransactionTask::BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions, uint32_t communityIdIndex) + : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransactions(std::move(rawTransactions)), mCommunityIdIndex(communityIdIndex) + { + } + + BatchDeserializeConfirmedTransactionTask::~BatchDeserializeConfirmedTransactionTask() + { + } + + int BatchDeserializeConfirmedTransactionTask::run() + { + mConfirmedTransactions.clear(); + mConfirmedTransactions.resize(mRawTransactions.size(), nullptr); + for (size_t i = 0; i < mRawTransactions.size(); i++) { + Deserializer deserializer(mRawTransactions[i], DeserializerType::CONFIRMED_TRANSACTION); + try { + deserializer.run(mCommunityIdIndex); + } + catch (std::exception& e) { + int zahl = 0; + LOG_F(ERROR, "error on deserialize: %s", e.what()); + return -1; + } + if (!deserializer.isConfirmedTransaction()) { + throw InvalidGradidoTransaction("invalid confirmed transaction", mRawTransactions[i]); + } + mConfirmedTransactions[i] = deserializer.getConfirmedTransaction(); + // printf("%d ", deserializer.getConfirmedTransaction()->getId()); + } + return 0; + } +} diff --git a/src/task/BatchDeserializeConfirmedTransactionTask.h b/src/task/BatchDeserializeConfirmedTransactionTask.h new file mode 100644 index 00000000..fd9e7787 --- /dev/null +++ b/src/task/BatchDeserializeConfirmedTransactionTask.h @@ -0,0 +1,39 @@ +#ifndef __GRADIDO_NODE_TASK_BATCH_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H +#define __GRADIDO_NODE_TASK_BATCH_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H + +#include "CPUTask.h" +#include +#include + +namespace gradido { + namespace data { + class ConfirmedTransaction; + } +} + +namespace memory { + class Block; + typedef std::shared_ptr ConstBlockPtr; +} + +namespace task { + + class BatchDeserializeConfirmedTransactionTask : public CPUTask { + public: + BatchDeserializeConfirmedTransactionTask(std::vector&& rawTransactions, uint32_t communityIdIndex); + virtual ~BatchDeserializeConfirmedTransactionTask(); + + const char* getResourceType() const override { return "BatchDeserializeConfirmedTransactionTask"; }; + int run() override; + + inline const std::vector>& getConfirmedTransactions() const { return mConfirmedTransactions; } + inline const std::vector& getRawTransactions() const { return mRawTransactions; } + + private: + std::vector mRawTransactions; + std::vector> mConfirmedTransactions; + uint32_t mCommunityIdIndex; + }; +} + +#endif // __GRADIDO_NODE_TASK_BATCH_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H \ No newline at end of file diff --git a/src/task/CPUSheduler.cpp b/src/task/CPUSheduler.cpp index c2e76744..1a8524a4 100644 --- a/src/task/CPUSheduler.cpp +++ b/src/task/CPUSheduler.cpp @@ -56,7 +56,7 @@ namespace task { // else put task to pending queue // printf("[CPUSheduler::sheduleTask] all %d threads in use \n", getThreadCount()); if (mFreeWorkerThreads.empty()) { - LOG_F(INFO, "sheduleTask in %s all %u threads in use, add to pending task list", mName.data(), getThreadCount()); + // LOG_F(INFO, "sheduleTask in %s all %u threads in use, add to pending task list", mName.data(), getThreadCount()); } { std::lock_guard _lock(mPendingTasksMutex); diff --git a/src/task/CPUShedulerThread.cpp b/src/task/CPUShedulerThread.cpp index 1d6159df..058e9d62 100644 --- a/src/task/CPUShedulerThread.cpp +++ b/src/task/CPUShedulerThread.cpp @@ -5,7 +5,7 @@ #include "../ServerGlobals.h" #ifdef _UNI_LIB_DEBUG -#include "gradido_blockchain/lib/Profiler.h" +#include "gradido_blockchain/lib/MonotonicTimer.h" #endif //_UNI_LIB_DEBUG #include "gradido_blockchain/Application.h" @@ -31,7 +31,7 @@ namespace task { { #ifdef _UNI_LIB_DEBUG - Profiler counter; + MonotonicTimer counter; //debug::CPUShedulerTasksLog* l = debug::CPUShedulerTasksLog::getInstance(); std::string name = mWaitingTask->getName(); //l->addTaskLogEntry((HASH)mWaitingTask.getResourcePtrHolder(), mWaitingTask->getResourceType(), mName.data(), name); diff --git a/src/task/DeserializeConfirmedTransactionTask.cpp b/src/task/DeserializeConfirmedTransactionTask.cpp new file mode 100644 index 00000000..3ffab56b --- /dev/null +++ b/src/task/DeserializeConfirmedTransactionTask.cpp @@ -0,0 +1,40 @@ +#include "gradido_blockchain/data/ConfirmedTransaction.h" +#include "gradido_blockchain/interaction/deserialize/Context.h" +#include "gradido_blockchain/interaction/deserialize/Type.h" +#include "gradido_blockchain/GradidoBlockchainException.h" + +#include "DeserializeConfirmedTransactionTask.h" +#include "../ServerGlobals.h" + +#include + +using gradido::data::ConfirmedTransaction; +using std::shared_ptr; +using Deserializer = gradido::interaction::deserialize::Context; +using DeserializerType = gradido::interaction::deserialize::Type; + +namespace task { + DeserializeConfirmedTransactionTask::DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction, uint32_t communityIdIndex) + : CPUTask(ServerGlobals::g_CPUScheduler), mRawTransaction(rawTransaction), mCommunityIdIndex(communityIdIndex) { + } + + DeserializeConfirmedTransactionTask::~DeserializeConfirmedTransactionTask() { + } + + int DeserializeConfirmedTransactionTask::run() { + Deserializer deserializer(mRawTransaction, DeserializerType::CONFIRMED_TRANSACTION); + try { + deserializer.run(mCommunityIdIndex); + } + catch (std::exception& e) { + int zahl = 0; + LOG_F(ERROR, "error on deserialize: %s", e.what()); + return -1; + } + if (!deserializer.isConfirmedTransaction()) { + throw InvalidGradidoTransaction("invalid confirmed transaction", mRawTransaction); + } + mConfirmedTransaction = deserializer.getConfirmedTransaction(); + return 0; + } +} diff --git a/src/task/DeserializeConfirmedTransactionTask.h b/src/task/DeserializeConfirmedTransactionTask.h new file mode 100644 index 00000000..ec14fdca --- /dev/null +++ b/src/task/DeserializeConfirmedTransactionTask.h @@ -0,0 +1,36 @@ +#ifndef __GRADIDO_NODE_TASK_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H +#define __GRADIDO_NODE_TASK_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H + +#include "CPUTask.h" +#include + +namespace gradido { + namespace data { + class ConfirmedTransaction; + } +} + +namespace memory { + class Block; + typedef std::shared_ptr ConstBlockPtr; +} + +namespace task { + class DeserializeConfirmedTransactionTask : public CPUTask { + public: + DeserializeConfirmedTransactionTask(memory::ConstBlockPtr rawTransaction, uint32_t communityIdIndex); + virtual ~DeserializeConfirmedTransactionTask(); + + const char* getResourceType() const override { return "DeserializeConfirmedTransactionTask"; }; + int run() override; + + inline std::shared_ptr getConfirmedTransaction() const { return mConfirmedTransaction; } + + private: + memory::ConstBlockPtr mRawTransaction; + std::shared_ptr mConfirmedTransaction; + uint32_t mCommunityIdIndex; + }; +} + +#endif // __GRADIDO_NODE_TASK_DESERIALIZE_CONFIRMED_TRANSACTIONTASK_H \ No newline at end of file diff --git a/src/task/HieroMessageToTransactionTask.cpp b/src/task/HieroMessageToTransactionTask.cpp index e135ec53..cdc6163c 100644 --- a/src/task/HieroMessageToTransactionTask.cpp +++ b/src/task/HieroMessageToTransactionTask.cpp @@ -1,7 +1,5 @@ #include "HieroMessageToTransactionTask.h" -#include "gradido_blockchain/lib/Profiler.h" - #include "../blockchain/FileBasedProvider.h" #include "../controller/SimpleOrderingManager.h" #include "gradido_blockchain/blockchain/Filter.h" @@ -56,7 +54,7 @@ namespace task { // deserialize deserialize::Context deserializer(mTransactionRaw, deserialize::Type::GRADIDO_TRANSACTION); - deserializer.run(); + deserializer.run(blockchain->getCommunityIdIndex()); if (deserializer.isGradidoTransaction()) { mTransaction = deserializer.getGradidoTransaction(); } @@ -81,11 +79,11 @@ namespace task { } // check if transaction already exist - // if this transaction doesn't belong to us, we can quit here + // if this transaction doesn't belong to us, we can quit here // also if we already have this transaction auto fileBasedBlockchain = std::dynamic_pointer_cast(blockchain); assert(fileBasedBlockchain); - if (fileBasedBlockchain->isTransactionExist(mTransaction)) { + if (fileBasedBlockchain->isTransactionExist(mTransaction, mConsensusTimestamp)) { LOG_F(INFO, "Transaction skipped (cached): %s", mConsensusTimestamp.toString().data()); return 0; } diff --git a/src/task/RebuildBlockIndexTask.cpp b/src/task/RebuildBlockIndexTask.cpp new file mode 100644 index 00000000..43ea2d1a --- /dev/null +++ b/src/task/RebuildBlockIndexTask.cpp @@ -0,0 +1,75 @@ +#include "RebuildBlockIndexTask.h" +#include "BatchDeserializeConfirmedTransactionTask.h" +#include "../ServerGlobals.h" +#include "../blockchain/FileBased.h" + +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain_core/data/wire/confirmed_transaction.h" +#include "gradido_blockchain_core/data/wire/transaction_body.h" +#include "gradido_blockchain/AppContext.h" +#include "gradido_blockchain/memory/Block.h" +#include "gradido_blockchain/serialization/toJsonString.h" + +#include "loguru/loguru.hpp" +#include "magic_enum/magic_enum.hpp" + +#include + +using gradido::g_appContext; +using gradido::blockchain::FileBased; +using gradido::data::compact::ConfirmedGradidoTx; +using namespace magic_enum; +using memory::Block; +using std::shared_ptr, std::make_shared, std::unique_lock; +using ServerGlobals::g_CPUScheduler; + +namespace task { + RebuildBlockIndexTask::RebuildBlockIndexTask( + std::shared_ptr blockIndex, + uint32_t communityIdIndex + ): task::CPUTask(g_CPUScheduler), mBlockIndex(blockIndex), mCommunityIdIndex(communityIdIndex), + mLastLineReaded(false) + { + grd_memory_init_arena_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + } + + int RebuildBlockIndexTask::run() + { + unique_lock lock(mWorkConfirmedMutex); + mConfirmedTxReadyCondition.wait(lock, [&] { return mLastLineReaded.load(); }); + return 0; + } + + void RebuildBlockIndexTask::finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) + { + grd_memory decodeMemoryAlloc; + grd_memory_init_arena_static(&decodeMemoryAlloc, mBuffers[1], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + grdw_confirmed_transaction tx{}; + grd_memory_block src = { .data = mBuffers[0], .size = size }; + auto result = grdw_confirmed_transaction_decode(&tx, &src, &decodeMemoryAlloc); + if (GRD_SUCCESS != result) { + LOG_F(ERROR, "decode error: %s", enum_name(result).data()); + throw GradidoNodeInvalidDataException("error deserialize confirmed transaction"); + } + + grdw_transaction_body body{}; + grd_memory_init_arena_static(&decodeMemoryAlloc, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + result = grdw_transaction_body_decode(&body, &tx.transaction.body_bytes, &decodeMemoryAlloc); + if (GRD_SUCCESS != result) { + LOG_F(ERROR, "body decode error: %s", enum_name(result).data()); + throw GradidoNodeInvalidDataException("error deserialize transaction body"); + } + + auto compactConfirmedTx = ConfirmedGradidoTx::fromGrdw(&tx, &body, mCommunityIdIndex, *g_appContext); + mBlockIndex->addIndicesForTransaction(compactConfirmedTx, g_appContext->getCommunityContext(mCommunityIdIndex).getBlockchain()->getPublicKeyDictionary()); + mBlockIndex->addFileCursorForTransaction(compactConfirmedTx.txNr, fileCursor); + + grd_memory_init_arena_static(&mReadInAllocator, mBuffers[0], REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE); + } + + void RebuildBlockIndexTask::flush() + { + mLastLineReaded = true; + mConfirmedTxReadyCondition.notify_one(); + } +} \ No newline at end of file diff --git a/src/task/RebuildBlockIndexTask.h b/src/task/RebuildBlockIndexTask.h new file mode 100644 index 00000000..f8e7817f --- /dev/null +++ b/src/task/RebuildBlockIndexTask.h @@ -0,0 +1,70 @@ +#ifndef __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H +#define __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H + +#include "CPUTask.h" +#include "../model/files/Block.h" +#include "gradido_blockchain_core/memory.h" +#include "gradido_blockchain/data/compact/ConfirmedGradidoTx.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" + +#include +#include +#include +#include +#include + + +namespace gradido { + namespace blockchain { + class FileBased; + class NodeTransactionEntry; + } +} + +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr; +} + +namespace cache { + class BlockIndex; +} + +// TODO: maybe put into config +#define REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE 2048 + +namespace task { + + //! remove dependencie to CPUTask because this isn't really a cpu task, more are result storage, + //! because it will start subsequent tasks of it own which will call back via command if finished + class RebuildBlockIndexTask : public CPUTask, public model::files::IBlockBufferRead + { + public: + RebuildBlockIndexTask( + std::shared_ptr blockIndex, + uint32_t communityIdIndex + ); + const char* getResourceType() const override { return "RebuildBlockIndexTask"; }; + + int run() override; + + virtual void finishedLine(uint16_t memStart, uint16_t size, int32_t fileCursor) override; + virtual void flush() override; + + inline grd_memory* getAlloc() { return &mReadInAllocator; } + + protected: + uint8_t mBuffers[2][REBUILD_BLOCK_INDEX_TASK_BUFFER_SIZE]; + + grd_memory mReadInAllocator; + std::mutex mWorkConfirmedMutex; + gradido::data::compact::ConfirmedGradidoTx mConfirmedTx; + std::condition_variable mConfirmedTxReadyCondition; + std::shared_ptr mBlockIndex; + uint32_t mCommunityIdIndex; + std::atomic mLastLineReaded; + }; + +} + +#endif // __GRADIDO_NODE_TASK_REBUILD_BLOCK_INDEX_TASK_H \ No newline at end of file diff --git a/src/task/SyncTopicOnStartup.cpp b/src/task/SyncTopic.cpp similarity index 78% rename from src/task/SyncTopicOnStartup.cpp rename to src/task/SyncTopic.cpp index 675bd690..4e0cd1fb 100644 --- a/src/task/SyncTopicOnStartup.cpp +++ b/src/task/SyncTopic.cpp @@ -1,26 +1,36 @@ -#include "SyncTopicOnStartup.h" +#include "SyncTopic.h" #include "../controller/SimpleOrderingManager.h" #include "../blockchain/FileBased.h" #include "../client/hiero/MirrorClient.h" #include "../ServerGlobals.h" +#include "gradido_blockchain/AppContext.h" #include "gradido_blockchain/blockchain/Filter.h" #include "gradido_blockchain/interaction/deserialize/Context.h" #include "gradido_blockchain/lib/DataTypeConverter.h" #include "gradido_blockchain/serialization/toJsonString.h" #include "loguru/loguru.hpp" +#include "magic_enum/magic_enum.hpp" + +#include +#include using namespace gradido; using namespace blockchain; using namespace interaction; +using namespace magic_enum; using namespace serialization; +using gradido::blockchain::FileBased; +using std::shared_ptr; +using std::string, std::to_string; + namespace task { - SyncTopicOnStartup::SyncTopicOnStartup( + SyncTopic::SyncTopic( uint64_t lastKnownSequenceNumber, hiero::TopicId lastKnowTopicId, - std::shared_ptr blockchain + shared_ptr blockchain ) : CPUTaskGRPCReactor(ServerGlobals::g_CPUScheduler), mLastKnownSequenceNumber(lastKnownSequenceNumber), mLastKnowTopicId(lastKnowTopicId), @@ -29,12 +39,12 @@ namespace task { } - SyncTopicOnStartup::~SyncTopicOnStartup() + SyncTopic::~SyncTopic() { } - int SyncTopicOnStartup::run() + int SyncTopic::run() { // check topic info const auto& topicInfo = mObject; @@ -45,10 +55,11 @@ namespace task { // for example: check previous transaction until one was found, // or check if maybe or block files get corrupted auto lastTransactionIdentical = checkLastTransaction(); + LOG_F(INFO, "last transaction state: %s, last known sequence number: %lu", enum_name(lastTransactionIdentical).data(), mLastKnownSequenceNumber); + + // (re-)start ordering manager + orderingManager->reinitialize(mLastKnownSequenceNumber); - // start ordering manager - orderingManager->init(mLastKnownSequenceNumber); - // load transactions from mirror node mConfirmedAtLastReadedTransaction = data::Timestamp(); if (LastTransactionState::IDENTICAL == lastTransactionIdentical) { @@ -81,7 +92,7 @@ namespace task { return 0; } - SyncTopicOnStartup::LastTransactionState SyncTopicOnStartup::checkLastTransaction() + SyncTopic::LastTransactionState SyncTopic::checkLastTransaction() { const auto& mirrorNode = ServerGlobals::g_HieroMirrorNode; @@ -108,16 +119,16 @@ namespace task { } deserialize::Context deserializer(lastTransactionMirrorRaw); - deserializer.run(); + deserializer.run(mBlockchain->getCommunityIdIndex()); if (!deserializer.isGradidoTransaction()) { LOG_F( - ERROR, + ERROR, "last transaction on mirrors is invalid Gradido Transaction. CommunityId: %s, lastKnownTopicId: %s, sequenceNumber: %lu, transactionNr: %lu, confirmedAt: %s", mBlockchain->getCommunityId().data(), mLastKnowTopicId.toString().data(), mLastKnownSequenceNumber, lastTransaction->getTransactionNr(), - lastTransaction->getConfirmedTransaction()->getConfirmedAt().toString().data() + confirmedAt.toString().data() ); return LastTransactionState::INVALID; } @@ -127,10 +138,10 @@ namespace task { bool prettyJson = true; LOG_F(ERROR, "own transaction: %s", toJsonString(*lastTransaction->getConfirmedTransaction(), prettyJson).data()); LOG_F( - ERROR, - "from topic: %s, sequenceNumber: %lu: %s", - mLastKnowTopicId.toString().data(), - mLastKnownSequenceNumber, + ERROR, + "from topic: %s, sequenceNumber: %lu: %s", + mLastKnowTopicId.toString().data(), + mLastKnownSequenceNumber, toJsonString(*lastTransactionMirror, prettyJson).data() ); return LastTransactionState::NOT_IDENTICAL; @@ -139,7 +150,7 @@ namespace task { return LastTransactionState::IDENTICAL; } - uint32_t SyncTopicOnStartup::loadTransactionsFromMirrorNode(hiero::TopicId topicId) + uint32_t SyncTopic::loadTransactionsFromMirrorNode(hiero::TopicId topicId) { const auto& mirrorNode = ServerGlobals::g_HieroMirrorNode; const auto& orderingManager = mBlockchain->getOrderingManager(); @@ -152,6 +163,15 @@ namespace task { auto responses = mirrorNode->listTopicMessagesById(topicId, mConfirmedAtLastReadedTransaction, limit, "asc"); if (responses.empty()) break; responsesCount = responses.size(); + if (responses.back().getConsensusTimestamp() <= mConfirmedAtLastReadedTransaction) { + LOG_F(WARNING, "unexpected response from mirror node, transaction has the same or lower consensus timestamp than last readed transaction. CommunityId: %s, topicId: %s, last readed consensus timestamp: %s, last readed sequence number: %lu, responses count: %u", + mBlockchain->getCommunityId().data(), + topicId.toString().data(), + mConfirmedAtLastReadedTransaction.toString().data(), + mLastKnownSequenceNumber, + responsesCount + ); + } mConfirmedAtLastReadedTransaction = responses.back().getConsensusTimestamp(); addedTransactionsSum += responsesCount; for (auto& response : responses) { diff --git a/src/task/SyncTopicOnStartup.h b/src/task/SyncTopic.h similarity index 89% rename from src/task/SyncTopicOnStartup.h rename to src/task/SyncTopic.h index 32065a0d..e47944c4 100644 --- a/src/task/SyncTopicOnStartup.h +++ b/src/task/SyncTopic.h @@ -15,14 +15,14 @@ namespace gradido { #define MAGIC_NUMBER_MIRROR_API_GET_TOPIC_MESSAGES_BULK_SIZE 25 namespace task { - class SyncTopicOnStartup: public CPUTaskGRPCReactor { + class SyncTopic: public CPUTaskGRPCReactor { public: - SyncTopicOnStartup( + SyncTopic( uint64_t lastKnownSequenceNumber, hiero::TopicId lastKnowTopicId, std::shared_ptr blockchain ); - virtual ~SyncTopicOnStartup(); + virtual ~SyncTopic(); int run() override; const char* getResourceType() const override { return "SyncTopicOnStartup"; } diff --git a/src/task/Task.h b/src/task/Task.h index 44d64389..d38a9407 100644 --- a/src/task/Task.h +++ b/src/task/Task.h @@ -46,6 +46,7 @@ namespace task { class Command { public: virtual ~Command() = default; + // TODO: check if int as return is needed and can be used or better void or better enum virtual int taskFinished(Task* task) = 0; }; diff --git a/src/task/WriteTransactionsToBlockTask.cpp b/src/task/WriteTransactionsToBlockTask.cpp index 0cd63a80..e42cf4d3 100644 --- a/src/task/WriteTransactionsToBlockTask.cpp +++ b/src/task/WriteTransactionsToBlockTask.cpp @@ -4,7 +4,9 @@ #include "../model/files/Block.h" #include "../ServerGlobals.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/GradidoBlockchainException.h" +#include "gradido_blockchain/memory/Block.h" #include "loguru/loguru.hpp" #include @@ -61,13 +63,16 @@ namespace task { return 0; } - void WriteTransactionsToBlockTask::addSerializedTransaction(std::shared_ptr transaction) + void WriteTransactionsToBlockTask::addSerializedTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary + ) { assert(!isTaskSheduled()); assert(!isTaskFinished()); std::lock_guard lock(mFastMutex); mTransactions.insert({transaction->getTransactionNr(), transaction}); - mBlockIndex->addIndicesForTransaction(transaction); + mBlockIndex->addTransactionIndices(transaction->convertToCompactConfirmedTx(), publicKeyDictionary); } std::shared_ptr WriteTransactionsToBlockTask::getTransaction(uint64_t nr) diff --git a/src/task/WriteTransactionsToBlockTask.h b/src/task/WriteTransactionsToBlockTask.h index 89af9ffe..fb713349 100644 --- a/src/task/WriteTransactionsToBlockTask.h +++ b/src/task/WriteTransactionsToBlockTask.h @@ -8,7 +8,11 @@ */ #include "CPUTask.h" +#include "gradido_blockchain/data/ByteArray.h" #include "gradido_blockchain/lib/MultithreadQueue.h" +#include "gradido_blockchain/lib/DictionaryInterface.h" + +#include namespace cache { class BlockIndex; @@ -26,6 +30,11 @@ namespace gradido { } } +namespace memory { + class Block; + using ConstBlockPtr = std::shared_ptr; +} + /*! * @author Dario Rekowski * @date 202-03-11 @@ -47,8 +56,11 @@ namespace task { //! no mutex lock, value doesn't change, set in WriteTransactionsToBlockTask() inline Timepoint getCreationDate() { return mCreationDate; } - void addSerializedTransaction(std::shared_ptr transaction); - + void addSerializedTransaction( + std::shared_ptr transaction, + IMutableDictionary& publicKeyDictionary + ); + //! return transaction by nr std::shared_ptr getTransaction(uint64_t nr);