From 28b769cd31edd69627c77c3e90d29b6ab2ec2933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Komor?= Date: Wed, 14 Oct 2026 10:09:10 +0200 Subject: [PATCH 1/2] Refactor Kafka transport interface and implementation - Updated the `ITransport` interface to return a `std::vector` from the `SendCommand` method, enhancing response handling. - Modified `TransportKafka` to align with the new interface, removing the previous response management system that relied on thread-specific storage. - Introduced new transport C ABI functions: `CreateTransport`, `DestroyTransport`, `TransportSendCommand`, and `TransportFreeBuffer` for better resource management. - Updated README documentation to reflect changes in transport function exports and usage. --- kafka/GraftcodePluginsInterfaces/ITransport.h | 4 +- .../TransportPluginAbi.h | 55 +++++++++ .../TransportPluginExports.h | 112 ++++++++++++++++++ kafka/KafkaPlugin/TransportKafka.cpp | 52 ++++---- kafka/KafkaPlugin/TransportKafka.h | 8 +- kafka/KafkaPluginTest/smoke.cpp | 26 ++-- kafka/Readme.md | 2 +- .../GraftcodePluginsInterfaces/ITransport.h | 4 +- .../TransportPluginAbi.h | 55 +++++++++ .../TransportPluginExports.h | 112 ++++++++++++++++++ pubsub/PubSubPlugin/TransportPubSub.cpp | 75 +++++------- pubsub/PubSubPlugin/TransportPubSub.h | 10 +- .../PubSubPluginTest/PubSubTransportTests.cpp | 46 ++++--- pubsub/PubSubPluginTest/smoke.cpp | 27 ++--- pubsub/Readme.md | 2 +- .../GraftcodePluginsInterfaces/ITransport.h | 15 +-- .../TransportPluginAbi.h | 55 +++++++++ .../TransportPluginExports.h | 112 ++++++++++++++++++ rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp | 67 ++++------- rabbitmq/RabbitmqPlugin/TransportRabbitMq.h | 8 +- .../RabbitMqTransportTests.cpp | 36 ++---- rabbitmq/Readme.md | 2 +- .../GraftcodePluginsInterfaces/ITransport.h | 15 +-- .../TransportPluginAbi.h | 55 +++++++++ .../TransportPluginExports.h | 112 ++++++++++++++++++ servicebus/Readme.md | 2 +- .../ServiceBusPlugin/TransportServiceBus.cpp | 67 ++++------- .../ServiceBusPlugin/TransportServiceBus.h | 8 +- .../ServiceBusTransportTests.cpp | 23 +++- sqs/GraftcodePluginsInterfaces/ITransport.h | 4 +- .../TransportPluginAbi.h | 55 +++++++++ .../TransportPluginExports.h | 112 ++++++++++++++++++ sqs/Readme.md | 2 +- sqs/SqsPlugin/TransportSqs.cpp | 74 +++++------- sqs/SqsPlugin/TransportSqs.h | 10 +- sqs/SqsPluginTest/SqsTransportTests.cpp | 46 ++++--- sqs/SqsPluginTest/smoke.cpp | 27 ++--- 37 files changed, 1113 insertions(+), 384 deletions(-) create mode 100644 kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h create mode 100644 kafka/GraftcodePluginsInterfaces/TransportPluginExports.h create mode 100644 pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h create mode 100644 pubsub/GraftcodePluginsInterfaces/TransportPluginExports.h create mode 100644 rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h create mode 100644 rabbitmq/GraftcodePluginsInterfaces/TransportPluginExports.h create mode 100644 servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h create mode 100644 servicebus/GraftcodePluginsInterfaces/TransportPluginExports.h create mode 100644 sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h create mode 100644 sqs/GraftcodePluginsInterfaces/TransportPluginExports.h diff --git a/kafka/GraftcodePluginsInterfaces/ITransport.h b/kafka/GraftcodePluginsInterfaces/ITransport.h index 3ab68b9..02c6334 100644 --- a/kafka/GraftcodePluginsInterfaces/ITransport.h +++ b/kafka/GraftcodePluginsInterfaces/ITransport.h @@ -1,12 +1,12 @@ #pragma once #include "_common.h" +#include namespace Hypertube::Native::Interfaces { class ITransport { public: virtual ~ITransport() = default; virtual int Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) = 0; - virtual int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; - virtual int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) = 0; + virtual std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; }; } diff --git a/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h b/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h new file mode 100644 index 0000000..05f27d3 --- /dev/null +++ b/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -0,0 +1,55 @@ +/** + * @file TransportPluginAbi.h + * @brief C ABI exported by transport plugin libraries. + * + * A plugin exports: + * - CreateTransport + * - DestroyTransport + * - TransportSendCommand + * - TransportFreeBuffer + * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. + */ + +#pragma once +#include +#ifndef __cplusplus +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRAFTCODE_TRANSPORT_OK 0 +#define GRAFTCODE_TRANSPORT_ERROR (-1) + +/** Opaque transport channel owned by the plugin. */ +typedef struct GraftcodeTransportChannel GraftcodeTransportChannel; + +/** + * Creates a transport channel. + * Returns null on failure; *errorMessage then receives a NUL-terminated message (or null). + */ +typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); + +/** Destroys a channel returned by CreateTransport. */ +typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); + +/** + * Sends a request and returns its response. + * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). + * Otherwise they hold a NUL-terminated error message and its length without the terminator. + */ +typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen); + +/** Releases a buffer returned by the plugin. */ +typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); + +/** Returns true when TransportSendCommand may be called concurrently on one channel. */ +typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); + +#ifdef __cplusplus +} +#endif diff --git a/kafka/GraftcodePluginsInterfaces/TransportPluginExports.h b/kafka/GraftcodePluginsInterfaces/TransportPluginExports.h new file mode 100644 index 0000000..0ea252d --- /dev/null +++ b/kafka/GraftcodePluginsInterfaces/TransportPluginExports.h @@ -0,0 +1,112 @@ +/** + * @file TransportPluginExports.h + * @brief Implements the transport plugin C ABI (TransportPluginAbi.h) on top of ITransport. + */ + +#pragma once +#include "ITransport.h" +#include "TransportPluginAbi.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::TransportAbi +{ + using Hypertube::Native::Interfaces::ITransport; + + // Allocates a NUL-terminated copy released with TransportFreeBuffer. + inline unsigned char* CopyBuffer(const void* data, std::size_t length) + { + auto* buffer = static_cast(std::malloc(length + 1)); + if (buffer == nullptr) { + return nullptr; + } + if (length > 0) { + std::memcpy(buffer, data, length); + } + buffer[length] = 0; + return buffer; + } + + inline char* CopyMessage(const std::string& message) + { + return reinterpret_cast(CopyBuffer(message.data(), message.size())); + } + + template + GraftcodeTransportChannel* CreateTransport(Factory&& factory, char** errorMessage) + { + if (errorMessage != nullptr) { + *errorMessage = nullptr; + } + std::string error; + try { + ITransport* transport = factory(); + return reinterpret_cast(transport); + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + if (errorMessage != nullptr) { + *errorMessage = CopyMessage(error); + } + return nullptr; + } + + inline void DestroyTransport(GraftcodeTransportChannel* channel) + { + delete reinterpret_cast(channel); + } + + inline int32_t SendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) + { + if (response == nullptr || responseLen == nullptr) { + return GRAFTCODE_TRANSPORT_ERROR; + } + *response = nullptr; + *responseLen = 0; + + std::string error; + try { + if (channel == nullptr) { + throw std::invalid_argument("transport channel is null"); + } + const std::vector result = reinterpret_cast(channel)->SendCommand( + const_cast(message), messageLen); + unsigned char* buffer = CopyBuffer(result.data(), result.size()); + if (buffer == nullptr) { + throw std::bad_alloc(); + } + *response = buffer; + *responseLen = static_cast(result.size()); + return GRAFTCODE_TRANSPORT_OK; + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + *response = CopyBuffer(error.data(), error.size()); + *responseLen = *response != nullptr ? static_cast(error.size()) : 0; + return GRAFTCODE_TRANSPORT_ERROR; + } + + inline void FreeBuffer(void* buffer) + { + std::free(buffer); + } +} diff --git a/kafka/KafkaPlugin/TransportKafka.cpp b/kafka/KafkaPlugin/TransportKafka.cpp index 0a2156a..e6e6332 100644 --- a/kafka/KafkaPlugin/TransportKafka.cpp +++ b/kafka/KafkaPlugin/TransportKafka.cpp @@ -1,10 +1,16 @@ #include "TransportKafka.h" #include "KafkaServer.h" +#include "TransportPluginExports.h" #include +#include #include +#include +#include +#include #include +#include #if defined(_WIN32) #define KAFKA_PLUGIN_EXPORT extern "C" __declspec(dllexport) @@ -65,42 +71,28 @@ TransportKafka::TransportKafka(const char*, unsigned short, const char* configJs int TransportKafka::Initialize(byte, byte, byte) { return 0; } -int TransportKafka::SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) { - std::vector response = - client_->call(messageByteArray, static_cast(messageByteArrayLen)); - const int responseSize = static_cast(response.size()); - std::unique_lock lock(responseByThreadMutex_); - responseByThread_[std::this_thread::get_id()] = std::move(response); - return responseSize; +std::vector TransportKafka::SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) { + std::vector response = client_->call(messageByteArray, static_cast(messageByteArrayLen)); + if (!response.empty() && response.front() == static_cast(255)) { + throw std::runtime_error(std::string(response.begin() + 1, response.end())); + } + return response; } -int TransportKafka::ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) { - std::vector response; - { - std::unique_lock lock(responseByThreadMutex_); - const auto it = responseByThread_.find(std::this_thread::get_id()); - if (it == responseByThread_.end()) { - return -1; - } - response = std::move(it->second); - responseByThread_.erase(it); - } - if (responseByteArrayLen < static_cast(response.size())) { - return -1; - } - if (!response.empty()) { - std::memcpy(responseByteArray, response.data(), response.size()); - } - return 0; +KAFKA_PLUGIN_EXPORT GraftcodeTransportChannel* CreateTransport(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage) { + return Graftcode::Plugins::TransportAbi::CreateTransport([&] { return new TransportKafka(ipAddress, port, configSource); }, errorMessage); +} + +KAFKA_PLUGIN_EXPORT void DestroyTransport(GraftcodeTransportChannel* channel) { + Graftcode::Plugins::TransportAbi::DestroyTransport(channel); } -KAFKA_PLUGIN_EXPORT Hypertube::Native::Interfaces::ITransport* -CreateTransportChannel(const char* ipAddress, const unsigned short port, const char* configSource) { - return new TransportKafka(ipAddress, port, configSource); +KAFKA_PLUGIN_EXPORT int32_t TransportSendCommand(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen) { + return Graftcode::Plugins::TransportAbi::SendCommand(channel, message, messageLen, response, responseLen); } -KAFKA_PLUGIN_EXPORT void DestroyTransportChannel(Hypertube::Native::Interfaces::ITransport* transport) { - delete transport; +KAFKA_PLUGIN_EXPORT void TransportFreeBuffer(void* buffer) { + Graftcode::Plugins::TransportAbi::FreeBuffer(buffer); } KAFKA_PLUGIN_EXPORT GraftcodeGateway::IServer* CreateServer() { diff --git a/kafka/KafkaPlugin/TransportKafka.h b/kafka/KafkaPlugin/TransportKafka.h index 6988b5b..56a4c04 100644 --- a/kafka/KafkaPlugin/TransportKafka.h +++ b/kafka/KafkaPlugin/TransportKafka.h @@ -1,21 +1,15 @@ #pragma once #include "ITransport.h" #include "KafkaClient.h" -#include #include -#include -#include #include class TransportKafka : public Hypertube::Native::Interfaces::ITransport { public: TransportKafka(const char* /*ip*/, unsigned short /*port*/, const char* configJson); int Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) override; - int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) override; - int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) override; + std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) override; private: std::unique_ptr client_; - std::map> responseByThread_; - std::shared_mutex responseByThreadMutex_; }; diff --git a/kafka/KafkaPluginTest/smoke.cpp b/kafka/KafkaPluginTest/smoke.cpp index b68b717..2db7bac 100644 --- a/kafka/KafkaPluginTest/smoke.cpp +++ b/kafka/KafkaPluginTest/smoke.cpp @@ -1,5 +1,5 @@ #include "IServer.h" -#include "ITransport.h" +#include "TransportPluginAbi.h" #include #include @@ -12,9 +12,10 @@ KAFKA_PLUGIN_IMPORT GraftcodeGateway::IServer* CreateServer(); KAFKA_PLUGIN_IMPORT void DestroyServer(GraftcodeGateway::IServer* server); -KAFKA_PLUGIN_IMPORT Hypertube::Native::Interfaces::ITransport* CreateTransportChannel( - const char* ipAddress, unsigned short port, const char* configSource); -KAFKA_PLUGIN_IMPORT void DestroyTransportChannel(Hypertube::Native::Interfaces::ITransport* transport); +KAFKA_PLUGIN_IMPORT GraftcodeTransportChannel* CreateTransport( + const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); +KAFKA_PLUGIN_IMPORT void DestroyTransport(GraftcodeTransportChannel* channel); +KAFKA_PLUGIN_IMPORT void TransportFreeBuffer(void* buffer); static bool g_processCalled = false; @@ -44,18 +45,15 @@ int main() { // Do not start() — that would spin a reconnect loop against a missing broker. DestroyServer(server); - Hypertube::Native::Interfaces::ITransport* transport = - CreateTransportChannel("127.0.0.1", 0, cfg); - if (!transport) { - std::puts("FAIL: CreateTransportChannel returned null"); + char* errorMessage = nullptr; + GraftcodeTransportChannel* transport = + CreateTransport("127.0.0.1", 0, cfg, &errorMessage); + if (transport == nullptr) { + std::printf("FAIL: CreateTransport returned null: %s\n", errorMessage != nullptr ? errorMessage : ""); + TransportFreeBuffer(errorMessage); return 1; } - if (transport->Initialize(1, 2, 1) != 0) { - std::puts("FAIL: Initialize returned non-zero"); - DestroyTransportChannel(transport); - return 1; - } - DestroyTransportChannel(transport); + DestroyTransport(transport); std::puts("KafkaPlugin smoke OK — factories linked; run against local Kafka to exercise RPC"); return 0; diff --git a/kafka/Readme.md b/kafka/Readme.md index 827e372..762b26d 100644 --- a/kafka/Readme.md +++ b/kafka/Readme.md @@ -1,6 +1,6 @@ # Kafka Plugin -This plugin carries Graftcode Gateway calls over Apache Kafka. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. +This plugin carries Graftcode Gateway calls over Apache Kafka. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports the transport C ABI `CreateTransport` / `DestroyTransport` / `TransportSendCommand` / `TransportFreeBuffer` (`GraftcodePluginsInterfaces/TransportPluginAbi.h`) and `CreateServer` / `DestroyServer`. It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0`, librdkafka `v2.8.0` (static `rdkafka++`), zlib `v1.3.1`, and zstd `v1.5.6`, all linked into the plugin. OpenSSL is static too: on Windows configure with the vcpkg toolchain so it is the `/MT` build from `kafka/vcpkg.json` (the OpenSSL in `Program Files` is `/MD`); on Linux and macOS the build uses the PIC static archive from the system or Homebrew. LZ4 and Snappy are the copies compiled into librdkafka. The first configure needs network access. diff --git a/pubsub/GraftcodePluginsInterfaces/ITransport.h b/pubsub/GraftcodePluginsInterfaces/ITransport.h index 335768c..c50114e 100644 --- a/pubsub/GraftcodePluginsInterfaces/ITransport.h +++ b/pubsub/GraftcodePluginsInterfaces/ITransport.h @@ -1,6 +1,7 @@ #pragma once #include "_common.h" +#include namespace Hypertube::Native::Interfaces { @@ -12,7 +13,6 @@ namespace Hypertube::Native::Interfaces byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) = 0; - virtual int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; - virtual int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) = 0; + virtual std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; }; } diff --git a/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h b/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h new file mode 100644 index 0000000..05f27d3 --- /dev/null +++ b/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -0,0 +1,55 @@ +/** + * @file TransportPluginAbi.h + * @brief C ABI exported by transport plugin libraries. + * + * A plugin exports: + * - CreateTransport + * - DestroyTransport + * - TransportSendCommand + * - TransportFreeBuffer + * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. + */ + +#pragma once +#include +#ifndef __cplusplus +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRAFTCODE_TRANSPORT_OK 0 +#define GRAFTCODE_TRANSPORT_ERROR (-1) + +/** Opaque transport channel owned by the plugin. */ +typedef struct GraftcodeTransportChannel GraftcodeTransportChannel; + +/** + * Creates a transport channel. + * Returns null on failure; *errorMessage then receives a NUL-terminated message (or null). + */ +typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); + +/** Destroys a channel returned by CreateTransport. */ +typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); + +/** + * Sends a request and returns its response. + * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). + * Otherwise they hold a NUL-terminated error message and its length without the terminator. + */ +typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen); + +/** Releases a buffer returned by the plugin. */ +typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); + +/** Returns true when TransportSendCommand may be called concurrently on one channel. */ +typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); + +#ifdef __cplusplus +} +#endif diff --git a/pubsub/GraftcodePluginsInterfaces/TransportPluginExports.h b/pubsub/GraftcodePluginsInterfaces/TransportPluginExports.h new file mode 100644 index 0000000..0ea252d --- /dev/null +++ b/pubsub/GraftcodePluginsInterfaces/TransportPluginExports.h @@ -0,0 +1,112 @@ +/** + * @file TransportPluginExports.h + * @brief Implements the transport plugin C ABI (TransportPluginAbi.h) on top of ITransport. + */ + +#pragma once +#include "ITransport.h" +#include "TransportPluginAbi.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::TransportAbi +{ + using Hypertube::Native::Interfaces::ITransport; + + // Allocates a NUL-terminated copy released with TransportFreeBuffer. + inline unsigned char* CopyBuffer(const void* data, std::size_t length) + { + auto* buffer = static_cast(std::malloc(length + 1)); + if (buffer == nullptr) { + return nullptr; + } + if (length > 0) { + std::memcpy(buffer, data, length); + } + buffer[length] = 0; + return buffer; + } + + inline char* CopyMessage(const std::string& message) + { + return reinterpret_cast(CopyBuffer(message.data(), message.size())); + } + + template + GraftcodeTransportChannel* CreateTransport(Factory&& factory, char** errorMessage) + { + if (errorMessage != nullptr) { + *errorMessage = nullptr; + } + std::string error; + try { + ITransport* transport = factory(); + return reinterpret_cast(transport); + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + if (errorMessage != nullptr) { + *errorMessage = CopyMessage(error); + } + return nullptr; + } + + inline void DestroyTransport(GraftcodeTransportChannel* channel) + { + delete reinterpret_cast(channel); + } + + inline int32_t SendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) + { + if (response == nullptr || responseLen == nullptr) { + return GRAFTCODE_TRANSPORT_ERROR; + } + *response = nullptr; + *responseLen = 0; + + std::string error; + try { + if (channel == nullptr) { + throw std::invalid_argument("transport channel is null"); + } + const std::vector result = reinterpret_cast(channel)->SendCommand( + const_cast(message), messageLen); + unsigned char* buffer = CopyBuffer(result.data(), result.size()); + if (buffer == nullptr) { + throw std::bad_alloc(); + } + *response = buffer; + *responseLen = static_cast(result.size()); + return GRAFTCODE_TRANSPORT_OK; + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + *response = CopyBuffer(error.data(), error.size()); + *responseLen = *response != nullptr ? static_cast(error.size()) : 0; + return GRAFTCODE_TRANSPORT_ERROR; + } + + inline void FreeBuffer(void* buffer) + { + std::free(buffer); + } +} diff --git a/pubsub/PubSubPlugin/TransportPubSub.cpp b/pubsub/PubSubPlugin/TransportPubSub.cpp index ac7ce2a..ecddfa4 100644 --- a/pubsub/PubSubPlugin/TransportPubSub.cpp +++ b/pubsub/PubSubPlugin/TransportPubSub.cpp @@ -1,4 +1,5 @@ #include "TransportPubSub.h" +#include "TransportPluginExports.h" #include "PubSubConfig.h" @@ -28,7 +29,7 @@ namespace Graftcode::Plugins::PubSub return 0; } - int TransportPubSub::SendCommand( + std::vector TransportPubSub::SendCommand( byte* messageByteArray, int32_t messageByteArrayLen) { @@ -43,42 +44,7 @@ namespace Graftcode::Plugins::PubSub static_cast(messageByteArrayLen)); RejectGatewayErrorPayload(response); - const int responseSize = static_cast(response.size()); - { - std::lock_guard lock(responsesMutex_); - responses_[std::this_thread::get_id()] = std::move(response); - } - return responseSize; - } - - int TransportPubSub::ReadResponse( - byte* responseByteArray, - int32_t responseByteArrayLen) - { - if (responseByteArrayLen < 0 || - (responseByteArrayLen > 0 && responseByteArray == nullptr)) { - throw std::invalid_argument( - "Pub/Sub transport: invalid response buffer"); - } - - std::lock_guard lock(responsesMutex_); - const auto response = responses_.find(std::this_thread::get_id()); - if (response == responses_.end()) { - throw std::runtime_error( - "Pub/Sub transport: response not found for calling thread"); - } - if (responseByteArrayLen != - static_cast(response->second.size())) { - throw std::runtime_error( - "Pub/Sub transport: response buffer length mismatch"); - } - - std::copy( - response->second.begin(), - response->second.end(), - responseByteArray); - responses_.erase(response); - return 0; + return response; } } @@ -88,17 +54,36 @@ namespace Graftcode::Plugins::PubSub #define PUBSUB_PLUGIN_EXPORT extern "C" #endif -PUBSUB_PLUGIN_EXPORT Hypertube::Native::Interfaces::ITransport* -CreateTransportChannel( +PUBSUB_PLUGIN_EXPORT GraftcodeTransportChannel* +CreateTransport( const char*, - const unsigned short, - const char* configSource) + unsigned short, + const char* configSource, + char** errorMessage) +{ + return Graftcode::Plugins::TransportAbi::CreateTransport( + [&] { return new Graftcode::Plugins::PubSub::TransportPubSub(configSource); }, + errorMessage); +} + +PUBSUB_PLUGIN_EXPORT void DestroyTransport( + GraftcodeTransportChannel* channel) +{ + Graftcode::Plugins::TransportAbi::DestroyTransport(channel); +} + +PUBSUB_PLUGIN_EXPORT int32_t TransportSendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) { - return new Graftcode::Plugins::PubSub::TransportPubSub(configSource); + return Graftcode::Plugins::TransportAbi::SendCommand( + channel, message, messageLen, response, responseLen); } -PUBSUB_PLUGIN_EXPORT void DestroyTransportChannel( - Hypertube::Native::Interfaces::ITransport* transport) +PUBSUB_PLUGIN_EXPORT void TransportFreeBuffer(void* buffer) { - delete transport; + Graftcode::Plugins::TransportAbi::FreeBuffer(buffer); } diff --git a/pubsub/PubSubPlugin/TransportPubSub.h b/pubsub/PubSubPlugin/TransportPubSub.h index bb1da37..b759e3a 100644 --- a/pubsub/PubSubPlugin/TransportPubSub.h +++ b/pubsub/PubSubPlugin/TransportPubSub.h @@ -3,10 +3,7 @@ #include "ITransport.h" #include "PubSubClient.h" -#include #include -#include -#include #include namespace Graftcode::Plugins::PubSub @@ -22,17 +19,12 @@ namespace Graftcode::Plugins::PubSub byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) override; - int SendCommand( + std::vector SendCommand( byte* messageByteArray, int32_t messageByteArrayLen) override; - int ReadResponse( - byte* responseByteArray, - int32_t responseByteArrayLen) override; private: std::unique_ptr client_; - std::map> responses_; - std::mutex responsesMutex_; }; void RejectGatewayErrorPayload(const std::vector& payload); diff --git a/pubsub/PubSubPluginTest/PubSubTransportTests.cpp b/pubsub/PubSubPluginTest/PubSubTransportTests.cpp index 018bf66..1aaf99b 100644 --- a/pubsub/PubSubPluginTest/PubSubTransportTests.cpp +++ b/pubsub/PubSubPluginTest/PubSubTransportTests.cpp @@ -2,6 +2,7 @@ #include "IServer.h" #include "ITransport.h" +#include "TransportPluginAbi.h" #include "PubSubClient.h" #include "PubSubConfig.h" #include "PubSubServer.h" @@ -24,13 +25,19 @@ #include #include -extern "C" Hypertube::Native::Interfaces::ITransport* -CreateTransportChannel( +extern "C" GraftcodeTransportChannel* CreateTransport( const char* ipAddress, unsigned short port, - const char* configSource); -extern "C" void DestroyTransportChannel( - Hypertube::Native::Interfaces::ITransport* transport); + const char* configSource, + char** errorMessage); +extern "C" void DestroyTransport(GraftcodeTransportChannel* channel); +extern "C" int32_t TransportSendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen); +extern "C" void TransportFreeBuffer(void* buffer); extern "C" GraftcodeGateway::IServer* CreateServer(); extern "C" void DestroyServer(GraftcodeGateway::IServer* server); @@ -492,17 +499,24 @@ TEST(PubSubAuth, ServiceAccountJwtIsRs256) TEST(PubSubTransport, FactoryCreatesAndDestroysTransport) { - auto* transport = CreateTransportChannel(nullptr, 0, kLocalConfig); - ASSERT_NE(transport, nullptr); - EXPECT_EQ(transport->Initialize(1, 2, 0), 0); - - EXPECT_THROW(transport->SendCommand(nullptr, -1), std::invalid_argument); - EXPECT_THROW(transport->SendCommand(nullptr, 4), std::invalid_argument); - - unsigned char buffer[4]{}; - EXPECT_THROW(transport->ReadResponse(nullptr, 4), std::invalid_argument); - EXPECT_THROW(transport->ReadResponse(buffer, 4), std::runtime_error); - DestroyTransportChannel(transport); + char* errorMessage = nullptr; + auto* channel = CreateTransport(nullptr, 0, kLocalConfig, &errorMessage); + ASSERT_NE(channel, nullptr); + EXPECT_EQ(errorMessage, nullptr); + + for (const int32_t length : {-1, 4}) { + unsigned char* response = nullptr; + int32_t responseLen = 0; + EXPECT_EQ( + TransportSendCommand(channel, nullptr, length, &response, &responseLen), + GRAFTCODE_TRANSPORT_ERROR); + ASSERT_NE(response, nullptr); + EXPECT_EQ( + std::string(reinterpret_cast(response), responseLen), + "Pub/Sub transport: invalid request payload"); + TransportFreeBuffer(response); + } + DestroyTransport(channel); } TEST(PubSubServer, FactoryConfigureAndStop) diff --git a/pubsub/PubSubPluginTest/smoke.cpp b/pubsub/PubSubPluginTest/smoke.cpp index e4df8a3..9794e88 100644 --- a/pubsub/PubSubPluginTest/smoke.cpp +++ b/pubsub/PubSubPluginTest/smoke.cpp @@ -1,5 +1,5 @@ #include "IServer.h" -#include "ITransport.h" +#include "TransportPluginAbi.h" #include @@ -11,12 +11,10 @@ PUBSUB_PLUGIN_IMPORT GraftcodeGateway::IServer* CreateServer(); PUBSUB_PLUGIN_IMPORT void DestroyServer(GraftcodeGateway::IServer* server); -PUBSUB_PLUGIN_IMPORT Hypertube::Native::Interfaces::ITransport* CreateTransportChannel( - const char* ipAddress, - unsigned short port, - const char* configSource); -PUBSUB_PLUGIN_IMPORT void DestroyTransportChannel( - Hypertube::Native::Interfaces::ITransport* transport); +PUBSUB_PLUGIN_IMPORT GraftcodeTransportChannel* CreateTransport( + const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); +PUBSUB_PLUGIN_IMPORT void DestroyTransport(GraftcodeTransportChannel* channel); +PUBSUB_PLUGIN_IMPORT void TransportFreeBuffer(void* buffer); static bool smokeProcess( const GraftcodeGateway::IServer::byte*, @@ -52,18 +50,15 @@ int main() server->stop(); DestroyServer(server); - Hypertube::Native::Interfaces::ITransport* transport = - CreateTransportChannel(nullptr, 0, config); + char* errorMessage = nullptr; + GraftcodeTransportChannel* transport = + CreateTransport(nullptr, 0, config, &errorMessage); if (transport == nullptr) { - std::puts("FAIL: CreateTransportChannel returned null"); + std::printf("FAIL: CreateTransport returned null: %s\n", errorMessage != nullptr ? errorMessage : ""); + TransportFreeBuffer(errorMessage); return 1; } - if (transport->Initialize(1, 2, 1) != 0) { - std::puts("FAIL: Initialize returned non-zero"); - DestroyTransportChannel(transport); - return 1; - } - DestroyTransportChannel(transport); + DestroyTransport(transport); std::puts("PubSubPlugin smoke OK"); return 0; diff --git a/pubsub/Readme.md b/pubsub/Readme.md index 50dbf74..321b4d4 100644 --- a/pubsub/Readme.md +++ b/pubsub/Readme.md @@ -1,6 +1,6 @@ # Google Cloud Pub/Sub plugin -This plugin carries Graftcode Gateway calls over Google Cloud Pub/Sub. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. +This plugin carries Graftcode Gateway calls over Google Cloud Pub/Sub. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports the transport C ABI `CreateTransport` / `DestroyTransport` / `TransportSendCommand` / `TransportFreeBuffer` (`GraftcodePluginsInterfaces/TransportPluginAbi.h`) and `CreateServer` / `DestroyServer`. It is written in C++. Dependencies are libcurl, OpenSSL, and nlohmann/json, installed with the same vcpkg toolchain file as the SQS and Service Bus plugins. They are linked into the plugin. diff --git a/rabbitmq/GraftcodePluginsInterfaces/ITransport.h b/rabbitmq/GraftcodePluginsInterfaces/ITransport.h index 0047ca9..6da92e5 100644 --- a/rabbitmq/GraftcodePluginsInterfaces/ITransport.h +++ b/rabbitmq/GraftcodePluginsInterfaces/ITransport.h @@ -5,6 +5,7 @@ #pragma once #include "_common.h" +#include namespace Hypertube::Native::Interfaces { @@ -27,20 +28,12 @@ namespace Hypertube::Native::Interfaces virtual int Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) = 0; /** - * @brief Sends a byte array to the transport. + * @brief Sends a byte array to the transport and returns its response. * @param messageByteArray The byte array to send. * @param messageByteArrayLen The length of the byte array. - * @return lenght of response byte array + * @return The response byte array. */ - virtual int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; - - /** - * @brief Reads the response byte array from the transport. - * @param responseByteArray The byte array to store the response. - * @param responseByteArrayLen The length of the byte array. - * @return Status code of the operation. - */ - virtual int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) = 0; + virtual std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; }; } diff --git a/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h b/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h new file mode 100644 index 0000000..05f27d3 --- /dev/null +++ b/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -0,0 +1,55 @@ +/** + * @file TransportPluginAbi.h + * @brief C ABI exported by transport plugin libraries. + * + * A plugin exports: + * - CreateTransport + * - DestroyTransport + * - TransportSendCommand + * - TransportFreeBuffer + * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. + */ + +#pragma once +#include +#ifndef __cplusplus +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRAFTCODE_TRANSPORT_OK 0 +#define GRAFTCODE_TRANSPORT_ERROR (-1) + +/** Opaque transport channel owned by the plugin. */ +typedef struct GraftcodeTransportChannel GraftcodeTransportChannel; + +/** + * Creates a transport channel. + * Returns null on failure; *errorMessage then receives a NUL-terminated message (or null). + */ +typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); + +/** Destroys a channel returned by CreateTransport. */ +typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); + +/** + * Sends a request and returns its response. + * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). + * Otherwise they hold a NUL-terminated error message and its length without the terminator. + */ +typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen); + +/** Releases a buffer returned by the plugin. */ +typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); + +/** Returns true when TransportSendCommand may be called concurrently on one channel. */ +typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); + +#ifdef __cplusplus +} +#endif diff --git a/rabbitmq/GraftcodePluginsInterfaces/TransportPluginExports.h b/rabbitmq/GraftcodePluginsInterfaces/TransportPluginExports.h new file mode 100644 index 0000000..0ea252d --- /dev/null +++ b/rabbitmq/GraftcodePluginsInterfaces/TransportPluginExports.h @@ -0,0 +1,112 @@ +/** + * @file TransportPluginExports.h + * @brief Implements the transport plugin C ABI (TransportPluginAbi.h) on top of ITransport. + */ + +#pragma once +#include "ITransport.h" +#include "TransportPluginAbi.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::TransportAbi +{ + using Hypertube::Native::Interfaces::ITransport; + + // Allocates a NUL-terminated copy released with TransportFreeBuffer. + inline unsigned char* CopyBuffer(const void* data, std::size_t length) + { + auto* buffer = static_cast(std::malloc(length + 1)); + if (buffer == nullptr) { + return nullptr; + } + if (length > 0) { + std::memcpy(buffer, data, length); + } + buffer[length] = 0; + return buffer; + } + + inline char* CopyMessage(const std::string& message) + { + return reinterpret_cast(CopyBuffer(message.data(), message.size())); + } + + template + GraftcodeTransportChannel* CreateTransport(Factory&& factory, char** errorMessage) + { + if (errorMessage != nullptr) { + *errorMessage = nullptr; + } + std::string error; + try { + ITransport* transport = factory(); + return reinterpret_cast(transport); + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + if (errorMessage != nullptr) { + *errorMessage = CopyMessage(error); + } + return nullptr; + } + + inline void DestroyTransport(GraftcodeTransportChannel* channel) + { + delete reinterpret_cast(channel); + } + + inline int32_t SendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) + { + if (response == nullptr || responseLen == nullptr) { + return GRAFTCODE_TRANSPORT_ERROR; + } + *response = nullptr; + *responseLen = 0; + + std::string error; + try { + if (channel == nullptr) { + throw std::invalid_argument("transport channel is null"); + } + const std::vector result = reinterpret_cast(channel)->SendCommand( + const_cast(message), messageLen); + unsigned char* buffer = CopyBuffer(result.data(), result.size()); + if (buffer == nullptr) { + throw std::bad_alloc(); + } + *response = buffer; + *responseLen = static_cast(result.size()); + return GRAFTCODE_TRANSPORT_OK; + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + *response = CopyBuffer(error.data(), error.size()); + *responseLen = *response != nullptr ? static_cast(error.size()) : 0; + return GRAFTCODE_TRANSPORT_ERROR; + } + + inline void FreeBuffer(void* buffer) + { + std::free(buffer); + } +} diff --git a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp index 36532b3..a3bdb9d 100644 --- a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp +++ b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp @@ -1,10 +1,9 @@ #include "TransportRabbitMq.h" +#include "TransportPluginExports.h" #include #include #include -#include -#include #include using namespace Graftcode::Plugins::Rabbitmq; @@ -26,7 +25,7 @@ int TransportRabbitMq::Initialize(byte callingRuntimeNumber, byte calledRuntimeN return 0; } -int TransportRabbitMq::SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) +std::vector TransportRabbitMq::SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) { if (messageByteArray == nullptr || messageByteArrayLen < 0) { throw std::runtime_error("invalid RabbitMQ payload"); @@ -47,39 +46,7 @@ int TransportRabbitMq::SendCommand(byte* messageByteArray, int32_t messageByteAr throw std::runtime_error(errorMessage); } - { - std::unique_lock lock(responseByThreadMutex_); - responseByThread_[std::this_thread::get_id()] = responsePayload; - } - - return static_cast(responsePayload.size()); -} - -int TransportRabbitMq::ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) -{ - if (responseByteArray == nullptr || responseByteArrayLen < 0) { - throw std::runtime_error("invalid response buffer"); - } - - std::vector responsePayload; - { - std::shared_lock lock(responseByThreadMutex_); - const auto it = responseByThread_.find(std::this_thread::get_id()); - if (it == responseByThread_.end()) { - throw std::runtime_error("RabbitMQ response not found for thread"); - } - responsePayload = it->second; - } - - if (static_cast(responseByteArrayLen) > responsePayload.size()) { - throw std::runtime_error("response buffer length mismatch"); - } - - for (int i = 0; i < responseByteArrayLen; i++) { - responseByteArray[i] = responsePayload[static_cast(i)]; - } - - return 0; + return responsePayload; } #if defined(_WIN32) @@ -88,16 +55,30 @@ int TransportRabbitMq::ReadResponse(byte* responseByteArray, int32_t responseByt #define RABBITMQ_PLUGIN_EXPORT extern "C" #endif -RABBITMQ_PLUGIN_EXPORT Hypertube::Native::Interfaces::ITransport* CreateTransportChannel(const char* ipAddress, const unsigned short port, const char* configSource) +RABBITMQ_PLUGIN_EXPORT GraftcodeTransportChannel* CreateTransport(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage) +{ + return Graftcode::Plugins::TransportAbi::CreateTransport( + [&] { + return new TransportRabbitMq( + ipAddress != nullptr ? std::string(ipAddress) : std::string(""), + port, + configSource != nullptr ? std::string(configSource) : std::string("")); + }, + errorMessage); +} + +RABBITMQ_PLUGIN_EXPORT void DestroyTransport(GraftcodeTransportChannel* channel) +{ + Graftcode::Plugins::TransportAbi::DestroyTransport(channel); +} + +RABBITMQ_PLUGIN_EXPORT int32_t TransportSendCommand(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen) { - return new TransportRabbitMq( - ipAddress != nullptr ? std::string(ipAddress) : std::string(""), - port, - configSource != nullptr ? std::string(configSource) : std::string("")); + return Graftcode::Plugins::TransportAbi::SendCommand(channel, message, messageLen, response, responseLen); } -RABBITMQ_PLUGIN_EXPORT void DestroyTransportChannel(Hypertube::Native::Interfaces::ITransport* transport) +RABBITMQ_PLUGIN_EXPORT void TransportFreeBuffer(void* buffer) { - delete transport; + Graftcode::Plugins::TransportAbi::FreeBuffer(buffer); } diff --git a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h index b200172..e01284b 100644 --- a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h +++ b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h @@ -3,10 +3,7 @@ #include "ITransport.h" #include "RabbitMqClient.h" -#include -#include #include -#include #include namespace Graftcode::Plugins::Rabbitmq @@ -18,13 +15,10 @@ namespace Graftcode::Plugins::Rabbitmq ~TransportRabbitMq() override = default; int Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) override; - int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) override; - int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) override; + std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) override; private: RabbitMqClientConfig config_; std::string configSource; - std::map> responseByThread_; - std::shared_mutex responseByThreadMutex_; }; } diff --git a/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp b/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp index a46dfb7..c8f3679 100644 --- a/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp +++ b/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp @@ -1,6 +1,7 @@ #include #include "ITransport.h" +#include "TransportPluginAbi.h" #include "RabbitMqClient.h" #include #include @@ -14,11 +15,19 @@ #include #include -extern "C" Hypertube::Native::Interfaces::ITransport* CreateTransportChannel( +extern "C" GraftcodeTransportChannel* CreateTransport( const char* ipAddress, unsigned short port, - const char* configSource); -extern "C" void DestroyTransportChannel(Hypertube::Native::Interfaces::ITransport* transport); + const char* configSource, + char** errorMessage); +extern "C" void DestroyTransport(GraftcodeTransportChannel* channel); +extern "C" int32_t TransportSendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen); +extern "C" void TransportFreeBuffer(void* buffer); namespace { const char* kValidClientConfig = R"({ @@ -74,21 +83,6 @@ TEST(RabbitMqTransport, SendCommandRejectsInvalidPayloadArguments) { ExpectInvalidPayload(transport, nullptr, 8); ExpectInvalidPayload(transport, &sample, -5); - byte buffer[4]{}; - try { - transport.ReadResponse(nullptr, 4); - FAIL() << "ReadResponse should reject a null buffer"; - } - catch (const std::runtime_error& error) { - EXPECT_STREQ(error.what(), "invalid response buffer"); - } - try { - transport.ReadResponse(buffer, 4); - FAIL() << "ReadResponse should fail when no response was stored"; - } - catch (const std::runtime_error& error) { - EXPECT_STREQ(error.what(), "RabbitMQ response not found for thread"); - } } TEST(RabbitMqTransport, LiveRoundTrip) { @@ -101,13 +95,9 @@ TEST(RabbitMqTransport, LiveRoundTrip) { 5672, kValidClientConfig); for (int i = 0; i < 300; ++i) { - const int responseSize = transport.SendCommand( + const std::vector actualResponse = transport.SendCommand( const_cast(kSamplePayload.data()), static_cast(kSamplePayload.size())); - std::vector actualResponse(static_cast(responseSize)); - transport.ReadResponse( - actualResponse.data(), - static_cast(actualResponse.size())); EXPECT_EQ(actualResponse, kReadResponsePayload); } } diff --git a/rabbitmq/Readme.md b/rabbitmq/Readme.md index 2786e2b..a3e9f6e 100644 --- a/rabbitmq/Readme.md +++ b/rabbitmq/Readme.md @@ -1,6 +1,6 @@ # RabbitMQ Plugin -This plugin carries Graftcode Gateway calls over RabbitMQ (AMQP 0-9-1). It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. +This plugin carries Graftcode Gateway calls over RabbitMQ (AMQP 0-9-1). It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports the transport C ABI `CreateTransport` / `DestroyTransport` / `TransportSendCommand` / `TransportFreeBuffer` (`GraftcodePluginsInterfaces/TransportPluginAbi.h`) and `CreateServer` / `DestroyServer`. It is written in C++. CMake FetchContent builds [AMQP-CPP](https://github.com/CopernicaMarketingSoftware/AMQP-CPP) `v4.3.27` and nlohmann/json `v3.12.0` into the plugin. There is no vcpkg manifest. diff --git a/servicebus/GraftcodePluginsInterfaces/ITransport.h b/servicebus/GraftcodePluginsInterfaces/ITransport.h index 0047ca9..6da92e5 100644 --- a/servicebus/GraftcodePluginsInterfaces/ITransport.h +++ b/servicebus/GraftcodePluginsInterfaces/ITransport.h @@ -5,6 +5,7 @@ #pragma once #include "_common.h" +#include namespace Hypertube::Native::Interfaces { @@ -27,20 +28,12 @@ namespace Hypertube::Native::Interfaces virtual int Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) = 0; /** - * @brief Sends a byte array to the transport. + * @brief Sends a byte array to the transport and returns its response. * @param messageByteArray The byte array to send. * @param messageByteArrayLen The length of the byte array. - * @return lenght of response byte array + * @return The response byte array. */ - virtual int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; - - /** - * @brief Reads the response byte array from the transport. - * @param responseByteArray The byte array to store the response. - * @param responseByteArrayLen The length of the byte array. - * @return Status code of the operation. - */ - virtual int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) = 0; + virtual std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; }; } diff --git a/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h b/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h new file mode 100644 index 0000000..05f27d3 --- /dev/null +++ b/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -0,0 +1,55 @@ +/** + * @file TransportPluginAbi.h + * @brief C ABI exported by transport plugin libraries. + * + * A plugin exports: + * - CreateTransport + * - DestroyTransport + * - TransportSendCommand + * - TransportFreeBuffer + * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. + */ + +#pragma once +#include +#ifndef __cplusplus +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRAFTCODE_TRANSPORT_OK 0 +#define GRAFTCODE_TRANSPORT_ERROR (-1) + +/** Opaque transport channel owned by the plugin. */ +typedef struct GraftcodeTransportChannel GraftcodeTransportChannel; + +/** + * Creates a transport channel. + * Returns null on failure; *errorMessage then receives a NUL-terminated message (or null). + */ +typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); + +/** Destroys a channel returned by CreateTransport. */ +typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); + +/** + * Sends a request and returns its response. + * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). + * Otherwise they hold a NUL-terminated error message and its length without the terminator. + */ +typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen); + +/** Releases a buffer returned by the plugin. */ +typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); + +/** Returns true when TransportSendCommand may be called concurrently on one channel. */ +typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); + +#ifdef __cplusplus +} +#endif diff --git a/servicebus/GraftcodePluginsInterfaces/TransportPluginExports.h b/servicebus/GraftcodePluginsInterfaces/TransportPluginExports.h new file mode 100644 index 0000000..0ea252d --- /dev/null +++ b/servicebus/GraftcodePluginsInterfaces/TransportPluginExports.h @@ -0,0 +1,112 @@ +/** + * @file TransportPluginExports.h + * @brief Implements the transport plugin C ABI (TransportPluginAbi.h) on top of ITransport. + */ + +#pragma once +#include "ITransport.h" +#include "TransportPluginAbi.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::TransportAbi +{ + using Hypertube::Native::Interfaces::ITransport; + + // Allocates a NUL-terminated copy released with TransportFreeBuffer. + inline unsigned char* CopyBuffer(const void* data, std::size_t length) + { + auto* buffer = static_cast(std::malloc(length + 1)); + if (buffer == nullptr) { + return nullptr; + } + if (length > 0) { + std::memcpy(buffer, data, length); + } + buffer[length] = 0; + return buffer; + } + + inline char* CopyMessage(const std::string& message) + { + return reinterpret_cast(CopyBuffer(message.data(), message.size())); + } + + template + GraftcodeTransportChannel* CreateTransport(Factory&& factory, char** errorMessage) + { + if (errorMessage != nullptr) { + *errorMessage = nullptr; + } + std::string error; + try { + ITransport* transport = factory(); + return reinterpret_cast(transport); + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + if (errorMessage != nullptr) { + *errorMessage = CopyMessage(error); + } + return nullptr; + } + + inline void DestroyTransport(GraftcodeTransportChannel* channel) + { + delete reinterpret_cast(channel); + } + + inline int32_t SendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) + { + if (response == nullptr || responseLen == nullptr) { + return GRAFTCODE_TRANSPORT_ERROR; + } + *response = nullptr; + *responseLen = 0; + + std::string error; + try { + if (channel == nullptr) { + throw std::invalid_argument("transport channel is null"); + } + const std::vector result = reinterpret_cast(channel)->SendCommand( + const_cast(message), messageLen); + unsigned char* buffer = CopyBuffer(result.data(), result.size()); + if (buffer == nullptr) { + throw std::bad_alloc(); + } + *response = buffer; + *responseLen = static_cast(result.size()); + return GRAFTCODE_TRANSPORT_OK; + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + *response = CopyBuffer(error.data(), error.size()); + *responseLen = *response != nullptr ? static_cast(error.size()) : 0; + return GRAFTCODE_TRANSPORT_ERROR; + } + + inline void FreeBuffer(void* buffer) + { + std::free(buffer); + } +} diff --git a/servicebus/Readme.md b/servicebus/Readme.md index 2d1cd80..47e2d80 100644 --- a/servicebus/Readme.md +++ b/servicebus/Readme.md @@ -3,7 +3,7 @@ This plugin is the Azure Service Bus counterpart of the RabbitMQ plugin. It implements the same graftcode plugin interfaces (`Hypertube::Native::Interfaces::ITransport` for the calling runtime and `GraftcodeGateway::IServer` for the gateway) and exposes the same exported -factory symbols (`CreateTransportChannel` / `DestroyTransportChannel` and +factory symbols (the transport C ABI `CreateTransport` / `DestroyTransport` / `TransportSendCommand` / `TransportFreeBuffer` (`GraftcodePluginsInterfaces/TransportPluginAbi.h`) and `CreateServer` / `DestroyServer`). It is written in C++ and talks to Azure Service Bus over its native AMQP 1.0 protocol using diff --git a/servicebus/ServiceBusPlugin/TransportServiceBus.cpp b/servicebus/ServiceBusPlugin/TransportServiceBus.cpp index 280f85a..df67676 100644 --- a/servicebus/ServiceBusPlugin/TransportServiceBus.cpp +++ b/servicebus/ServiceBusPlugin/TransportServiceBus.cpp @@ -1,10 +1,9 @@ #include "TransportServiceBus.h" +#include "TransportPluginExports.h" #include #include #include -#include -#include #include using namespace Graftcode::Plugins::ServiceBus; @@ -26,7 +25,7 @@ int TransportServiceBus::Initialize(byte callingRuntimeNumber, byte calledRuntim return 0; } -int TransportServiceBus::SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) +std::vector TransportServiceBus::SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) { if (messageByteArray == nullptr || messageByteArrayLen < 0) { throw std::runtime_error("invalid Service Bus payload"); @@ -47,39 +46,7 @@ int TransportServiceBus::SendCommand(byte* messageByteArray, int32_t messageByte throw std::runtime_error(errorMessage); } - { - std::unique_lock lock(responseByThreadMutex_); - responseByThread_[std::this_thread::get_id()] = responsePayload; - } - - return static_cast(responsePayload.size()); -} - -int TransportServiceBus::ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) -{ - if (responseByteArray == nullptr || responseByteArrayLen < 0) { - throw std::runtime_error("invalid response buffer"); - } - - std::vector responsePayload; - { - std::shared_lock lock(responseByThreadMutex_); - const auto it = responseByThread_.find(std::this_thread::get_id()); - if (it == responseByThread_.end()) { - throw std::runtime_error("Service Bus response not found for thread"); - } - responsePayload = it->second; - } - - if (static_cast(responseByteArrayLen) > responsePayload.size()) { - throw std::runtime_error("response buffer length mismatch"); - } - - for (int i = 0; i < responseByteArrayLen; i++) { - responseByteArray[i] = responsePayload[static_cast(i)]; - } - - return 0; + return responsePayload; } #if defined(_WIN32) @@ -88,15 +55,29 @@ int TransportServiceBus::ReadResponse(byte* responseByteArray, int32_t responseB #define SERVICEBUS_PLUGIN_EXPORT extern "C" #endif -SERVICEBUS_PLUGIN_EXPORT Hypertube::Native::Interfaces::ITransport* CreateTransportChannel(const char* ipAddress, const unsigned short port, const char* configSource) +SERVICEBUS_PLUGIN_EXPORT GraftcodeTransportChannel* CreateTransport(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage) +{ + return Graftcode::Plugins::TransportAbi::CreateTransport( + [&] { + return new TransportServiceBus( + ipAddress != nullptr ? std::string(ipAddress) : std::string(""), + port, + configSource != nullptr ? std::string(configSource) : std::string("")); + }, + errorMessage); +} + +SERVICEBUS_PLUGIN_EXPORT void DestroyTransport(GraftcodeTransportChannel* channel) +{ + Graftcode::Plugins::TransportAbi::DestroyTransport(channel); +} + +SERVICEBUS_PLUGIN_EXPORT int32_t TransportSendCommand(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen) { - return new TransportServiceBus( - ipAddress != nullptr ? std::string(ipAddress) : std::string(""), - port, - configSource != nullptr ? std::string(configSource) : std::string("")); + return Graftcode::Plugins::TransportAbi::SendCommand(channel, message, messageLen, response, responseLen); } -SERVICEBUS_PLUGIN_EXPORT void DestroyTransportChannel(Hypertube::Native::Interfaces::ITransport* transport) +SERVICEBUS_PLUGIN_EXPORT void TransportFreeBuffer(void* buffer) { - delete transport; + Graftcode::Plugins::TransportAbi::FreeBuffer(buffer); } diff --git a/servicebus/ServiceBusPlugin/TransportServiceBus.h b/servicebus/ServiceBusPlugin/TransportServiceBus.h index 410a178..693d6ac 100644 --- a/servicebus/ServiceBusPlugin/TransportServiceBus.h +++ b/servicebus/ServiceBusPlugin/TransportServiceBus.h @@ -3,10 +3,7 @@ #include "ITransport.h" #include "ServiceBusClient.h" -#include -#include #include -#include #include namespace Graftcode::Plugins::ServiceBus @@ -18,13 +15,10 @@ namespace Graftcode::Plugins::ServiceBus ~TransportServiceBus() override = default; int Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) override; - int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) override; - int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) override; + std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) override; private: ServiceBusClientConfig config_; std::string configSource; - std::map> responseByThread_; - std::shared_mutex responseByThreadMutex_; }; } diff --git a/servicebus/ServiceBusPluginTest/ServiceBusTransportTests.cpp b/servicebus/ServiceBusPluginTest/ServiceBusTransportTests.cpp index 78978b4..febbaac 100644 --- a/servicebus/ServiceBusPluginTest/ServiceBusTransportTests.cpp +++ b/servicebus/ServiceBusPluginTest/ServiceBusTransportTests.cpp @@ -1,6 +1,7 @@ #include #include "ITransport.h" +#include "TransportPluginAbi.h" #include "IServer.h" #include "ServiceBusClient.h" #include "ServiceBusServer.h" @@ -16,11 +17,19 @@ #include #include -extern "C" Hypertube::Native::Interfaces::ITransport* CreateTransportChannel( +extern "C" GraftcodeTransportChannel* CreateTransport( const char* ipAddress, unsigned short port, - const char* configSource); -extern "C" void DestroyTransportChannel(Hypertube::Native::Interfaces::ITransport* transport); + const char* configSource, + char** errorMessage); +extern "C" void DestroyTransport(GraftcodeTransportChannel* channel); +extern "C" int32_t TransportSendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen); +extern "C" void TransportFreeBuffer(void* buffer); using Graftcode::Plugins::ServiceBus::ServiceBusClient; using Graftcode::Plugins::ServiceBus::ServiceBusClientConfig; @@ -148,9 +157,11 @@ TEST(ServiceBusClient, PublishOneWayFailsFastWithoutTopic) { } TEST(ServiceBusTransport, FactoryCreatesAndDestroysTransport) { - auto* transport = CreateTransportChannel("", 0, kConnectionStringConfig); - ASSERT_NE(transport, nullptr); - DestroyTransportChannel(transport); + char* errorMessage = nullptr; + auto* channel = CreateTransport("", 0, kConnectionStringConfig, &errorMessage); + ASSERT_NE(channel, nullptr); + EXPECT_EQ(errorMessage, nullptr); + DestroyTransport(channel); } // Self-contained RPC round-trip against a live Service Bus namespace: starts the diff --git a/sqs/GraftcodePluginsInterfaces/ITransport.h b/sqs/GraftcodePluginsInterfaces/ITransport.h index 335768c..c50114e 100644 --- a/sqs/GraftcodePluginsInterfaces/ITransport.h +++ b/sqs/GraftcodePluginsInterfaces/ITransport.h @@ -1,6 +1,7 @@ #pragma once #include "_common.h" +#include namespace Hypertube::Native::Interfaces { @@ -12,7 +13,6 @@ namespace Hypertube::Native::Interfaces byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) = 0; - virtual int SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; - virtual int ReadResponse(byte* responseByteArray, int32_t responseByteArrayLen) = 0; + virtual std::vector SendCommand(byte* messageByteArray, int32_t messageByteArrayLen) = 0; }; } diff --git a/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h b/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h new file mode 100644 index 0000000..05f27d3 --- /dev/null +++ b/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -0,0 +1,55 @@ +/** + * @file TransportPluginAbi.h + * @brief C ABI exported by transport plugin libraries. + * + * A plugin exports: + * - CreateTransport + * - DestroyTransport + * - TransportSendCommand + * - TransportFreeBuffer + * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. + */ + +#pragma once +#include +#ifndef __cplusplus +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRAFTCODE_TRANSPORT_OK 0 +#define GRAFTCODE_TRANSPORT_ERROR (-1) + +/** Opaque transport channel owned by the plugin. */ +typedef struct GraftcodeTransportChannel GraftcodeTransportChannel; + +/** + * Creates a transport channel. + * Returns null on failure; *errorMessage then receives a NUL-terminated message (or null). + */ +typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); + +/** Destroys a channel returned by CreateTransport. */ +typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); + +/** + * Sends a request and returns its response. + * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). + * Otherwise they hold a NUL-terminated error message and its length without the terminator. + */ +typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* channel, const unsigned char* message, int32_t messageLen, unsigned char** response, int32_t* responseLen); + +/** Releases a buffer returned by the plugin. */ +typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); + +/** Returns true when TransportSendCommand may be called concurrently on one channel. */ +typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); + +#ifdef __cplusplus +} +#endif diff --git a/sqs/GraftcodePluginsInterfaces/TransportPluginExports.h b/sqs/GraftcodePluginsInterfaces/TransportPluginExports.h new file mode 100644 index 0000000..0ea252d --- /dev/null +++ b/sqs/GraftcodePluginsInterfaces/TransportPluginExports.h @@ -0,0 +1,112 @@ +/** + * @file TransportPluginExports.h + * @brief Implements the transport plugin C ABI (TransportPluginAbi.h) on top of ITransport. + */ + +#pragma once +#include "ITransport.h" +#include "TransportPluginAbi.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::TransportAbi +{ + using Hypertube::Native::Interfaces::ITransport; + + // Allocates a NUL-terminated copy released with TransportFreeBuffer. + inline unsigned char* CopyBuffer(const void* data, std::size_t length) + { + auto* buffer = static_cast(std::malloc(length + 1)); + if (buffer == nullptr) { + return nullptr; + } + if (length > 0) { + std::memcpy(buffer, data, length); + } + buffer[length] = 0; + return buffer; + } + + inline char* CopyMessage(const std::string& message) + { + return reinterpret_cast(CopyBuffer(message.data(), message.size())); + } + + template + GraftcodeTransportChannel* CreateTransport(Factory&& factory, char** errorMessage) + { + if (errorMessage != nullptr) { + *errorMessage = nullptr; + } + std::string error; + try { + ITransport* transport = factory(); + return reinterpret_cast(transport); + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + if (errorMessage != nullptr) { + *errorMessage = CopyMessage(error); + } + return nullptr; + } + + inline void DestroyTransport(GraftcodeTransportChannel* channel) + { + delete reinterpret_cast(channel); + } + + inline int32_t SendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) + { + if (response == nullptr || responseLen == nullptr) { + return GRAFTCODE_TRANSPORT_ERROR; + } + *response = nullptr; + *responseLen = 0; + + std::string error; + try { + if (channel == nullptr) { + throw std::invalid_argument("transport channel is null"); + } + const std::vector result = reinterpret_cast(channel)->SendCommand( + const_cast(message), messageLen); + unsigned char* buffer = CopyBuffer(result.data(), result.size()); + if (buffer == nullptr) { + throw std::bad_alloc(); + } + *response = buffer; + *responseLen = static_cast(result.size()); + return GRAFTCODE_TRANSPORT_OK; + } + catch (const std::exception& e) { + error = e.what(); + } + catch (...) { + error = "unknown transport plugin error"; + } + *response = CopyBuffer(error.data(), error.size()); + *responseLen = *response != nullptr ? static_cast(error.size()) : 0; + return GRAFTCODE_TRANSPORT_ERROR; + } + + inline void FreeBuffer(void* buffer) + { + std::free(buffer); + } +} diff --git a/sqs/Readme.md b/sqs/Readme.md index 45726dc..8ecbe6b 100644 --- a/sqs/Readme.md +++ b/sqs/Readme.md @@ -1,6 +1,6 @@ # Amazon SQS Plugin -This plugin carries Graftcode Gateway calls over Amazon Simple Queue Service. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. +This plugin carries Graftcode Gateway calls over Amazon Simple Queue Service. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports the transport C ABI `CreateTransport` / `DestroyTransport` / `TransportSendCommand` / `TransportFreeBuffer` (`GraftcodePluginsInterfaces/TransportPluginAbi.h`) and `CreateServer` / `DestroyServer`. It is written in C++ and uses the AWS SDK for C++ (`aws-sdk-cpp[sqs]`) plus `nlohmann-json`, both from vcpkg. Both are linked into the plugin. diff --git a/sqs/SqsPlugin/TransportSqs.cpp b/sqs/SqsPlugin/TransportSqs.cpp index d95083f..c61e00e 100644 --- a/sqs/SqsPlugin/TransportSqs.cpp +++ b/sqs/SqsPlugin/TransportSqs.cpp @@ -1,4 +1,5 @@ #include "TransportSqs.h" +#include "TransportPluginExports.h" #include "SqsConfig.h" @@ -34,7 +35,7 @@ int TransportSqs::Initialize(byte, byte, byte) return 0; } -int TransportSqs::SendCommand( +std::vector TransportSqs::SendCommand( byte* messageByteArray, int32_t messageByteArrayLen) { @@ -48,41 +49,7 @@ int TransportSqs::SendCommand( static_cast(messageByteArrayLen)); RejectGatewayErrorPayload(response); - const int responseSize = static_cast(response.size()); - { - std::lock_guard lock(responsesMutex_); - responses_[std::this_thread::get_id()] = std::move(response); - } - return responseSize; -} - -int TransportSqs::ReadResponse( - byte* responseByteArray, - int32_t responseByteArrayLen) -{ - if (responseByteArrayLen < 0 || - (responseByteArrayLen > 0 && responseByteArray == nullptr)) { - throw std::invalid_argument("SQS transport: invalid response buffer"); - } - - std::lock_guard lock(responsesMutex_); - const auto response = responses_.find(std::this_thread::get_id()); - if (response == responses_.end()) { - throw std::runtime_error( - "SQS transport: response not found for calling thread"); - } - if (responseByteArrayLen != - static_cast(response->second.size())) { - throw std::runtime_error( - "SQS transport: response buffer length mismatch"); - } - - std::copy( - response->second.begin(), - response->second.end(), - responseByteArray); - responses_.erase(response); - return 0; + return response; } #if defined(_WIN32) @@ -91,17 +58,36 @@ int TransportSqs::ReadResponse( #define SQS_PLUGIN_EXPORT extern "C" #endif -SQS_PLUGIN_EXPORT Hypertube::Native::Interfaces::ITransport* -CreateTransportChannel( +SQS_PLUGIN_EXPORT GraftcodeTransportChannel* +CreateTransport( const char*, - const unsigned short, - const char* configSource) + unsigned short, + const char* configSource, + char** errorMessage) +{ + return Graftcode::Plugins::TransportAbi::CreateTransport( + [&] { return new TransportSqs(configSource); }, + errorMessage); +} + +SQS_PLUGIN_EXPORT void DestroyTransport( + GraftcodeTransportChannel* channel) +{ + Graftcode::Plugins::TransportAbi::DestroyTransport(channel); +} + +SQS_PLUGIN_EXPORT int32_t TransportSendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen) { - return new TransportSqs(configSource); + return Graftcode::Plugins::TransportAbi::SendCommand( + channel, message, messageLen, response, responseLen); } -SQS_PLUGIN_EXPORT void DestroyTransportChannel( - Hypertube::Native::Interfaces::ITransport* transport) +SQS_PLUGIN_EXPORT void TransportFreeBuffer(void* buffer) { - delete transport; + Graftcode::Plugins::TransportAbi::FreeBuffer(buffer); } diff --git a/sqs/SqsPlugin/TransportSqs.h b/sqs/SqsPlugin/TransportSqs.h index 62cd254..b54270c 100644 --- a/sqs/SqsPlugin/TransportSqs.h +++ b/sqs/SqsPlugin/TransportSqs.h @@ -3,10 +3,7 @@ #include "ITransport.h" #include "SqsClient.h" -#include #include -#include -#include #include namespace Graftcode::Plugins::Sqs @@ -22,17 +19,12 @@ namespace Graftcode::Plugins::Sqs byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) override; - int SendCommand( + std::vector SendCommand( byte* messageByteArray, int32_t messageByteArrayLen) override; - int ReadResponse( - byte* responseByteArray, - int32_t responseByteArrayLen) override; private: std::unique_ptr client_; - std::map> responses_; - std::mutex responsesMutex_; }; void RejectGatewayErrorPayload(const std::vector& payload); diff --git a/sqs/SqsPluginTest/SqsTransportTests.cpp b/sqs/SqsPluginTest/SqsTransportTests.cpp index cda11bc..0d6d129 100644 --- a/sqs/SqsPluginTest/SqsTransportTests.cpp +++ b/sqs/SqsPluginTest/SqsTransportTests.cpp @@ -2,6 +2,7 @@ #include "IServer.h" #include "ITransport.h" +#include "TransportPluginAbi.h" #include "SqsClient.h" #include "SqsConfig.h" #include "SqsServer.h" @@ -22,13 +23,19 @@ #include #include -extern "C" Hypertube::Native::Interfaces::ITransport* -CreateTransportChannel( +extern "C" GraftcodeTransportChannel* CreateTransport( const char* ipAddress, unsigned short port, - const char* configSource); -extern "C" void DestroyTransportChannel( - Hypertube::Native::Interfaces::ITransport* transport); + const char* configSource, + char** errorMessage); +extern "C" void DestroyTransport(GraftcodeTransportChannel* channel); +extern "C" int32_t TransportSendCommand( + GraftcodeTransportChannel* channel, + const unsigned char* message, + int32_t messageLen, + unsigned char** response, + int32_t* responseLen); +extern "C" void TransportFreeBuffer(void* buffer); extern "C" GraftcodeGateway::IServer* CreateServer(); extern "C" void DestroyServer(GraftcodeGateway::IServer* server); @@ -342,17 +349,24 @@ TEST(SqsFifo, SetsGroupAndDedupOnlyForFifoQueues) TEST(SqsTransport, FactoryCreatesAndDestroysTransport) { - auto* transport = CreateTransportChannel(nullptr, 0, kLocalConfig); - ASSERT_NE(transport, nullptr); - EXPECT_EQ(transport->Initialize(1, 2, 0), 0); - - EXPECT_THROW(transport->SendCommand(nullptr, -1), std::invalid_argument); - EXPECT_THROW(transport->SendCommand(nullptr, 4), std::invalid_argument); - - unsigned char buffer[4]{}; - EXPECT_THROW(transport->ReadResponse(nullptr, 4), std::invalid_argument); - EXPECT_THROW(transport->ReadResponse(buffer, 4), std::runtime_error); - DestroyTransportChannel(transport); + char* errorMessage = nullptr; + auto* channel = CreateTransport(nullptr, 0, kLocalConfig, &errorMessage); + ASSERT_NE(channel, nullptr); + EXPECT_EQ(errorMessage, nullptr); + + for (const int32_t length : {-1, 4}) { + unsigned char* response = nullptr; + int32_t responseLen = 0; + EXPECT_EQ( + TransportSendCommand(channel, nullptr, length, &response, &responseLen), + GRAFTCODE_TRANSPORT_ERROR); + ASSERT_NE(response, nullptr); + EXPECT_EQ( + std::string(reinterpret_cast(response), responseLen), + "SQS transport: invalid request payload"); + TransportFreeBuffer(response); + } + DestroyTransport(channel); } TEST(SqsServer, FactoryConfigureAndStop) diff --git a/sqs/SqsPluginTest/smoke.cpp b/sqs/SqsPluginTest/smoke.cpp index 5bf7e85..bb03403 100644 --- a/sqs/SqsPluginTest/smoke.cpp +++ b/sqs/SqsPluginTest/smoke.cpp @@ -1,5 +1,5 @@ #include "IServer.h" -#include "ITransport.h" +#include "TransportPluginAbi.h" #include @@ -11,12 +11,10 @@ SQS_PLUGIN_IMPORT GraftcodeGateway::IServer* CreateServer(); SQS_PLUGIN_IMPORT void DestroyServer(GraftcodeGateway::IServer* server); -SQS_PLUGIN_IMPORT Hypertube::Native::Interfaces::ITransport* CreateTransportChannel( - const char* ipAddress, - unsigned short port, - const char* configSource); -SQS_PLUGIN_IMPORT void DestroyTransportChannel( - Hypertube::Native::Interfaces::ITransport* transport); +SQS_PLUGIN_IMPORT GraftcodeTransportChannel* CreateTransport( + const char* ipAddress, unsigned short port, const char* configSource, char** errorMessage); +SQS_PLUGIN_IMPORT void DestroyTransport(GraftcodeTransportChannel* channel); +SQS_PLUGIN_IMPORT void TransportFreeBuffer(void* buffer); static bool smokeProcess( const GraftcodeGateway::IServer::byte*, @@ -53,18 +51,15 @@ int main() server->stop(); DestroyServer(server); - Hypertube::Native::Interfaces::ITransport* transport = - CreateTransportChannel(nullptr, 0, config); + char* errorMessage = nullptr; + GraftcodeTransportChannel* transport = + CreateTransport(nullptr, 0, config, &errorMessage); if (transport == nullptr) { - std::puts("FAIL: CreateTransportChannel returned null"); + std::printf("FAIL: CreateTransport returned null: %s\n", errorMessage != nullptr ? errorMessage : ""); + TransportFreeBuffer(errorMessage); return 1; } - if (transport->Initialize(1, 2, 1) != 0) { - std::puts("FAIL: Initialize returned non-zero"); - DestroyTransportChannel(transport); - return 1; - } - DestroyTransportChannel(transport); + DestroyTransport(transport); std::puts("SqsPlugin smoke OK"); return 0; From 80ad8c0231c237f930ca7609787c3b4bc5aee3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Komor?= Date: Wed, 14 Oct 2026 11:27:33 +0200 Subject: [PATCH 2/2] Refactor RabbitMQ transport implementation for concurrent handling - Removed the need for a dedicated reply queue by allowing clients to declare their own exclusive reply queues, simplifying the configuration. - Updated the RabbitMQ client to support concurrent calls, enabling multiple requests to be processed simultaneously. - Enhanced the README documentation to reflect the new configuration requirements and usage instructions. - Cleaned up the transport interface to ensure thread safety and improved response handling. --- README.md | 1 - .../PendingReplies.h | 190 +++++++++ .../TransportPluginAbi.h | 8 +- kafka/KafkaPlugin/KafkaClient.cpp | 129 +++--- kafka/KafkaPlugin/KafkaClient.h | 16 +- .../PendingReplies.h | 190 +++++++++ .../TransportPluginAbi.h | 8 +- pubsub/PubSubPlugin/PubSubClient.cpp | 192 +++++---- pubsub/PubSubPlugin/PubSubClient.h | 19 +- pubsub/Readme.md | 2 +- .../PendingReplies.h | 190 +++++++++ .../TransportPluginAbi.h | 8 +- rabbitmq/RabbitmqPlugin/RabbitMqClient.cpp | 400 +++++++++++------- rabbitmq/RabbitmqPlugin/RabbitMqClient.h | 29 +- rabbitmq/RabbitmqPlugin/RabbitMqServer.cpp | 2 - rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp | 12 +- rabbitmq/RabbitmqPlugin/TransportRabbitMq.h | 2 + .../RabbitMqTransportTests.cpp | 1 - rabbitmq/Readme.md | 6 +- samples/rabbitmq/middleware/definitions.json | 7 - .../PendingReplies.h | 190 +++++++++ .../TransportPluginAbi.h | 8 +- .../ServiceBusPlugin/ServiceBusClient.cpp | 358 +++++++++------- .../ServiceBusPlugin/ServiceBusClient.h | 30 ++ .../ServiceBusPlugin/TransportServiceBus.cpp | 12 +- .../ServiceBusPlugin/TransportServiceBus.h | 2 + .../PendingReplies.h | 190 +++++++++ .../TransportPluginAbi.h | 8 +- sqs/Readme.md | 2 +- sqs/SqsPlugin/SqsClient.cpp | 152 ++++--- sqs/SqsPlugin/SqsClient.h | 12 +- 31 files changed, 1803 insertions(+), 573 deletions(-) create mode 100644 kafka/GraftcodePluginsInterfaces/PendingReplies.h create mode 100644 pubsub/GraftcodePluginsInterfaces/PendingReplies.h create mode 100644 rabbitmq/GraftcodePluginsInterfaces/PendingReplies.h create mode 100644 servicebus/GraftcodePluginsInterfaces/PendingReplies.h create mode 100644 sqs/GraftcodePluginsInterfaces/PendingReplies.h diff --git a/README.md b/README.md index fa0cbd8..b4117fd 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ RabbitMQ (`rabbitmq/Readme.md`): "host": "localhost", "port": 5672, "queue": "myqueue", - "replyQueue": "myqueue.reply", "user": "guest", "password": "guest", "vhost": "/", diff --git a/kafka/GraftcodePluginsInterfaces/PendingReplies.h b/kafka/GraftcodePluginsInterfaces/PendingReplies.h new file mode 100644 index 0000000..e9462d2 --- /dev/null +++ b/kafka/GraftcodePluginsInterfaces/PendingReplies.h @@ -0,0 +1,190 @@ +/** + * @file PendingReplies.h + * @brief Correlates concurrent requests with replies delivered by a single consumer. + * + * A caller registers its correlation id before sending, then waits on the returned ticket. + * The consumer completes or fails entries by id. The ticket removes its entry when it goes + * out of scope, so timed-out and failed requests leave nothing behind. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::Concurrency +{ + class PendingReplies + { + struct Slot + { + std::condition_variable ready; + bool done{ false }; + bool failed{ false }; + std::vector payload; + std::string error; + }; + + public: + using Payload = std::vector; + + class Ticket + { + public: + Ticket(PendingReplies& owner, std::string id, std::shared_ptr slot) + : owner_(&owner), id_(std::move(id)), slot_(std::move(slot)) + { + } + + ~Ticket() + { + if (owner_ != nullptr) { + owner_->Remove(id_); + } + } + + Ticket(Ticket&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), + id_(std::move(other.id_)), + slot_(std::move(other.slot_)) + { + } + + Ticket(const Ticket&) = delete; + Ticket& operator=(const Ticket&) = delete; + Ticket& operator=(Ticket&&) = delete; + + const std::string& Id() const { return id_; } + + // Returns false on timeout; throws when the request was failed. + bool Wait(std::chrono::milliseconds timeout, Payload& response) + { + std::unique_lock lock(owner_->mutex_); + if (!slot_->ready.wait_for(lock, timeout, [this] { return slot_->done; })) { + return false; + } + if (slot_->failed) { + throw std::runtime_error(slot_->error); + } + response = std::move(slot_->payload); + return true; + } + + private: + PendingReplies* owner_; + std::string id_; + std::shared_ptr slot_; + }; + + PendingReplies() = default; + PendingReplies(const PendingReplies&) = delete; + PendingReplies& operator=(const PendingReplies&) = delete; + + Ticket Register(const std::string& id) + { + auto slot = std::make_shared(); + { + std::lock_guard lock(mutex_); + if (closed_) { + throw std::runtime_error(closedError_); + } + if (!slots_.emplace(id, slot).second) { + throw std::runtime_error("duplicate correlation id " + id); + } + } + pendingChanged_.notify_all(); + return Ticket(*this, id, std::move(slot)); + } + + // Returns false when no request is waiting for this id. + bool Complete(const std::string& id, Payload payload) + { + return Finish(id, false, std::move(payload), std::string()); + } + + bool Fail(const std::string& id, const std::string& error) + { + return Finish(id, true, Payload(), error); + } + + void FailAll(const std::string& error) + { + std::lock_guard lock(mutex_); + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + + // Fails every waiting request and rejects new registrations. + void Close(const std::string& error) + { + { + std::lock_guard lock(mutex_); + closed_ = true; + closedError_ = error; + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + pendingChanged_.notify_all(); + } + + bool Empty() const + { + std::lock_guard lock(mutex_); + return slots_.empty(); + } + + // Blocks until a request is registered, the set is closed, or the timeout elapses. + // Returns true when at least one request is waiting. + bool WaitForPending(std::chrono::milliseconds timeout) + { + std::unique_lock lock(mutex_); + pendingChanged_.wait_for(lock, timeout, [this] { return closed_ || !slots_.empty(); }); + return !closed_ && !slots_.empty(); + } + + private: + static void MarkDone(Slot& slot, bool failed, Payload payload, const std::string& error) + { + if (slot.done) { + return; + } + slot.done = true; + slot.failed = failed; + slot.payload = std::move(payload); + slot.error = error; + slot.ready.notify_all(); + } + + bool Finish(const std::string& id, bool failed, Payload payload, const std::string& error) + { + std::lock_guard lock(mutex_); + const auto it = slots_.find(id); + if (it == slots_.end()) { + return false; + } + MarkDone(*it->second, failed, std::move(payload), error); + return true; + } + + void Remove(const std::string& id) + { + std::lock_guard lock(mutex_); + slots_.erase(id); + } + + mutable std::mutex mutex_; + std::condition_variable pendingChanged_; + std::map> slots_; + bool closed_{ false }; + std::string closedError_; + }; +} diff --git a/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h b/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h index 05f27d3..f4a486b 100644 --- a/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h +++ b/kafka/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -7,7 +7,8 @@ * - DestroyTransport * - TransportSendCommand * - TransportFreeBuffer - * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Plugins must be thread-safe: TransportSendCommand may be called concurrently on one channel. * * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. */ @@ -38,7 +39,7 @@ typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipA typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); /** - * Sends a request and returns its response. + * Sends a request and returns its response. May be called concurrently on one channel. * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). * Otherwise they hold a NUL-terminated error message and its length without the terminator. */ @@ -47,9 +48,6 @@ typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* ch /** Releases a buffer returned by the plugin. */ typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); -/** Returns true when TransportSendCommand may be called concurrently on one channel. */ -typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); - #ifdef __cplusplus } #endif diff --git a/kafka/KafkaPlugin/KafkaClient.cpp b/kafka/KafkaPlugin/KafkaClient.cpp index fd7718b..7c4f1b4 100644 --- a/kafka/KafkaPlugin/KafkaClient.cpp +++ b/kafka/KafkaPlugin/KafkaClient.cpp @@ -1,9 +1,9 @@ #include "KafkaClient.h" #include "KafkaUtil.h" -#include #include #include +#include #include #if __has_include() @@ -28,6 +28,23 @@ void drainProducer(RdKafka::Producer* producer, int timeoutMs) { } // namespace +class KafkaClient::DeliveryReports : public RdKafka::DeliveryReportCb { +public: + explicit DeliveryReports(Graftcode::Plugins::Concurrency::PendingReplies& pending) + : pending_(pending) {} + + void dr_cb(RdKafka::Message& message) override { + std::unique_ptr correlationId(static_cast(message.msg_opaque())); + if (correlationId && message.err() != RdKafka::ERR_NO_ERROR) { + pending_.Fail(*correlationId, "Kafka delivery failed for correlation-id=" + + *correlationId + ": " + message.errstr()); + } + } + +private: + Graftcode::Plugins::Concurrency::PendingReplies& pending_; +}; + KafkaClient::KafkaClient(Config cfg) : cfg_(std::move(cfg)) { instanceId_ = KafkaPluginUtil::makeUuidV4(); // Unique group so concurrent clients sharing one reply topic each get a full @@ -38,7 +55,11 @@ KafkaClient::KafkaClient(Config cfg) : cfg_(std::move(cfg)) { } KafkaClient::~KafkaClient() { - std::lock_guard lock(callMutex_); + stopping_.store(true, std::memory_order_release); + if (dispatcher_.joinable()) { + dispatcher_.join(); + } + pending_.Close("Kafka client is shutting down"); if (consumer_) { consumer_->close(); consumer_.reset(); @@ -69,7 +90,11 @@ void KafkaClient::applySecurity(RdKafka::Conf* conf) const { } void KafkaClient::ensureStarted() { - if (started_) { + if (started_.load(std::memory_order_acquire)) { + return; + } + std::lock_guard lock(startMutex_); + if (started_.load(std::memory_order_acquire)) { return; } @@ -82,6 +107,12 @@ void KafkaClient::ensureStarted() { setConfOrThrow(pconf.get(), "client.id", "graft-kafka-client-producer-" + instanceId_); setConfOrThrow(pconf.get(), "message.timeout.ms", std::to_string(cfg_.rpcTimeoutMs)); applySecurity(pconf.get()); + if (!deliveryReports_) { + deliveryReports_ = std::make_unique(pending_); + } + if (pconf->set("dr_cb", deliveryReports_.get(), errstr) != RdKafka::Conf::CONF_OK) { + throw std::runtime_error("Failed to set Kafka delivery report callback: " + errstr); + } RdKafka::Producer* raw = RdKafka::Producer::create(pconf.release(), errstr); if (!raw) { @@ -122,17 +153,37 @@ void KafkaClient::ensureStarted() { continue; } if (msg->err() == RdKafka::ERR_NO_ERROR) { - // Unexpected early message — leave for the next call() filter. break; } - if (msg->err() != RdKafka::ERR__TIMED_OUT && - msg->err() != RdKafka::ERR__PARTITION_EOF) { - // Keep warming on transient errors. - } } } - started_ = true; + dispatcher_ = std::thread(&KafkaClient::dispatchReplies, this); + started_.store(true, std::memory_order_release); +} + +void KafkaClient::dispatchReplies() { + while (!stopping_.load(std::memory_order_acquire)) { + producer_->poll(0); + + std::unique_ptr msg(consumer_->consume(100)); + if (!msg || msg->err() != RdKafka::ERR_NO_ERROR) { + continue; + } + + const std::string correlationId = + KafkaPluginUtil::headerValue(msg->headers(), "correlation-id"); + if (correlationId.empty()) { + continue; + } + + std::vector payload; + if (msg->payload() != nullptr && msg->len() > 0) { + const auto* bytes = static_cast(msg->payload()); + payload.assign(bytes, bytes + msg->len()); + } + pending_.Complete(correlationId, std::move(payload)); + } } std::vector KafkaClient::call(const unsigned char* data, std::size_t len) { @@ -140,12 +191,13 @@ std::vector KafkaClient::call(const unsigned char* data, std::siz throw std::invalid_argument("KafkaClient::call: null data with non-zero length"); } - std::lock_guard lock(callMutex_); ensureStarted(); const std::string correlationId = KafkaPluginUtil::makeUuidV4(); + auto ticket = pending_.Register(correlationId); RdKafka::Headers* headers = KafkaPluginUtil::makeRpcHeaders(correlationId, cfg_.replyTopic); + auto* deliveryTag = new std::string(correlationId); const RdKafka::ErrorCode produceErr = producer_->produce( cfg_.requestTopic, @@ -157,61 +209,20 @@ std::vector KafkaClient::call(const unsigned char* data, std::siz /*key_len=*/0, /*timestamp=*/0, headers, - /*msg_opaque=*/nullptr); + /*msg_opaque=*/deliveryTag); if (produceErr != RdKafka::ERR_NO_ERROR) { - // produce() takes ownership of headers only on success. + // produce() takes ownership of headers and the opaque tag only on success. delete headers; + delete deliveryTag; throw std::runtime_error("Kafka produce to '" + cfg_.requestTopic + "' failed: " + RdKafka::err2str(produceErr)); } - producer_->poll(0); - // Wait until the request is handed off to the broker (best-effort). - if (producer_->flush(std::min(cfg_.rpcTimeoutMs, 10000)) != RdKafka::ERR_NO_ERROR) { - throw std::runtime_error("Kafka produce flush timed out for correlation-id=" + - correlationId); - } - - const auto deadline = - std::chrono::steady_clock::now() + std::chrono::milliseconds(cfg_.rpcTimeoutMs); - - while (std::chrono::steady_clock::now() < deadline) { - const auto remaining = std::chrono::duration_cast( - deadline - std::chrono::steady_clock::now()) - .count(); - const int pollMs = static_cast(std::min(remaining, 200)); - - std::unique_ptr msg(consumer_->consume(pollMs)); - if (!msg) { - continue; - } - - if (msg->err() == RdKafka::ERR__TIMED_OUT || - msg->err() == RdKafka::ERR__PARTITION_EOF) { - continue; - } - if (msg->err() != RdKafka::ERR_NO_ERROR) { - throw std::runtime_error("Kafka consume error while waiting for reply: " + - msg->errstr()); - } - - const std::string gotId = - KafkaPluginUtil::headerValue(msg->headers(), "correlation-id"); - if (gotId != correlationId) { - // Shared reply topic / other clients — skip. - continue; - } - - const void* payload = msg->payload(); - const std::size_t payloadLen = static_cast(msg->len()); - if (!payload || payloadLen == 0) { - return {}; - } - const auto* bytes = static_cast(payload); - return std::vector(bytes, bytes + payloadLen); + std::vector response; + if (!ticket.Wait(std::chrono::milliseconds(cfg_.rpcTimeoutMs), response)) { + throw std::runtime_error("Kafka RPC timed out after " + std::to_string(cfg_.rpcTimeoutMs) + + " ms waiting for correlation-id=" + correlationId); } - - throw std::runtime_error("Kafka RPC timed out after " + std::to_string(cfg_.rpcTimeoutMs) + - " ms waiting for correlation-id=" + correlationId); + return response; } diff --git a/kafka/KafkaPlugin/KafkaClient.h b/kafka/KafkaPlugin/KafkaClient.h index 8c3cc7f..288eba9 100644 --- a/kafka/KafkaPlugin/KafkaClient.h +++ b/kafka/KafkaPlugin/KafkaClient.h @@ -1,9 +1,13 @@ #pragma once +#include "PendingReplies.h" + +#include #include #include #include #include +#include #include namespace RdKafka { @@ -13,6 +17,7 @@ class KafkaConsumer; } // namespace RdKafka // Client-side RPC over Kafka: produce to request topic, consume correlated reply. +// call() may run concurrently; one consumer thread dispatches replies by correlation id. class KafkaClient { public: struct Config { @@ -39,14 +44,21 @@ class KafkaClient { std::vector call(const unsigned char* data, std::size_t len); private: + class DeliveryReports; + void applySecurity(RdKafka::Conf* conf) const; void ensureStarted(); + void dispatchReplies(); Config cfg_; std::string instanceId_; std::string consumerGroupId_; + Graftcode::Plugins::Concurrency::PendingReplies pending_; + std::unique_ptr deliveryReports_; std::unique_ptr producer_; std::unique_ptr consumer_; - std::mutex callMutex_; - bool started_ = false; + std::thread dispatcher_; + std::mutex startMutex_; + std::atomic started_{false}; + std::atomic stopping_{false}; }; diff --git a/pubsub/GraftcodePluginsInterfaces/PendingReplies.h b/pubsub/GraftcodePluginsInterfaces/PendingReplies.h new file mode 100644 index 0000000..e9462d2 --- /dev/null +++ b/pubsub/GraftcodePluginsInterfaces/PendingReplies.h @@ -0,0 +1,190 @@ +/** + * @file PendingReplies.h + * @brief Correlates concurrent requests with replies delivered by a single consumer. + * + * A caller registers its correlation id before sending, then waits on the returned ticket. + * The consumer completes or fails entries by id. The ticket removes its entry when it goes + * out of scope, so timed-out and failed requests leave nothing behind. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::Concurrency +{ + class PendingReplies + { + struct Slot + { + std::condition_variable ready; + bool done{ false }; + bool failed{ false }; + std::vector payload; + std::string error; + }; + + public: + using Payload = std::vector; + + class Ticket + { + public: + Ticket(PendingReplies& owner, std::string id, std::shared_ptr slot) + : owner_(&owner), id_(std::move(id)), slot_(std::move(slot)) + { + } + + ~Ticket() + { + if (owner_ != nullptr) { + owner_->Remove(id_); + } + } + + Ticket(Ticket&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), + id_(std::move(other.id_)), + slot_(std::move(other.slot_)) + { + } + + Ticket(const Ticket&) = delete; + Ticket& operator=(const Ticket&) = delete; + Ticket& operator=(Ticket&&) = delete; + + const std::string& Id() const { return id_; } + + // Returns false on timeout; throws when the request was failed. + bool Wait(std::chrono::milliseconds timeout, Payload& response) + { + std::unique_lock lock(owner_->mutex_); + if (!slot_->ready.wait_for(lock, timeout, [this] { return slot_->done; })) { + return false; + } + if (slot_->failed) { + throw std::runtime_error(slot_->error); + } + response = std::move(slot_->payload); + return true; + } + + private: + PendingReplies* owner_; + std::string id_; + std::shared_ptr slot_; + }; + + PendingReplies() = default; + PendingReplies(const PendingReplies&) = delete; + PendingReplies& operator=(const PendingReplies&) = delete; + + Ticket Register(const std::string& id) + { + auto slot = std::make_shared(); + { + std::lock_guard lock(mutex_); + if (closed_) { + throw std::runtime_error(closedError_); + } + if (!slots_.emplace(id, slot).second) { + throw std::runtime_error("duplicate correlation id " + id); + } + } + pendingChanged_.notify_all(); + return Ticket(*this, id, std::move(slot)); + } + + // Returns false when no request is waiting for this id. + bool Complete(const std::string& id, Payload payload) + { + return Finish(id, false, std::move(payload), std::string()); + } + + bool Fail(const std::string& id, const std::string& error) + { + return Finish(id, true, Payload(), error); + } + + void FailAll(const std::string& error) + { + std::lock_guard lock(mutex_); + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + + // Fails every waiting request and rejects new registrations. + void Close(const std::string& error) + { + { + std::lock_guard lock(mutex_); + closed_ = true; + closedError_ = error; + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + pendingChanged_.notify_all(); + } + + bool Empty() const + { + std::lock_guard lock(mutex_); + return slots_.empty(); + } + + // Blocks until a request is registered, the set is closed, or the timeout elapses. + // Returns true when at least one request is waiting. + bool WaitForPending(std::chrono::milliseconds timeout) + { + std::unique_lock lock(mutex_); + pendingChanged_.wait_for(lock, timeout, [this] { return closed_ || !slots_.empty(); }); + return !closed_ && !slots_.empty(); + } + + private: + static void MarkDone(Slot& slot, bool failed, Payload payload, const std::string& error) + { + if (slot.done) { + return; + } + slot.done = true; + slot.failed = failed; + slot.payload = std::move(payload); + slot.error = error; + slot.ready.notify_all(); + } + + bool Finish(const std::string& id, bool failed, Payload payload, const std::string& error) + { + std::lock_guard lock(mutex_); + const auto it = slots_.find(id); + if (it == slots_.end()) { + return false; + } + MarkDone(*it->second, failed, std::move(payload), error); + return true; + } + + void Remove(const std::string& id) + { + std::lock_guard lock(mutex_); + slots_.erase(id); + } + + mutable std::mutex mutex_; + std::condition_variable pendingChanged_; + std::map> slots_; + bool closed_{ false }; + std::string closedError_; + }; +} diff --git a/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h b/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h index 05f27d3..f4a486b 100644 --- a/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h +++ b/pubsub/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -7,7 +7,8 @@ * - DestroyTransport * - TransportSendCommand * - TransportFreeBuffer - * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Plugins must be thread-safe: TransportSendCommand may be called concurrently on one channel. * * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. */ @@ -38,7 +39,7 @@ typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipA typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); /** - * Sends a request and returns its response. + * Sends a request and returns its response. May be called concurrently on one channel. * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). * Otherwise they hold a NUL-terminated error message and its length without the terminator. */ @@ -47,9 +48,6 @@ typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* ch /** Releases a buffer returned by the plugin. */ typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); -/** Returns true when TransportSendCommand may be called concurrently on one channel. */ -typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); - #ifdef __cplusplus } #endif diff --git a/pubsub/PubSubPlugin/PubSubClient.cpp b/pubsub/PubSubPlugin/PubSubClient.cpp index ff9c092..2417bf8 100644 --- a/pubsub/PubSubPlugin/PubSubClient.cpp +++ b/pubsub/PubSubPlugin/PubSubClient.cpp @@ -3,7 +3,10 @@ #include "PubSubUtil.h" #include +#include +#include #include +#include #include #include @@ -24,25 +27,118 @@ namespace Graftcode::Plugins::PubSub ValidateClientConfig(config_); } - PubSubClient::~PubSubClient() = default; + PubSubClient::~PubSubClient() + { + stopping_.store(true, std::memory_order_release); + pending_.Close("Pub/Sub client is shutting down"); + if (dispatcher_.joinable()) { + dispatcher_.join(); + } + } + + std::unique_ptr PubSubClient::AcquireSession() + { + { + std::lock_guard lock(sessionsMutex_); + if (!idleSessions_.empty()) { + auto session = std::move(idleSessions_.back()); + idleSessions_.pop_back(); + return session; + } + } + return std::make_unique(config_); + } + + void PubSubClient::ReleaseSession(std::unique_ptr session) + { + std::lock_guard lock(sessionsMutex_); + idleSessions_.push_back(std::move(session)); + } - void PubSubClient::EnsureResources() + void PubSubClient::EnsureStarted() { - if (session_ == nullptr) { - session_ = std::make_unique(config_); + if (started_.load(std::memory_order_acquire)) { + return; } - if (resourcesReady_ || !config_.autoCreate) { + std::lock_guard lock(startMutex_); + if (started_.load(std::memory_order_acquire)) { return; } - session_->EnsureTopic(config_.requestTopic); + auto session = AcquireSession(); + if (config_.autoCreate) { + session->EnsureTopic(config_.requestTopic); + if (!config_.oneWay) { + session->EnsureTopic(config_.replyTopic); + session->EnsureSubscription( + config_.replySubscription, + config_.replyTopic); + } + } + ReleaseSession(std::move(session)); + if (!config_.oneWay) { - session_->EnsureTopic(config_.replyTopic); - session_->EnsureSubscription( - config_.replySubscription, - config_.replyTopic); + replySession_ = std::make_unique(config_); + dispatcher_ = std::thread(&PubSubClient::DispatchReplies, this); + } + started_.store(true, std::memory_order_release); + } + + void PubSubClient::DispatchReplies() + { + while (!stopping_.load(std::memory_order_acquire)) { + if (!pending_.WaitForPending(std::chrono::milliseconds(200))) { + continue; + } + + std::vector batch; + try { + batch = replySession_->Pull( + config_.replySubscription, + config_.maxMessages); + } + catch (const PubSubHttpError& error) { + if (error.status() != 429 && error.status() < 500 && + error.status() != 0) { + pending_.FailAll(error.what()); + } + SleepBriefly(); + continue; + } + catch (const std::exception& error) { + pending_.FailAll(error.what()); + SleepBriefly(); + continue; + } + + if (batch.empty()) { + SleepBriefly(); + continue; + } + + // Replies nobody waits for are acknowledged too, so they are not redelivered. + std::vector ackIds; + for (const auto& message : batch) { + if (!message.ackId.empty()) { + ackIds.push_back(message.ackId); + } + } + if (!ackIds.empty()) { + try { + replySession_->Acknowledge(config_.replySubscription, ackIds); + } + catch (const std::exception&) { + } + } + + for (auto& message : batch) { + const auto correlation = message.attributes.find( + CorrelationIdAttribute); + if (correlation != message.attributes.end()) { + pending_.Complete(correlation->second, std::move(message.data)); + } + } } - resourcesReady_ = true; } std::vector PubSubClient::Call( @@ -58,24 +154,23 @@ namespace Graftcode::Plugins::PubSub "Pub/Sub client: null payload with non-zero length"); } - // A reply subscription is dedicated to this transport. Calls are - // serialized so an unrelated reply can be acked instead of being - // delivered to a second in-flight call. - std::lock_guard lock(callMutex_); - EnsureResources(); + EnsureStarted(); const std::string correlationId = NewCorrelationId(); + std::optional ticket; std::map attributes; attributes.emplace(CorrelationIdAttribute, correlationId); if (!config_.oneWay) { attributes.emplace(ReplyToAttribute, config_.replyTopic); + ticket.emplace(pending_.Register(correlationId)); } + auto session = AcquireSession(); const auto publishDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); for (;;) { try { - session_->Publish( + session->Publish( config_.requestTopic, data, size, @@ -91,66 +186,19 @@ namespace Graftcode::Plugins::PubSub SleepBriefly(); } } + ReleaseSession(std::move(session)); if (config_.oneWay) { return {}; } - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(config_.rpcTimeoutMs); - while (std::chrono::steady_clock::now() < deadline) { - std::vector batch; - try { - batch = session_->Pull( - config_.replySubscription, - config_.maxMessages); - } - catch (const PubSubHttpError& error) { - if (error.status() == 429 || error.status() >= 500 || - error.status() == 0) { - SleepBriefly(); - continue; - } - throw; - } - - std::vector ackIds; - std::vector response; - bool found = false; - for (auto& message : batch) { - if (message.ackId.empty()) { - continue; - } - ackIds.push_back(message.ackId); - const auto correlation = message.attributes.find( - CorrelationIdAttribute); - if (!found && - correlation != message.attributes.end() && - correlation->second == correlationId) { - response = std::move(message.data); - found = true; - } - } - - if (!ackIds.empty()) { - try { - session_->Acknowledge(config_.replySubscription, ackIds); - } - catch (const std::exception&) { - if (!found) { - throw; - } - } - } - if (found) { - return response; - } - SleepBriefly(); + std::vector response; + if (!ticket->Wait(std::chrono::milliseconds(config_.rpcTimeoutMs), response)) { + throw std::runtime_error( + "Pub/Sub RPC timed out after " + + std::to_string(config_.rpcTimeoutMs) + + " ms waiting for correlation id " + correlationId); } - - throw std::runtime_error( - "Pub/Sub RPC timed out after " + - std::to_string(config_.rpcTimeoutMs) + - " ms waiting for correlation id " + correlationId); + return response; } } diff --git a/pubsub/PubSubPlugin/PubSubClient.h b/pubsub/PubSubPlugin/PubSubClient.h index 59d6e0d..2271d3b 100644 --- a/pubsub/PubSubPlugin/PubSubClient.h +++ b/pubsub/PubSubPlugin/PubSubClient.h @@ -1,10 +1,13 @@ #pragma once +#include "PendingReplies.h" #include "PubSubConfig.h" +#include #include #include #include +#include #include namespace Graftcode::Plugins::PubSub @@ -25,11 +28,19 @@ namespace Graftcode::Plugins::PubSub std::size_t size); private: - void EnsureResources(); + void EnsureStarted(); + void DispatchReplies(); + std::unique_ptr AcquireSession(); + void ReleaseSession(std::unique_ptr session); PubSubConfig config_; - std::unique_ptr session_; - bool resourcesReady_{ false }; - std::mutex callMutex_; + Concurrency::PendingReplies pending_; + std::unique_ptr replySession_; + std::vector> idleSessions_; + std::mutex sessionsMutex_; + std::thread dispatcher_; + std::mutex startMutex_; + std::atomic started_{ false }; + std::atomic stopping_{ false }; }; } diff --git a/pubsub/Readme.md b/pubsub/Readme.md index 321b4d4..956b251 100644 --- a/pubsub/Readme.md +++ b/pubsub/Readme.md @@ -30,7 +30,7 @@ Pub/Sub is at-least-once. The gateway acks a request only after `processMessage` A request that is missing `correlationId` or a valid `replyTo` is also left unacked, so a dead-letter policy on the request subscription can take over. Configure `deadLetterPolicy` and `retryPolicy` on the subscription in Google Cloud when a poison message should stop being retried. The plugin does not change those policies. Make called operations idempotent: a crash after the reply is published and before the ack produces a second reply. The client drops replies whose correlation id does not match. -Pulls use the REST `:pull` method, which returns immediately when the subscription is empty. The client polls until `rpcTimeoutMs`. Calls through one transport instance are serialized. +Pulls use the REST `:pull` method, which returns immediately when the subscription is empty. The client polls until `rpcTimeoutMs`. Calls through one transport instance may run concurrently; one background puller dispatches replies by correlation id. Decoded payloads larger than 10 MiB are rejected. The JSON API carries message data as standard base64; that encoding is part of the Pub/Sub protocol, not an extra wrapper. diff --git a/rabbitmq/GraftcodePluginsInterfaces/PendingReplies.h b/rabbitmq/GraftcodePluginsInterfaces/PendingReplies.h new file mode 100644 index 0000000..e9462d2 --- /dev/null +++ b/rabbitmq/GraftcodePluginsInterfaces/PendingReplies.h @@ -0,0 +1,190 @@ +/** + * @file PendingReplies.h + * @brief Correlates concurrent requests with replies delivered by a single consumer. + * + * A caller registers its correlation id before sending, then waits on the returned ticket. + * The consumer completes or fails entries by id. The ticket removes its entry when it goes + * out of scope, so timed-out and failed requests leave nothing behind. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::Concurrency +{ + class PendingReplies + { + struct Slot + { + std::condition_variable ready; + bool done{ false }; + bool failed{ false }; + std::vector payload; + std::string error; + }; + + public: + using Payload = std::vector; + + class Ticket + { + public: + Ticket(PendingReplies& owner, std::string id, std::shared_ptr slot) + : owner_(&owner), id_(std::move(id)), slot_(std::move(slot)) + { + } + + ~Ticket() + { + if (owner_ != nullptr) { + owner_->Remove(id_); + } + } + + Ticket(Ticket&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), + id_(std::move(other.id_)), + slot_(std::move(other.slot_)) + { + } + + Ticket(const Ticket&) = delete; + Ticket& operator=(const Ticket&) = delete; + Ticket& operator=(Ticket&&) = delete; + + const std::string& Id() const { return id_; } + + // Returns false on timeout; throws when the request was failed. + bool Wait(std::chrono::milliseconds timeout, Payload& response) + { + std::unique_lock lock(owner_->mutex_); + if (!slot_->ready.wait_for(lock, timeout, [this] { return slot_->done; })) { + return false; + } + if (slot_->failed) { + throw std::runtime_error(slot_->error); + } + response = std::move(slot_->payload); + return true; + } + + private: + PendingReplies* owner_; + std::string id_; + std::shared_ptr slot_; + }; + + PendingReplies() = default; + PendingReplies(const PendingReplies&) = delete; + PendingReplies& operator=(const PendingReplies&) = delete; + + Ticket Register(const std::string& id) + { + auto slot = std::make_shared(); + { + std::lock_guard lock(mutex_); + if (closed_) { + throw std::runtime_error(closedError_); + } + if (!slots_.emplace(id, slot).second) { + throw std::runtime_error("duplicate correlation id " + id); + } + } + pendingChanged_.notify_all(); + return Ticket(*this, id, std::move(slot)); + } + + // Returns false when no request is waiting for this id. + bool Complete(const std::string& id, Payload payload) + { + return Finish(id, false, std::move(payload), std::string()); + } + + bool Fail(const std::string& id, const std::string& error) + { + return Finish(id, true, Payload(), error); + } + + void FailAll(const std::string& error) + { + std::lock_guard lock(mutex_); + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + + // Fails every waiting request and rejects new registrations. + void Close(const std::string& error) + { + { + std::lock_guard lock(mutex_); + closed_ = true; + closedError_ = error; + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + pendingChanged_.notify_all(); + } + + bool Empty() const + { + std::lock_guard lock(mutex_); + return slots_.empty(); + } + + // Blocks until a request is registered, the set is closed, or the timeout elapses. + // Returns true when at least one request is waiting. + bool WaitForPending(std::chrono::milliseconds timeout) + { + std::unique_lock lock(mutex_); + pendingChanged_.wait_for(lock, timeout, [this] { return closed_ || !slots_.empty(); }); + return !closed_ && !slots_.empty(); + } + + private: + static void MarkDone(Slot& slot, bool failed, Payload payload, const std::string& error) + { + if (slot.done) { + return; + } + slot.done = true; + slot.failed = failed; + slot.payload = std::move(payload); + slot.error = error; + slot.ready.notify_all(); + } + + bool Finish(const std::string& id, bool failed, Payload payload, const std::string& error) + { + std::lock_guard lock(mutex_); + const auto it = slots_.find(id); + if (it == slots_.end()) { + return false; + } + MarkDone(*it->second, failed, std::move(payload), error); + return true; + } + + void Remove(const std::string& id) + { + std::lock_guard lock(mutex_); + slots_.erase(id); + } + + mutable std::mutex mutex_; + std::condition_variable pendingChanged_; + std::map> slots_; + bool closed_{ false }; + std::string closedError_; + }; +} diff --git a/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h b/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h index 05f27d3..f4a486b 100644 --- a/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h +++ b/rabbitmq/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -7,7 +7,8 @@ * - DestroyTransport * - TransportSendCommand * - TransportFreeBuffer - * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Plugins must be thread-safe: TransportSendCommand may be called concurrently on one channel. * * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. */ @@ -38,7 +39,7 @@ typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipA typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); /** - * Sends a request and returns its response. + * Sends a request and returns its response. May be called concurrently on one channel. * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). * Otherwise they hold a NUL-terminated error message and its length without the terminator. */ @@ -47,9 +48,6 @@ typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* ch /** Releases a buffer returned by the plugin. */ typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); -/** Returns true when TransportSendCommand may be called concurrently on one channel. */ -typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); - #ifdef __cplusplus } #endif diff --git a/rabbitmq/RabbitmqPlugin/RabbitMqClient.cpp b/rabbitmq/RabbitmqPlugin/RabbitMqClient.cpp index ab8fc5e..a68d628 100644 --- a/rabbitmq/RabbitmqPlugin/RabbitMqClient.cpp +++ b/rabbitmq/RabbitmqPlugin/RabbitMqClient.cpp @@ -29,9 +29,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -147,19 +149,6 @@ namespace std::atomic_bool connectionError{ false }; }; - struct RpcState - { - std::atomic_bool channelError{ false }; - std::atomic_bool queueDeclared{ false }; - std::atomic_bool consumerReady{ false }; - std::atomic_bool publishAttempted{ false }; - std::atomic_bool publishAccepted{ false }; - std::atomic_bool responseReceived{ false }; - std::string replyQueueName{}; - std::string correlationId{}; - std::vector responsePayload{}; - }; - class SocketConnectionHandler final : public AMQP::ConnectionHandler { public: @@ -216,20 +205,17 @@ namespace std::mutex sendMutex_; }; + std::atomic correlationCounter{ 0 }; + std::string BuildCorrelationId() { const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + const auto sequence = correlationCounter.fetch_add(1, std::memory_order_relaxed); std::ostringstream oss; - oss << std::this_thread::get_id() << "-" << now; + oss << std::this_thread::get_id() << "-" << now << "-" << sequence; return oss.str(); } - std::string GetSessionConsumerTag() - { - static const std::string sessionConsumerTag = "hypertube-rpc-" + BuildCorrelationId(); - return sessionConsumerTag; - } - std::optional ParseJsonConfigSource(const std::string& configSource) { if (configSource.empty()) { @@ -272,7 +258,6 @@ namespace } const bool hasDedicatedRabbitKeys = node.contains("queue") - || node.contains("replyQueue") || node.contains("user") || node.contains("password") || node.contains("vhost") @@ -331,11 +316,6 @@ namespace config.queue = queueIt->get(); } - const auto replyQueueIt = rabbitNode.find("replyQueue"); - if (replyQueueIt != rabbitNode.end() && replyQueueIt->is_string()) { - config.replyQueue = replyQueueIt->get(); - } - const auto userIt = rabbitNode.find("user"); if (userIt != rabbitNode.end() && userIt->is_string()) { config.user = userIt->get(); @@ -404,163 +384,279 @@ bool RabbitMqClient::SendRpc( std::vector& responsePayload) { responsePayload.clear(); - - if (config.queue.empty()) { - return false; - } - if (config.replyQueue.empty()) { - return false; + try { + RabbitMqRpcClient client(config); + responsePayload = client.Call(payload, payloadSize); + return true; } - if (payloadSize > 0 && payload == nullptr) { + catch (...) { + responsePayload.clear(); return false; } +} +// One AMQP connection and channel with a private, exclusive reply queue. AMQP-CPP is not +// thread safe, so every call into the connection or channel happens under amqpMutex_; the +// I/O thread waits for socket data without holding it and dispatches replies by correlation id. +class RabbitMqRpcClient::Link +{ +public: + Link(const RabbitMqClientConfig& config, Concurrency::PendingReplies& pending) + : pending_(pending), queue_(config.queue) + { #ifdef _WIN32 - WSAData wsaData{}; - if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) { - return false; - } + WSAData wsaData{}; + if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) { + throw std::runtime_error("RabbitMQ: WSAStartup failed"); + } #endif + socket_ = ConnectSocket(config); + if (socket_ == InvalidSocket) { + Shutdown(); + throw std::runtime_error( + "RabbitMQ: cannot connect to " + config.host + ":" + std::to_string(config.port)); + } + + handler_ = std::make_unique(socket_, runtimeState_); + connection_ = std::make_unique( + handler_.get(), + AMQP::Login(config.user, config.password), + config.vhost.empty() ? "/" : config.vhost); + channel_ = std::make_unique(connection_.get()); + channel_->onError([this](const char* message) { + Disconnect(std::string("RabbitMQ channel error: ") + (message != nullptr ? message : "")); + }); + channel_->declareQueue(AMQP::exclusive | AMQP::autodelete) + .onSuccess([this](const std::string& name, uint32_t, uint32_t) { + StartConsumer(name); + }); - SocketHandle socket = ConnectSocket(config); - if (socket == InvalidSocket) { -#ifdef _WIN32 - WSACleanup(); -#endif - return false; + io_ = std::thread(&Link::Run, this); + + const auto deadline = std::chrono::steady_clock::now() + ConnectionReadyTimeout; + while (!consumerReady_.load(std::memory_order_acquire) && + !broken_.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + if (!consumerReady_.load(std::memory_order_acquire)) { + const std::string error = broken_.load(std::memory_order_acquire) + ? BrokenError() + : std::string("RabbitMQ: timed out waiting for the reply consumer"); + Shutdown(); + throw std::runtime_error(error); + } + } + + ~Link() + { + Shutdown(); + } + + Link(const Link&) = delete; + Link& operator=(const Link&) = delete; + + bool Broken() const + { + return broken_.load(std::memory_order_acquire); + } + + void Publish(const std::string& correlationId, const byte* payload, std::size_t payloadSize) + { + std::lock_guard lock(amqpMutex_); + if (broken_.load(std::memory_order_acquire)) { + throw std::runtime_error(BrokenError()); + } + const char* body = payloadSize == 0 ? "" : reinterpret_cast(payload); + AMQP::Envelope envelope(body, payloadSize); + envelope.setContentType("application/octet-stream"); + envelope.setReplyTo(replyQueue_); + envelope.setCorrelationID(correlationId); + if (!channel_->publish("", queue_, envelope)) { + throw std::runtime_error("RabbitMQ: failed to publish request to queue " + queue_); + } + } + +private: + void StartConsumer(const std::string& replyQueue) + { + replyQueue_ = replyQueue; + channel_->consume(replyQueue_) + .onSuccess([this](const std::string&) { + consumerReady_.store(true, std::memory_order_release); + }) + .onReceived([this](const AMQP::Message& message, uint64_t deliveryTag, bool) { + std::vector body(message.bodySize()); + if (!body.empty()) { + std::memcpy(body.data(), message.body(), body.size()); + } + channel_->ack(deliveryTag); + if (message.hasCorrelationID()) { + pending_.Complete(message.correlationID(), std::move(body)); + } + }); } - RuntimeState runtimeState{}; - RpcState rpcState{}; - rpcState.correlationId = BuildCorrelationId(); - - SocketConnectionHandler handler(socket, runtimeState); - AMQP::Connection connection( - &handler, - AMQP::Login(config.user, config.password), - config.vhost.empty() ? "/" : config.vhost); - AMQP::Channel channel(&connection); - - channel.onError([&](const char*) { - rpcState.channelError.store(true, std::memory_order_release); - }); - - rpcState.replyQueueName = config.replyQueue; - rpcState.queueDeclared.store(true, std::memory_order_release); - - auto& consumer = channel.consume(config.replyQueue, GetSessionConsumerTag()); - consumer.onSuccess([&](const std::string&) { - rpcState.consumerReady.store(true, std::memory_order_release); - }).onReceived([&](const AMQP::Message& message, uint64_t deliveryTag, bool) { - if (!message.hasCorrelationID() || message.correlationID() != rpcState.correlationId) { - channel.reject(deliveryTag, AMQP::requeue); + void Run() + { + std::vector receiveBuffer(ReadBufferSize); + std::vector pendingBytes; + pendingBytes.reserve(ReadBufferSize); + + while (!stopping_.load(std::memory_order_acquire) && !broken_.load(std::memory_order_acquire)) { + if (runtimeState_.connectionError.load(std::memory_order_acquire) || + runtimeState_.connectionClosed.load(std::memory_order_acquire)) { + Disconnect("RabbitMQ: connection lost"); + return; + } + + const WaitStatus waitStatus = WaitForSocketReadable(socket_, 200); + if (waitStatus == WaitStatus::Timeout) { + continue; + } + if (waitStatus == WaitStatus::Error) { + Disconnect("RabbitMQ: socket error"); return; } - const std::size_t bodySize = message.bodySize(); - rpcState.responsePayload.resize(bodySize); - if (bodySize > 0) { - std::memcpy(rpcState.responsePayload.data(), message.body(), bodySize); +#ifdef _WIN32 + const int bytesRead = ::recv(socket_, receiveBuffer.data(), static_cast(receiveBuffer.size()), 0); +#else + const ssize_t bytesRead = ::recv(socket_, receiveBuffer.data(), receiveBuffer.size(), 0); +#endif + if (bytesRead == 0) { + Disconnect("RabbitMQ: connection closed by broker"); + return; + } + if (bytesRead < 0) { + if (IsWouldBlockError()) { + continue; + } + Disconnect("RabbitMQ: socket read failed"); + return; } - rpcState.responseReceived.store(true, std::memory_order_release); - channel.ack(deliveryTag); - }).onError([&](const char*) { - rpcState.channelError.store(true, std::memory_order_release); - }); + pendingBytes.insert( + pendingBytes.end(), + receiveBuffer.begin(), + receiveBuffer.begin() + static_cast(bytesRead)); - std::vector receiveBuffer(ReadBufferSize); - std::vector pendingBytes; - pendingBytes.reserve(ReadBufferSize); - const auto startedAt = std::chrono::steady_clock::now(); - const auto rpcTimeout = std::chrono::milliseconds(config.rpcTimeoutMs == 0 ? 30000u : config.rpcTimeoutMs); - - while (!runtimeState.connectionClosed.load(std::memory_order_acquire) && - !runtimeState.connectionError.load(std::memory_order_acquire) && - !rpcState.channelError.load(std::memory_order_acquire)) { - - if (runtimeState.connectionReady.load(std::memory_order_acquire) && - rpcState.queueDeclared.load(std::memory_order_acquire) && - rpcState.consumerReady.load(std::memory_order_acquire) && - !rpcState.publishAttempted.load(std::memory_order_acquire)) { - const char* body = payloadSize == 0 ? "" : reinterpret_cast(payload); - AMQP::Envelope envelope(body, payloadSize); - envelope.setContentType("application/octet-stream"); - envelope.setReplyTo(config.replyQueue); - envelope.setCorrelationID(rpcState.correlationId); - - const bool publishAccepted = channel.publish("", config.queue, envelope); - rpcState.publishAttempted.store(true, std::memory_order_release); - rpcState.publishAccepted.store(publishAccepted, std::memory_order_release); - if (!publishAccepted) { - break; + std::lock_guard lock(amqpMutex_); + while (!pendingBytes.empty()) { + const std::uint64_t parsed = connection_->parse(pendingBytes.data(), pendingBytes.size()); + if (parsed == 0 || parsed > pendingBytes.size()) { + break; + } + pendingBytes.erase( + pendingBytes.begin(), + pendingBytes.begin() + static_cast(parsed)); } } + } - if ((std::chrono::steady_clock::now() - startedAt) > ConnectionReadyTimeout && - !runtimeState.connectionReady.load(std::memory_order_acquire)) { - break; + void Disconnect(const std::string& error) + { + { + std::lock_guard lock(errorMutex_); + if (brokenError_.empty()) { + brokenError_ = error; + } } + broken_.store(true, std::memory_order_release); + pending_.FailAll(error); + } - if ((std::chrono::steady_clock::now() - startedAt) > rpcTimeout) { - break; - } + std::string BrokenError() const + { + std::lock_guard lock(errorMutex_); + return brokenError_.empty() ? std::string("RabbitMQ: connection is not available") : brokenError_; + } - if (rpcState.responseReceived.load(std::memory_order_acquire)) { - responsePayload = rpcState.responsePayload; - break; + void Shutdown() + { + stopping_.store(true, std::memory_order_release); + if (io_.joinable()) { + io_.join(); } - - const WaitStatus waitStatus = WaitForSocketReadable(socket, 500); - if (waitStatus == WaitStatus::Timeout) { - continue; + if (connection_) { + connection_->close(); + } + channel_.reset(); + connection_.reset(); + handler_.reset(); + if (socket_ != InvalidSocket) { + CloseSocket(socket_); + socket_ = InvalidSocket; } - if (waitStatus == WaitStatus::Error) { - break; +#ifdef _WIN32 + if (!wsaCleaned_) { + wsaCleaned_ = true; + WSACleanup(); } +#endif + } + Concurrency::PendingReplies& pending_; + std::string queue_; + std::string replyQueue_; + SocketHandle socket_{ InvalidSocket }; + RuntimeState runtimeState_{}; + std::unique_ptr handler_; + std::unique_ptr connection_; + std::unique_ptr channel_; + std::mutex amqpMutex_; + mutable std::mutex errorMutex_; + std::string brokenError_; + std::thread io_; + std::atomic_bool consumerReady_{ false }; + std::atomic_bool broken_{ false }; + std::atomic_bool stopping_{ false }; #ifdef _WIN32 - const int bytesRead = ::recv(socket, receiveBuffer.data(), static_cast(receiveBuffer.size()), 0); -#else - const ssize_t bytesRead = ::recv(socket, receiveBuffer.data(), receiveBuffer.size(), 0); + bool wsaCleaned_{ false }; #endif - if (bytesRead == 0) { - runtimeState.connectionClosed.store(true, std::memory_order_release); - break; - } - if (bytesRead < 0) { - if (IsWouldBlockError()) { - continue; - } - break; - } +}; - const std::size_t chunkSize = static_cast(bytesRead); - pendingBytes.insert( - pendingBytes.end(), - receiveBuffer.begin(), - receiveBuffer.begin() + static_cast(chunkSize)); +RabbitMqRpcClient::RabbitMqRpcClient(RabbitMqClientConfig config) + : config_(std::move(config)) +{ +} - while (!pendingBytes.empty()) { - const std::uint64_t parsed = connection.parse(pendingBytes.data(), pendingBytes.size()); - if (parsed == 0 || parsed > pendingBytes.size()) { - break; - } - pendingBytes.erase( - pendingBytes.begin(), - pendingBytes.begin() + static_cast(parsed)); - } +RabbitMqRpcClient::~RabbitMqRpcClient() +{ + pending_.Close("RabbitMQ client is shutting down"); + std::lock_guard lock(linkMutex_); + link_.reset(); +} + +std::shared_ptr RabbitMqRpcClient::CurrentLink() +{ + std::lock_guard lock(linkMutex_); + if (!link_ || link_->Broken()) { + link_.reset(); + link_ = std::make_shared(config_, pending_); } + return link_; +} - connection.close(); - CloseSocket(socket); +std::vector RabbitMqRpcClient::Call(const byte* payload, std::size_t payloadSize) +{ + if (config_.queue.empty()) { + throw std::runtime_error("RabbitMQ queue is not configured"); + } + if (payloadSize > 0 && payload == nullptr) { + throw std::invalid_argument("RabbitMQ client: null payload with non-zero length"); + } -#ifdef _WIN32 - WSACleanup(); -#endif + const std::shared_ptr link = CurrentLink(); + const std::string correlationId = BuildCorrelationId(); + auto ticket = pending_.Register(correlationId); + link->Publish(correlationId, payload, payloadSize); - return rpcState.publishAccepted.load(std::memory_order_acquire) && - rpcState.responseReceived.load(std::memory_order_acquire) && - !runtimeState.connectionError.load(std::memory_order_acquire) && - !rpcState.channelError.load(std::memory_order_acquire); + const std::uint32_t timeoutMs = config_.rpcTimeoutMs == 0 ? 30000u : config_.rpcTimeoutMs; + std::vector response; + if (!ticket.Wait(std::chrono::milliseconds(timeoutMs), response)) { + throw std::runtime_error("RabbitMQ RPC timed out after " + std::to_string(timeoutMs) + + " ms waiting for correlation id " + correlationId); + } + return response; } diff --git a/rabbitmq/RabbitmqPlugin/RabbitMqClient.h b/rabbitmq/RabbitmqPlugin/RabbitMqClient.h index fcbe6a6..bff7aaf 100644 --- a/rabbitmq/RabbitmqPlugin/RabbitMqClient.h +++ b/rabbitmq/RabbitmqPlugin/RabbitMqClient.h @@ -1,7 +1,11 @@ #pragma once +#include "PendingReplies.h" + #include #include +#include +#include #include #include @@ -14,7 +18,6 @@ namespace Graftcode::Plugins::Rabbitmq std::string host{ "" }; std::uint16_t port{ }; std::string queue{ "" }; - std::string replyQueue{ "" }; std::string user{ "" }; std::string password{ "" }; std::string vhost{ "" }; @@ -40,4 +43,28 @@ namespace Graftcode::Plugins::Rabbitmq std::size_t payloadSize, std::vector& responsePayload); }; + + // Request/reply client that keeps one connection open and lets Call run concurrently. + // Replies arrive on a private reply queue and are matched by correlation id. + class RabbitMqRpcClient + { + public: + explicit RabbitMqRpcClient(RabbitMqClientConfig config); + ~RabbitMqRpcClient(); + + RabbitMqRpcClient(const RabbitMqRpcClient&) = delete; + RabbitMqRpcClient& operator=(const RabbitMqRpcClient&) = delete; + + std::vector Call(const byte* payload, std::size_t payloadSize); + + private: + class Link; + + std::shared_ptr CurrentLink(); + + RabbitMqClientConfig config_; + Concurrency::PendingReplies pending_; + std::mutex linkMutex_; + std::shared_ptr link_; + }; } diff --git a/rabbitmq/RabbitmqPlugin/RabbitMqServer.cpp b/rabbitmq/RabbitmqPlugin/RabbitMqServer.cpp index 43e1fd0..279693a 100644 --- a/rabbitmq/RabbitmqPlugin/RabbitMqServer.cpp +++ b/rabbitmq/RabbitmqPlugin/RabbitMqServer.cpp @@ -51,7 +51,6 @@ namespace { std::string host{ "" }; std::uint16_t port{ 0 }; std::string queue{}; - std::string replyQueue{}; std::string user{ "" }; std::string password{ "" }; std::string vhost{ "" }; @@ -215,7 +214,6 @@ namespace { config.name = getOptionalString(parsed, "name", config.name); config.host = getOptionalString(parsed, "host", config.host); config.queue = getOptionalString(parsed, "queue", config.queue); - config.replyQueue = getOptionalString(parsed, "replyQueue", config.replyQueue); config.user = getOptionalString(parsed, "user", config.user); config.password = getOptionalString(parsed, "password", config.password); config.vhost = getOptionalString(parsed, "vhost", config.vhost); diff --git a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp index a3bdb9d..3771eaa 100644 --- a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp +++ b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.cpp @@ -2,6 +2,7 @@ #include "TransportPluginExports.h" #include +#include #include #include #include @@ -15,6 +16,7 @@ TransportRabbitMq::TransportRabbitMq(std::string host, unsigned short port, std: config_.port = static_cast(port); this->configSource = configSource; RabbitMqClient::ApplyConfigSource(configSource, config_); + client_ = std::make_unique(config_); } int TransportRabbitMq::Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) @@ -31,15 +33,9 @@ std::vector TransportRabbitMq::SendCommand(byte* messageByteArray, int32_t throw std::runtime_error("invalid RabbitMQ payload"); } - std::vector responsePayload; - const bool sent = RabbitMqClient::SendRpc( - config_, + std::vector responsePayload = client_->Call( messageByteArray, - static_cast(messageByteArrayLen), - responsePayload); - if (!sent) { - throw std::runtime_error("failed RabbitMQ request/response"); - } + static_cast(messageByteArrayLen)); if (!responsePayload.empty() && responsePayload[0] == static_cast(255)) { const std::string errorMessage(responsePayload.begin() + 1, responsePayload.end()); diff --git a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h index e01284b..50be2bf 100644 --- a/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h +++ b/rabbitmq/RabbitmqPlugin/TransportRabbitMq.h @@ -3,6 +3,7 @@ #include "ITransport.h" #include "RabbitMqClient.h" +#include #include #include @@ -20,5 +21,6 @@ namespace Graftcode::Plugins::Rabbitmq private: RabbitMqClientConfig config_; std::string configSource; + std::unique_ptr client_; }; } diff --git a/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp b/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp index c8f3679..cf3409d 100644 --- a/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp +++ b/rabbitmq/RabbitmqPluginTest/RabbitMqTransportTests.cpp @@ -34,7 +34,6 @@ namespace { "host": "127.0.0.1", "port": 5672, "queue": "gg", - "replyQueue": "gg.reply", "user": "guest", "password": "guest", "vhost": "/", diff --git a/rabbitmq/Readme.md b/rabbitmq/Readme.md index a3e9f6e..4f6017c 100644 --- a/rabbitmq/Readme.md +++ b/rabbitmq/Readme.md @@ -40,7 +40,7 @@ docker build -t graftcode-rabbitmq . docker run -d --name graftcode-rabbitmq -p 5672:5672 -p 15672:15672 graftcode-rabbitmq ``` -Create `myqueue` and `myqueue.reply`. +Create `myqueue`. Each client declares its own exclusive reply queue and names it in `reply-to`, so no reply queue needs to be created. Management UI at `http://localhost:15672` (`guest` / `guest`): **Queues and Streams** → **Add a new queue**. @@ -48,7 +48,6 @@ Or: ```bash docker exec -it graftcode-rabbitmq rabbitmqadmin declare queue name=myqueue durable=true -docker exec -it graftcode-rabbitmq rabbitmqadmin declare queue name=myqueue.reply durable=true ``` ## Gateway @@ -61,7 +60,6 @@ docker exec -it graftcode-rabbitmq rabbitmqadmin declare queue name=myqueue.repl "host": "localhost", "port": 5672, "queue": "myqueue", - "replyQueue": "myqueue.reply", "user": "guest", "password": "guest", "vhost": "/", @@ -88,7 +86,6 @@ A .NET client can call the gateway through the Hypertube SDK. The client config "host": "localhost", "port": 5673, "queue": "myqueue", - "replyQueue": "myqueue.reply", "user": "graftcode", "password": "graftcode", "vhost": "/", @@ -119,7 +116,6 @@ var result = rtmCtx.GetType("netapp.Calculator").CreateInstance() | `host` | yes | Broker hostname. | | `port` | yes | AMQP port. `5672` for the Docker image. | | `queue` | yes | Request queue. | -| `replyQueue` | yes (client) | Queue named by `reply-to`. The gateway reads it from the message. | | `user` | yes | AMQP username. | | `password` | no | AMQP password. | | `vhost` | no | Virtual host. Empty means `/`. | diff --git a/samples/rabbitmq/middleware/definitions.json b/samples/rabbitmq/middleware/definitions.json index c056251..665dd0c 100644 --- a/samples/rabbitmq/middleware/definitions.json +++ b/samples/rabbitmq/middleware/definitions.json @@ -28,13 +28,6 @@ "durable": true, "auto_delete": false, "arguments": {} - }, - { - "name": "myqueue.reply", - "vhost": "/", - "durable": true, - "auto_delete": false, - "arguments": {} } ] } diff --git a/servicebus/GraftcodePluginsInterfaces/PendingReplies.h b/servicebus/GraftcodePluginsInterfaces/PendingReplies.h new file mode 100644 index 0000000..e9462d2 --- /dev/null +++ b/servicebus/GraftcodePluginsInterfaces/PendingReplies.h @@ -0,0 +1,190 @@ +/** + * @file PendingReplies.h + * @brief Correlates concurrent requests with replies delivered by a single consumer. + * + * A caller registers its correlation id before sending, then waits on the returned ticket. + * The consumer completes or fails entries by id. The ticket removes its entry when it goes + * out of scope, so timed-out and failed requests leave nothing behind. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::Concurrency +{ + class PendingReplies + { + struct Slot + { + std::condition_variable ready; + bool done{ false }; + bool failed{ false }; + std::vector payload; + std::string error; + }; + + public: + using Payload = std::vector; + + class Ticket + { + public: + Ticket(PendingReplies& owner, std::string id, std::shared_ptr slot) + : owner_(&owner), id_(std::move(id)), slot_(std::move(slot)) + { + } + + ~Ticket() + { + if (owner_ != nullptr) { + owner_->Remove(id_); + } + } + + Ticket(Ticket&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), + id_(std::move(other.id_)), + slot_(std::move(other.slot_)) + { + } + + Ticket(const Ticket&) = delete; + Ticket& operator=(const Ticket&) = delete; + Ticket& operator=(Ticket&&) = delete; + + const std::string& Id() const { return id_; } + + // Returns false on timeout; throws when the request was failed. + bool Wait(std::chrono::milliseconds timeout, Payload& response) + { + std::unique_lock lock(owner_->mutex_); + if (!slot_->ready.wait_for(lock, timeout, [this] { return slot_->done; })) { + return false; + } + if (slot_->failed) { + throw std::runtime_error(slot_->error); + } + response = std::move(slot_->payload); + return true; + } + + private: + PendingReplies* owner_; + std::string id_; + std::shared_ptr slot_; + }; + + PendingReplies() = default; + PendingReplies(const PendingReplies&) = delete; + PendingReplies& operator=(const PendingReplies&) = delete; + + Ticket Register(const std::string& id) + { + auto slot = std::make_shared(); + { + std::lock_guard lock(mutex_); + if (closed_) { + throw std::runtime_error(closedError_); + } + if (!slots_.emplace(id, slot).second) { + throw std::runtime_error("duplicate correlation id " + id); + } + } + pendingChanged_.notify_all(); + return Ticket(*this, id, std::move(slot)); + } + + // Returns false when no request is waiting for this id. + bool Complete(const std::string& id, Payload payload) + { + return Finish(id, false, std::move(payload), std::string()); + } + + bool Fail(const std::string& id, const std::string& error) + { + return Finish(id, true, Payload(), error); + } + + void FailAll(const std::string& error) + { + std::lock_guard lock(mutex_); + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + + // Fails every waiting request and rejects new registrations. + void Close(const std::string& error) + { + { + std::lock_guard lock(mutex_); + closed_ = true; + closedError_ = error; + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + pendingChanged_.notify_all(); + } + + bool Empty() const + { + std::lock_guard lock(mutex_); + return slots_.empty(); + } + + // Blocks until a request is registered, the set is closed, or the timeout elapses. + // Returns true when at least one request is waiting. + bool WaitForPending(std::chrono::milliseconds timeout) + { + std::unique_lock lock(mutex_); + pendingChanged_.wait_for(lock, timeout, [this] { return closed_ || !slots_.empty(); }); + return !closed_ && !slots_.empty(); + } + + private: + static void MarkDone(Slot& slot, bool failed, Payload payload, const std::string& error) + { + if (slot.done) { + return; + } + slot.done = true; + slot.failed = failed; + slot.payload = std::move(payload); + slot.error = error; + slot.ready.notify_all(); + } + + bool Finish(const std::string& id, bool failed, Payload payload, const std::string& error) + { + std::lock_guard lock(mutex_); + const auto it = slots_.find(id); + if (it == slots_.end()) { + return false; + } + MarkDone(*it->second, failed, std::move(payload), error); + return true; + } + + void Remove(const std::string& id) + { + std::lock_guard lock(mutex_); + slots_.erase(id); + } + + mutable std::mutex mutex_; + std::condition_variable pendingChanged_; + std::map> slots_; + bool closed_{ false }; + std::string closedError_; + }; +} diff --git a/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h b/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h index 05f27d3..f4a486b 100644 --- a/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h +++ b/servicebus/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -7,7 +7,8 @@ * - DestroyTransport * - TransportSendCommand * - TransportFreeBuffer - * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Plugins must be thread-safe: TransportSendCommand may be called concurrently on one channel. * * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. */ @@ -38,7 +39,7 @@ typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipA typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); /** - * Sends a request and returns its response. + * Sends a request and returns its response. May be called concurrently on one channel. * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). * Otherwise they hold a NUL-terminated error message and its length without the terminator. */ @@ -47,9 +48,6 @@ typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* ch /** Releases a buffer returned by the plugin. */ typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); -/** Returns true when TransportSendCommand may be called concurrently on one channel. */ -typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); - #ifdef __cplusplus } #endif diff --git a/servicebus/ServiceBusPlugin/ServiceBusClient.cpp b/servicebus/ServiceBusPlugin/ServiceBusClient.cpp index 0bde8c0..67377d4 100644 --- a/servicebus/ServiceBusPlugin/ServiceBusClient.cpp +++ b/servicebus/ServiceBusPlugin/ServiceBusClient.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ namespace // specific session. Matches the value used by the official .NET/Java SDKs. constexpr const char* SessionFilterName = "com.microsoft:session-filter"; - std::recursive_mutex callMutex_; std::atomic correlationCounter_{ 0 }; std::string BuildCorrelationId() @@ -258,65 +258,13 @@ bool ServiceBusClient::PublishOneWay( const byte* payload, std::size_t payloadSize) { - std::lock_guard callLock(callMutex_); if (config.topic.empty()) { return false; } - if (payloadSize > 0 && payload == nullptr) { - return false; - } - - const std::string connectionString = BuildConnectionString(config); - if (connectionString.empty()) { - return false; - } - try { - auto credential = std::make_shared(connectionString); - - Amqp::ConnectionOptions connectionOptions; - connectionOptions.ContainerId = "graftcode-servicebus-client"; - connectionOptions.EnableTrace = false; - const std::uint16_t credentialPort = credential->GetPort(); - connectionOptions.Port = credentialPort != 0 ? credentialPort : Amqp::AmqpTlsPort; - - Amqp::Connection connection(credential->GetHostName(), credential, connectionOptions); - - Amqp::SessionOptions sessionOptions; - sessionOptions.InitialIncomingWindowSize = (std::numeric_limits::max)(); - sessionOptions.InitialOutgoingWindowSize = (std::numeric_limits::max)(); - Amqp::Session session(connection.CreateSession(sessionOptions)); - - Amqp::MessageSenderOptions senderOptions; - senderOptions.Name = "graftcode-oneway-publish"; - senderOptions.MessageSource = "graftcode-oneway-publish-source"; - senderOptions.SettleMode = Amqp::SenderSettleMode::Settled; - senderOptions.MaxMessageSize = MaxMessageSize; - Amqp::MessageSender sender(session.CreateMessageSender(config.topic, senderOptions, nullptr)); - - if (auto openError = sender.Open()) { - (void)openError; - return false; - } - - AmqpModels::AmqpMessage message; - std::vector body; - if (payloadSize > 0) { - body.assign(payload, payload + payloadSize); - } - message.SetBody(AmqpModels::AmqpBinaryData(body)); - message.Properties.MessageId = AmqpModels::AmqpValue(BuildCorrelationId()); - message.Properties.ContentType = std::string("application/octet-stream"); - - bool sendSucceeded = false; -#if defined(ENABLE_RUST_AMQP) && ENABLE_RUST_AMQP - sendSucceeded = !static_cast(sender.Send(message)); -#else - const auto sendResult = sender.Send(message); - sendSucceeded = std::get<0>(sendResult) == Amqp::MessageSendStatus::Ok; -#endif - sender.Close(); - return sendSucceeded; + ServiceBusRpcClient client(config); + client.Call(payload, payloadSize); + return true; } catch (...) { return false; @@ -338,151 +286,247 @@ bool ServiceBusClient::SendRpc( std::size_t payloadSize, std::vector& responsePayload) { - std::lock_guard callLock(callMutex_); responsePayload.clear(); - - // One-way (topic) mode: publish and return immediately without a reply. - // Used for void methods that produce no result. - if (!config.topic.empty()) { - return PublishOneWay(config, payload, payloadSize); - } - - if (config.queue.empty()) { - return false; - } - if (config.replyQueue.empty()) { - return false; + try { + ServiceBusRpcClient client(config); + responsePayload = client.Call(payload, payloadSize); + return true; } - if (payloadSize > 0 && payload == nullptr) { + catch (...) { + responsePayload.clear(); return false; } +} - const std::string connectionString = BuildConnectionString(config); - if (connectionString.empty()) { - return false; - } +// One AMQP connection with a request sender and, in RPC mode, a reply receiver bound to the +// client's reply session. A background thread dispatches replies by correlation id. +class ServiceBusRpcClient::Link +{ +public: + Link(const ServiceBusClientConfig& config, const std::string& replySessionId, Concurrency::PendingReplies& pending) + : pending_(pending) + { + const std::string connectionString = ServiceBusClient::BuildConnectionString(config); + if (connectionString.empty()) { + throw std::runtime_error("Service Bus connection string is not configured"); + } + const bool oneWay = !config.topic.empty(); + if (!oneWay && (config.queue.empty() || config.replyQueue.empty())) { + throw std::runtime_error("Service Bus queue and replyQueue must be configured"); + } - try { auto credential = std::make_shared(connectionString); - Amqp::ConnectionOptions connectionOptions; connectionOptions.ContainerId = "graftcode-servicebus-client"; connectionOptions.EnableTrace = false; const std::uint16_t credentialPort = credential->GetPort(); connectionOptions.Port = credentialPort != 0 ? credentialPort : Amqp::AmqpTlsPort; - - Amqp::Connection connection(credential->GetHostName(), credential, connectionOptions); + connection_ = std::make_unique(credential->GetHostName(), credential, connectionOptions); Amqp::SessionOptions sessionOptions; sessionOptions.InitialIncomingWindowSize = (std::numeric_limits::max)(); sessionOptions.InitialOutgoingWindowSize = (std::numeric_limits::max)(); - Amqp::Session session(connection.CreateSession(sessionOptions)); - - const std::string correlationId = BuildCorrelationId(); - // Each request gets a unique reply session id. The reply queue is session - // enabled, so the broker routes the matching response back to this exact - // client even when many clients share the same reply queue. - const std::string replySessionId = correlationId; - - // Attach the reply receiver with a Service Bus session filter so it only - // ever sees the response that belongs to this request's session. - AmqpModels::_internal::MessageSourceOptions replySourceOptions; - replySourceOptions.Address = AmqpModels::AmqpValue(config.replyQueue); - AmqpModels::AmqpMap sessionFilter; - sessionFilter[AmqpModels::AmqpSymbol(SessionFilterName).AsAmqpValue()] - = AmqpModels::AmqpValue(replySessionId); - replySourceOptions.Filter = sessionFilter; - AmqpModels::_internal::MessageSource replySource(replySourceOptions); - - Amqp::MessageReceiverOptions receiverOptions; - receiverOptions.Name = "graftcode-rpc-reply"; - receiverOptions.MessageTarget = "graftcode-rpc-reply-target"; - receiverOptions.SettleMode = Amqp::ReceiverSettleMode::First; - receiverOptions.MaxMessageSize = MaxMessageSize; - receiverOptions.MaxLinkCredit = 10; - Amqp::MessageReceiver receiver(session.CreateMessageReceiver(replySource, receiverOptions, nullptr)); + session_ = std::make_unique(connection_->CreateSession(sessionOptions)); + + if (!oneWay) { + // The reply queue is session enabled; the server stamps the reply session id on every + // response, so this receiver sees exactly the replies addressed to this client. + AmqpModels::_internal::MessageSourceOptions replySourceOptions; + replySourceOptions.Address = AmqpModels::AmqpValue(config.replyQueue); + AmqpModels::AmqpMap sessionFilter; + sessionFilter[AmqpModels::AmqpSymbol(SessionFilterName).AsAmqpValue()] + = AmqpModels::AmqpValue(replySessionId); + replySourceOptions.Filter = sessionFilter; + AmqpModels::_internal::MessageSource replySource(replySourceOptions); + + Amqp::MessageReceiverOptions receiverOptions; + receiverOptions.Name = "graftcode-rpc-reply"; + receiverOptions.MessageTarget = "graftcode-rpc-reply-target"; + receiverOptions.SettleMode = Amqp::ReceiverSettleMode::First; + receiverOptions.MaxMessageSize = MaxMessageSize; + receiverOptions.MaxLinkCredit = 100; + receiver_ = std::make_unique( + session_->CreateMessageReceiver(replySource, receiverOptions, nullptr)); + receiver_->Open(); + } Amqp::MessageSenderOptions senderOptions; - senderOptions.Name = "graftcode-rpc-request"; - senderOptions.MessageSource = "graftcode-rpc-request-source"; + senderOptions.Name = oneWay ? "graftcode-oneway-publish" : "graftcode-rpc-request"; + senderOptions.MessageSource = oneWay ? "graftcode-oneway-publish-source" : "graftcode-rpc-request-source"; senderOptions.SettleMode = Amqp::SenderSettleMode::Settled; senderOptions.MaxMessageSize = MaxMessageSize; - Amqp::MessageSender sender(session.CreateMessageSender(config.queue, senderOptions, nullptr)); - - receiver.Open(); - if (auto openError = sender.Open()) { + sender_ = std::make_unique( + session_->CreateMessageSender(oneWay ? config.topic : config.queue, senderOptions, nullptr)); + if (auto openError = sender_->Open()) { (void)openError; - receiver.Close(); - return false; + if (receiver_) { + receiver_->Close(); + } + throw std::runtime_error("failed to open Service Bus sender"); } - AmqpModels::AmqpMessage message; - std::vector body; - if (payloadSize > 0) { - body.assign(payload, payload + payloadSize); + if (receiver_) { + dispatcher_ = std::thread(&Link::DispatchReplies, this); } - message.SetBody(AmqpModels::AmqpBinaryData(body)); - message.Properties.MessageId = AmqpModels::AmqpValue(correlationId); - message.Properties.ReplyTo = AmqpModels::AmqpValue(config.replyQueue); - // ReplyToGroupId maps to the Service Bus ReplyToSessionId: it tells the - // server which session id to stamp on the response. - message.Properties.ReplyToGroupId = replySessionId; - message.Properties.ContentType = std::string("application/octet-stream"); + } + + ~Link() + { + stopping_.store(true, std::memory_order_release); + if (dispatcher_.joinable()) { + dispatcher_.join(); + } + try { + if (receiver_) { + receiver_->Close(); + } + sender_->Close(); + } + catch (...) { + } + } + + Link(const Link&) = delete; + Link& operator=(const Link&) = delete; + + bool Broken() const + { + return broken_.load(std::memory_order_acquire); + } + void Send(const AmqpModels::AmqpMessage& message) + { bool sendSucceeded = false; + { + std::lock_guard lock(sendMutex_); #if defined(ENABLE_RUST_AMQP) && ENABLE_RUST_AMQP - sendSucceeded = !static_cast(sender.Send(message)); + sendSucceeded = !static_cast(sender_->Send(message)); #else - const auto sendResult = sender.Send(message); - sendSucceeded = std::get<0>(sendResult) == Amqp::MessageSendStatus::Ok; + const auto sendResult = sender_->Send(message); + sendSucceeded = std::get<0>(sendResult) == Amqp::MessageSendStatus::Ok; #endif + } if (!sendSucceeded) { - receiver.Close(); - sender.Close(); - return false; + broken_.store(true, std::memory_order_release); + throw std::runtime_error("failed to send Service Bus message"); } + } - const std::uint32_t timeoutMs = config.rpcTimeoutMs == 0 ? DefaultRpcTimeoutMs : config.rpcTimeoutMs; - const auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(timeoutMs); - Azure::Core::Context context = Azure::Core::Context{}.WithDeadline(Azure::DateTime(deadline)); - - const AmqpModels::AmqpValue expectedCorrelation(correlationId); - bool received = false; - while (std::chrono::system_clock::now() < deadline) { - std::pair, Azure::Core::Amqp::Models::_internal::AmqpError> incoming; +private: + void DispatchReplies() + { + while (!stopping_.load(std::memory_order_acquire)) { + const auto deadline = std::chrono::system_clock::now() + ReceivePollInterval; + Azure::Core::Context context = Azure::Core::Context{}.WithDeadline(Azure::DateTime(deadline)); + std::pair, AmqpModels::_internal::AmqpError> incoming; try { - incoming = receiver.WaitForIncomingMessage(context); + incoming = receiver_->WaitForIncomingMessage(context); } - catch (...) { - break; + catch (const Azure::Core::OperationCancelledException&) { + continue; + } + catch (const std::exception& error) { + Disconnect(std::string("Service Bus reply receiver failed: ") + error.what()); + return; } - if (incoming.second) { - break; + Disconnect("Service Bus reply receiver failed: " + incoming.second.Description); + return; } - const auto& incomingMessage = incoming.first; - if (!incomingMessage) { + if (!incoming.first) { continue; } - const bool correlationMatches = - incomingMessage->Properties.CorrelationId == expectedCorrelation - || incomingMessage->Properties.MessageId == expectedCorrelation; - if (!correlationMatches) { + const auto& properties = incoming.first->Properties; + const auto& correlation = properties.CorrelationId.IsNull() ? properties.MessageId : properties.CorrelationId; + if (correlation.IsNull()) { continue; } - - responsePayload = ExtractMessageBody(*incomingMessage); - received = true; - break; + try { + pending_.Complete(static_cast(correlation), ExtractMessageBody(*incoming.first)); + } + catch (...) { + } } + } - receiver.Close(); - sender.Close(); - return received; + void Disconnect(const std::string& error) + { + broken_.store(true, std::memory_order_release); + pending_.FailAll(error); } - catch (...) { - responsePayload.clear(); - return false; + + static constexpr std::chrono::milliseconds ReceivePollInterval{ 500 }; + + Concurrency::PendingReplies& pending_; + std::unique_ptr connection_; + std::unique_ptr session_; + std::unique_ptr receiver_; + std::unique_ptr sender_; + std::mutex sendMutex_; + std::thread dispatcher_; + std::atomic_bool stopping_{ false }; + std::atomic_bool broken_{ false }; +}; + +ServiceBusRpcClient::ServiceBusRpcClient(ServiceBusClientConfig config) + : config_(std::move(config)), replySessionId_(BuildCorrelationId()) +{ +} + +ServiceBusRpcClient::~ServiceBusRpcClient() +{ + pending_.Close("Service Bus client is shutting down"); + std::lock_guard lock(linkMutex_); + link_.reset(); +} + +std::shared_ptr ServiceBusRpcClient::CurrentLink() +{ + std::lock_guard lock(linkMutex_); + if (!link_ || link_->Broken()) { + link_.reset(); + link_ = std::make_shared(config_, replySessionId_, pending_); + } + return link_; +} + +std::vector ServiceBusRpcClient::Call(const byte* payload, std::size_t payloadSize) +{ + if (payloadSize > 0 && payload == nullptr) { + throw std::invalid_argument("Service Bus client: null payload with non-zero length"); + } + + const std::shared_ptr link = CurrentLink(); + const bool oneWay = !config_.topic.empty(); + const std::string correlationId = BuildCorrelationId(); + + AmqpModels::AmqpMessage message; + std::vector body; + if (payloadSize > 0) { + body.assign(payload, payload + payloadSize); + } + message.SetBody(AmqpModels::AmqpBinaryData(body)); + message.Properties.MessageId = AmqpModels::AmqpValue(correlationId); + message.Properties.ContentType = std::string("application/octet-stream"); + + if (oneWay) { + link->Send(message); + return {}; + } + + message.Properties.ReplyTo = AmqpModels::AmqpValue(config_.replyQueue); + // ReplyToGroupId maps to the Service Bus ReplyToSessionId: the server stamps it on the response. + message.Properties.ReplyToGroupId = replySessionId_; + + auto ticket = pending_.Register(correlationId); + link->Send(message); + + const std::uint32_t timeoutMs = config_.rpcTimeoutMs == 0 ? DefaultRpcTimeoutMs : config_.rpcTimeoutMs; + std::vector response; + if (!ticket.Wait(std::chrono::milliseconds(timeoutMs), response)) { + throw std::runtime_error("Service Bus RPC timed out after " + std::to_string(timeoutMs) + + " ms waiting for correlation id " + correlationId); } + return response; } diff --git a/servicebus/ServiceBusPlugin/ServiceBusClient.h b/servicebus/ServiceBusPlugin/ServiceBusClient.h index a218c2b..d6a4d7c 100644 --- a/servicebus/ServiceBusPlugin/ServiceBusClient.h +++ b/servicebus/ServiceBusPlugin/ServiceBusClient.h @@ -1,7 +1,11 @@ #pragma once +#include "PendingReplies.h" + #include #include +#include +#include #include #include @@ -60,4 +64,30 @@ namespace Graftcode::Plugins::ServiceBus // Builds a full Service Bus connection string from the config fields. static std::string BuildConnectionString(const ServiceBusClientConfig& config); }; + + // Request/reply client that keeps one connection open and lets Call run concurrently. + // Replies come back on a reply session owned by this client and are matched by correlation id. + class ServiceBusRpcClient + { + public: + explicit ServiceBusRpcClient(ServiceBusClientConfig config); + ~ServiceBusRpcClient(); + + ServiceBusRpcClient(const ServiceBusRpcClient&) = delete; + ServiceBusRpcClient& operator=(const ServiceBusRpcClient&) = delete; + + // Sends the payload and returns the reply. In topic mode it publishes and returns an empty reply. + std::vector Call(const byte* payload, std::size_t payloadSize); + + private: + class Link; + + std::shared_ptr CurrentLink(); + + ServiceBusClientConfig config_; + std::string replySessionId_; + Concurrency::PendingReplies pending_; + std::mutex linkMutex_; + std::shared_ptr link_; + }; } diff --git a/servicebus/ServiceBusPlugin/TransportServiceBus.cpp b/servicebus/ServiceBusPlugin/TransportServiceBus.cpp index df67676..03b34c8 100644 --- a/servicebus/ServiceBusPlugin/TransportServiceBus.cpp +++ b/servicebus/ServiceBusPlugin/TransportServiceBus.cpp @@ -2,6 +2,7 @@ #include "TransportPluginExports.h" #include +#include #include #include #include @@ -15,6 +16,7 @@ TransportServiceBus::TransportServiceBus(std::string host, unsigned short port, config_.host = host; this->configSource = configSource; ServiceBusClient::ApplyConfigSource(configSource, config_); + client_ = std::make_unique(config_); } int TransportServiceBus::Initialize(byte callingRuntimeNumber, byte calledRuntimeNumber, byte calledRuntimeVersion) @@ -31,15 +33,9 @@ std::vector TransportServiceBus::SendCommand(byte* messageByteArray, int32 throw std::runtime_error("invalid Service Bus payload"); } - std::vector responsePayload; - const bool sent = ServiceBusClient::SendRpc( - config_, + std::vector responsePayload = client_->Call( messageByteArray, - static_cast(messageByteArrayLen), - responsePayload); - if (!sent) { - throw std::runtime_error("failed Service Bus request/response"); - } + static_cast(messageByteArrayLen)); if (!responsePayload.empty() && responsePayload[0] == static_cast(255)) { const std::string errorMessage(responsePayload.begin() + 1, responsePayload.end()); diff --git a/servicebus/ServiceBusPlugin/TransportServiceBus.h b/servicebus/ServiceBusPlugin/TransportServiceBus.h index 693d6ac..d6d7f67 100644 --- a/servicebus/ServiceBusPlugin/TransportServiceBus.h +++ b/servicebus/ServiceBusPlugin/TransportServiceBus.h @@ -3,6 +3,7 @@ #include "ITransport.h" #include "ServiceBusClient.h" +#include #include #include @@ -20,5 +21,6 @@ namespace Graftcode::Plugins::ServiceBus private: ServiceBusClientConfig config_; std::string configSource; + std::unique_ptr client_; }; } diff --git a/sqs/GraftcodePluginsInterfaces/PendingReplies.h b/sqs/GraftcodePluginsInterfaces/PendingReplies.h new file mode 100644 index 0000000..e9462d2 --- /dev/null +++ b/sqs/GraftcodePluginsInterfaces/PendingReplies.h @@ -0,0 +1,190 @@ +/** + * @file PendingReplies.h + * @brief Correlates concurrent requests with replies delivered by a single consumer. + * + * A caller registers its correlation id before sending, then waits on the returned ticket. + * The consumer completes or fails entries by id. The ticket removes its entry when it goes + * out of scope, so timed-out and failed requests leave nothing behind. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Graftcode::Plugins::Concurrency +{ + class PendingReplies + { + struct Slot + { + std::condition_variable ready; + bool done{ false }; + bool failed{ false }; + std::vector payload; + std::string error; + }; + + public: + using Payload = std::vector; + + class Ticket + { + public: + Ticket(PendingReplies& owner, std::string id, std::shared_ptr slot) + : owner_(&owner), id_(std::move(id)), slot_(std::move(slot)) + { + } + + ~Ticket() + { + if (owner_ != nullptr) { + owner_->Remove(id_); + } + } + + Ticket(Ticket&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), + id_(std::move(other.id_)), + slot_(std::move(other.slot_)) + { + } + + Ticket(const Ticket&) = delete; + Ticket& operator=(const Ticket&) = delete; + Ticket& operator=(Ticket&&) = delete; + + const std::string& Id() const { return id_; } + + // Returns false on timeout; throws when the request was failed. + bool Wait(std::chrono::milliseconds timeout, Payload& response) + { + std::unique_lock lock(owner_->mutex_); + if (!slot_->ready.wait_for(lock, timeout, [this] { return slot_->done; })) { + return false; + } + if (slot_->failed) { + throw std::runtime_error(slot_->error); + } + response = std::move(slot_->payload); + return true; + } + + private: + PendingReplies* owner_; + std::string id_; + std::shared_ptr slot_; + }; + + PendingReplies() = default; + PendingReplies(const PendingReplies&) = delete; + PendingReplies& operator=(const PendingReplies&) = delete; + + Ticket Register(const std::string& id) + { + auto slot = std::make_shared(); + { + std::lock_guard lock(mutex_); + if (closed_) { + throw std::runtime_error(closedError_); + } + if (!slots_.emplace(id, slot).second) { + throw std::runtime_error("duplicate correlation id " + id); + } + } + pendingChanged_.notify_all(); + return Ticket(*this, id, std::move(slot)); + } + + // Returns false when no request is waiting for this id. + bool Complete(const std::string& id, Payload payload) + { + return Finish(id, false, std::move(payload), std::string()); + } + + bool Fail(const std::string& id, const std::string& error) + { + return Finish(id, true, Payload(), error); + } + + void FailAll(const std::string& error) + { + std::lock_guard lock(mutex_); + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + + // Fails every waiting request and rejects new registrations. + void Close(const std::string& error) + { + { + std::lock_guard lock(mutex_); + closed_ = true; + closedError_ = error; + for (auto& entry : slots_) { + MarkDone(*entry.second, true, Payload(), error); + } + } + pendingChanged_.notify_all(); + } + + bool Empty() const + { + std::lock_guard lock(mutex_); + return slots_.empty(); + } + + // Blocks until a request is registered, the set is closed, or the timeout elapses. + // Returns true when at least one request is waiting. + bool WaitForPending(std::chrono::milliseconds timeout) + { + std::unique_lock lock(mutex_); + pendingChanged_.wait_for(lock, timeout, [this] { return closed_ || !slots_.empty(); }); + return !closed_ && !slots_.empty(); + } + + private: + static void MarkDone(Slot& slot, bool failed, Payload payload, const std::string& error) + { + if (slot.done) { + return; + } + slot.done = true; + slot.failed = failed; + slot.payload = std::move(payload); + slot.error = error; + slot.ready.notify_all(); + } + + bool Finish(const std::string& id, bool failed, Payload payload, const std::string& error) + { + std::lock_guard lock(mutex_); + const auto it = slots_.find(id); + if (it == slots_.end()) { + return false; + } + MarkDone(*it->second, failed, std::move(payload), error); + return true; + } + + void Remove(const std::string& id) + { + std::lock_guard lock(mutex_); + slots_.erase(id); + } + + mutable std::mutex mutex_; + std::condition_variable pendingChanged_; + std::map> slots_; + bool closed_{ false }; + std::string closedError_; + }; +} diff --git a/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h b/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h index 05f27d3..f4a486b 100644 --- a/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h +++ b/sqs/GraftcodePluginsInterfaces/TransportPluginAbi.h @@ -7,7 +7,8 @@ * - DestroyTransport * - TransportSendCommand * - TransportFreeBuffer - * - SupportsConcurrentRequests (optional; when missing or false, requests on one channel are serialized) + * + * Plugins must be thread-safe: TransportSendCommand may be called concurrently on one channel. * * Every buffer returned by the plugin is allocated by the plugin and released with TransportFreeBuffer. */ @@ -38,7 +39,7 @@ typedef GraftcodeTransportChannel* (*GraftcodeCreateTransportFn)(const char* ipA typedef void (*GraftcodeDestroyTransportFn)(GraftcodeTransportChannel* channel); /** - * Sends a request and returns its response. + * Sends a request and returns its response. May be called concurrently on one channel. * On GRAFTCODE_TRANSPORT_OK, *response and *responseLen hold the response (*response may be null when the length is 0). * Otherwise they hold a NUL-terminated error message and its length without the terminator. */ @@ -47,9 +48,6 @@ typedef int32_t (*GraftcodeTransportSendCommandFn)(GraftcodeTransportChannel* ch /** Releases a buffer returned by the plugin. */ typedef void (*GraftcodeTransportFreeBufferFn)(void* buffer); -/** Returns true when TransportSendCommand may be called concurrently on one channel. */ -typedef bool (*GraftcodeSupportsConcurrentRequestsFn)(void); - #ifdef __cplusplus } #endif diff --git a/sqs/Readme.md b/sqs/Readme.md index 8ecbe6b..2a5edc5 100644 --- a/sqs/Readme.md +++ b/sqs/Readme.md @@ -14,7 +14,7 @@ Binary Graft payloads are Base64-encoded in the SQS message body with the prefix ## Pitfalls - SQS is at-least-once. The gateway deletes a request only after `processMessage` succeeds and, for RPC, the reply has been sent. A failure leaves the message for the visibility timeout, then another receive. Make called operations idempotent, and attach a dead-letter queue for messages that never succeed. -- SQS cannot filter a receive by message attribute. Give each client process its own reply queue. Do not share one reply queue between independent processes. Calls through one transport instance are serialized. +- SQS cannot filter a receive by message attribute. Give each client process its own reply queue. Do not share one reply queue between independent processes. Calls through one transport instance may run concurrently; one background receiver dispatches replies by correlation id. - Encoded bodies larger than 1 MiB are rejected. Some LocalStack versions and older queues still enforce 256 KiB. - Queue URLs and the SDK endpoint are different. `endpointOverride` is where the SDK connects (`http://localhost:4566` for LocalStack). `requestQueueUrl` / `replyQueueUrl` must be the URL the broker returns. LocalStack's working form is `http://sqs..localhost.localstack.cloud:4566/000000000000/`, not `http://localhost:4566/000000000000/`. diff --git a/sqs/SqsPlugin/SqsClient.cpp b/sqs/SqsPlugin/SqsClient.cpp index 384c7da..0816477 100644 --- a/sqs/SqsPlugin/SqsClient.cpp +++ b/sqs/SqsPlugin/SqsClient.cpp @@ -8,11 +8,11 @@ #include #include -#include #include -#include +#include #include #include +#include #include namespace Graftcode::Plugins::Sqs @@ -46,7 +46,77 @@ namespace Graftcode::Plugins::Sqs client_ = CreateAwsSqsClient(config_); } - SqsClient::~SqsClient() = default; + SqsClient::~SqsClient() + { + stopping_.store(true, std::memory_order_release); + pending_.Close("SQS client is shutting down"); + if (dispatcher_.joinable()) { + dispatcher_.join(); + } + } + + void SqsClient::EnsureDispatcher() + { + if (started_.load(std::memory_order_acquire)) { + return; + } + std::lock_guard lock(startMutex_); + if (started_.load(std::memory_order_acquire)) { + return; + } + dispatcher_ = std::thread(&SqsClient::DispatchReplies, this); + started_.store(true, std::memory_order_release); + } + + void SqsClient::DispatchReplies() + { + while (!stopping_.load(std::memory_order_acquire)) { + if (!pending_.WaitForPending(std::chrono::milliseconds(200))) { + continue; + } + + Aws::SQS::Model::ReceiveMessageRequest receive; + receive.SetQueueUrl(config_.replyQueueUrl.c_str()); + receive.SetMaxNumberOfMessages(10); + receive.SetWaitTimeSeconds(static_cast(config_.waitTimeSeconds)); + receive.SetVisibilityTimeout(static_cast( + config_.visibilityTimeoutSeconds)); + receive.AddMessageAttributeNames("All"); + + const auto receiveOutcome = client_->ReceiveMessage(receive); + if (!receiveOutcome.IsSuccess()) { + pending_.FailAll( + AwsError("ReceiveMessage", receiveOutcome.GetError()).what()); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + continue; + } + + for (const auto& message : receiveOutcome.GetResult().GetMessages()) { + const std::string correlationId = + GetStringAttribute(message, CorrelationIdAttribute); + + // Reply queues are client-instance-dedicated, so replies nobody + // waits for are stale and are removed to keep FIFO groups moving. + Aws::SQS::Model::DeleteMessageRequest remove; + remove.SetQueueUrl(config_.replyQueueUrl.c_str()); + remove.SetReceiptHandle(message.GetReceiptHandle()); + const auto deleteOutcome = client_->DeleteMessage(remove); + if (!deleteOutcome.IsSuccess()) { + pending_.Fail( + correlationId, + AwsError("DeleteMessage", deleteOutcome.GetError()).what()); + continue; + } + + try { + pending_.Complete(correlationId, DecodePayload(message.GetBody())); + } + catch (const std::exception& error) { + pending_.Fail(correlationId, error.what()); + } + } + } + } std::vector SqsClient::Call( const unsigned char* data, @@ -57,13 +127,13 @@ namespace Graftcode::Plugins::Sqs "SQS client: null payload with non-zero length"); } - // SQS cannot broker-filter a shared reply queue by correlation id. - // Serializing calls keeps one configured reply queue deterministic for - // this transport instance. Separate runtime instances should use - // separate reply queues. - std::lock_guard lock(callMutex_); - const std::string correlationId = NewCorrelationId(); + std::optional ticket; + if (!config_.oneWay) { + EnsureDispatcher(); + ticket.emplace(pending_.Register(correlationId)); + } + Aws::SQS::Model::SendMessageRequest send; send.SetQueueUrl(config_.requestQueueUrl.c_str()); send.SetMessageBody(EncodePayload(data, size)); @@ -90,63 +160,13 @@ namespace Graftcode::Plugins::Sqs return {}; } - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(config_.rpcTimeoutMs); - while (std::chrono::steady_clock::now() < deadline) { - const auto remainingMs = - std::chrono::duration_cast( - deadline - std::chrono::steady_clock::now()).count(); - const auto remainingSeconds = - static_cast(std::max( - 0, - remainingMs / 1000)); - - Aws::SQS::Model::ReceiveMessageRequest receive; - receive.SetQueueUrl(config_.replyQueueUrl.c_str()); - receive.SetMaxNumberOfMessages(10); - receive.SetWaitTimeSeconds(static_cast( - (std::min)(config_.waitTimeSeconds, remainingSeconds))); - receive.SetVisibilityTimeout(static_cast( - config_.visibilityTimeoutSeconds)); - receive.AddMessageAttributeNames("All"); - - const auto receiveOutcome = client_->ReceiveMessage(receive); - if (!receiveOutcome.IsSuccess()) { - throw AwsError("ReceiveMessage", receiveOutcome.GetError()); - } - - for (const auto& message : receiveOutcome.GetResult().GetMessages()) { - const auto deleteReply = [&]() { - Aws::SQS::Model::DeleteMessageRequest remove; - remove.SetQueueUrl(config_.replyQueueUrl.c_str()); - remove.SetReceiptHandle(message.GetReceiptHandle()); - const auto deleteOutcome = client_->DeleteMessage(remove); - if (!deleteOutcome.IsSuccess()) { - throw AwsError( - "DeleteMessage", - deleteOutcome.GetError()); - } - }; - - if (GetStringAttribute( - message, - CorrelationIdAttribute) != correlationId) { - // Reply queues are required to be client-instance-dedicated. - // Remove stale responses so they cannot block a FIFO group. - deleteReply(); - continue; - } - - std::vector response = - DecodePayload(message.GetBody()); - deleteReply(); - return response; - } + std::vector response; + if (!ticket->Wait(std::chrono::milliseconds(config_.rpcTimeoutMs), response)) { + throw std::runtime_error( + "SQS RPC timed out after " + + std::to_string(config_.rpcTimeoutMs) + + " ms waiting for correlation id " + correlationId); } - - throw std::runtime_error( - "SQS RPC timed out after " + - std::to_string(config_.rpcTimeoutMs) + - " ms waiting for correlation id " + correlationId); + return response; } } diff --git a/sqs/SqsPlugin/SqsClient.h b/sqs/SqsPlugin/SqsClient.h index 26bdc47..3349ca9 100644 --- a/sqs/SqsPlugin/SqsClient.h +++ b/sqs/SqsPlugin/SqsClient.h @@ -1,10 +1,13 @@ #pragma once +#include "PendingReplies.h" #include "SqsConfig.h" +#include #include #include #include +#include #include namespace Aws::SQS @@ -28,8 +31,15 @@ namespace Graftcode::Plugins::Sqs std::size_t size); private: + void EnsureDispatcher(); + void DispatchReplies(); + SqsConfig config_; std::unique_ptr client_; - std::mutex callMutex_; + Concurrency::PendingReplies pending_; + std::thread dispatcher_; + std::mutex startMutex_; + std::atomic started_{ false }; + std::atomic stopping_{ false }; }; }