From 87b2f23e04f55bc362a91a3c5a9b94dd3f8f756d Mon Sep 17 00:00:00 2001 From: mrabine Date: Mon, 23 Mar 2026 22:45:51 +0100 Subject: [PATCH 1/7] optional io_uring backend --- CMakeLists.txt | 1 + CMakePresets.json | 3 ++- README.md | 6 +++++- core/CMakeLists.txt | 18 ++++++++++++++++++ core/include/join/proactor.hpp | 32 ++++++++++++++++++++++++++++++++ core/src/proactor.cpp | 26 ++++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 core/include/join/proactor.hpp create mode 100644 core/src/proactor.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 875e2402..5f34c43e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ option(JOIN_ENABLE_CRYPTO "Enable crypto." ON) option(JOIN_ENABLE_DATA "Enable data." ON) option(JOIN_ENABLE_FABRIC "Enable fabric." ON) option(JOIN_ENABLE_SERVICES "Enable services." ON) +option(JOIN_ENABLE_IO_URING "Enable io_uring backend." OFF) option(JOIN_ENABLE_NUMA "Enable NUMA support." OFF) option(JOIN_ENABLE_SAMPLES "Build samples" OFF) option(JOIN_ENABLE_TESTS "Enable tests." OFF) diff --git a/CMakePresets.json b/CMakePresets.json index 181a321e..11da8911 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -8,6 +8,7 @@ "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", "CMAKE_VERBOSE_MAKEFILE": "ON", + "JOIN_ENABLE_IO_URING": "ON", "JOIN_ENABLE_NUMA": "ON", "JOIN_ENABLE_SAMPLES": "ON", "JOIN_ENABLE_TESTS": "ON" @@ -120,4 +121,4 @@ } } ] -} +} \ No newline at end of file diff --git a/README.md b/README.md index 1f2cf3fa..caf0ce9b 100644 --- a/README.md +++ b/README.md @@ -83,17 +83,19 @@ Install required libraries and test dependencies: sudo apt install libssl-dev zlib1g-dev libgtest-dev libgmock-dev ``` -> **Compilers:** Both GCC and Clang are supported. Clang requires `libclang-rt-dev` for coverage instrumentation (`--coverage`). +> **Compilers:** Both GCC and Clang are supported. Clang requires `libclang-rt-dev` for coverage instrumentation. > **OpenSSL** provides the core TLS runtime. ### Optional Dependencies | Option | Library | Default | Description | | :--- | :--- | :---: | :--- | +| `JOIN_ENABLE_IO_URING` | `liburing-dev` | `OFF` | Enables the io_uring based proactor backend for async I/O. | | `JOIN_ENABLE_NUMA` | `libnuma-dev` | `OFF` | Enables NUMA aware memory binding for `LocalMem` and `ShmMem`. | Install as needed: ```bash +sudo apt install liburing-dev # for JOIN_ENABLE_IO_URING sudo apt install libnuma-dev # for JOIN_ENABLE_NUMA ``` @@ -108,6 +110,7 @@ cmake --build build With optional backends: ```bash cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DJOIN_ENABLE_IO_URING=ON \ -DJOIN_ENABLE_NUMA=ON \ -DJOIN_ENABLE_TESTS=ON cmake --build build @@ -122,6 +125,7 @@ cmake --build build | `JOIN_ENABLE_DATA` | `ON` | Build the data module. | | `JOIN_ENABLE_FABRIC` | `ON` | Build the fabric module. | | `JOIN_ENABLE_SERVICES` | `ON` | Build the services module (requires crypto, data, fabric). | +| `JOIN_ENABLE_IO_URING` | `OFF` | Enable io_uring backend (requires `liburing-dev`). | | `JOIN_ENABLE_NUMA` | `OFF` | Enable NUMA support (requires `libnuma-dev`). | | `JOIN_ENABLE_SAMPLES` | `OFF` | Build sample programs. | | `JOIN_ENABLE_TESTS` | `OFF` | Build the test suite. | diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index aa668f23..929964c2 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -1,5 +1,9 @@ cmake_minimum_required(VERSION 3.22.1) +if(JOIN_ENABLE_IO_URING) + pkg_check_modules(LIBURING REQUIRED liburing) +endif() + if(JOIN_ENABLE_NUMA) pkg_check_modules(NUMA REQUIRED numa) endif() @@ -35,6 +39,10 @@ set(PUBLIC_HEADERS include/join/statistics.hpp ) +if(JOIN_ENABLE_IO_URING) + list(APPEND PUBLIC_HEADERS include/join/proactor.hpp) +endif() + set(SOURCES src/error.cpp src/reactor.cpp @@ -50,6 +58,10 @@ set(SOURCES src/cpu.cpp ) +if(JOIN_ENABLE_IO_URING) + list(APPEND SOURCES src/proactor.cpp) +endif() + add_library(${JOIN_CORE} ${SOURCES}) add_library(join::core ALIAS ${JOIN_CORE}) @@ -63,6 +75,7 @@ target_include_directories(${JOIN_CORE} PUBLIC $ $ + $<$:${LIBURING_INCLUDE_DIRS}> $<$:${NUMA_INCLUDE_DIRS}> PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src @@ -73,9 +86,14 @@ target_link_libraries(${JOIN_CORE} OpenSSL::SSL OpenSSL::Crypto Threads::Threads + $<$:${LIBURING_LIBRARIES}> $<$:${NUMA_LIBRARIES}> ) +if(JOIN_ENABLE_IO_URING) + target_compile_definitions(${JOIN_CORE} PUBLIC JOIN_HAS_IO_URING) +endif() + if(JOIN_ENABLE_NUMA) target_compile_definitions(${JOIN_CORE} PUBLIC JOIN_HAS_NUMA) endif() diff --git a/core/include/join/proactor.hpp b/core/include/join/proactor.hpp new file mode 100644 index 00000000..486560cc --- /dev/null +++ b/core/include/join/proactor.hpp @@ -0,0 +1,32 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef JOIN_CORE_PROACTOR_HPP +#define JOIN_CORE_PROACTOR_HPP + +namespace join +{ +} + +#endif diff --git a/core/src/proactor.cpp b/core/src/proactor.cpp new file mode 100644 index 00000000..ac176d02 --- /dev/null +++ b/core/src/proactor.cpp @@ -0,0 +1,26 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include From 11747df7be6c7f4803ac36661a028a3f9eb040e1 Mon Sep 17 00:00:00 2001 From: mrabine Date: Sat, 28 Mar 2026 01:24:27 +0100 Subject: [PATCH 2/7] add proactor --- core/include/join/proactor.hpp | 599 ++++++++++++++++++++++++++ core/src/proactor.cpp | 742 +++++++++++++++++++++++++++++++++ 2 files changed, 1341 insertions(+) diff --git a/core/include/join/proactor.hpp b/core/include/join/proactor.hpp index 486560cc..a9c7c9d7 100644 --- a/core/include/join/proactor.hpp +++ b/core/include/join/proactor.hpp @@ -25,8 +25,607 @@ #ifndef JOIN_CORE_PROACTOR_HPP #define JOIN_CORE_PROACTOR_HPP +// libjoin. +#include +#include + +// C. +#include +#include + namespace join { + // forward declaration. + struct IoOperation; + + /** + * @brief completion handler interface class. + */ + class CompletionHandler + { + public: + /** + * @brief create instance. + */ + CompletionHandler () = default; + + /** + * @brief copy constructor. + * @param other other object to copy. + */ + CompletionHandler (const CompletionHandler& other) = default; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + * @return current object. + */ + CompletionHandler& operator= (const CompletionHandler& other) = default; + + /** + * @brief move constructor. + * @param other other object to move. + */ + CompletionHandler (CompletionHandler&& other) = default; + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return current object. + */ + CompletionHandler& operator= (CompletionHandler&& other) = default; + + /** + * @brief destroy instance. + */ + virtual ~CompletionHandler () = default; + + protected: + /** + * @brief method called when an operation completes successfully. + * @param op completed operation. + * @param result number of bytes transferred, or operation-specific value. + */ + virtual void onComplete ([[maybe_unused]] IoOperation* op, [[maybe_unused]] int result) + { + // do nothing. + } + + /** + * @brief method called when an operation is cancelled. + * @param op cancelled operation. + * @param result -ECANCELED, or other negative errno on failure. + */ + virtual void onCancel ([[maybe_unused]] IoOperation* op, [[maybe_unused]] int result) + { + // do nothing. + } + + /// friendship with proactor. + friend class Proactor; + }; + + /** + * @brief Describes a single asynchronous operation submitted to the Proactor. + */ + struct alignas (64) IoOperation + { + /** + * @brief operation lifecycle state. + */ + enum class State : uint8_t + { + Idle, + Submitted, + Cancelling, + }; + + /** + * @brief payload for IORING_OP_TIMEOUT. + */ + struct TimeoutData + { + /// absolute or relative kernel timespec. + __kernel_timespec ts; + + /// 0 or IORING_TIMEOUT_ABS. + uint32_t flags; + }; + + /** + * @brief payload for IORING_OP_READ / IORING_OP_READ_FIXED / IORING_OP_WRITE / IORING_OP_WRITE_FIXED. + */ + struct RwData + { + /// file descriptor. + int fd; + + /// buffer. + void* buf; + + /// number of bytes to proceed. + uint32_t len; + + /// registered buffer index (FIXED only, ignored otherwise). + uint16_t index; + + /// use FIXED if true. + bool fixed; + }; + + /** + * @brief payload for IORING_OP_SENDMSG / IORING_OP_RECVMSG. + */ + struct MsgData + { + /// file descriptor. + int fd; + + /// message header. + msghdr* msg; + + /// send/recv flags. + uint32_t flags; + }; + + /** + * @brief payload for IORING_OP_ACCEPT. + */ + struct AcceptData + { + /// listening file descriptor. + int fd; + + /// peer address output buffer. + sockaddr* addr; + + /// peer address length. + socklen_t* addrlen; + + /// accept flags. + uint32_t flags; + }; + + /** + * @brief payload for IORING_OP_ASYNC_CANCEL. + */ + // struct CancelData + // { + // /// operation to cancel, identified by its address as user_data. + // IoOperation* target; + // }; + + union Data + { + TimeoutData timeout; + RwData rw; + MsgData msg; + AcceptData accept; + // CancelData cancel; + }; + + /** + * @brief build a relative or absolute timeout operation. + * @param ts kernel timespec (relative duration or absolute deadline). + * @param flags 0 for relative, IORING_TIMEOUT_ABS for absolute. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeTimeout (const __kernel_timespec& ts, uint32_t flags, + CompletionHandler* handler) noexcept; + + /** + * @brief build a regular (non-registered) read operation. + * @param fd file descriptor to read from. + * @param buf destination buffer. + * @param len number of bytes to read. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept; + + /** + * @brief build a fixed-buffer read operation (requires registered buffers). + * @param fd file descriptor to read from. + * @param buf destination buffer (must belong to a registered region). + * @param len number of bytes to read. + * @param index index of the registered buffer. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept; + + /** + * @brief build a regular (non-registered) write operation. + * @param fd file descriptor to write to. + * @param buf source buffer. + * @param len number of bytes to write. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept; + + /** + * @brief build a fixed-buffer write operation (requires registered buffers). + * @param fd file descriptor to write to. + * @param buf source buffer (must belong to a registered region). + * @param len number of bytes to write. + * @param index index of the registered buffer. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept; + /** + * @brief build a send-message operation. + * @param fd socket file descriptor. + * @param msg message header. + * @param flags send flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeSendmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept; + + /** + * @brief build a receive-message operation. + * @param fd socket file descriptor. + * @param msg message header. + * @param flags recv flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeRecvmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept; + + /** + * @brief build an accept operation. + * @param fd listening socket file descriptor. + * @param addr peer address output buffer (may be nullptr). + * @param addrlen peer address length output (may be nullptr). + * @param flags accept flags (e.g. SOCK_NONBLOCK | SOCK_CLOEXEC). + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, uint32_t flags, + CompletionHandler* handler) noexcept; + + /** + * @brief build a cancellation operation targeting another in-flight operation. + * @param target operation to cancel. + * @param handler handler to notify when the cancellation itself completes. + * @return initialized IoOperation. + */ + // static IoOperation makeCancel (IoOperation* target, CompletionHandler* handler) noexcept; + + /// io_uring opcode (IORING_OP_TIMEOUT, IORING_OP_READ, ...). + uint8_t code = 0; + + /// state. + State state = State::Idle; + + /// handler to dispatch to on completion or cancellation. + CompletionHandler* handler = nullptr; + + /// opcode-specific payload. + Data data = {}; + }; + + /** + * @brief Proactor class. + */ + class Proactor + { + public: + /** + * @brief create instance. + */ + Proactor (); + + /** + * @brief copy constructor. + * @param other other object to copy. + */ + Proactor (const Proactor& other) = delete; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + * @return current object. + */ + Proactor& operator= (const Proactor& other) = delete; + + /** + * @brief move constructor. + * @param other other object to move. + */ + Proactor (Proactor&& other) = delete; + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return current object. + */ + Proactor& operator= (Proactor&& other) = delete; + + /** + * @brief destroy instance. + */ + ~Proactor () noexcept; + + /** + * @brief submit an asynchronous operation to the proactor. + * @param op operation to submit. + * @param sync wait for submission acknowledgment if true. + * @return 0 on success, -1 on failure. + */ + int submit (IoOperation* op, bool sync = true) noexcept; + + /** + * @brief cancel an in-flight asynchronous operation. + * @param op operation to cancel. + * @param sync wait for cancellation acknowledgment if true. + * @return 0 on success, -1 on failure. + */ + int cancel (IoOperation* op, bool sync = true) noexcept; + + /** + * @brief run the event loop (blocking). + */ + void run (); + + /** + * @brief stop the event loop. + * @param sync wait for loop termination if true. + */ + void stop (bool sync = true) noexcept; + +#ifdef JOIN_HAS_NUMA + /** + * @brief bind command queue memory to a NUMA node. + * @param numa NUMA node ID. + * @return 0 on success, -1 on failure. + */ + int mbind (int numa) const noexcept; +#endif + + /** + * @brief lock command queue memory in RAM. + * @return 0 on success, -1 on failure. + */ + int mlock () const noexcept; + + /** + * @brief check if the calling thread is the proactor thread. + * @return true if called from the proactor thread. + */ + bool isProactorThread () const noexcept; + + private: + /// queue size. + static constexpr size_t _queueSize = 1024; + + /// io_uring submission/completion ring size. + static constexpr size_t _ringSize = 1024; + + /** + * @brief Command type for proactor dispatcher. + */ + enum class CommandType + { + Submit, + Cancel, + Stop + }; + + /** + * @brief Command for proactor dispatcher. + */ + struct alignas (64) Command + { + CommandType type; + IoOperation* op; + std::atomic* done; + std::atomic* errc; + }; + + /** + * @brief get a free SQE from the ring, flushing if necessary. + * @return pointer to an SQE, or nullptr on failure. + */ + io_uring_sqe* getSqe () noexcept; + + /** + * @brief submit op directly onto the ring. + * @param op operation to submit. + * @return 0 on success, -1 on failure. + */ + int submitOperation (IoOperation* op) noexcept; + + /** + * @brief cancel op directly on the ring. + * @param op operation to cancel. + * @return 0 on success, -1 on failure. + */ + int cancelOperation (IoOperation* op) noexcept; + + /** + * @brief (re)submit the internal wakeup read on _wakeup eventfd. + */ + void rearmWakeup () noexcept; + + /** + * @brief write command to queue and wake dispatcher. + * @param cmd command to write. + * @return 0 on success, -1 on failure. + */ + int writeCommand (const Command& cmd) noexcept; + + /** + * @brief process a single command. + * @param cmd command to process. + */ + void processCommand (const Command& cmd) noexcept; + + /** + * @brief read and process all pending commands from queue. + */ + void readCommands () noexcept; + + /** + * @brief dispatch a single CQE to its completion handler. + * @param cqe completed queue entry. + */ + void dispatchCqe (io_uring_cqe* cqe) noexcept; + + /** + * @brief fill an SQE from an operation. + * @param op source operation. + * @param sqe submission queue entry to populate. + */ + void prepareSqe (const IoOperation& op, io_uring_sqe* sqe) const noexcept; + + /** + * @brief dispatch a completed operation to its handler. + * @param op operation whose CQE has arrived. + * @param result cqe->res. + */ + void dispatchOperation (IoOperation* op, int result) noexcept; + + /** + * @brief main event loop running in dispatcher thread. + */ + void eventLoop (); + + /// eventfd descriptor. + int _wakeup = -1; + + /// read buffer for the wakeup eventfd. + uint64_t _wakeupBuf = 0; + + /// internal IoOperation that keeps a persistent READ on _wakeup in the ring. + IoOperation _wakeupOp; + + /// io_uring instance. + io_uring _ring = {}; + + /// command queue. + LocalMem::Mpsc::Queue _commands; + + /// running flag for dispatcher thread. + std::atomic _running{false}; + + /// event loop thread ID. + std::atomic _threadId{0}; + }; + + /** + * @brief Convenience class that owns a Proactor running on a dedicated background thread. + */ + class ProactorThread + { + public: + /** + * @brief get the global Proactor instance. + * @return pointer to the singleton Proactor. + */ + static Proactor* proactor (); + + /** + * @brief set proactor thread affinity. + * @param core thread core affinity (-1 to disable pinning). + * @return 0 on success, -1 on failure. + */ + static int affinity (int core); + + /** + * @brief get proactor thread affinity. + * @return affinity or -1 if not pinned. + */ + static int affinity (); + + /** + * @brief set proactor thread priority. + * @param prio thread priority (0 = SCHED_OTHER, 1-99 = SCHED_FIFO). + * @return 0 on success, -1 on failure. + */ + static int priority (int prio); + + /** + * @brief get proactor thread priority. + * @return priority. + */ + static int priority (); + + /** + * @brief get the handle of the proactor thread. + * @return proactor thread handle. + */ + static pthread_t handle (); + +#ifdef JOIN_HAS_NUMA + /** + * @brief bind command queue memory to a NUMA node. + * @param numa NUMA node ID. + * @return 0 on success, -1 on failure. + */ + static int mbind (int numa); +#endif + + /** + * @brief lock command queue memory in RAM. + * @return 0 on success, -1 on failure. + */ + static int mlock (); + + private: + /** + * @brief get the singleton ProactorThread instance. + * @return reference to the singleton ProactorThread. + */ + static ProactorThread& instance (); + + /** + * @brief construct the ProactorThread and start the event loop thread. + */ + ProactorThread (); + + /** + * @brief copy constructor. + * @param other other object to copy. + */ + ProactorThread (const ProactorThread& other) = delete; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + * @return current object. + */ + ProactorThread& operator= (const ProactorThread& other) = delete; + + /** + * @brief move constructor. + * @param other other object to move. + */ + ProactorThread (ProactorThread&& other) noexcept = delete; + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return current object. + */ + ProactorThread& operator= (ProactorThread&& other) noexcept = delete; + + /** + * @brief destroy the ProactorThread and cleanly shut down the event loop. + */ + ~ProactorThread (); + + /// Proactor instance. + Proactor _proactor; + + /// Background thread. + Thread _dispatcher; + }; } #endif diff --git a/core/src/proactor.cpp b/core/src/proactor.cpp index ac176d02..1ac6edbb 100644 --- a/core/src/proactor.cpp +++ b/core/src/proactor.cpp @@ -24,3 +24,745 @@ // libjoin. #include +#include + +// C. +#include +#include +#include +#include + +using join::CompletionHandler; +using join::IoOperation; +using join::Proactor; +using join::ProactorThread; + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeTimeout +// ========================================================================= +IoOperation IoOperation::makeTimeout (const __kernel_timespec& ts, uint32_t flags, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_TIMEOUT; + op.handler = handler; + op.data.timeout.ts = ts; + op.data.timeout.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeRead +// ========================================================================= +IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_READ; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = buf; + op.data.rw.len = len; + op.data.rw.index = 0; + op.data.rw.fixed = false; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeReadFixed +// ========================================================================= +IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_t buf_index, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_READ_FIXED; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = buf; + op.data.rw.len = len; + op.data.rw.index = buf_index; + op.data.rw.fixed = true; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeWrite +// ========================================================================= +IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_WRITE; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = const_cast (buf); + op.data.rw.len = len; + op.data.rw.index = 0; + op.data.rw.fixed = false; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeWriteFixed +// ========================================================================= +IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_WRITE_FIXED; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = const_cast (buf); + op.data.rw.len = len; + op.data.rw.index = index; + op.data.rw.fixed = true; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeSendmsg +// ========================================================================= +IoOperation IoOperation::makeSendmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_SENDMSG; + op.handler = handler; + op.data.msg.fd = fd; + op.data.msg.msg = msg; + op.data.msg.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeRecvmsg +// ========================================================================= +IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_RECVMSG; + op.handler = handler; + op.data.msg.fd = fd; + op.data.msg.msg = msg; + op.data.msg.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeAccept +// ========================================================================= +IoOperation IoOperation::makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, uint32_t flags, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = IORING_OP_ACCEPT; + op.handler = handler; + op.data.accept.fd = fd; + op.data.accept.addr = addr; + op.data.accept.addrlen = addrlen; + op.data.accept.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeCancel +// ========================================================================= +// IoOperation IoOperation::makeCancel (IoOperation* target, CompletionHandler* handler) noexcept +// { +// IoOperation op; +// op.code = IORING_OP_ASYNC_CANCEL; +// op.handler = handler; +// op.data.cancel.target = target; +// return op; +// } + +// ========================================================================= +// CLASS : Proactor +// METHOD : Proactor +// ========================================================================= +Proactor::Proactor () +: _wakeup (eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC)) +, _commands (_queueSize) +{ + if (_wakeup == -1) + { + throw std::system_error (errno, std::system_category (), "eventfd failed"); + } + + if (io_uring_queue_init (_ringSize, &_ring, 0) != 0) + { + int err = errno; + ::close (_wakeup); + throw std::system_error (err, std::system_category (), "io_uring_queue_init failed"); + } + + _wakeupOp = IoOperation::makeRead (_wakeup, &_wakeupBuf, sizeof (_wakeupBuf), nullptr); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : ~Proactor +// ========================================================================= +Proactor::~Proactor () noexcept +{ + stop (); + + io_uring_queue_exit (&_ring); + + ::close (_wakeup); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : submit +// ========================================================================= +int Proactor::submit (IoOperation* op, bool sync) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (isProactorThread ()) + { + return submitOperation (op); + } + + std::atomic done{false}, *pdone = nullptr; + std::atomic errc{0}, *perrc = nullptr; + + if (JOIN_LIKELY (sync)) + { + pdone = &done; + perrc = &errc; + } + + if (JOIN_UNLIKELY (writeCommand ({CommandType::Submit, op, pdone, perrc}) == -1)) + { + return -1; + } + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + + int err = errc.load (std::memory_order_acquire); + if (JOIN_UNLIKELY (err != 0)) + { + lastError = std::make_error_code (static_cast (err)); + return -1; + } + } + + return 0; +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : cancel +// ========================================================================= +int Proactor::cancel (IoOperation* op, bool sync) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (isProactorThread ()) + { + return cancelOperation (op); + } + + std::atomic done{false}, *pdone = nullptr; + std::atomic errc{0}, *perrc = nullptr; + + if (JOIN_LIKELY (sync)) + { + pdone = &done; + perrc = &errc; + } + + if (JOIN_UNLIKELY (writeCommand ({CommandType::Cancel, op, pdone, perrc}) == -1)) + { + return -1; + } + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + + int err = errc.load (std::memory_order_acquire); + if (JOIN_UNLIKELY (err != 0)) + { + lastError = std::make_error_code (static_cast (err)); + return -1; + } + } + + return 0; +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : run +// ========================================================================= +void Proactor::run () +{ + _threadId.store (pthread_self (), std::memory_order_release); + + _running.store (true, std::memory_order_release); + rearmWakeup (); + eventLoop (); + + _threadId.store (0, std::memory_order_release); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : stop +// ========================================================================= +void Proactor::stop (bool sync) noexcept +{ + _running.store (false, std::memory_order_release); + + if (isProactorThread ()) + { + return; + } + + writeCommand ({CommandType::Stop, nullptr, nullptr, nullptr}); + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (_threadId.load (std::memory_order_acquire) != 0) + { + backoff (); + } + } +} + +#ifdef JOIN_HAS_NUMA +// ========================================================================= +// CLASS : Proactor +// METHOD : mbind +// ========================================================================= +int Proactor::mbind (int numa) const noexcept +{ + return _commands.mbind (numa); +} +#endif + +// ========================================================================= +// CLASS : Proactor +// METHOD : mlock +// ========================================================================= +int Proactor::mlock () const noexcept +{ + return _commands.mlock (); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : isProactorThread +// ========================================================================= +bool Proactor::isProactorThread () const noexcept +{ + return _threadId.load (std::memory_order_acquire) == pthread_self (); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : getSqe +// ========================================================================= +io_uring_sqe* Proactor::getSqe () noexcept +{ + io_uring_sqe* sqe = io_uring_get_sqe (&_ring); + + if (JOIN_UNLIKELY (sqe == nullptr)) + { + io_uring_submit (&_ring); + sqe = io_uring_get_sqe (&_ring); + } + + return sqe; +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : submitOperation +// ========================================================================= +int Proactor::submitOperation (IoOperation* op) noexcept +{ + io_uring_sqe* sqe = getSqe (); + + if (JOIN_UNLIKELY (sqe == nullptr)) + { + lastError = std::make_error_code (std::errc::resource_unavailable_try_again); + return -1; + } + + prepareSqe (*op, sqe); + op->state = IoOperation::State::Submitted; + + return 0; +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : cancelOperation +// ========================================================================= +int Proactor::cancelOperation (IoOperation* op) noexcept +{ + if (op->state != IoOperation::State::Submitted) + { + return 0; + } + + io_uring_sqe* sqe = getSqe (); + + if (JOIN_UNLIKELY (sqe == nullptr)) + { + lastError = std::make_error_code (std::errc::resource_unavailable_try_again); + return -1; + } + + io_uring_prep_cancel (sqe, op, 0); + io_uring_sqe_set_data (sqe, nullptr); + op->state = IoOperation::State::Cancelling; + + return 0; +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : rearmWakeup +// ========================================================================= +void Proactor::rearmWakeup () noexcept +{ + io_uring_sqe* sqe = getSqe (); + + if (JOIN_UNLIKELY (sqe == nullptr)) + { + return; + } + + _wakeupOp.state = IoOperation::State::Submitted; + prepareSqe (_wakeupOp, sqe); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : writeCommand +// ========================================================================= +int Proactor::writeCommand (const Command& cmd) noexcept +{ + if (_commands.push (cmd) == -1) + { + return -1; + } + + uint64_t value = 1; + [[maybe_unused]] ssize_t bytes = ::write (_wakeup, &value, sizeof (uint64_t)); + + return 0; +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : processCommand +// ========================================================================= +void Proactor::processCommand (const Command& cmd) noexcept +{ + int err = 0; + + switch (cmd.type) + { + case CommandType::Submit: + err = submitOperation (cmd.op); + break; + + case CommandType::Cancel: + err = cancelOperation (cmd.op); + break; + + case CommandType::Stop: + break; + } + + if (JOIN_UNLIKELY (cmd.done)) + { + if (cmd.errc && (err != 0)) + { + cmd.errc->store (lastError.value (), std::memory_order_release); + } + cmd.done->store (true, std::memory_order_release); + } +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : readCommands +// ========================================================================= +void Proactor::readCommands () noexcept +{ + Command cmd; + + while (_commands.tryPop (cmd) == 0) + { + processCommand (cmd); + } + + if (_running.load (std::memory_order_acquire)) + { + rearmWakeup (); + } +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : dispatchCqe +// ========================================================================= +void Proactor::dispatchCqe (io_uring_cqe* cqe) noexcept +{ + auto* op = reinterpret_cast (io_uring_cqe_get_data (cqe)); + + if (op == &_wakeupOp) + { + _wakeupOp.state = IoOperation::State::Idle; + readCommands (); + return; + } + + if (op != nullptr) + { + dispatchOperation (op, cqe->res); + } +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : prepareSqe +// ========================================================================= +void Proactor::prepareSqe (const IoOperation& op, io_uring_sqe* sqe) const noexcept +{ + assert (sqe != nullptr); + + switch (op.code) + { + case IORING_OP_TIMEOUT: + { + auto* ts = const_cast<__kernel_timespec*> (&op.data.timeout.ts); + io_uring_prep_timeout (sqe, ts, 0, op.data.timeout.flags); + } + break; + + case IORING_OP_READ: + io_uring_prep_read (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0); + break; + + case IORING_OP_READ_FIXED: + io_uring_prep_read_fixed (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0, op.data.rw.index); + break; + + case IORING_OP_WRITE: + io_uring_prep_write (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0); + break; + + case IORING_OP_WRITE_FIXED: + io_uring_prep_write_fixed (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0, op.data.rw.index); + break; + + case IORING_OP_SENDMSG: + io_uring_prep_sendmsg (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags); + break; + + case IORING_OP_RECVMSG: + io_uring_prep_recvmsg (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags); + break; + + case IORING_OP_ACCEPT: + io_uring_prep_accept (sqe, op.data.accept.fd, op.data.accept.addr, op.data.accept.addrlen, + static_cast (op.data.accept.flags)); + break; + + // case IORING_OP_ASYNC_CANCEL: + // io_uring_prep_cancel (sqe, reinterpret_cast (op.data.cancel.target), 0); + // break; + + default: + break; + } + + io_uring_sqe_set_data (sqe, static_cast (const_cast (&op))); +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : dispatchOperation +// ========================================================================= +void Proactor::dispatchOperation (IoOperation* op, int result) noexcept +{ + const IoOperation::State prev = op->state; + op->state = IoOperation::State::Idle; + + if (op->handler == nullptr) + { + return; + } + + if (result == -ECANCELED || prev == IoOperation::State::Cancelling) + { + op->handler->onCancel (op, result); + } + else + { + op->handler->onComplete (op, result); + } +} + +// ========================================================================= +// CLASS : Proactor +// METHOD : eventLoop +// ========================================================================= +void Proactor::eventLoop () +{ + while (_running.load (std::memory_order_acquire)) + { + io_uring_submit (&_ring); + + io_uring_cqe* cqe = nullptr; + int ret = io_uring_wait_cqe (&_ring, &cqe); + if (JOIN_UNLIKELY (ret < 0)) + { + continue; + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + } +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : proactor +// ========================================================================= +Proactor* ProactorThread::proactor () +{ + return &instance ()._proactor; +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : affinity (setter) +// ========================================================================= +int ProactorThread::affinity (int core) +{ + return instance ()._dispatcher.affinity (core); +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : affinity (getter) +// ========================================================================= +int ProactorThread::affinity () +{ + return instance ()._dispatcher.affinity (); +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : priority (setter) +// ========================================================================= +int ProactorThread::priority (int prio) +{ + return instance ()._dispatcher.priority (prio); +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : priority (getter) +// ========================================================================= +int ProactorThread::priority () +{ + return instance ()._dispatcher.priority (); +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : handle +// ========================================================================= +pthread_t ProactorThread::handle () +{ + return instance ()._dispatcher.handle (); +} + +#ifdef JOIN_HAS_NUMA +// ========================================================================= +// CLASS : ProactorThread +// METHOD : mbind +// ========================================================================= +int ProactorThread::mbind (int numa) +{ + return instance ()._proactor.mbind (numa); +} +#endif + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : mlock +// ========================================================================= +int ProactorThread::mlock () +{ + return instance ()._proactor.mlock (); +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : instance +// ========================================================================= +ProactorThread& ProactorThread::instance () +{ + static ProactorThread proactorThread; + return proactorThread; +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : ProactorThread +// ========================================================================= +ProactorThread::ProactorThread () +{ + _dispatcher = Thread ([this] () { + _proactor.run (); + }); +} + +// ========================================================================= +// CLASS : ProactorThread +// METHOD : ~ProactorThread +// ========================================================================= +ProactorThread::~ProactorThread () +{ + _proactor.stop (); + _dispatcher.join (); +} From 0db76f37dddf1245a76f8104bb2364e2b4f4d083 Mon Sep 17 00:00:00 2001 From: mrabine Date: Sat, 28 Mar 2026 01:33:32 +0100 Subject: [PATCH 3/7] proactor format --- core/src/proactor.cpp | 82 +++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/core/src/proactor.cpp b/core/src/proactor.cpp index 1ac6edbb..08d9afd6 100644 --- a/core/src/proactor.cpp +++ b/core/src/proactor.cpp @@ -44,9 +44,9 @@ using join::ProactorThread; IoOperation IoOperation::makeTimeout (const __kernel_timespec& ts, uint32_t flags, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_TIMEOUT; - op.handler = handler; - op.data.timeout.ts = ts; + op.code = IORING_OP_TIMEOUT; + op.handler = handler; + op.data.timeout.ts = ts; op.data.timeout.flags = flags; return op; } @@ -58,11 +58,11 @@ IoOperation IoOperation::makeTimeout (const __kernel_timespec& ts, uint32_t flag IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_READ; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = buf; - op.data.rw.len = len; + op.code = IORING_OP_READ; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = buf; + op.data.rw.len = len; op.data.rw.index = 0; op.data.rw.fixed = false; return op; @@ -76,11 +76,11 @@ IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_ CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_READ_FIXED; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = buf; - op.data.rw.len = len; + op.code = IORING_OP_READ_FIXED; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = buf; + op.data.rw.len = len; op.data.rw.index = buf_index; op.data.rw.fixed = true; return op; @@ -93,11 +93,11 @@ IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_ IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_WRITE; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = const_cast (buf); - op.data.rw.len = len; + op.code = IORING_OP_WRITE; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = const_cast (buf); + op.data.rw.len = len; op.data.rw.index = 0; op.data.rw.fixed = false; return op; @@ -111,11 +111,11 @@ IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_WRITE_FIXED; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = const_cast (buf); - op.data.rw.len = len; + op.code = IORING_OP_WRITE_FIXED; + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = const_cast (buf); + op.data.rw.len = len; op.data.rw.index = index; op.data.rw.fixed = true; return op; @@ -128,10 +128,10 @@ IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, IoOperation IoOperation::makeSendmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_SENDMSG; - op.handler = handler; - op.data.msg.fd = fd; - op.data.msg.msg = msg; + op.code = IORING_OP_SENDMSG; + op.handler = handler; + op.data.msg.fd = fd; + op.data.msg.msg = msg; op.data.msg.flags = flags; return op; } @@ -143,10 +143,10 @@ IoOperation IoOperation::makeSendmsg (int fd, msghdr* msg, uint32_t flags, Compl IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_RECVMSG; - op.handler = handler; - op.data.msg.fd = fd; - op.data.msg.msg = msg; + op.code = IORING_OP_RECVMSG; + op.handler = handler; + op.data.msg.fd = fd; + op.data.msg.msg = msg; op.data.msg.flags = flags; return op; } @@ -159,12 +159,12 @@ IoOperation IoOperation::makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, CompletionHandler* handler) noexcept { IoOperation op; - op.code = IORING_OP_ACCEPT; - op.handler = handler; - op.data.accept.fd = fd; - op.data.accept.addr = addr; + op.code = IORING_OP_ACCEPT; + op.handler = handler; + op.data.accept.fd = fd; + op.data.accept.addr = addr; op.data.accept.addrlen = addrlen; - op.data.accept.flags = flags; + op.data.accept.flags = flags; return op; } @@ -235,7 +235,7 @@ int Proactor::submit (IoOperation* op, bool sync) noexcept } std::atomic done{false}, *pdone = nullptr; - std::atomic errc{0}, *perrc = nullptr; + std::atomic errc{0}, *perrc = nullptr; if (JOIN_LIKELY (sync)) { @@ -285,7 +285,7 @@ int Proactor::cancel (IoOperation* op, bool sync) noexcept } std::atomic done{false}, *pdone = nullptr; - std::atomic errc{0}, *perrc = nullptr; + std::atomic errc{0}, *perrc = nullptr; if (JOIN_LIKELY (sync)) { @@ -477,7 +477,7 @@ int Proactor::writeCommand (const Command& cmd) noexcept return -1; } - uint64_t value = 1; + uint64_t value = 1; [[maybe_unused]] ssize_t bytes = ::write (_wakeup, &value, sizeof (uint64_t)); return 0; @@ -619,7 +619,7 @@ void Proactor::prepareSqe (const IoOperation& op, io_uring_sqe* sqe) const noexc void Proactor::dispatchOperation (IoOperation* op, int result) noexcept { const IoOperation::State prev = op->state; - op->state = IoOperation::State::Idle; + op->state = IoOperation::State::Idle; if (op->handler == nullptr) { @@ -647,7 +647,7 @@ void Proactor::eventLoop () io_uring_submit (&_ring); io_uring_cqe* cqe = nullptr; - int ret = io_uring_wait_cqe (&_ring, &cqe); + int ret = io_uring_wait_cqe (&_ring, &cqe); if (JOIN_UNLIKELY (ret < 0)) { continue; From 42d8d8113b0c967025617f68b21147b8bd5001b5 Mon Sep 17 00:00:00 2001 From: mrabine Date: Wed, 27 May 2026 23:53:17 +0200 Subject: [PATCH 4/7] refactoring --- CMakePresets.json | 4 +- README.md | 4 +- core/CMakeLists.txt | 11 +- core/include/join/allocator.hpp | 20 + core/include/join/io_operation.hpp | 299 ++++++ core/include/join/io_policy.hpp | 116 +++ core/include/join/proactor.hpp | 1309 +++++++++++++++----------- core/include/join/proactor_epoll.inl | 526 +++++++++++ core/include/join/proactor_uring.inl | 748 +++++++++++++++ core/include/join/reactor.hpp | 6 + core/src/io_operation.cpp | 224 +++++ core/src/proactor.cpp | 768 --------------- core/src/reactor.cpp | 13 +- core/tests/CMakeLists.txt | 24 + core/tests/hybrid_proactor_test.cpp | 752 +++++++++++++++ core/tests/io_operation_test.cpp | 268 ++++++ core/tests/io_policy_test.cpp | 108 +++ core/tests/proactor_test.cpp | 287 +++--- core/tests/reactor_test.cpp | 24 + core/tests/sqpoll_proactor_test.cpp | 752 +++++++++++++++ 20 files changed, 4783 insertions(+), 1480 deletions(-) create mode 100644 core/include/join/io_operation.hpp create mode 100644 core/include/join/io_policy.hpp create mode 100644 core/include/join/proactor_epoll.inl create mode 100644 core/include/join/proactor_uring.inl create mode 100644 core/src/io_operation.cpp delete mode 100644 core/src/proactor.cpp create mode 100644 core/tests/hybrid_proactor_test.cpp create mode 100644 core/tests/io_operation_test.cpp create mode 100644 core/tests/io_policy_test.cpp create mode 100644 core/tests/sqpoll_proactor_test.cpp diff --git a/CMakePresets.json b/CMakePresets.json index 11da8911..b3c4596a 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -8,7 +8,6 @@ "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", "CMAKE_VERBOSE_MAKEFILE": "ON", - "JOIN_ENABLE_IO_URING": "ON", "JOIN_ENABLE_NUMA": "ON", "JOIN_ENABLE_SAMPLES": "ON", "JOIN_ENABLE_TESTS": "ON" @@ -20,7 +19,8 @@ "inherits": "base", "cacheVariables": { "CMAKE_C_COMPILER": "gcc", - "CMAKE_CXX_COMPILER": "g++" + "CMAKE_CXX_COMPILER": "g++", + "JOIN_ENABLE_IO_URING": "ON" } }, { diff --git a/README.md b/README.md index 83796c07..9c59ff2e 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Install required libraries and test dependencies: sudo apt install pkg-config libssl-dev zlib1g-dev libgtest-dev libgmock-dev ``` -> **Compilers:** Both GCC and Clang are supported. Clang requires `libclang-rt-dev` for coverage instrumentation. +> **Compilers:** Both GCC and Clang are supported. Clang requires `libclang-rt-dev` for coverage instrumentation (`--coverage`). > **OpenSSL** provides the core TLS runtime. ### Optional Dependencies @@ -125,7 +125,7 @@ cmake --build build | `JOIN_ENABLE_DATA` | `ON` | Build the data module. | | `JOIN_ENABLE_FABRIC` | `ON` | Build the fabric module. | | `JOIN_ENABLE_SERVICES` | `ON` | Build the services module (requires crypto, data, fabric). | -| `JOIN_ENABLE_IO_URING` | `OFF` | Enable io_uring backend (requires `liburing-dev`). | +| `JOIN_ENABLE_IO_URING` | `OFF` | Enable io_uring based proactor backend (requires `liburing-dev`). | | `JOIN_ENABLE_NUMA` | `OFF` | Enable NUMA support (requires `libnuma-dev`). | | `JOIN_ENABLE_SAMPLES` | `OFF` | Build sample programs. | | `JOIN_ENABLE_TESTS` | `OFF` | Build the test suite. | diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 45d9088a..2953241d 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -16,6 +16,7 @@ set(PUBLIC_HEADERS include/join/traits.hpp include/join/variant.hpp include/join/reactor.hpp + include/join/io_operation.hpp include/join/proactor.hpp include/join/memory.hpp include/join/allocator.hpp @@ -43,13 +44,15 @@ set(PUBLIC_HEADERS ) if(JOIN_ENABLE_IO_URING) - list(APPEND PUBLIC_HEADERS include/join/proactor.hpp) + list(APPEND PUBLIC_HEADERS include/join/io_policy.hpp include/join/proactor_uring.inl) +else() + list(APPEND PUBLIC_HEADERS include/join/proactor_epoll.inl) endif() set(SOURCES src/error.cpp src/reactor.cpp - src/proactor.cpp + src/io_operation.cpp src/mutex.cpp src/condition.cpp src/semaphore.cpp @@ -62,10 +65,6 @@ set(SOURCES src/cpu.cpp ) -if(JOIN_ENABLE_IO_URING) - list(APPEND SOURCES src/proactor.cpp) -endif() - add_library(${JOIN_CORE} ${SOURCES}) add_library(join::core ALIAS ${JOIN_CORE}) diff --git a/core/include/join/allocator.hpp b/core/include/join/allocator.hpp index b0e9ba5c..567440eb 100644 --- a/core/include/join/allocator.hpp +++ b/core/include/join/allocator.hpp @@ -262,6 +262,26 @@ namespace join return ((ptr >= base) && (ptr < end)); } + /** + * @brief get the index of a chunk in the pool. + * @param p pointer to the chunk (must be owned by this pool). + * @return index of the chunk. + */ + uint32_t getIndex (void* p) const noexcept + { + return static_cast (reinterpret_cast (p) - _segment->_chunks); + } + + /** + * @brief get the pointer to a chunk by index. + * @param idx index of the chunk (must be in range). + * @return pointer to the chunk. + */ + void* getPtr (uint32_t idx) const noexcept + { + return &_segment->_chunks[idx]; + } + /// pointer into the mapped region. Segment* _segment = nullptr; }; diff --git a/core/include/join/io_operation.hpp b/core/include/join/io_operation.hpp new file mode 100644 index 00000000..ecb43035 --- /dev/null +++ b/core/include/join/io_operation.hpp @@ -0,0 +1,299 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef JOIN_CORE_IO_OPERATION_HPP +#define JOIN_CORE_IO_OPERATION_HPP + +// C. +#include +#include + +namespace join +{ + // forward declaration. + class CompletionHandler; + + /** + * @brief Describes a single asynchronous operation submitted to the Proactor. + */ + struct alignas (64) IoOperation + { + /** + * @brief operation lifecycle state. + */ + enum class State : uint8_t + { + Idle, /**< operation is not in flight. */ + Submitted, /**< operation has been submitted and is awaiting completion. */ + Cancelling, /**< operation has been canceled and is awaiting completion. */ + }; + + /** + * @brief operation code. + */ + enum class Opcode : uint8_t + { + Accept, /**< accept an incoming connection. */ + Connect, /**< initiate an outgoing connection. */ + Read, /**< read from a file descriptor. */ + Write, /**< write to a file descriptor. */ + ReadFixed, /**< read using a registered buffer. */ + WriteFixed, /**< write using a registered buffer. */ + RecvMsg, /**< receive a message with ancillary data. */ + SendMsg, /**< send a message with ancillary data. */ + Recv, /**< receive data from a socket. */ + Send, /**< send data on a socket. */ + }; + + /** + * @brief payload for accept. + */ + struct AcceptData + { + /// file descriptor. + int fd; + + /// peer address. + sockaddr* addr; + + /// peer address length. + socklen_t* addrlen; + + /// accept flags. + int flags; + }; + + /** + * @brief build an accept operation. + * @param fd listening socket file descriptor. + * @param addr peer address. + * @param addrlen peer address length. + * @param flags accept flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, int flags, + CompletionHandler* handler) noexcept; + + /** + * @brief payload for connect. + */ + struct ConnectData + { + /// file descriptor. + int fd; + + /// remote address. + const sockaddr* addr; + + /// address length. + socklen_t addrlen; + }; + + /** + * @brief build a connect operation. + * @param fd socket file descriptor. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeConnect (int fd, const sockaddr* addr, socklen_t addrlen, + CompletionHandler* handler) noexcept; + + /** + * @brief payload for read / read fixed / write / write fixed. + */ + struct RwData + { + /// file descriptor. + int fd; + + /// buffer. + void* buf; + + /// number of bytes to proceed. + unsigned long len; + + /// registered buffer index (ignored with reactor backend). + uint16_t index; + + /// use registered buffer (ignored with reactor backend). + bool fixed; + }; + + /** + * @brief build a regular read operation. + * @param fd file descriptor to read from. + * @param buf destination buffer. + * @param len number of bytes to read. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept; + + /** + * @brief build a regular write operation. + * @param fd file descriptor to write to. + * @param buf source buffer. + * @param len number of bytes to write. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept; + + /** + * @brief build a regular read operation. + * @param fd file descriptor to read from. + * @param buf destination buffer. + * @param len number of bytes to read. + * @param index registered buffer index. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept; + + /** + * @brief build a regular write operation. + * @param fd file descriptor to write to. + * @param buf source buffer. + * @param len number of bytes to write. + * @param index registered buffer index. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept; + + /** + * @brief payload for sendmsg / recvmsg. + */ + struct MsgData + { + /// file descriptor. + int fd; + + /// message header. + msghdr* msg; + + /// sendmsg / recvmsg flags. + int flags; + }; + + /** + * @brief build a receive-message operation. + * @param fd socket file descriptor. + * @param msg message header. + * @param flags recv flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeRecvmsg (int fd, msghdr* msg, int flags, CompletionHandler* handler) noexcept; + + /** + * @brief build a send-message operation. + * @param fd socket file descriptor. + * @param msg message header. + * @param flags send flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeSendmsg (int fd, const msghdr* msg, int flags, CompletionHandler* handler) noexcept; + + /** + * @brief payload for recv / send. + */ + struct StreamData + { + /// file descriptor. + int fd; + + /// buffer. + void* buf; + + /// number of bytes to proceed. + unsigned long len; + + /// send / recv flags. + int flags; + }; + + /** + * @brief build a receive-message operation. + * @param fd socket file descriptor. + * @param msg message header. + * @param flags recv flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeRecv (int fd, void* buf, uint32_t len, int flags, CompletionHandler* handler) noexcept; + + /** + * @brief build a send-message operation. + * @param fd socket file descriptor. + * @param msg message header. + * @param flags send flags. + * @param handler handler to notify on completion. + * @return initialized IoOperation. + */ + static IoOperation makeSend (int fd, const void* buf, uint32_t len, int flags, + CompletionHandler* handler) noexcept; + + union Data + { + AcceptData accept; + ConnectData connect; + RwData rw; + MsgData msg; + StreamData stream; + }; + + /** + * @brief return the file descriptor associated with this operation. + * @return file descriptor, or -1 if opcode is unknown. + */ + int fd () const noexcept; + + /// operation code. + uint8_t code = 0; + + /// operation state. + State state = State::Idle; + +#ifdef JOIN_HAS_IO_URING + /// index of this operation in the proactor pending ops vector. + uint32_t slot = 0; + + /// link this SQE to the next one (next executes only if this succeeds). + bool linked = false; +#endif + + /// handler to dispatch to on completion. + CompletionHandler* handler = nullptr; + + /// operation code specific payload. + Data data = {}; + }; +} + +#endif diff --git a/core/include/join/io_policy.hpp b/core/include/join/io_policy.hpp new file mode 100644 index 00000000..77573556 --- /dev/null +++ b/core/include/join/io_policy.hpp @@ -0,0 +1,116 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef JOIN_CORE_IO_POLICY_HPP +#define JOIN_CORE_IO_POLICY_HPP + +// C. +#include +#include + +namespace join +{ + struct IoDefaultPolicy + { + static constexpr uint32_t sqEntries = 1024; + static constexpr uint32_t flags = 0; + }; + + struct IoHybridPolicy + { + static constexpr uint32_t sqEntries = 1024; + static constexpr uint32_t flags = 0; + static constexpr uint32_t spin = 200; + }; + + struct IoSqpollPolicy + { + static constexpr uint32_t sqEntries = 1024; + static constexpr uint32_t flags = IORING_SETUP_SQPOLL; + static constexpr uint32_t spin = 200; + static constexpr uint32_t sqThreadIdle = 2000; + static constexpr uint32_t sqThreadCpu = 0; + }; + + template + using void_t = void; + + template + struct has_spin : std::false_type + { + }; + + template + struct has_spin> : std::true_type + { + }; + + template + struct has_sqpoll : std::false_type + { + }; + + template + struct has_sqpoll> + : std::integral_constant + { + }; + + template + struct is_default : std::integral_constant::value && !has_sqpoll::value> + { + }; + + template + struct has_cq_entries : std::false_type + { + }; + + template + struct has_cq_entries> : std::true_type + { + }; + + template + struct has_sq_thread_idle : std::false_type + { + }; + + template + struct has_sq_thread_idle> : std::true_type + { + }; + + template + struct has_sq_thread_cpu : std::false_type + { + }; + + template + struct has_sq_thread_cpu> : std::true_type + { + }; +} + +#endif diff --git a/core/include/join/proactor.hpp b/core/include/join/proactor.hpp index a9c7c9d7..5c1f4a86 100644 --- a/core/include/join/proactor.hpp +++ b/core/include/join/proactor.hpp @@ -26,606 +26,817 @@ #define JOIN_CORE_PROACTOR_HPP // libjoin. +#include +#ifdef JOIN_HAS_IO_URING +#include +#else +#include +#endif +#include #include #include +// C++. +#include +#include + // C. -#include -#include +#include +#include namespace join { - // forward declaration. - struct IoOperation; + class CompletionHandler; +#ifdef JOIN_HAS_IO_URING + template + class BasicProactor; + template + class BasicProactorThread; + + using Proactor = BasicProactor; + using HybridProactor = BasicProactor; + using SqpollProactor = BasicProactor; + using ProactorThread = BasicProactorThread; + using HybridProactorThread = BasicProactorThread; + using SqpollProactorThread = BasicProactorThread; +#else + class BasicProactor; + class BasicProactorThread; + + using Proactor = BasicProactor; + using ProactorThread = BasicProactorThread; +#endif +} + +/** + * @brief completion handler interface class. + */ +class join::CompletionHandler +{ +public: + /** + * @brief create instance. + */ + CompletionHandler () = default; /** - * @brief completion handler interface class. + * @brief copy constructor. + * @param other other object to copy. */ - class CompletionHandler + CompletionHandler (const CompletionHandler& other) = default; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + * @return current object. + */ + CompletionHandler& operator= (const CompletionHandler& other) = default; + + /** + * @brief move constructor. + * @param other other object to move. + */ + CompletionHandler (CompletionHandler&& other) = default; + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return current object. + */ + CompletionHandler& operator= (CompletionHandler&& other) = default; + + /** + * @brief destroy instance. + */ + ~CompletionHandler () = default; + +protected: + /** + * @brief method called when an operation completes successfully. + * @param op completed operation. + * @param result number of bytes transferred, or operation-specific value. + */ + virtual void onComplete ([[maybe_unused]] IoOperation* op, [[maybe_unused]] int result) { - public: - /** - * @brief create instance. - */ - CompletionHandler () = default; - - /** - * @brief copy constructor. - * @param other other object to copy. - */ - CompletionHandler (const CompletionHandler& other) = default; - - /** - * @brief copy assignment operator. - * @param other other object to copy. - * @return current object. - */ - CompletionHandler& operator= (const CompletionHandler& other) = default; - - /** - * @brief move constructor. - * @param other other object to move. - */ - CompletionHandler (CompletionHandler&& other) = default; - - /** - * @brief move assignment operator. - * @param other other object to move. - * @return current object. - */ - CompletionHandler& operator= (CompletionHandler&& other) = default; - - /** - * @brief destroy instance. - */ - virtual ~CompletionHandler () = default; - - protected: - /** - * @brief method called when an operation completes successfully. - * @param op completed operation. - * @param result number of bytes transferred, or operation-specific value. - */ - virtual void onComplete ([[maybe_unused]] IoOperation* op, [[maybe_unused]] int result) - { - // do nothing. - } + // do nothing. + } - /** - * @brief method called when an operation is cancelled. - * @param op cancelled operation. - * @param result -ECANCELED, or other negative errno on failure. - */ - virtual void onCancel ([[maybe_unused]] IoOperation* op, [[maybe_unused]] int result) - { - // do nothing. - } + /** + * @brief method called when an operation is cancelled. + * @param op cancelled operation. + * @param result -ECANCELED, or other negative errno on failure. + */ + virtual void onCancel ([[maybe_unused]] IoOperation* op, [[maybe_unused]] int result) + { + // do nothing. + } + + /// friendship with proactor. +#ifdef JOIN_HAS_IO_URING + template + friend class join::BasicProactor; +#else + friend class join::BasicProactor; +#endif +}; + +/** + * @brief basic proactor class. + */ +#ifdef JOIN_HAS_IO_URING +template +class join::BasicProactor +#else +class join::BasicProactor : public join::EventHandler +#endif +{ +public: + /** + * @brief construct with the given ring / queue size. + */ + explicit BasicProactor (); + + /** + * @brief copy constructor. + * @param other other object to copy. + */ + BasicProactor (const BasicProactor& other) = delete; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + * @return current object. + */ + BasicProactor& operator= (const BasicProactor& other) = delete; + + /** + * @brief move constructor. + * @param other other object to move. + */ + BasicProactor (BasicProactor&& other) = delete; + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return current object. + */ + BasicProactor& operator= (BasicProactor&& other) = delete; + + /** + * @brief destroy instance. + */ + ~BasicProactor () noexcept; + + /** + * @brief submit an asynchronous operation to the proactor. + * @param op operation to submit. + * @param flush call io_uring_submit after pushing the SQE if true (io_uring only). + * @param sync wait for submission acknowledgment if true. + * @return 0 on success, -1 on failure. + */ + int submit (IoOperation* op, bool flush = false, bool sync = true) noexcept; + + /** + * @brief cancel an in-flight operation. + * @param op operation to cancel. + * @param flush call io_uring_submit after pushing the cancel SQE if true (io_uring only). + * @param sync block until the cancellation has been acknowledged. + * @return 0 on success, -1 on failure (lastError set). + */ + int cancel (IoOperation* op, bool flush = false, bool sync = true) noexcept; + +#ifdef JOIN_HAS_IO_URING + /** + * @brief flush pending submissions to the kernel. + * @param sync block until the flush has been acknowledged if true. + * @return 0 on success, -1 on failure (lastError set). + */ + int flush (bool sync = true) noexcept; +#endif + + /** + * @brief run the event loop (blocking). + */ + void run (); + + /** + * @brief stop the event loop. + * @param sync wait for loop termination if true. + */ + void stop (bool sync = true) noexcept; + +#ifdef JOIN_HAS_IO_URING + /** + * @brief register fixed buffers with the io_uring instance. + * @param iovecs list of buffers to register. + * @return 0 on success, -1 on failure. + */ + int registerBuffers (const std::vector& iovecs); + + /** + * @brief unregister previously registered fixed buffers from the io_uring instance. + * @return 0 on success, -1 on failure. + */ + int unregisterBuffers (); +#endif + +#ifdef JOIN_HAS_NUMA + /** + * @brief bind reactor command queue memory to a NUMA node. + * @param numa NUMA node ID. + * @return 0 on success, -1 on failure. + */ + int mbind (int numa) const noexcept; +#endif + + /** + * @brief lock reactor command queue memory in RAM. + * @return 0 on success, -1 on failure. + */ + int mlock () const noexcept; + + /** + * @brief check if the event loop is running. + * @return true if the event loop is running. + */ + bool isRunning () const noexcept; + + /** + * @brief check if the calling thread is the proactor thread. + * @return true if called from the proactor thread. + */ + bool isProactorThread () const noexcept; + +protected: +#ifdef JOIN_HAS_IO_URING + /** + * @brief init wakeup eventfd descriptor. + * @return eventfd descriptor. + */ + int initWakeup (std::true_type) noexcept; + + /** + * @brief init wakeup eventfd descriptor. + * @return eventfd descriptor. + */ + int initWakeup (std::false_type) noexcept; + + /** + * @brief init wakeup operation. + */ + void initWakeupOp (std::true_type) noexcept; + + /** + * @brief init wakeup eventfd descriptor. + */ + void initWakeupOp (std::false_type) noexcept; + + /** + * @brief init completion queue entries. + * @param params parameters. + */ + void initCqEntries (io_uring_params&, std::false_type) noexcept; + + /** + * @brief init completion queue entries. + * @param params parameters. + */ + void initCqEntries (io_uring_params& params, std::true_type) noexcept; + + /** + * @brief init sqpoll thread idle timeout. + * @param params parameters. + */ + void initSqThreadIdle (io_uring_params&, std::false_type) noexcept; + + /** + * @brief init sqpoll thread idle timeout. + * @param params parameters. + */ + void initSqThreadIdle (io_uring_params& params, std::true_type) noexcept; - /// friendship with proactor. - friend class Proactor; + /** + * @brief init sqpoll thread cpu affinity. + * @param params parameters. + */ + void initSqThreadCpu (io_uring_params&, std::false_type) noexcept; + + /** + * @brief init sqpoll thread cpu affinity. + * @param params parameters. + */ + void initSqThreadCpu (io_uring_params& params, std::true_type) noexcept; +#endif + + /** + * @brief command type for proactor dispatcher. + */ + enum class CommandType + { + Submit, /**< submit an operation. */ + Cancel, /**< cancel an in-flight operation. */ + Stop, /**< stop the event loop. */ +#ifdef JOIN_HAS_IO_URING + Flush, /**< flush pending submissions to the kernel. */ +#endif }; /** - * @brief Describes a single asynchronous operation submitted to the Proactor. + * @brief command for proactor dispatcher. + */ + struct alignas (64) Command + { + CommandType type; /**< command type. */ + IoOperation* op; /**< target operation, or nullptr for Stop/Flush. */ + bool flush; /**< call io_uring_submit after processing. */ + std::atomic* done; /**< set to true when the command is processed. */ + std::error_code* errc; /**< filled with the error code on failure. */ + }; + + /** + * @brief write command to queue and wake dispatcher. + * @param cmd command to write. + * @return 0 on success, -1 on failure. + */ + int writeCommand (const Command& cmd) noexcept; + +#ifdef JOIN_HAS_IO_URING + /** + * @brief write command to queue and wake dispatcher. + * @param cmd command to write. + * @return 0 on success, -1 on failure. + */ + int writeCommand (const Command& cmd, std::true_type) noexcept; + + /** + * @brief write command to queue and wake dispatcher. + * @param cmd command to write. + * @return 0 on success, -1 on failure. + */ + int writeCommand (const Command& cmd, std::false_type) noexcept; +#endif + + /** + * @brief read and process all pending commands from queue. + */ + void readCommands () noexcept; + + /** + * @brief process a single command. + * @param cmd command to process. + */ + void processCommand (const Command& cmd) noexcept; + + /** + * @brief submit op directly onto the ring. + * @param op operation to submit. + * @return 0 on success, -1 on failure. + */ + int submitOperation (IoOperation* op, bool flush = true) noexcept; + + /** + * @brief cancel op directly on the ring. + * @param op operation to cancel. + * @return 0 on success, -1 on failure. + */ + int cancelOperation (IoOperation* op, bool flush = true) noexcept; + + /** + * @brief dispatch completion callback and reset operation state. + * @param op operation to dispatch, may be nullptr. + * @param result negative errno or bytes-transferred result. + * @param cancelled if true, dispatch to onCancel; otherwise to onComplete. */ - struct alignas (64) IoOperation + void dispatchOperation (IoOperation* op, int result, bool cancelled) noexcept; + + /** + * @brief end an operation, dispatching onCancel or onComplete. + * @param op operation to end, may be nullptr. + * @param result negative errno or bytes-transferred result. + * @param cancelled if true, dispatch to onCancel; otherwise to onComplete. + */ + void endOperation (IoOperation* op, int result, bool cancelled = false) noexcept; + +#ifdef JOIN_HAS_IO_URING + /** + * @brief get a free submission queue entry, submitting pending entries if the ring is full. + * @return pointer to sqe, or nullptr if unavailable. + */ + io_uring_sqe* getSqe () noexcept; + + /** + * @brief prepare a submission queue entry for the given operation and attach it as user data. + * @param sqe submission queue entry. + * @param op operation to prepare. + */ + void prepareSqe (io_uring_sqe* sqe, IoOperation* op) noexcept; + + /** + * @brief re-arm the wakeup eventfd read on the ring. + */ + void rearmWakeup () noexcept; + + /** + * @brief dispatch a completion queue entry to the appropriate handler. + * @param cqe completion queue entry. + */ + void dispatchCqe (io_uring_cqe* cqe) noexcept; + + /** + * @brief dispatch a completion queue entry to the appropriate handler. + * @param cqe completion queue entry. + */ + void dispatchCqe (io_uring_cqe* cqe, std::true_type) noexcept; + + /** + * @brief dispatch a completion queue entry to the appropriate handler. + * @param cqe completion queue entry. + */ + void dispatchCqe (io_uring_cqe* cqe, std::false_type) noexcept; + + /** + * @brief run the backend event loop until stop() is called. + */ + void eventLoop () noexcept; + + /** + * @brief run the epoll backend event loop until stop() is called. + */ + void eventLoop (std::false_type, std::false_type) noexcept; + + /** + * @brief run the hybrid backend event loop until stop() is called. + */ + void eventLoop (std::true_type, std::false_type) noexcept; + + /** + * @brief run the sqpoll backend event loop until stop() is called. + */ + void eventLoop (std::true_type, std::true_type) noexcept; +#else + /** + * @brief return true if opcode requires EPOLLOUT. + * @param code raw opcode value. + * @return true for Write, WriteFixed, SendMsg. + */ + static bool isWriteOp (uint8_t code) noexcept; + + /** + * @brief execute the syscall described by op. + * @param op operation to execute. + * @return bytes transferred (>= 0) or -errno (< 0). + */ + static int executeOp (IoOperation* op) noexcept; + + /** + * @brief method called when data are ready to be read on handle. + * @param fd file descriptor. + */ + void onReadable (int fd) override; + + /** + * @brief method called when data are ready to be written on handle. + * @param fd file descriptor. + */ + void onWriteable (int fd) override; + + /** + * @brief method called when handle was closed by the peer. + * @param fd file descriptor. + */ + void onClose (int fd) override; + + /** + * @brief method called when an error occurred on handle. + * @param fd file descriptor. + */ + void onError (int fd) override; +#endif + + /// command queue size. + static constexpr size_t _queueSize = 1024; + + /// command queue. + LocalMem::Mpsc::Queue _commands; + + /// stop waiter: non-null while a sync stop() is in progress. + std::atomic*> _stopDone{nullptr}; + + /// eventfd descriptor. + int _wakeup = -1; + +#ifdef JOIN_HAS_IO_URING + /// buffer for the wakeup eventfd read. + uint64_t _wakeupBuf = 0; + + /// internal operation used to watch the wakeup eventfd. + IoOperation _wakeupOp = {}; + + /// io_uring instance. + io_uring _ring = {}; + + /// in-flight operations. + std::vector _pendingOps; + + /// invalid thread id sentinel. + static constexpr pthread_t _invalidThreadId = static_cast (-1); + + /// proactor thread id. + std::atomic _threadId{_invalidThreadId}; + + /// running flag. + std::atomic _running{false}; +#else + /// pending read operations. + std::vector _readOps; + + /// pending write operations. + std::vector _writeOps; + + /// reactor instance. + Reactor _reactor; +#endif +}; + +#ifdef JOIN_HAS_IO_URING +template +int join::BasicProactor::submit (IoOperation* op, bool flush, bool sync) noexcept +#else +inline int join::BasicProactor::submit (IoOperation* op, bool flush, bool sync) noexcept +#endif +{ + if (isProactorThread ()) { - /** - * @brief operation lifecycle state. - */ - enum class State : uint8_t + return submitOperation (op, flush); + } + + std::atomic done{false}, *pdone = nullptr; + std::error_code errc, *perrc = nullptr; + + if (JOIN_LIKELY (sync)) + { + pdone = &done; + perrc = &errc; + } + + if (JOIN_UNLIKELY (writeCommand ({CommandType::Submit, op, flush, pdone, perrc}) == -1)) + { + return -1; // LCOV_EXCL_LINE + } + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) { - Idle, - Submitted, - Cancelling, - }; - - /** - * @brief payload for IORING_OP_TIMEOUT. - */ - struct TimeoutData + backoff (); + } + + if (JOIN_UNLIKELY (errc)) { - /// absolute or relative kernel timespec. - __kernel_timespec ts; + lastError = errc; + return -1; + } + } - /// 0 or IORING_TIMEOUT_ABS. - uint32_t flags; - }; + return 0; +} - /** - * @brief payload for IORING_OP_READ / IORING_OP_READ_FIXED / IORING_OP_WRITE / IORING_OP_WRITE_FIXED. - */ - struct RwData - { - /// file descriptor. - int fd; +#ifdef JOIN_HAS_IO_URING +template +int join::BasicProactor::cancel (IoOperation* op, bool flush, bool sync) noexcept +#else +inline int join::BasicProactor::cancel (IoOperation* op, bool flush, bool sync) noexcept +#endif +{ + if (isProactorThread ()) + { + return cancelOperation (op, flush); + } - /// buffer. - void* buf; + std::atomic done{false}, *pdone = nullptr; + std::error_code errc, *perrc = nullptr; - /// number of bytes to proceed. - uint32_t len; + if (JOIN_LIKELY (sync)) + { + pdone = &done; + perrc = &errc; + } - /// registered buffer index (FIXED only, ignored otherwise). - uint16_t index; + if (JOIN_UNLIKELY (writeCommand ({CommandType::Cancel, op, flush, pdone, perrc}) == -1)) + { + return -1; // LCOV_EXCL_LINE + } - /// use FIXED if true. - bool fixed; - }; + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } - /** - * @brief payload for IORING_OP_SENDMSG / IORING_OP_RECVMSG. - */ - struct MsgData + if (JOIN_UNLIKELY (errc)) { - /// file descriptor. - int fd; + lastError = errc; + return -1; + } + } - /// message header. - msghdr* msg; + return 0; +} - /// send/recv flags. - uint32_t flags; - }; +#ifdef JOIN_HAS_IO_URING +template +void join::BasicProactor::dispatchOperation (IoOperation* op, int result, bool cancelled) noexcept +#else +inline void join::BasicProactor::dispatchOperation (IoOperation* op, int result, bool cancelled) noexcept +#endif +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } - /** - * @brief payload for IORING_OP_ACCEPT. - */ - struct AcceptData + if (op->handler) + { + if (cancelled) { - /// listening file descriptor. - int fd; + op->handler->onCancel (op, result); + } + else + { + op->handler->onComplete (op, result); + } + } - /// peer address output buffer. - sockaddr* addr; + op->state = IoOperation::State::Idle; +} - /// peer address length. - socklen_t* addrlen; +#ifdef JOIN_HAS_IO_URING +#include "proactor_uring.inl" +#else +#include "proactor_epoll.inl" +#endif - /// accept flags. - uint32_t flags; - }; +/** + * @brief Convenience class that owns a Proactor running on a dedicated background thread. + */ +#ifdef JOIN_HAS_IO_URING +template +class join::BasicProactorThread +#else +class join::BasicProactorThread +#endif +{ +public: + /** + * @brief get the global Proactor instance. + * @return reference to the singleton Proactor. + */ +#ifdef JOIN_HAS_IO_URING + static BasicProactor& proactor () +#else + static BasicProactor& proactor () +#endif + { + return instance ()._proactor; + } - /** - * @brief payload for IORING_OP_ASYNC_CANCEL. - */ - // struct CancelData - // { - // /// operation to cancel, identified by its address as user_data. - // IoOperation* target; - // }; + /** + * @brief set proactor thread affinity. + * @param core CPU core (-1 to disable pinning). + * @return 0 on success, -1 on failure. + */ + static int affinity (int core) + { + return instance ()._dispatcher.affinity (core); + } - union Data - { - TimeoutData timeout; - RwData rw; - MsgData msg; - AcceptData accept; - // CancelData cancel; - }; - - /** - * @brief build a relative or absolute timeout operation. - * @param ts kernel timespec (relative duration or absolute deadline). - * @param flags 0 for relative, IORING_TIMEOUT_ABS for absolute. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeTimeout (const __kernel_timespec& ts, uint32_t flags, - CompletionHandler* handler) noexcept; - - /** - * @brief build a regular (non-registered) read operation. - * @param fd file descriptor to read from. - * @param buf destination buffer. - * @param len number of bytes to read. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept; - - /** - * @brief build a fixed-buffer read operation (requires registered buffers). - * @param fd file descriptor to read from. - * @param buf destination buffer (must belong to a registered region). - * @param len number of bytes to read. - * @param index index of the registered buffer. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept; - - /** - * @brief build a regular (non-registered) write operation. - * @param fd file descriptor to write to. - * @param buf source buffer. - * @param len number of bytes to write. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept; - - /** - * @brief build a fixed-buffer write operation (requires registered buffers). - * @param fd file descriptor to write to. - * @param buf source buffer (must belong to a registered region). - * @param len number of bytes to write. - * @param index index of the registered buffer. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept; - /** - * @brief build a send-message operation. - * @param fd socket file descriptor. - * @param msg message header. - * @param flags send flags. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeSendmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept; - - /** - * @brief build a receive-message operation. - * @param fd socket file descriptor. - * @param msg message header. - * @param flags recv flags. - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeRecvmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept; - - /** - * @brief build an accept operation. - * @param fd listening socket file descriptor. - * @param addr peer address output buffer (may be nullptr). - * @param addrlen peer address length output (may be nullptr). - * @param flags accept flags (e.g. SOCK_NONBLOCK | SOCK_CLOEXEC). - * @param handler handler to notify on completion. - * @return initialized IoOperation. - */ - static IoOperation makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, uint32_t flags, - CompletionHandler* handler) noexcept; - - /** - * @brief build a cancellation operation targeting another in-flight operation. - * @param target operation to cancel. - * @param handler handler to notify when the cancellation itself completes. - * @return initialized IoOperation. - */ - // static IoOperation makeCancel (IoOperation* target, CompletionHandler* handler) noexcept; - - /// io_uring opcode (IORING_OP_TIMEOUT, IORING_OP_READ, ...). - uint8_t code = 0; - - /// state. - State state = State::Idle; - - /// handler to dispatch to on completion or cancellation. - CompletionHandler* handler = nullptr; - - /// opcode-specific payload. - Data data = {}; - }; + /** + * @brief get proactor thread affinity. + * @return core index, or -1 if not pinned. + */ + static int affinity () + { + return instance ()._dispatcher.affinity (); + } /** - * @brief Proactor class. + * @brief set proactor thread scheduling priority. + * @param prio 0 = SCHED_OTHER, 1-99 = SCHED_FIFO. + * @return 0 on success, -1 on failure. */ - class Proactor + static int priority (int prio) { - public: - /** - * @brief create instance. - */ - Proactor (); - - /** - * @brief copy constructor. - * @param other other object to copy. - */ - Proactor (const Proactor& other) = delete; - - /** - * @brief copy assignment operator. - * @param other other object to copy. - * @return current object. - */ - Proactor& operator= (const Proactor& other) = delete; - - /** - * @brief move constructor. - * @param other other object to move. - */ - Proactor (Proactor&& other) = delete; - - /** - * @brief move assignment operator. - * @param other other object to move. - * @return current object. - */ - Proactor& operator= (Proactor&& other) = delete; - - /** - * @brief destroy instance. - */ - ~Proactor () noexcept; - - /** - * @brief submit an asynchronous operation to the proactor. - * @param op operation to submit. - * @param sync wait for submission acknowledgment if true. - * @return 0 on success, -1 on failure. - */ - int submit (IoOperation* op, bool sync = true) noexcept; - - /** - * @brief cancel an in-flight asynchronous operation. - * @param op operation to cancel. - * @param sync wait for cancellation acknowledgment if true. - * @return 0 on success, -1 on failure. - */ - int cancel (IoOperation* op, bool sync = true) noexcept; - - /** - * @brief run the event loop (blocking). - */ - void run (); - - /** - * @brief stop the event loop. - * @param sync wait for loop termination if true. - */ - void stop (bool sync = true) noexcept; + return instance ()._dispatcher.priority (prio); + } + + /** + * @brief get proactor thread scheduling priority. + * @return current priority. + */ + static int priority () + { + return instance ()._dispatcher.priority (); + } + + /** + * @brief get the native handle of the proactor thread. + * @return pthread_t handle. + */ + static pthread_t handle () + { + return instance ()._dispatcher.handle (); + } #ifdef JOIN_HAS_NUMA - /** - * @brief bind command queue memory to a NUMA node. - * @param numa NUMA node ID. - * @return 0 on success, -1 on failure. - */ - int mbind (int numa) const noexcept; + /** + * @brief bind reactor command queue memory to a NUMA node. + * @param numa NUMA node ID. + * @return 0 on success, -1 on failure. + */ + static int mbind (int numa) + { + return instance ()._proactor.mbind (numa); + } #endif - /** - * @brief lock command queue memory in RAM. - * @return 0 on success, -1 on failure. - */ - int mlock () const noexcept; - - /** - * @brief check if the calling thread is the proactor thread. - * @return true if called from the proactor thread. - */ - bool isProactorThread () const noexcept; - - private: - /// queue size. - static constexpr size_t _queueSize = 1024; - - /// io_uring submission/completion ring size. - static constexpr size_t _ringSize = 1024; - - /** - * @brief Command type for proactor dispatcher. - */ - enum class CommandType - { - Submit, - Cancel, - Stop - }; - - /** - * @brief Command for proactor dispatcher. - */ - struct alignas (64) Command - { - CommandType type; - IoOperation* op; - std::atomic* done; - std::atomic* errc; - }; - - /** - * @brief get a free SQE from the ring, flushing if necessary. - * @return pointer to an SQE, or nullptr on failure. - */ - io_uring_sqe* getSqe () noexcept; - - /** - * @brief submit op directly onto the ring. - * @param op operation to submit. - * @return 0 on success, -1 on failure. - */ - int submitOperation (IoOperation* op) noexcept; - - /** - * @brief cancel op directly on the ring. - * @param op operation to cancel. - * @return 0 on success, -1 on failure. - */ - int cancelOperation (IoOperation* op) noexcept; - - /** - * @brief (re)submit the internal wakeup read on _wakeup eventfd. - */ - void rearmWakeup () noexcept; - - /** - * @brief write command to queue and wake dispatcher. - * @param cmd command to write. - * @return 0 on success, -1 on failure. - */ - int writeCommand (const Command& cmd) noexcept; - - /** - * @brief process a single command. - * @param cmd command to process. - */ - void processCommand (const Command& cmd) noexcept; - - /** - * @brief read and process all pending commands from queue. - */ - void readCommands () noexcept; - - /** - * @brief dispatch a single CQE to its completion handler. - * @param cqe completed queue entry. - */ - void dispatchCqe (io_uring_cqe* cqe) noexcept; - - /** - * @brief fill an SQE from an operation. - * @param op source operation. - * @param sqe submission queue entry to populate. - */ - void prepareSqe (const IoOperation& op, io_uring_sqe* sqe) const noexcept; - - /** - * @brief dispatch a completed operation to its handler. - * @param op operation whose CQE has arrived. - * @param result cqe->res. - */ - void dispatchOperation (IoOperation* op, int result) noexcept; - - /** - * @brief main event loop running in dispatcher thread. - */ - void eventLoop (); - - /// eventfd descriptor. - int _wakeup = -1; - - /// read buffer for the wakeup eventfd. - uint64_t _wakeupBuf = 0; - - /// internal IoOperation that keeps a persistent READ on _wakeup in the ring. - IoOperation _wakeupOp; - - /// io_uring instance. - io_uring _ring = {}; - - /// command queue. - LocalMem::Mpsc::Queue _commands; - - /// running flag for dispatcher thread. - std::atomic _running{false}; - - /// event loop thread ID. - std::atomic _threadId{0}; - }; + /** + * @brief lock reactor command queue memory in RAM. + * @return 0 on success, -1 on failure. + */ + static int mlock () + { + return instance ()._proactor.mlock (); + } +private: /** - * @brief Convenience class that owns a Proactor running on a dedicated background thread. + * @brief get the singleton ProactorThread instance. + * @return reference to the singleton ProactorThread. */ - class ProactorThread + static BasicProactorThread& instance () { - public: - /** - * @brief get the global Proactor instance. - * @return pointer to the singleton Proactor. - */ - static Proactor* proactor (); - - /** - * @brief set proactor thread affinity. - * @param core thread core affinity (-1 to disable pinning). - * @return 0 on success, -1 on failure. - */ - static int affinity (int core); - - /** - * @brief get proactor thread affinity. - * @return affinity or -1 if not pinned. - */ - static int affinity (); - - /** - * @brief set proactor thread priority. - * @param prio thread priority (0 = SCHED_OTHER, 1-99 = SCHED_FIFO). - * @return 0 on success, -1 on failure. - */ - static int priority (int prio); - - /** - * @brief get proactor thread priority. - * @return priority. - */ - static int priority (); - - /** - * @brief get the handle of the proactor thread. - * @return proactor thread handle. - */ - static pthread_t handle (); + static BasicProactorThread proactorThread; + return proactorThread; + } -#ifdef JOIN_HAS_NUMA - /** - * @brief bind command queue memory to a NUMA node. - * @param numa NUMA node ID. - * @return 0 on success, -1 on failure. - */ - static int mbind (int numa); + /** + * @brief construct the ProactorThread and start the event loop thread. + */ + BasicProactorThread () + { + _dispatcher = Thread ([this] () { + _proactor.run (); + }); + } + + /** + * @brief copy constructor. + * @param other other object to copy. + */ + BasicProactorThread (const BasicProactorThread&) = delete; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + * @return current object. + */ + BasicProactorThread& operator= (const BasicProactorThread&) = delete; + + /** + * @brief move constructor. + * @param other other object to move. + */ + BasicProactorThread (BasicProactorThread&&) = delete; + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return current object. + */ + BasicProactorThread& operator= (BasicProactorThread&&) = delete; + + /** + * @brief destroy the ProactorThread and cleanly shut down the event loop. + */ + ~BasicProactorThread () + { + _proactor.stop (); + _dispatcher.join (); + } + +#ifdef JOIN_HAS_IO_URING + /// owned Proactor instance. + BasicProactor _proactor; +#else + /// owned Proactor instance. + BasicProactor _proactor; #endif - /** - * @brief lock command queue memory in RAM. - * @return 0 on success, -1 on failure. - */ - static int mlock (); - - private: - /** - * @brief get the singleton ProactorThread instance. - * @return reference to the singleton ProactorThread. - */ - static ProactorThread& instance (); - - /** - * @brief construct the ProactorThread and start the event loop thread. - */ - ProactorThread (); - - /** - * @brief copy constructor. - * @param other other object to copy. - */ - ProactorThread (const ProactorThread& other) = delete; - - /** - * @brief copy assignment operator. - * @param other other object to copy. - * @return current object. - */ - ProactorThread& operator= (const ProactorThread& other) = delete; - - /** - * @brief move constructor. - * @param other other object to move. - */ - ProactorThread (ProactorThread&& other) noexcept = delete; - - /** - * @brief move assignment operator. - * @param other other object to move. - * @return current object. - */ - ProactorThread& operator= (ProactorThread&& other) noexcept = delete; - - /** - * @brief destroy the ProactorThread and cleanly shut down the event loop. - */ - ~ProactorThread (); - - /// Proactor instance. - Proactor _proactor; - - /// Background thread. - Thread _dispatcher; - }; -} + /// background dispatcher thread. + Thread _dispatcher; +}; #endif diff --git a/core/include/join/proactor_epoll.inl b/core/include/join/proactor_epoll.inl new file mode 100644 index 00000000..66725d90 --- /dev/null +++ b/core/include/join/proactor_epoll.inl @@ -0,0 +1,526 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +namespace join +{ + inline BasicProactor::BasicProactor () + : _commands (_queueSize) + , _wakeup (eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC)) + , _readOps (256, nullptr) + , _writeOps (256, nullptr) + { + if (_wakeup == -1) + { + throw std::system_error (errno, std::system_category (), "eventfd failed"); // LCOV_EXCL_LINE + } + + _reactor.addHandler (_wakeup, this, true, false, false); + } + + inline BasicProactor::~BasicProactor () noexcept + { + stop (); + + ::close (_wakeup); + } + + inline void BasicProactor::run () + { + _reactor.run (); + } + + inline void BasicProactor::stop (bool sync) noexcept + { + if (isProactorThread ()) + { + for (size_t fd = 0; fd < _readOps.size (); ++fd) + { + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (rOp || wOp) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, -ECANCELED, true); + dispatchOperation (wOp, -ECANCELED, true); + } + _reactor.stop (false); + return; + } + + if (!_reactor.isRunning ()) + { + return; + } + + std::atomic done{false}; + + if (JOIN_LIKELY (sync)) + { + std::atomic* expected = nullptr; + if (!_stopDone.compare_exchange_strong (expected, &done, std::memory_order_acq_rel)) + { + Backoff backoff; + while (isRunning ()) + { + backoff (); + } + return; + } + } + + writeCommand ({CommandType::Stop, nullptr, sync, sync ? &done : nullptr, nullptr}); + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + _stopDone.store (nullptr, std::memory_order_release); + } + + _reactor.stop (sync); + } + +#ifdef JOIN_HAS_NUMA + inline int BasicProactor::mbind (int numa) const noexcept + { + if (_commands.mbind (numa) == -1) + { + return -1; + } + + return _reactor.mbind (numa); + } +#endif + + inline int BasicProactor::mlock () const noexcept + { + if (_commands.mlock () == -1) + { + return -1; + } + + return _reactor.mlock (); + } + + inline bool BasicProactor::isRunning () const noexcept + { + return _reactor.isRunning (); + } + + inline bool BasicProactor::isProactorThread () const noexcept + { + return _reactor.isReactorThread (); + } + + inline int join::BasicProactor::writeCommand (const Command& cmd) noexcept + { + if (JOIN_UNLIKELY (_commands.push (cmd) == -1)) + { + return -1; // LCOV_EXCL_LINE + } + + uint64_t value = 1; + if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1)) + { + // LCOV_EXCL_START + lastError = std::error_code (errno, std::system_category ()); + return -1; + // LCOV_EXCL_STOP + } + + return 0; + } + + inline void BasicProactor::readCommands () noexcept + { + uint64_t count; + if (JOIN_UNLIKELY (::read (_wakeup, &count, sizeof (count)) == -1)) + { + return; // LCOV_EXCL_LINE + } + + Command cmd; + while (_commands.tryPop (cmd) == 0) + { + processCommand (cmd); + } + } + + inline void BasicProactor::processCommand (const Command& cmd) noexcept + { + int err = 0; + + switch (cmd.type) + { + case CommandType::Submit: + err = submitOperation (cmd.op); + break; + + case CommandType::Cancel: + err = cancelOperation (cmd.op); + break; + + case CommandType::Stop: + { + for (size_t fd = 0; fd < _readOps.size (); ++fd) + { + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (rOp || wOp) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, -ECANCELED, true); + dispatchOperation (wOp, -ECANCELED, true); + } + break; + } + + default: + break; + } + + if (JOIN_UNLIKELY (cmd.done)) + { + if (cmd.errc && (err != 0)) + { + *cmd.errc = lastError; + } + cmd.done->store (true, std::memory_order_release); + } + } + + inline int BasicProactor::submitOperation (IoOperation* op, [[maybe_unused]] bool flush) noexcept + { + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Idle)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + if (static_cast (op->code) == IoOperation::Opcode::Connect) + { + if (::connect (op->data.connect.fd, op->data.connect.addr, op->data.connect.addrlen) == -1 && + errno != EINPROGRESS) + { + lastError = std::error_code (errno, std::system_category ()); + return -1; + } + } + + if (JOIN_UNLIKELY (static_cast (op->fd ()) >= _readOps.size ())) + { + size_t newSize = static_cast (op->fd ()) + 1; + _readOps.resize (newSize, nullptr); + _writeOps.resize (newSize, nullptr); + } + + bool isWrite = isWriteOp (op->code); + + if (JOIN_UNLIKELY ((isWrite && (_writeOps[op->fd ()] != nullptr)) || + (!isWrite && (_readOps[op->fd ()] != nullptr)))) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (isWrite) + { + _writeOps[op->fd ()] = op; + } + else + { + _readOps[op->fd ()] = op; + } + + op->state = IoOperation::State::Submitted; + + return _reactor.addHandler (op->fd (), this, _readOps[op->fd ()] != nullptr, _writeOps[op->fd ()] != nullptr); + } + + inline int BasicProactor::cancelOperation (IoOperation* op, [[maybe_unused]] bool flush) noexcept + { + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Submitted)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + if (JOIN_UNLIKELY (static_cast (op->fd ()) >= _readOps.size ())) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + op->state = IoOperation::State::Cancelling; + bool isWrite = isWriteOp (op->code); + + if (JOIN_UNLIKELY ((isWrite && (_writeOps[op->fd ()] != op)) || (!isWrite && (_readOps[op->fd ()] != op)))) + { + op->state = IoOperation::State::Submitted; + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (isWrite) + { + _writeOps[op->fd ()] = nullptr; + } + else + { + _readOps[op->fd ()] = nullptr; + } + + int ret = 0; + + if (_readOps[op->fd ()] == nullptr && _writeOps[op->fd ()] == nullptr) + { + ret = _reactor.delHandler (op->fd ()); + } + else + { + ret = + _reactor.addHandler (op->fd (), this, _readOps[op->fd ()] != nullptr, _writeOps[op->fd ()] != nullptr); + } + + if (op->handler) + { + op->handler->onCancel (op, -ECANCELED); + } + + op->state = IoOperation::State::Idle; + + return ret; + } + + inline void BasicProactor::endOperation (IoOperation* op, int result, bool cancelled) noexcept + { + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } + + int fd = op->fd (); + + if (JOIN_UNLIKELY (fd < 0 || static_cast (fd) >= _readOps.size ())) + { + return; // LCOV_EXCL_LINE + } + + if (isWriteOp (op->code)) + { + _writeOps[fd] = nullptr; + } + else + { + _readOps[fd] = nullptr; + } + + if (_readOps[fd] == nullptr && _writeOps[fd] == nullptr) + { + _reactor.delHandler (fd); + } + else + { + _reactor.addHandler (fd, this, _readOps[fd] != nullptr, _writeOps[fd] != nullptr); + } + + dispatchOperation (op, result, cancelled); + } + + inline bool BasicProactor::isWriteOp (uint8_t code) noexcept + { + return code == static_cast (IoOperation::Opcode::Connect) || + code == static_cast (IoOperation::Opcode::Write) || + code == static_cast (IoOperation::Opcode::WriteFixed) || + code == static_cast (IoOperation::Opcode::SendMsg) || + code == static_cast (IoOperation::Opcode::Send); + } + + inline int BasicProactor::executeOp (IoOperation* op) noexcept + { + for (;;) + { + switch (static_cast (op->code)) + { + case IoOperation::Opcode::Accept: + { + int fd = ::accept4 (op->data.accept.fd, op->data.accept.addr, op->data.accept.addrlen, + op->data.accept.flags); + if ((fd == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (fd == -1) ? -errno : fd; + } + + case IoOperation::Opcode::Connect: + { + int err = 0; + socklen_t len = sizeof (err); + if (::getsockopt (op->data.connect.fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1) + { + return -errno; + } + return err ? -err : 0; + } + + case IoOperation::Opcode::Read: + case IoOperation::Opcode::ReadFixed: + { + ssize_t n = ::read (op->data.rw.fd, op->data.rw.buf, op->data.rw.len); + if ((n == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::Write: + case IoOperation::Opcode::WriteFixed: + { + ssize_t n = ::write (op->data.rw.fd, op->data.rw.buf, op->data.rw.len); + if ((n == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::RecvMsg: + { + ssize_t n = ::recvmsg (op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + if ((n == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::SendMsg: + { + ssize_t n = ::sendmsg (op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + if ((n == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::Recv: + { + ssize_t n = ::recv (op->data.stream.fd, op->data.stream.buf, op->data.stream.len, + op->data.stream.flags); + if ((n == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::Send: + { + ssize_t n = ::send (op->data.stream.fd, op->data.stream.buf, op->data.stream.len, + op->data.stream.flags); + if ((n == -1) && (errno == EINTR)) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + default: + return -EINVAL; + } + } + } + + inline void BasicProactor::onReadable (int fd) + { + if (JOIN_UNLIKELY (fd == _wakeup)) + { + readCommands (); + return; + } + + endOperation (_readOps[fd], executeOp (_readOps[fd]), false); + } + + inline void BasicProactor::onWriteable (int fd) + { + endOperation (_writeOps[fd], executeOp (_writeOps[fd]), false); + } + + inline void BasicProactor::onClose (int fd) + { + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (rOp || wOp) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, 0, false); + dispatchOperation (wOp, 0, false); + } + + inline void BasicProactor::onError (int fd) + { + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (rOp || wOp) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, -ECONNRESET, false); + dispatchOperation (wOp, -ECONNRESET, false); + } +} diff --git a/core/include/join/proactor_uring.inl b/core/include/join/proactor_uring.inl new file mode 100644 index 00000000..0d25804c --- /dev/null +++ b/core/include/join/proactor_uring.inl @@ -0,0 +1,748 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +namespace join +{ + template + BasicProactor::BasicProactor () + : _commands (_queueSize) + , _wakeup (initWakeup (is_default{})) + { + static_assert (has_spin::value || !has_sqpoll::value, "spin resuired for sq poll policy"); + + io_uring_params params{}; + params.flags = Policy::flags; + initCqEntries (params, has_cq_entries{}); + initSqThreadIdle (params, has_sq_thread_idle{}); + initSqThreadCpu (params, has_sq_thread_cpu{}); + + if (io_uring_queue_init_params (Policy::sqEntries, &_ring, ¶ms) < 0) + { + // LCOV_EXCL_START + ::close (_wakeup); + throw std::system_error (errno, std::system_category (), "io_uring_queue_init failed"); + // LCOV_EXCL_STOP + } + + initWakeupOp (is_default{}); + _pendingOps.reserve (Policy::sqEntries); + } + + template + BasicProactor::~BasicProactor () noexcept + { + stop (); + + io_uring_queue_exit (&_ring); + + if (_wakeup != -1) + { + ::close (_wakeup); + } + } + + template + int BasicProactor::flush (bool sync) noexcept + { + if (isProactorThread ()) + { + if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) + { + lastError = std::error_code (errno, std::system_category ()); + return -1; + } + return 0; + } + + std::atomic done{false}, *pdone = nullptr; + std::error_code errc, *perrc = nullptr; + + if (JOIN_LIKELY (sync)) + { + pdone = &done; + perrc = &errc; + } + + if (JOIN_UNLIKELY (writeCommand ({CommandType::Flush, nullptr, false, pdone, perrc}) == -1)) + { + return -1; // LCOV_EXCL_LINE + } + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + + if (JOIN_UNLIKELY (errc)) + { + lastError = errc; + return -1; + } + } + + return 0; + } + + template + void BasicProactor::run () + { + _threadId.store (pthread_self (), std::memory_order_release); + + _running.store (true, std::memory_order_release); + eventLoop (); + + _threadId.store (_invalidThreadId, std::memory_order_release); + } + + template + void BasicProactor::stop (bool sync) noexcept + { + if (isProactorThread ()) + { + _running.store (false, std::memory_order_release); + for (IoOperation* op : _pendingOps) + { + cancelOperation (op, false); + } + eventLoop (); + return; + } + + if (!isRunning ()) + { + return; + } + + std::atomic done{false}; + + if (JOIN_LIKELY (sync)) + { + std::atomic* expected = nullptr; + if (!_stopDone.compare_exchange_strong (expected, &done, std::memory_order_acq_rel)) + { + Backoff backoff; + while (isRunning ()) + { + backoff (); + } + return; + } + } + + writeCommand ({CommandType::Stop, nullptr, sync, sync ? &done : nullptr, nullptr}); + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + _stopDone.store (nullptr, std::memory_order_release); + } + } + + template + int BasicProactor::registerBuffers (const std::vector& iovecs) + { + int ret = io_uring_register_buffers (&_ring, iovecs.data (), iovecs.size ()); + if (ret < 0) + { + lastError = std::error_code (-ret, std::system_category ()); + return -1; + } + + return 0; + } + + template + int BasicProactor::unregisterBuffers () + { + int ret = io_uring_unregister_buffers (&_ring); + if (ret < 0) + { + lastError = std::error_code (-ret, std::system_category ()); + return -1; + } + + return 0; + } + +#ifdef JOIN_HAS_NUMA + template + int BasicProactor::mbind (int numa) const noexcept + { + return _commands.mbind (numa); + } +#endif + + template + int BasicProactor::mlock () const noexcept + { + return _commands.mlock (); + } + + template + bool BasicProactor::isRunning () const noexcept + { + return _running.load (std::memory_order_acquire); + } + + template + bool BasicProactor::isProactorThread () const noexcept + { + return _threadId.load (std::memory_order_acquire) == pthread_self (); + } + + template + int BasicProactor::initWakeup (std::true_type) noexcept + { + return eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC); + } + + template + int BasicProactor::initWakeup (std::false_type) noexcept + { + return -1; + } + + template + void BasicProactor::initWakeupOp (std::true_type) noexcept + { + _wakeupOp = IoOperation::makeRead (_wakeup, &_wakeupBuf, sizeof (_wakeupBuf), nullptr); + } + + template + void BasicProactor::initWakeupOp (std::false_type) noexcept + { + // no-op. + } + + template + void BasicProactor::initCqEntries (io_uring_params&, std::false_type) noexcept + { + // no-op. + } + + template + void BasicProactor::initCqEntries (io_uring_params& params, std::true_type) noexcept + { + params.flags |= IORING_SETUP_CQSIZE; + params.cq_entries = Policy::cqEntries; + } + + template + void BasicProactor::initSqThreadIdle (io_uring_params&, std::false_type) noexcept + { + // no-op. + } + + template + void BasicProactor::initSqThreadIdle (io_uring_params& params, std::true_type) noexcept + { + params.sq_thread_idle = Policy::sqThreadIdle; + } + + template + void BasicProactor::initSqThreadCpu (io_uring_params&, std::false_type) noexcept + { + // no-op. + } + + template + void BasicProactor::initSqThreadCpu (io_uring_params& params, std::true_type) noexcept + { + params.sq_thread_cpu = Policy::sqThreadCpu; + } + + template + int BasicProactor::writeCommand (const Command& cmd) noexcept + { + return writeCommand (cmd, is_default{}); + } + + template + int BasicProactor::writeCommand (const Command& cmd, std::true_type) noexcept + { + if (JOIN_UNLIKELY (_commands.push (cmd) == -1)) + { + return -1; + } + + uint64_t value = 1; + if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1)) + { + lastError = std::error_code (errno, std::system_category ()); + return -1; + } + + return 0; + } + + template + int BasicProactor::writeCommand (const Command& cmd, std::false_type) noexcept + { + return _commands.push (cmd); + } + + template + void BasicProactor::readCommands () noexcept + { + Command cmd; + while (_commands.tryPop (cmd) == 0) + { + processCommand (cmd); + } + } + + template + void BasicProactor::processCommand (const Command& cmd) noexcept + { + int err = 0; + + switch (cmd.type) + { + case CommandType::Submit: + err = submitOperation (cmd.op, cmd.flush); + break; + + case CommandType::Cancel: + err = cancelOperation (cmd.op, cmd.flush); + break; + + case CommandType::Stop: + { + _running.store (false, std::memory_order_release); + for (IoOperation* op : _pendingOps) + { + cancelOperation (op, false); + } + if (cmd.flush) + { + io_uring_submit (&_ring); + } + break; + } + + case CommandType::Flush: + if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) + { + lastError = std::error_code (errno, std::system_category ()); + err = -1; + } + break; + + default: + break; + } + + if (JOIN_UNLIKELY (cmd.done && (cmd.type != CommandType::Stop || _pendingOps.empty ()))) + { + if (cmd.errc && (err != 0)) + { + *cmd.errc = lastError; + } + cmd.done->store (true, std::memory_order_release); + } + } + + template + int BasicProactor::submitOperation (IoOperation* op, bool flush) noexcept + { + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Idle)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + io_uring_sqe* sqe = getSqe (); + + if (JOIN_UNLIKELY (sqe == nullptr)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + prepareSqe (sqe, op); + op->state = IoOperation::State::Submitted; + op->slot = static_cast (_pendingOps.size ()); + _pendingOps.push_back (op); + + if (flush) + { + io_uring_submit (&_ring); + } + + return 0; + } + + template + int BasicProactor::cancelOperation (IoOperation* op, bool flush) noexcept + { + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Submitted)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + if (JOIN_UNLIKELY (op->slot >= _pendingOps.size () || _pendingOps[op->slot] != op)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + io_uring_sqe* sqe = getSqe (); + if (JOIN_UNLIKELY (sqe == nullptr)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + op->state = IoOperation::State::Cancelling; + io_uring_prep_cancel (sqe, op, 0); + io_uring_sqe_set_data (sqe, nullptr); + + if (flush) + { + io_uring_submit (&_ring); + } + + return 0; + } + + template + void BasicProactor::endOperation (IoOperation* op, int result, bool cancelled) noexcept + { + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } + + if (op->slot < _pendingOps.size () && _pendingOps[op->slot] == op) + { + IoOperation* last = _pendingOps.back (); + _pendingOps[op->slot] = last; + last->slot = op->slot; + _pendingOps.pop_back (); + } + + dispatchOperation (op, result, cancelled); + + if (JOIN_UNLIKELY (_pendingOps.empty ())) + { + auto* p = _stopDone.load (std::memory_order_acquire); + if (p) + p->store (true, std::memory_order_release); + } + } + + template + io_uring_sqe* BasicProactor::getSqe () noexcept + { + io_uring_sqe* sqe = io_uring_get_sqe (&_ring); + + if (JOIN_UNLIKELY (sqe == nullptr)) + { + io_uring_submit (&_ring); + sqe = io_uring_get_sqe (&_ring); + } + + return sqe; + } + + template + void BasicProactor::prepareSqe (io_uring_sqe* sqe, IoOperation* op) noexcept + { + switch (static_cast (op->code)) + { + case IoOperation::Opcode::Accept: + io_uring_prep_accept (sqe, op->data.accept.fd, op->data.accept.addr, op->data.accept.addrlen, + op->data.accept.flags); + break; + + case IoOperation::Opcode::Connect: + io_uring_prep_connect (sqe, op->data.connect.fd, op->data.connect.addr, op->data.connect.addrlen); + break; + + case IoOperation::Opcode::Read: + io_uring_prep_read (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0); + break; + + case IoOperation::Opcode::Write: + io_uring_prep_write (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0); + break; + + case IoOperation::Opcode::ReadFixed: + io_uring_prep_read_fixed (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0, op->data.rw.index); + break; + + case IoOperation::Opcode::WriteFixed: + io_uring_prep_write_fixed (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0, op->data.rw.index); + break; + + case IoOperation::Opcode::RecvMsg: + io_uring_prep_recvmsg (sqe, op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + break; + + case IoOperation::Opcode::SendMsg: + io_uring_prep_sendmsg (sqe, op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + break; + + case IoOperation::Opcode::Recv: + io_uring_prep_recv (sqe, op->data.stream.fd, op->data.stream.buf, op->data.stream.len, + op->data.stream.flags); + break; + + case IoOperation::Opcode::Send: + io_uring_prep_send (sqe, op->data.stream.fd, op->data.stream.buf, op->data.stream.len, + op->data.stream.flags); + break; + + default: + io_uring_prep_nop (sqe); + break; + } + + io_uring_sqe_set_data (sqe, op); + + if (op->linked) + { + sqe->flags |= IOSQE_IO_LINK; + } + } + + template + void BasicProactor::rearmWakeup () noexcept + { + io_uring_sqe* sqe = getSqe (); + + if (JOIN_LIKELY (sqe != nullptr)) + { + prepareSqe (sqe, &_wakeupOp); + _wakeupOp.state = IoOperation::State::Submitted; + io_uring_submit (&_ring); + } + } + + template + void BasicProactor::dispatchCqe (io_uring_cqe* cqe) noexcept + { + dispatchCqe (cqe, is_default{}); + } + + template + void BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::true_type) noexcept + { + IoOperation* op = static_cast (io_uring_cqe_get_data (cqe)); + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } + + if (op == &_wakeupOp) + { + _wakeupOp.state = IoOperation::State::Idle; + readCommands (); + if (_running.load (std::memory_order_acquire)) + { + rearmWakeup (); + } + return; + } + + if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) + { + return; + } + + int result = cqe->res; + bool cancelled = (result < 0) && (result == -ECANCELED || op->state == IoOperation::State::Cancelling); + endOperation (op, result, cancelled); + } + + template + void BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::false_type) noexcept + { + IoOperation* op = static_cast (io_uring_cqe_get_data (cqe)); + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } + + if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) + { + return; + } + + int result = cqe->res; + bool cancelled = (result < 0) && (result == -ECANCELED || op->state == IoOperation::State::Cancelling); + endOperation (op, result, cancelled); + } + + template + void BasicProactor::eventLoop () noexcept + { + eventLoop (has_spin{}, has_sqpoll{}); + } + + template + void BasicProactor::eventLoop (std::false_type, std::false_type) noexcept + { + if (_running.load (std::memory_order_acquire)) + { + rearmWakeup (); + } + + while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) + { + io_uring_cqe* cqe = nullptr; + + if (_running.load (std::memory_order_acquire)) + { + if (JOIN_UNLIKELY (io_uring_wait_cqe (&_ring, &cqe) < 0)) + { + continue; + } + } + else + { + io_uring_submit (&_ring); + if (io_uring_peek_cqe (&_ring, &cqe) != 0) + { + continue; + } + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + } + } + + template + void BasicProactor::eventLoop (std::true_type, std::false_type) noexcept + { + Backoff backoff (Policy::spin); + + while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) + { + bool running = _running.load (std::memory_order_acquire); + if (running) + { + readCommands (); + } + else + { + io_uring_submit (&_ring); + } + + io_uring_cqe* cqe = nullptr; + if (io_uring_peek_cqe (&_ring, &cqe) != 0) + { + backoff (); + continue; + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + + backoff.reset (); + } + } + + template + void BasicProactor::eventLoop (std::true_type, std::true_type) noexcept + { + Backoff backoff (Policy::spin); + + while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) + { + bool running = _running.load (std::memory_order_acquire); + + if (running) + { + readCommands (); + + if (IO_URING_READ_ONCE (*_ring.sq.kflags) & IORING_SQ_NEED_WAKEUP) + { + io_uring_enter (_ring.ring_fd, 0, 0, IORING_ENTER_SQ_WAKEUP, nullptr); + } + } + else + { + io_uring_submit (&_ring); + } + + io_uring_cqe* cqe = nullptr; + if (io_uring_peek_cqe (&_ring, &cqe) != 0) + { + backoff (); + continue; + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + + backoff.reset (); + } + } +} diff --git a/core/include/join/reactor.hpp b/core/include/join/reactor.hpp index 80123c2b..73105e09 100644 --- a/core/include/join/reactor.hpp +++ b/core/include/join/reactor.hpp @@ -210,6 +210,12 @@ namespace join */ int mlock () const noexcept; + /** + * @brief check if the event loop is running. + * @return true if the event loop is running. + */ + bool isRunning () const noexcept; + /** * @brief check if the calling thread is the reactor thread. * @return true if called from the reactor thread. diff --git a/core/src/io_operation.cpp b/core/src/io_operation.cpp new file mode 100644 index 00000000..4deeaa8c --- /dev/null +++ b/core/src/io_operation.cpp @@ -0,0 +1,224 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include +#include + +using join::IoOperation; +using join::CompletionHandler; + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeAccept +// ========================================================================= +IoOperation IoOperation::makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, int flags, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::Accept); + op.handler = handler; + op.data.accept.fd = fd; + op.data.accept.addr = addr; + op.data.accept.addrlen = addrlen; + op.data.accept.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeConnect +// ========================================================================= +IoOperation IoOperation::makeConnect (int fd, const sockaddr* addr, socklen_t addrlen, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::Connect); + op.handler = handler; + op.data.connect.fd = fd; + op.data.connect.addr = addr; + op.data.connect.addrlen = addrlen; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeRead +// ========================================================================= +IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::Read); + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = buf; + op.data.rw.len = len; + op.data.rw.index = 0; + op.data.rw.fixed = false; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeWrite +// ========================================================================= +IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::Write); + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = const_cast (buf); + op.data.rw.len = len; + op.data.rw.index = 0; + op.data.rw.fixed = false; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeReadFixed +// ========================================================================= +IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::ReadFixed); + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = buf; + op.data.rw.len = len; + op.data.rw.index = index; + op.data.rw.fixed = true; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeWriteFixed +// ========================================================================= +IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::WriteFixed); + op.handler = handler; + op.data.rw.fd = fd; + op.data.rw.buf = const_cast (buf); + op.data.rw.len = len; + op.data.rw.index = index; + op.data.rw.fixed = true; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeRecvmsg +// ========================================================================= +IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, int flags, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::RecvMsg); + op.handler = handler; + op.data.msg.fd = fd; + op.data.msg.msg = msg; + op.data.msg.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeSendmsg +// ========================================================================= +IoOperation IoOperation::makeSendmsg (int fd, const msghdr* msg, int flags, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::SendMsg); + op.handler = handler; + op.data.msg.fd = fd; + op.data.msg.msg = const_cast (msg); + op.data.msg.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeRecv +// ========================================================================= +IoOperation IoOperation::makeRecv (int fd, void* buf, uint32_t len, int flags, CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::Recv); + op.handler = handler; + op.data.stream.fd = fd; + op.data.stream.buf = buf; + op.data.stream.len = len; + op.data.stream.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : makeSend +// ========================================================================= +IoOperation IoOperation::makeSend (int fd, const void* buf, uint32_t len, int flags, + CompletionHandler* handler) noexcept +{ + IoOperation op; + op.code = static_cast (IoOperation::Opcode::Send); + op.handler = handler; + op.data.stream.fd = fd; + op.data.stream.buf = const_cast (buf); + op.data.stream.len = len; + op.data.stream.flags = flags; + return op; +} + +// ========================================================================= +// CLASS : IoOperation +// METHOD : fd +// ========================================================================= +int IoOperation::fd () const noexcept +{ + switch (static_cast (code)) + { + case Opcode::Accept: + return data.accept.fd; + case Opcode::Connect: + return data.connect.fd; + case Opcode::Read: + case Opcode::ReadFixed: + case Opcode::Write: + case Opcode::WriteFixed: + return data.rw.fd; + case Opcode::RecvMsg: + case Opcode::SendMsg: + return data.msg.fd; + case Opcode::Recv: + case Opcode::Send: + return data.stream.fd; + default: + return -1; + } +} diff --git a/core/src/proactor.cpp b/core/src/proactor.cpp deleted file mode 100644 index 08d9afd6..00000000 --- a/core/src/proactor.cpp +++ /dev/null @@ -1,768 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2026 Mathieu Rabine - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -// libjoin. -#include -#include - -// C. -#include -#include -#include -#include - -using join::CompletionHandler; -using join::IoOperation; -using join::Proactor; -using join::ProactorThread; - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeTimeout -// ========================================================================= -IoOperation IoOperation::makeTimeout (const __kernel_timespec& ts, uint32_t flags, CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_TIMEOUT; - op.handler = handler; - op.data.timeout.ts = ts; - op.data.timeout.flags = flags; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeRead -// ========================================================================= -IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_READ; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = buf; - op.data.rw.len = len; - op.data.rw.index = 0; - op.data.rw.fixed = false; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeReadFixed -// ========================================================================= -IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_t buf_index, - CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_READ_FIXED; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = buf; - op.data.rw.len = len; - op.data.rw.index = buf_index; - op.data.rw.fixed = true; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeWrite -// ========================================================================= -IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_WRITE; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = const_cast (buf); - op.data.rw.len = len; - op.data.rw.index = 0; - op.data.rw.fixed = false; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeWriteFixed -// ========================================================================= -IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_WRITE_FIXED; - op.handler = handler; - op.data.rw.fd = fd; - op.data.rw.buf = const_cast (buf); - op.data.rw.len = len; - op.data.rw.index = index; - op.data.rw.fixed = true; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeSendmsg -// ========================================================================= -IoOperation IoOperation::makeSendmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_SENDMSG; - op.handler = handler; - op.data.msg.fd = fd; - op.data.msg.msg = msg; - op.data.msg.flags = flags; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeRecvmsg -// ========================================================================= -IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, uint32_t flags, CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_RECVMSG; - op.handler = handler; - op.data.msg.fd = fd; - op.data.msg.msg = msg; - op.data.msg.flags = flags; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeAccept -// ========================================================================= -IoOperation IoOperation::makeAccept (int fd, sockaddr* addr, socklen_t* addrlen, uint32_t flags, - CompletionHandler* handler) noexcept -{ - IoOperation op; - op.code = IORING_OP_ACCEPT; - op.handler = handler; - op.data.accept.fd = fd; - op.data.accept.addr = addr; - op.data.accept.addrlen = addrlen; - op.data.accept.flags = flags; - return op; -} - -// ========================================================================= -// CLASS : IoOperation -// METHOD : makeCancel -// ========================================================================= -// IoOperation IoOperation::makeCancel (IoOperation* target, CompletionHandler* handler) noexcept -// { -// IoOperation op; -// op.code = IORING_OP_ASYNC_CANCEL; -// op.handler = handler; -// op.data.cancel.target = target; -// return op; -// } - -// ========================================================================= -// CLASS : Proactor -// METHOD : Proactor -// ========================================================================= -Proactor::Proactor () -: _wakeup (eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC)) -, _commands (_queueSize) -{ - if (_wakeup == -1) - { - throw std::system_error (errno, std::system_category (), "eventfd failed"); - } - - if (io_uring_queue_init (_ringSize, &_ring, 0) != 0) - { - int err = errno; - ::close (_wakeup); - throw std::system_error (err, std::system_category (), "io_uring_queue_init failed"); - } - - _wakeupOp = IoOperation::makeRead (_wakeup, &_wakeupBuf, sizeof (_wakeupBuf), nullptr); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : ~Proactor -// ========================================================================= -Proactor::~Proactor () noexcept -{ - stop (); - - io_uring_queue_exit (&_ring); - - ::close (_wakeup); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : submit -// ========================================================================= -int Proactor::submit (IoOperation* op, bool sync) noexcept -{ - if (JOIN_UNLIKELY (op == nullptr)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (isProactorThread ()) - { - return submitOperation (op); - } - - std::atomic done{false}, *pdone = nullptr; - std::atomic errc{0}, *perrc = nullptr; - - if (JOIN_LIKELY (sync)) - { - pdone = &done; - perrc = &errc; - } - - if (JOIN_UNLIKELY (writeCommand ({CommandType::Submit, op, pdone, perrc}) == -1)) - { - return -1; - } - - if (JOIN_LIKELY (sync)) - { - Backoff backoff; - while (!done.load (std::memory_order_acquire)) - { - backoff (); - } - - int err = errc.load (std::memory_order_acquire); - if (JOIN_UNLIKELY (err != 0)) - { - lastError = std::make_error_code (static_cast (err)); - return -1; - } - } - - return 0; -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : cancel -// ========================================================================= -int Proactor::cancel (IoOperation* op, bool sync) noexcept -{ - if (JOIN_UNLIKELY (op == nullptr)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (isProactorThread ()) - { - return cancelOperation (op); - } - - std::atomic done{false}, *pdone = nullptr; - std::atomic errc{0}, *perrc = nullptr; - - if (JOIN_LIKELY (sync)) - { - pdone = &done; - perrc = &errc; - } - - if (JOIN_UNLIKELY (writeCommand ({CommandType::Cancel, op, pdone, perrc}) == -1)) - { - return -1; - } - - if (JOIN_LIKELY (sync)) - { - Backoff backoff; - while (!done.load (std::memory_order_acquire)) - { - backoff (); - } - - int err = errc.load (std::memory_order_acquire); - if (JOIN_UNLIKELY (err != 0)) - { - lastError = std::make_error_code (static_cast (err)); - return -1; - } - } - - return 0; -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : run -// ========================================================================= -void Proactor::run () -{ - _threadId.store (pthread_self (), std::memory_order_release); - - _running.store (true, std::memory_order_release); - rearmWakeup (); - eventLoop (); - - _threadId.store (0, std::memory_order_release); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : stop -// ========================================================================= -void Proactor::stop (bool sync) noexcept -{ - _running.store (false, std::memory_order_release); - - if (isProactorThread ()) - { - return; - } - - writeCommand ({CommandType::Stop, nullptr, nullptr, nullptr}); - - if (JOIN_LIKELY (sync)) - { - Backoff backoff; - while (_threadId.load (std::memory_order_acquire) != 0) - { - backoff (); - } - } -} - -#ifdef JOIN_HAS_NUMA -// ========================================================================= -// CLASS : Proactor -// METHOD : mbind -// ========================================================================= -int Proactor::mbind (int numa) const noexcept -{ - return _commands.mbind (numa); -} -#endif - -// ========================================================================= -// CLASS : Proactor -// METHOD : mlock -// ========================================================================= -int Proactor::mlock () const noexcept -{ - return _commands.mlock (); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : isProactorThread -// ========================================================================= -bool Proactor::isProactorThread () const noexcept -{ - return _threadId.load (std::memory_order_acquire) == pthread_self (); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : getSqe -// ========================================================================= -io_uring_sqe* Proactor::getSqe () noexcept -{ - io_uring_sqe* sqe = io_uring_get_sqe (&_ring); - - if (JOIN_UNLIKELY (sqe == nullptr)) - { - io_uring_submit (&_ring); - sqe = io_uring_get_sqe (&_ring); - } - - return sqe; -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : submitOperation -// ========================================================================= -int Proactor::submitOperation (IoOperation* op) noexcept -{ - io_uring_sqe* sqe = getSqe (); - - if (JOIN_UNLIKELY (sqe == nullptr)) - { - lastError = std::make_error_code (std::errc::resource_unavailable_try_again); - return -1; - } - - prepareSqe (*op, sqe); - op->state = IoOperation::State::Submitted; - - return 0; -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : cancelOperation -// ========================================================================= -int Proactor::cancelOperation (IoOperation* op) noexcept -{ - if (op->state != IoOperation::State::Submitted) - { - return 0; - } - - io_uring_sqe* sqe = getSqe (); - - if (JOIN_UNLIKELY (sqe == nullptr)) - { - lastError = std::make_error_code (std::errc::resource_unavailable_try_again); - return -1; - } - - io_uring_prep_cancel (sqe, op, 0); - io_uring_sqe_set_data (sqe, nullptr); - op->state = IoOperation::State::Cancelling; - - return 0; -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : rearmWakeup -// ========================================================================= -void Proactor::rearmWakeup () noexcept -{ - io_uring_sqe* sqe = getSqe (); - - if (JOIN_UNLIKELY (sqe == nullptr)) - { - return; - } - - _wakeupOp.state = IoOperation::State::Submitted; - prepareSqe (_wakeupOp, sqe); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : writeCommand -// ========================================================================= -int Proactor::writeCommand (const Command& cmd) noexcept -{ - if (_commands.push (cmd) == -1) - { - return -1; - } - - uint64_t value = 1; - [[maybe_unused]] ssize_t bytes = ::write (_wakeup, &value, sizeof (uint64_t)); - - return 0; -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : processCommand -// ========================================================================= -void Proactor::processCommand (const Command& cmd) noexcept -{ - int err = 0; - - switch (cmd.type) - { - case CommandType::Submit: - err = submitOperation (cmd.op); - break; - - case CommandType::Cancel: - err = cancelOperation (cmd.op); - break; - - case CommandType::Stop: - break; - } - - if (JOIN_UNLIKELY (cmd.done)) - { - if (cmd.errc && (err != 0)) - { - cmd.errc->store (lastError.value (), std::memory_order_release); - } - cmd.done->store (true, std::memory_order_release); - } -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : readCommands -// ========================================================================= -void Proactor::readCommands () noexcept -{ - Command cmd; - - while (_commands.tryPop (cmd) == 0) - { - processCommand (cmd); - } - - if (_running.load (std::memory_order_acquire)) - { - rearmWakeup (); - } -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : dispatchCqe -// ========================================================================= -void Proactor::dispatchCqe (io_uring_cqe* cqe) noexcept -{ - auto* op = reinterpret_cast (io_uring_cqe_get_data (cqe)); - - if (op == &_wakeupOp) - { - _wakeupOp.state = IoOperation::State::Idle; - readCommands (); - return; - } - - if (op != nullptr) - { - dispatchOperation (op, cqe->res); - } -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : prepareSqe -// ========================================================================= -void Proactor::prepareSqe (const IoOperation& op, io_uring_sqe* sqe) const noexcept -{ - assert (sqe != nullptr); - - switch (op.code) - { - case IORING_OP_TIMEOUT: - { - auto* ts = const_cast<__kernel_timespec*> (&op.data.timeout.ts); - io_uring_prep_timeout (sqe, ts, 0, op.data.timeout.flags); - } - break; - - case IORING_OP_READ: - io_uring_prep_read (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0); - break; - - case IORING_OP_READ_FIXED: - io_uring_prep_read_fixed (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0, op.data.rw.index); - break; - - case IORING_OP_WRITE: - io_uring_prep_write (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0); - break; - - case IORING_OP_WRITE_FIXED: - io_uring_prep_write_fixed (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0, op.data.rw.index); - break; - - case IORING_OP_SENDMSG: - io_uring_prep_sendmsg (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags); - break; - - case IORING_OP_RECVMSG: - io_uring_prep_recvmsg (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags); - break; - - case IORING_OP_ACCEPT: - io_uring_prep_accept (sqe, op.data.accept.fd, op.data.accept.addr, op.data.accept.addrlen, - static_cast (op.data.accept.flags)); - break; - - // case IORING_OP_ASYNC_CANCEL: - // io_uring_prep_cancel (sqe, reinterpret_cast (op.data.cancel.target), 0); - // break; - - default: - break; - } - - io_uring_sqe_set_data (sqe, static_cast (const_cast (&op))); -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : dispatchOperation -// ========================================================================= -void Proactor::dispatchOperation (IoOperation* op, int result) noexcept -{ - const IoOperation::State prev = op->state; - op->state = IoOperation::State::Idle; - - if (op->handler == nullptr) - { - return; - } - - if (result == -ECANCELED || prev == IoOperation::State::Cancelling) - { - op->handler->onCancel (op, result); - } - else - { - op->handler->onComplete (op, result); - } -} - -// ========================================================================= -// CLASS : Proactor -// METHOD : eventLoop -// ========================================================================= -void Proactor::eventLoop () -{ - while (_running.load (std::memory_order_acquire)) - { - io_uring_submit (&_ring); - - io_uring_cqe* cqe = nullptr; - int ret = io_uring_wait_cqe (&_ring, &cqe); - if (JOIN_UNLIKELY (ret < 0)) - { - continue; - } - - do - { - dispatchCqe (cqe); - io_uring_cqe_seen (&_ring, cqe); - } - while (io_uring_peek_cqe (&_ring, &cqe) == 0); - } -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : proactor -// ========================================================================= -Proactor* ProactorThread::proactor () -{ - return &instance ()._proactor; -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : affinity (setter) -// ========================================================================= -int ProactorThread::affinity (int core) -{ - return instance ()._dispatcher.affinity (core); -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : affinity (getter) -// ========================================================================= -int ProactorThread::affinity () -{ - return instance ()._dispatcher.affinity (); -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : priority (setter) -// ========================================================================= -int ProactorThread::priority (int prio) -{ - return instance ()._dispatcher.priority (prio); -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : priority (getter) -// ========================================================================= -int ProactorThread::priority () -{ - return instance ()._dispatcher.priority (); -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : handle -// ========================================================================= -pthread_t ProactorThread::handle () -{ - return instance ()._dispatcher.handle (); -} - -#ifdef JOIN_HAS_NUMA -// ========================================================================= -// CLASS : ProactorThread -// METHOD : mbind -// ========================================================================= -int ProactorThread::mbind (int numa) -{ - return instance ()._proactor.mbind (numa); -} -#endif - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : mlock -// ========================================================================= -int ProactorThread::mlock () -{ - return instance ()._proactor.mlock (); -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : instance -// ========================================================================= -ProactorThread& ProactorThread::instance () -{ - static ProactorThread proactorThread; - return proactorThread; -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : ProactorThread -// ========================================================================= -ProactorThread::ProactorThread () -{ - _dispatcher = Thread ([this] () { - _proactor.run (); - }); -} - -// ========================================================================= -// CLASS : ProactorThread -// METHOD : ~ProactorThread -// ========================================================================= -ProactorThread::~ProactorThread () -{ - _proactor.stop (); - _dispatcher.join (); -} diff --git a/core/src/reactor.cpp b/core/src/reactor.cpp index 1b2313ec..d677874f 100644 --- a/core/src/reactor.cpp +++ b/core/src/reactor.cpp @@ -211,10 +211,9 @@ void Reactor::run () // ========================================================================= void Reactor::stop (bool sync) noexcept { - _running.store (false, std::memory_order_release); - if (isReactorThread ()) { + _running.store (false, std::memory_order_release); return; } @@ -250,6 +249,15 @@ int Reactor::mlock () const noexcept return _commands.mlock (); } +// ========================================================================= +// CLASS : Reactor +// METHOD : isRunning +// ========================================================================= +bool Reactor::isRunning () const noexcept +{ + return _running.load (std::memory_order_acquire); +} + // ========================================================================= // CLASS : Reactor // METHOD : isReactorThread @@ -380,6 +388,7 @@ void Reactor::processCommand (const Command& cmd) noexcept break; case CommandType::Stop: + _running.store (false, std::memory_order_release); break; } diff --git a/core/tests/CMakeLists.txt b/core/tests/CMakeLists.txt index 60553412..4610cb8c 100644 --- a/core/tests/CMakeLists.txt +++ b/core/tests/CMakeLists.txt @@ -32,11 +32,35 @@ target_link_libraries(reactor.gtest ${JOIN_CORE} GTest::gtest_main) add_test(NAME reactor.gtest COMMAND reactor.gtest) install(TARGETS reactor.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) +add_executable(io_operation.gtest io_operation_test.cpp) +target_link_libraries(io_operation.gtest ${JOIN_CORE} GTest::gtest_main) +add_test(NAME io_operation.gtest COMMAND io_operation.gtest) +install(TARGETS io_operation.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) + +if(JOIN_ENABLE_IO_URING) + add_executable(io_policy.gtest io_policy_test.cpp) + target_link_libraries(io_policy.gtest ${JOIN_CORE} GTest::gtest_main) + add_test(NAME io_policy.gtest COMMAND io_policy.gtest) + install(TARGETS io_policy.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) +endif() + add_executable(proactor.gtest proactor_test.cpp) target_link_libraries(proactor.gtest ${JOIN_CORE} GTest::gtest_main) add_test(NAME proactor.gtest COMMAND proactor.gtest) install(TARGETS proactor.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) +if(JOIN_ENABLE_IO_URING) + add_executable(hybrid_proactor.gtest hybrid_proactor_test.cpp) + target_link_libraries(hybrid_proactor.gtest ${JOIN_CORE} GTest::gtest_main) + add_test(NAME hybrid_proactor.gtest COMMAND hybrid_proactor.gtest) + install(TARGETS hybrid_proactor.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) + + add_executable(sqpoll_proactor.gtest sqpoll_proactor_test.cpp) + target_link_libraries(sqpoll_proactor.gtest ${JOIN_CORE} GTest::gtest_main) + add_test(NAME sqpoll_proactor.gtest COMMAND sqpoll_proactor.gtest) + install(TARGETS sqpoll_proactor.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) +endif() + add_executable(realtimetimer.gtest realtimetimer_test.cpp) target_link_libraries(realtimetimer.gtest ${JOIN_CORE} GTest::gtest_main) add_test(NAME realtimetimer.gtest COMMAND realtimetimer.gtest) diff --git a/core/tests/hybrid_proactor_test.cpp b/core/tests/hybrid_proactor_test.cpp new file mode 100644 index 00000000..b5b25c84 --- /dev/null +++ b/core/tests/hybrid_proactor_test.cpp @@ -0,0 +1,752 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include +#include +#include + +// Libraries. +#include + +using join::Errc; +using join::Mutex; +using join::Condition; +using join::ScopedLock; +using join::Thread; +using join::HybridProactor; +using join::HybridProactorThread; +using join::IoOperation; +using join::CompletionHandler; +using join::Tcp; + +/** + * @brief Class used to test HybridProactor. + */ +class HybridProactorTest : public CompletionHandler, public ::testing::Test +{ +protected: + /** + * @brief Sets up the test fixture. + */ + void SetUp () override + { + ASSERT_EQ (_acceptor.create ({_host, _port}), 0) << join::lastError.message (); + } + + /** + * @brief Tears down the test fixture. + */ + void TearDown () override + { + _server.close (); + _client.close (); + _acceptor.close (); + } + + /** + * @brief method called when an operation completes. + * @param op completed operation. + * @param result bytes transferred or negative errno. + */ + void onComplete (IoOperation* op, int result) override + { + HybridProactorThread::proactor ().submit (op, true); + HybridProactorThread::proactor ().cancel (op, true); + + { + ScopedLock lock (_mut); + _result = result; + _op = op; + CompletionHandler::onComplete (op, result); + } + + _cond.signal (); + } + + /** + * @brief method called when an operation is cancelled. + * @param op cancelled operation. + * @param result negative errno. + */ + void onCancel (IoOperation* op, int result) override + { + HybridProactorThread::proactor ().submit (op, true); + HybridProactorThread::proactor ().cancel (op, true); + + { + ScopedLock lock (_mut); + _result = result; + _op = op; + CompletionHandler::onCancel (op, result); + } + + _cond.signal (); + } + + /// server acceptor. + static Tcp::Acceptor _acceptor; + + /// client socket. + static Tcp::Socket _client; + + /// server socket. + static Tcp::Socket _server; + + /// host. + static std::string _host; + + /// port. + static uint16_t _port; + + /// timeout. + static const int _timeout; + + /// condition variable. + static Condition _cond; + + /// condition mutex. + static Mutex _mut; + + /// last completed operation. + static IoOperation* _op; + + /// last operation result. + static int _result; + + /// read buffer. + static char _buf[256]; +}; + +Tcp::Acceptor HybridProactorTest::_acceptor; +Tcp::Socket HybridProactorTest::_client (Tcp::Socket::Blocking); +Tcp::Socket HybridProactorTest::_server; +std::string HybridProactorTest::_host = "127.0.0.1"; +uint16_t HybridProactorTest::_port = 5001; +const int HybridProactorTest::_timeout = 1000; +Condition HybridProactorTest::_cond; +Mutex HybridProactorTest::_mut; +IoOperation* HybridProactorTest::_op = nullptr; +int HybridProactorTest::_result = 0; +char HybridProactorTest::_buf[256] = {}; + +/** + * @brief Test stop. + */ +TEST_F (HybridProactorTest, stop) +{ + HybridProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op, true), 0) << join::lastError.message (); + + proactor.stop (); + th.join (); + + { + ScopedLock lock (_mut); + ASSERT_EQ (_op, &op); + ASSERT_EQ (_result, -ECANCELED); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test submit. + */ +TEST_F (HybridProactorTest, submit) +{ + HybridProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (proactor.submit (nullptr, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); + + auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + op1.state = IoOperation::State::Submitted; + ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (join::lastError, Errc::OperationFailed); + + op1.state = IoOperation::State::Idle; + ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + +#ifndef JOIN_HAS_IO_URING + auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op2, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); +#endif + + ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op1 && _result == -ECANCELED; + })); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test cancel. + */ +TEST_F (HybridProactorTest, cancel) +{ + HybridProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (proactor.cancel (nullptr, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); + + auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (join::lastError, Errc::OperationFailed); + + ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + + auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + op2.state = IoOperation::State::Submitted; + ASSERT_EQ (proactor.cancel (&op2, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); + + ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op1 && _result == -ECANCELED; + })); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test flush. + */ +TEST_F (HybridProactorTest, flush) +{ + HybridProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op, false), 0) << join::lastError.message (); + ASSERT_EQ (proactor.flush (), 0) << join::lastError.message (); + + ASSERT_EQ (_client.writeExactly ("flush", strlen ("flush"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "flush"); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test SQE chaining. + */ +TEST_F (HybridProactorTest, chain) +{ + HybridProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this); + auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + writeOp.linked = true; + + ASSERT_EQ (proactor.submit (&writeOp, false), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&readOp, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &writeOp && _result == 4; + })); + _op = nullptr; + _result = 0; + } + + char tmp[4]; + ASSERT_EQ (_client.readExactly (tmp, 4, _timeout), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("pong", 4, _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &readOp && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "pong"); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +#ifdef JOIN_HAS_NUMA +/** + * @brief Test mbind. + */ +TEST_F (HybridProactorTest, mbind) +{ + HybridProactor proactor; + + ASSERT_EQ (proactor.mbind (0), 0) << join::lastError.message (); +} +#endif + +/** + * @brief Test mlock. + */ +TEST_F (HybridProactorTest, mlock) +{ + HybridProactor proactor; + + ASSERT_EQ (proactor.mlock (), 0) << join::lastError.message (); +} + +/** + * @brief Test isRunning. + */ +TEST_F (HybridProactorTest, isRunning) +{ + HybridProactor proactor; + + ASSERT_FALSE (proactor.isRunning ()); + + Thread th ([&proactor] () { + proactor.run (); + }); + while (!proactor.isRunning ()) + { + } + + ASSERT_TRUE (proactor.isRunning ()); + + proactor.stop (); + th.join (); + + ASSERT_FALSE (proactor.isRunning ()); +} + +#ifdef JOIN_HAS_IO_URING +/** + * @brief Test registerBuffers and unregisterBuffers. + */ +TEST_F (HybridProactorTest, registerBuffers) +{ + HybridProactor proactor; + + std::vector buf (4096); + iovec iov = {buf.data (), buf.size ()}; + std::vector iovecs = {iov}; + + ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); + ASSERT_EQ (proactor.registerBuffers (iovecs), -1); + ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); + ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); +} +#endif + +/** + * @brief Test async accept. + */ +TEST_F (HybridProactorTest, asyncAccept) +{ + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + sockaddr_storage addr = {}; + socklen_t addrlen = sizeof (addr); + auto op = IoOperation::makeAccept (_acceptor.handle (), reinterpret_cast (&addr), &addrlen, + SOCK_NONBLOCK | SOCK_CLOEXEC, this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result >= 0; + })); + ::close (_result); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test async connect. + */ +TEST_F (HybridProactorTest, asyncConnect) +{ + Tcp::Endpoint endpoint{_host, _port}; + + ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); + _client.setMode (Tcp::Socket::NonBlocking); + + auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } + + _client.setMode (Tcp::Socket::Blocking); +} + +/** + * @brief Test async read. + */ +TEST_F (HybridProactorTest, asyncRead) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRead", strlen ("asyncRead"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRead"); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test async write. + */ +TEST_F (HybridProactorTest, asyncWrite) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + const char* msg = "asyncWrite"; + auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (_result, static_cast (strlen (msg))); + _op = nullptr; + _result = 0; + } + + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async read and write. + */ +TEST_F (HybridProactorTest, asyncReadWrite) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + const char* msg = "asyncReadWrite"; + auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + auto writeOp = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&readOp, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&writeOp, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &writeOp && _result > 0; + })); + ASSERT_EQ (_result, static_cast (strlen (msg))); + _op = nullptr; + _result = 0; + } + + ASSERT_EQ (_client.writeExactly (msg, strlen (msg), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &readOp && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), msg); + _op = nullptr; + _result = 0; + } + + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async recvmsg. + */ +TEST_F (HybridProactorTest, asyncRecvmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + iovec iov = {.iov_base = _buf, .iov_len = sizeof (_buf)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeRecvmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRecvmsg", strlen ("asyncRecvmsg"), _timeout), 0) + << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRecvmsg"); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test async sendmsg. + */ +TEST_F (HybridProactorTest, asyncSendmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + const char* payload = "asyncSendmsg"; + iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (_result, static_cast (strlen (payload))); + _op = nullptr; + _result = 0; + } + + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); +} + +/** + * @brief Test onClose. + */ +TEST_F (HybridProactorTest, onClose) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + _client.close (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test onError. + */ +TEST_F (HybridProactorTest, onError) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + linger sl{.l_onoff = 1, .l_linger = 0}; + ASSERT_EQ (setsockopt (_client.handle (), SOL_SOCKET, SO_LINGER, &sl, sizeof (sl)), 0) << strerror (errno); + _client.close (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == -ECONNRESET; + })); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief main function. + */ +int main (int argc, char** argv) +{ + testing::InitGoogleTest (&argc, argv); + return RUN_ALL_TESTS (); +} diff --git a/core/tests/io_operation_test.cpp b/core/tests/io_operation_test.cpp new file mode 100644 index 00000000..533a369d --- /dev/null +++ b/core/tests/io_operation_test.cpp @@ -0,0 +1,268 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include +#include + +// Libraries. +#include + +using join::IoOperation; +using join::CompletionHandler; +using join::Tcp; + +static const std::string host = "127.0.0.1"; +static const uint16_t port = 5001; +static char buffer[256] = {}; + +/** + * @brief Test makeAccept. + */ +TEST (IoOperation, makeAccept) +{ + sockaddr_storage addr = {}; + socklen_t addrlen = sizeof (addr); + + auto op = IoOperation::makeAccept (8, reinterpret_cast (&addr), &addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC, + nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Accept)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.accept.fd, 8); + ASSERT_EQ (op.data.accept.addr, reinterpret_cast (&addr)); + ASSERT_EQ (op.data.accept.addrlen, &addrlen); + ASSERT_EQ (op.data.accept.flags, SOCK_NONBLOCK | SOCK_CLOEXEC); +} + +/** + * @brief Test makeConnect. + */ +TEST (IoOperation, makeConnect) +{ + Tcp::Endpoint endpoint{host, port}; + + auto op = IoOperation::makeConnect (8, endpoint.addr (), endpoint.length (), nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Connect)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.connect.fd, 8); + ASSERT_EQ (op.data.connect.addr, endpoint.addr ()); + ASSERT_EQ (op.data.connect.addrlen, endpoint.length ()); +} + +/** + * @brief Test makeRead. + */ +TEST (IoOperation, makeRead) +{ + auto op = IoOperation::makeRead (8, buffer, sizeof (buffer), nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Read)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.rw.fd, 8); + ASSERT_EQ (op.data.rw.buf, buffer); + ASSERT_EQ (op.data.rw.len, sizeof (buffer)); + ASSERT_EQ (op.data.rw.index, 0); + ASSERT_EQ (op.data.rw.fixed, false); +} + +/** + * @brief Test makeWrite. + */ +TEST (IoOperation, makeWrite) +{ + auto op = IoOperation::makeWrite (8, buffer, sizeof (buffer), nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Write)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.rw.fd, 8); + ASSERT_EQ (op.data.rw.buf, buffer); + ASSERT_EQ (op.data.rw.len, sizeof (buffer)); + ASSERT_EQ (op.data.rw.index, 0); + ASSERT_EQ (op.data.rw.fixed, false); +} + +/** + * @brief Test makeReadFixed. + */ +TEST (IoOperation, makeReadFixed) +{ + auto op = IoOperation::makeReadFixed (8, buffer, sizeof (buffer), 6, nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::ReadFixed)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.rw.fd, 8); + ASSERT_EQ (op.data.rw.buf, buffer); + ASSERT_EQ (op.data.rw.len, sizeof (buffer)); + ASSERT_EQ (op.data.rw.index, 6); + ASSERT_EQ (op.data.rw.fixed, true); +} + +/** + * @brief Test makeWriteFixed. + */ +TEST (IoOperation, makeWriteFixed) +{ + auto op = IoOperation::makeWriteFixed (8, buffer, sizeof (buffer), 6, nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::WriteFixed)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.rw.fd, 8); + ASSERT_EQ (op.data.rw.buf, buffer); + ASSERT_EQ (op.data.rw.len, sizeof (buffer)); + ASSERT_EQ (op.data.rw.index, 6); + ASSERT_EQ (op.data.rw.fixed, true); +} + +/** + * @brief Test makeRecvmsg. + */ +TEST (IoOperation, makeRecvmsg) +{ + iovec iov = {}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + auto op = IoOperation::makeRecvmsg (8, &msg, MSG_DONTWAIT, nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::RecvMsg)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.msg.fd, 8); + ASSERT_EQ (op.data.msg.msg, &msg); + ASSERT_EQ (op.data.msg.flags, MSG_DONTWAIT); +} + +/** + * @brief Test makeSendmsg. + */ +TEST (IoOperation, makeSendmsg) +{ + const char* payload = "makeSendmsg"; + iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + auto op = IoOperation::makeSendmsg (8, &msg, MSG_DONTWAIT, nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::SendMsg)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.msg.fd, 8); + ASSERT_EQ (op.data.msg.msg, &msg); + ASSERT_EQ (op.data.msg.flags, MSG_DONTWAIT); +} + +/** + * @brief Test makeRecv. + */ +TEST (IoOperation, makeRecv) +{ + auto op = IoOperation::makeRecv (8, buffer, sizeof (buffer), MSG_DONTWAIT, nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Recv)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.stream.fd, 8); + ASSERT_EQ (op.data.stream.buf, buffer); + ASSERT_EQ (op.data.stream.len, sizeof (buffer)); + ASSERT_EQ (op.data.stream.flags, MSG_DONTWAIT); +} + +/** + * @brief Test makeSend. + */ +TEST (IoOperation, makeSend) +{ + const char* payload = "makeSend"; + + auto op = IoOperation::makeSend (8, payload, strlen (payload), MSG_DONTWAIT, nullptr); + + ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Send)); + ASSERT_EQ (op.handler, nullptr); + ASSERT_EQ (op.data.stream.fd, 8); + ASSERT_EQ (op.data.stream.buf, payload); + ASSERT_EQ (op.data.stream.len, strlen (payload)); + ASSERT_EQ (op.data.stream.flags, MSG_DONTWAIT); +} + +/** + * @brief Test fd. + */ +TEST (IoOperation, fd) +{ + IoOperation op; + + op.code = static_cast (IoOperation::Opcode::Accept); + op.data.accept.fd = 0; + ASSERT_EQ (op.fd (), 0); + + op.code = static_cast (IoOperation::Opcode::Connect); + op.data.connect.fd = 1; + ASSERT_EQ (op.fd (), 1); + + op.code = static_cast (IoOperation::Opcode::Read); + op.data.rw.fd = 2; + ASSERT_EQ (op.fd (), 2); + + op.code = static_cast (IoOperation::Opcode::ReadFixed); + op.data.rw.fd = 3; + ASSERT_EQ (op.fd (), 3); + + op.code = static_cast (IoOperation::Opcode::Write); + op.data.rw.fd = 4; + ASSERT_EQ (op.fd (), 4); + + op.code = static_cast (IoOperation::Opcode::WriteFixed); + op.data.rw.fd = 5; + ASSERT_EQ (op.fd (), 5); + + op.code = static_cast (IoOperation::Opcode::RecvMsg); + op.data.msg.fd = 6; + ASSERT_EQ (op.fd (), 6); + + op.code = static_cast (IoOperation::Opcode::SendMsg); + op.data.msg.fd = 7; + ASSERT_EQ (op.fd (), 7); + + op.code = static_cast (IoOperation::Opcode::Recv); + op.data.stream.fd = 8; + ASSERT_EQ (op.fd (), 8); + + op.code = static_cast (IoOperation::Opcode::Send); + op.data.stream.fd = 9; + ASSERT_EQ (op.fd (), 9); + + op.code = 255; + ASSERT_EQ (op.fd (), -1); +} + +/** + * @brief main function. + */ +int main (int argc, char** argv) +{ + testing::InitGoogleTest (&argc, argv); + return RUN_ALL_TESTS (); +} diff --git a/core/tests/io_policy_test.cpp b/core/tests/io_policy_test.cpp new file mode 100644 index 00000000..963f6311 --- /dev/null +++ b/core/tests/io_policy_test.cpp @@ -0,0 +1,108 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include + +// Libraries. +#include + +using join::IoDefaultPolicy; +using join::IoHybridPolicy; +using join::IoSqpollPolicy; +using join::has_spin; +using join::has_sqpoll; +using join::is_default; +using join::has_cq_entries; +using join::has_sq_thread_idle; +using join::has_sq_thread_cpu; + +/** + * @brief Test has_spin trait. + */ +TEST (IoPolicy, has_spin) +{ + ASSERT_FALSE (has_spin::value); + ASSERT_TRUE (has_spin::value); + ASSERT_TRUE (has_spin::value); +} + +/** + * @brief Test has_sqpoll trait. + */ +TEST (IoPolicy, has_sqpoll) +{ + ASSERT_FALSE (has_sqpoll::value); + ASSERT_FALSE (has_sqpoll::value); + ASSERT_TRUE (has_sqpoll::value); +} + +/** + * @brief Test is_default trait. + */ +TEST (IoPolicy, is_default) +{ + ASSERT_TRUE (is_default::value); + ASSERT_FALSE (is_default::value); + ASSERT_FALSE (is_default::value); +} + +/** + * @brief Test has_cq_entries trait. + */ +TEST (IoPolicy, has_cq_entries) +{ + ASSERT_FALSE (has_cq_entries::value); + ASSERT_FALSE (has_cq_entries::value); + ASSERT_FALSE (has_cq_entries::value); +} + +/** + * @brief Test has_sq_thread_idle trait. + */ +TEST (IoPolicy, has_sq_thread_idle) +{ + ASSERT_FALSE (has_sq_thread_idle::value); + ASSERT_FALSE (has_sq_thread_idle::value); + ASSERT_TRUE (has_sq_thread_idle::value); +} + +/** + * @brief Test has_sq_thread_cpu trait. + */ +TEST (IoPolicy, has_sq_thread_cpu) +{ + ASSERT_FALSE (has_sq_thread_cpu::value); + ASSERT_FALSE (has_sq_thread_cpu::value); + ASSERT_TRUE (has_sq_thread_cpu::value); +} + +/** + * @brief main function. + */ +int main (int argc, char** argv) +{ + testing::InitGoogleTest (&argc, argv); + return RUN_ALL_TESTS (); +} diff --git a/core/tests/proactor_test.cpp b/core/tests/proactor_test.cpp index 2ad28954..a27c5751 100644 --- a/core/tests/proactor_test.cpp +++ b/core/tests/proactor_test.cpp @@ -152,143 +152,9 @@ int ProactorTest::_result = 0; char ProactorTest::_buf[256] = {}; /** - * @brief Test makeAccept. + * @brief Test stop. */ -TEST_F (ProactorTest, makeAccept) -{ - sockaddr_storage addr = {}; - socklen_t addrlen = sizeof (addr); - - auto op = - IoOperation::makeAccept (8, reinterpret_cast (&addr), &addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC, this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Accept)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.accept.addr, reinterpret_cast (&addr)); - ASSERT_EQ (op.data.accept.addrlen, &addrlen); - ASSERT_EQ (op.data.accept.flags, SOCK_NONBLOCK | SOCK_CLOEXEC); -} - -/** - * @brief Test makeConnect. - */ -TEST_F (ProactorTest, makeConnect) -{ - auto op = IoOperation::makeConnect (8, this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Connect)); - ASSERT_EQ (op.handler, this); -} - -/** - * @brief Test makeRead. - */ -TEST_F (ProactorTest, makeRead) -{ - auto op = IoOperation::makeRead (8, _buf, sizeof (_buf), this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Read)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.rw.buf, _buf); - ASSERT_EQ (op.data.rw.len, sizeof (_buf)); - ASSERT_EQ (op.data.rw.index, 0); - ASSERT_EQ (op.data.rw.fixed, false); -} - -/** - * @brief Test makeWrite. - */ -TEST_F (ProactorTest, makeWrite) -{ - auto op = IoOperation::makeWrite (8, _buf, sizeof (_buf), this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::Write)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.rw.buf, _buf); - ASSERT_EQ (op.data.rw.len, sizeof (_buf)); - ASSERT_EQ (op.data.rw.index, 0); - ASSERT_EQ (op.data.rw.fixed, false); -} - -/** - * @brief Test makeReadFixed. - */ -TEST_F (ProactorTest, makeReadFixed) -{ - auto op = IoOperation::makeReadFixed (8, _buf, sizeof (_buf), 6, this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::ReadFixed)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.rw.buf, _buf); - ASSERT_EQ (op.data.rw.len, sizeof (_buf)); - ASSERT_EQ (op.data.rw.index, 6); - ASSERT_EQ (op.data.rw.fixed, true); -} - -/** - * @brief Test makeWriteFixed. - */ -TEST_F (ProactorTest, makeWriteFixed) -{ - auto op = IoOperation::makeWriteFixed (8, _buf, sizeof (_buf), 6, this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::WriteFixed)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.rw.buf, _buf); - ASSERT_EQ (op.data.rw.len, sizeof (_buf)); - ASSERT_EQ (op.data.rw.index, 6); - ASSERT_EQ (op.data.rw.fixed, true); -} - -/** - * @brief Test makeRecvmsg. - */ -TEST_F (ProactorTest, makeRecvmsg) -{ - iovec iov = {}; - msghdr msg = {}; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - - auto op = IoOperation::makeRecvmsg (8, &msg, MSG_DONTWAIT, this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::RecvMsg)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.msg.msg, &msg); - ASSERT_EQ (op.data.msg.flags, MSG_DONTWAIT); -} - -/** - * @brief Test makeSendmsg. - */ -TEST_F (ProactorTest, makeSendmsg) -{ - const char* payload = "makeSendmsg"; - iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; - msghdr msg = {}; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - - auto op = IoOperation::makeSendmsg (8, &msg, MSG_DONTWAIT, this); - - ASSERT_EQ (op.fd, 8); - ASSERT_EQ (op.code, static_cast (IoOperation::Opcode::SendMsg)); - ASSERT_EQ (op.handler, this); - ASSERT_EQ (op.data.msg.msg, &msg); - ASSERT_EQ (op.data.msg.flags, MSG_DONTWAIT); -} - -/** - * @brief Test flush. - */ -TEST_F (ProactorTest, flush) +TEST_F (ProactorTest, stop) { Proactor proactor; Thread th ([&proactor] () { @@ -330,10 +196,6 @@ TEST_F (ProactorTest, submit) ASSERT_EQ (proactor.submit (&op1), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); - op1 = IoOperation::makeRead (2048, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op1), -1); - ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); - ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); @@ -345,9 +207,11 @@ TEST_F (ProactorTest, submit) op1.state = IoOperation::State::Idle; ASSERT_EQ (proactor.submit (&op1), 0) << join::lastError.message (); +#ifndef JOIN_HAS_IO_URING auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); ASSERT_EQ (proactor.submit (&op2), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); +#endif ASSERT_EQ (proactor.cancel (&op1), 0) << join::lastError.message (); { @@ -380,10 +244,6 @@ TEST_F (ProactorTest, cancel) ASSERT_EQ (proactor.cancel (&op1), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); - op1 = IoOperation::makeRead (2048, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op1), -1); - ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); - ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); @@ -412,6 +272,88 @@ TEST_F (ProactorTest, cancel) th.join (); } +#ifdef JOIN_HAS_IO_URING +/** + * @brief Test flush. + */ +TEST_F (ProactorTest, flush) +{ + Proactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op, false), 0) << join::lastError.message (); + ASSERT_EQ (proactor.flush (), 0) << join::lastError.message (); + + ASSERT_EQ (_client.writeExactly ("flush", strlen ("flush"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "flush"); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test SQE chaining. + */ +TEST_F (ProactorTest, chain) +{ + Proactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this); + auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + writeOp.linked = true; + + ASSERT_EQ (proactor.submit (&writeOp, false), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&readOp, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &writeOp && _result == 4; + })); + _op = nullptr; + _result = 0; + } + + char tmp[4]; + ASSERT_EQ (_client.readExactly (tmp, 4, _timeout), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("pong", 4, _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &readOp && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "pong"); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} +#endif + #ifdef JOIN_HAS_NUMA /** * @brief Test mbind. @@ -434,6 +376,50 @@ TEST_F (ProactorTest, mlock) ASSERT_EQ (proactor.mlock (), 0) << join::lastError.message (); } +/** + * @brief Test isRunning. + */ +TEST_F (ProactorTest, isRunning) +{ + Proactor proactor; + + ASSERT_FALSE (proactor.isRunning ()); + + Thread th ([&proactor] () { + proactor.run (); + }); + while (!proactor.isRunning ()) + { + } + + ASSERT_TRUE (proactor.isRunning ()); + + proactor.stop (); + th.join (); + + ASSERT_FALSE (proactor.isRunning ()); +} + +#ifdef JOIN_HAS_IO_URING +/** + * @brief Test registerBuffers and unregisterBuffers. + */ +TEST_F (ProactorTest, registerBuffers) +{ + Proactor proactor; + + std::vector buf (4096); + iovec iov = {buf.data (), buf.size ()}; + std::vector iovecs = {iov}; + + ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); + ASSERT_EQ (proactor.registerBuffers (iovecs), -1); + ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); + ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); +} +#endif + /** * @brief Test async accept. */ @@ -473,14 +459,13 @@ TEST_F (ProactorTest, asyncAccept) */ TEST_F (ProactorTest, asyncConnect) { + Tcp::Endpoint endpoint{_host, _port}; + ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); _client.setMode (Tcp::Socket::NonBlocking); - ASSERT_EQ (_client.connect ({_host, _port}), -1); - ASSERT_EQ (join::lastError, Errc::TemporaryError); - - auto op = IoOperation::makeConnect (_client.handle (), this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0); + auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); + ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); @@ -717,7 +702,7 @@ TEST_F (ProactorTest, onClose) { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &op && _result == -ECONNRESET; + return _op == &op && _result == 0; })); _op = nullptr; _result = 0; @@ -752,7 +737,7 @@ TEST_F (ProactorTest, onError) { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &op && _result == -EIO; + return _op == &op && _result == -ECONNRESET; })); _op = nullptr; _result = 0; diff --git a/core/tests/reactor_test.cpp b/core/tests/reactor_test.cpp index a2effe19..c68a9af2 100644 --- a/core/tests/reactor_test.cpp +++ b/core/tests/reactor_test.cpp @@ -312,6 +312,30 @@ TEST_F (ReactorTest, mlock) ASSERT_EQ (reactor.mlock (), 0) << join::lastError.message (); } +/** + * @brief Test isRunning. + */ +TEST_F (ReactorTest, isRunning) +{ + Reactor reactor; + + ASSERT_FALSE (reactor.isRunning ()); + + Thread th ([&reactor] () { + reactor.run (); + }); + while (!reactor.isRunning ()) + { + } + + ASSERT_TRUE (reactor.isRunning ()); + + reactor.stop (); + th.join (); + + ASSERT_FALSE (reactor.isRunning ()); +} + /** * @brief Test onReadable. */ diff --git a/core/tests/sqpoll_proactor_test.cpp b/core/tests/sqpoll_proactor_test.cpp new file mode 100644 index 00000000..2279a3d1 --- /dev/null +++ b/core/tests/sqpoll_proactor_test.cpp @@ -0,0 +1,752 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include +#include +#include + +// Libraries. +#include + +using join::Errc; +using join::Mutex; +using join::Condition; +using join::ScopedLock; +using join::Thread; +using join::SqpollProactor; +using join::SqpollProactorThread; +using join::IoOperation; +using join::CompletionHandler; +using join::Tcp; + +/** + * @brief Class used to test SqpollProactor. + */ +class SqpollProactorTest : public CompletionHandler, public ::testing::Test +{ +protected: + /** + * @brief Sets up the test fixture. + */ + void SetUp () override + { + ASSERT_EQ (_acceptor.create ({_host, _port}), 0) << join::lastError.message (); + } + + /** + * @brief Tears down the test fixture. + */ + void TearDown () override + { + _server.close (); + _client.close (); + _acceptor.close (); + } + + /** + * @brief method called when an operation completes. + * @param op completed operation. + * @param result bytes transferred or negative errno. + */ + void onComplete (IoOperation* op, int result) override + { + SqpollProactorThread::proactor ().submit (op, true); + SqpollProactorThread::proactor ().cancel (op, true); + + { + ScopedLock lock (_mut); + _result = result; + _op = op; + CompletionHandler::onComplete (op, result); + } + + _cond.signal (); + } + + /** + * @brief method called when an operation is cancelled. + * @param op cancelled operation. + * @param result negative errno. + */ + void onCancel (IoOperation* op, int result) override + { + SqpollProactorThread::proactor ().submit (op, true); + SqpollProactorThread::proactor ().cancel (op, true); + + { + ScopedLock lock (_mut); + _result = result; + _op = op; + CompletionHandler::onCancel (op, result); + } + + _cond.signal (); + } + + /// server acceptor. + static Tcp::Acceptor _acceptor; + + /// client socket. + static Tcp::Socket _client; + + /// server socket. + static Tcp::Socket _server; + + /// host. + static std::string _host; + + /// port. + static uint16_t _port; + + /// timeout. + static const int _timeout; + + /// condition variable. + static Condition _cond; + + /// condition mutex. + static Mutex _mut; + + /// last completed operation. + static IoOperation* _op; + + /// last operation result. + static int _result; + + /// read buffer. + static char _buf[256]; +}; + +Tcp::Acceptor SqpollProactorTest::_acceptor; +Tcp::Socket SqpollProactorTest::_client (Tcp::Socket::Blocking); +Tcp::Socket SqpollProactorTest::_server; +std::string SqpollProactorTest::_host = "127.0.0.1"; +uint16_t SqpollProactorTest::_port = 5001; +const int SqpollProactorTest::_timeout = 1000; +Condition SqpollProactorTest::_cond; +Mutex SqpollProactorTest::_mut; +IoOperation* SqpollProactorTest::_op = nullptr; +int SqpollProactorTest::_result = 0; +char SqpollProactorTest::_buf[256] = {}; + +/** + * @brief Test stop. + */ +TEST_F (SqpollProactorTest, stop) +{ + SqpollProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op, true), 0) << join::lastError.message (); + + proactor.stop (); + th.join (); + + { + ScopedLock lock (_mut); + ASSERT_EQ (_op, &op); + ASSERT_EQ (_result, -ECANCELED); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test submit. + */ +TEST_F (SqpollProactorTest, submit) +{ + SqpollProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (proactor.submit (nullptr, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); + + auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + op1.state = IoOperation::State::Submitted; + ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (join::lastError, Errc::OperationFailed); + + op1.state = IoOperation::State::Idle; + ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + +#ifndef JOIN_HAS_IO_URING + auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op2, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); +#endif + + ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op1 && _result == -ECANCELED; + })); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test cancel. + */ +TEST_F (SqpollProactorTest, cancel) +{ + SqpollProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (proactor.cancel (nullptr, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); + + auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (join::lastError, Errc::OperationFailed); + + ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + + auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + op2.state = IoOperation::State::Submitted; + ASSERT_EQ (proactor.cancel (&op2, true), -1); + ASSERT_EQ (join::lastError, Errc::InvalidParam); + + ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op1 && _result == -ECANCELED; + })); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test flush. + */ +TEST_F (SqpollProactorTest, flush) +{ + SqpollProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + ASSERT_EQ (proactor.submit (&op, false), 0) << join::lastError.message (); + ASSERT_EQ (proactor.flush (), 0) << join::lastError.message (); + + ASSERT_EQ (_client.writeExactly ("flush", strlen ("flush"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "flush"); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +/** + * @brief Test SQE chaining. + */ +TEST_F (SqpollProactorTest, chain) +{ + SqpollProactor proactor; + Thread th ([&proactor] () { + proactor.run (); + }); + + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this); + auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + writeOp.linked = true; + + ASSERT_EQ (proactor.submit (&writeOp, false), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&readOp, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &writeOp && _result == 4; + })); + _op = nullptr; + _result = 0; + } + + char tmp[4]; + ASSERT_EQ (_client.readExactly (tmp, 4, _timeout), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("pong", 4, _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &readOp && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "pong"); + _op = nullptr; + _result = 0; + } + + proactor.stop (); + th.join (); +} + +#ifdef JOIN_HAS_NUMA +/** + * @brief Test mbind. + */ +TEST_F (SqpollProactorTest, mbind) +{ + SqpollProactor proactor; + + ASSERT_EQ (proactor.mbind (0), 0) << join::lastError.message (); +} +#endif + +/** + * @brief Test mlock. + */ +TEST_F (SqpollProactorTest, mlock) +{ + SqpollProactor proactor; + + ASSERT_EQ (proactor.mlock (), 0) << join::lastError.message (); +} + +/** + * @brief Test isRunning. + */ +TEST_F (SqpollProactorTest, isRunning) +{ + SqpollProactor proactor; + + ASSERT_FALSE (proactor.isRunning ()); + + Thread th ([&proactor] () { + proactor.run (); + }); + while (!proactor.isRunning ()) + { + } + + ASSERT_TRUE (proactor.isRunning ()); + + proactor.stop (); + th.join (); + + ASSERT_FALSE (proactor.isRunning ()); +} + +#ifdef JOIN_HAS_IO_URING +/** + * @brief Test registerBuffers and unregisterBuffers. + */ +TEST_F (SqpollProactorTest, registerBuffers) +{ + SqpollProactor proactor; + + std::vector buf (4096); + iovec iov = {buf.data (), buf.size ()}; + std::vector iovecs = {iov}; + + ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); + ASSERT_EQ (proactor.registerBuffers (iovecs), -1); + ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); + ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); +} +#endif + +/** + * @brief Test async accept. + */ +TEST_F (SqpollProactorTest, asyncAccept) +{ + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + sockaddr_storage addr = {}; + socklen_t addrlen = sizeof (addr); + auto op = IoOperation::makeAccept (_acceptor.handle (), reinterpret_cast (&addr), &addrlen, + SOCK_NONBLOCK | SOCK_CLOEXEC, this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result >= 0; + })); + ::close (_result); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test async connect. + */ +TEST_F (SqpollProactorTest, asyncConnect) +{ + Tcp::Endpoint endpoint{_host, _port}; + + ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); + _client.setMode (Tcp::Socket::NonBlocking); + + auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } + + _client.setMode (Tcp::Socket::Blocking); +} + +/** + * @brief Test async read. + */ +TEST_F (SqpollProactorTest, asyncRead) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRead", strlen ("asyncRead"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRead"); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test async write. + */ +TEST_F (SqpollProactorTest, asyncWrite) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + const char* msg = "asyncWrite"; + auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (_result, static_cast (strlen (msg))); + _op = nullptr; + _result = 0; + } + + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async read and write. + */ +TEST_F (SqpollProactorTest, asyncReadWrite) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + const char* msg = "asyncReadWrite"; + auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + auto writeOp = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&readOp, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&writeOp, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &writeOp && _result > 0; + })); + ASSERT_EQ (_result, static_cast (strlen (msg))); + _op = nullptr; + _result = 0; + } + + ASSERT_EQ (_client.writeExactly (msg, strlen (msg), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &readOp && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), msg); + _op = nullptr; + _result = 0; + } + + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async recvmsg. + */ +TEST_F (SqpollProactorTest, asyncRecvmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + iovec iov = {.iov_base = _buf, .iov_len = sizeof (_buf)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeRecvmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRecvmsg", strlen ("asyncRecvmsg"), _timeout), 0) + << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRecvmsg"); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test async sendmsg. + */ +TEST_F (SqpollProactorTest, asyncSendmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + const char* payload = "asyncSendmsg"; + iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (_result, static_cast (strlen (payload))); + _op = nullptr; + _result = 0; + } + + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); +} + +/** + * @brief Test onClose. + */ +TEST_F (SqpollProactorTest, onClose) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + _client.close (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief Test onError. + */ +TEST_F (SqpollProactorTest, onError) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + linger sl{.l_onoff = 1, .l_linger = 0}; + ASSERT_EQ (setsockopt (_client.handle (), SOL_SOCKET, SO_LINGER, &sl, sizeof (sl)), 0) << strerror (errno); + _client.close (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == -ECONNRESET; + })); + _op = nullptr; + _result = 0; + } +} + +/** + * @brief main function. + */ +int main (int argc, char** argv) +{ + testing::InitGoogleTest (&argc, argv); + return RUN_ALL_TESTS (); +} From a2b23ebb56be8f7bf0d4e024ac55fc4f7a514fdb Mon Sep 17 00:00:00 2001 From: mrabine Date: Fri, 29 May 2026 18:06:24 +0200 Subject: [PATCH 5/7] coverage --- core/CMakeLists.txt | 4 +- core/include/join/io_operation.hpp | 61 +- core/include/join/proactor.hpp | 135 ++-- core/include/join/proactor_epoll.inl | 526 ------------- core/include/join/proactor_epoll_impl.hpp | 594 ++++++++++++++ core/include/join/proactor_uring.inl | 748 ------------------ core/include/join/proactor_uring_impl.hpp | 902 ++++++++++++++++++++++ core/src/io_operation.cpp | 31 +- core/tests/hybrid_proactor_test.cpp | 255 +++--- core/tests/proactor_test.cpp | 261 ++++--- core/tests/sqpoll_proactor_test.cpp | 255 +++--- 11 files changed, 2141 insertions(+), 1631 deletions(-) delete mode 100644 core/include/join/proactor_epoll.inl create mode 100644 core/include/join/proactor_epoll_impl.hpp delete mode 100644 core/include/join/proactor_uring.inl create mode 100644 core/include/join/proactor_uring_impl.hpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 2953241d..5809a341 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -44,9 +44,9 @@ set(PUBLIC_HEADERS ) if(JOIN_ENABLE_IO_URING) - list(APPEND PUBLIC_HEADERS include/join/io_policy.hpp include/join/proactor_uring.inl) + list(APPEND PUBLIC_HEADERS include/join/io_policy.hpp include/join/proactor_uring_impl.hpp) else() - list(APPEND PUBLIC_HEADERS include/join/proactor_epoll.inl) + list(APPEND PUBLIC_HEADERS include/join/proactor_epoll_impl.hpp) endif() set(SOURCES diff --git a/core/include/join/io_operation.hpp b/core/include/join/io_operation.hpp index ecb43035..99b17cc2 100644 --- a/core/include/join/io_operation.hpp +++ b/core/include/join/io_operation.hpp @@ -114,6 +114,8 @@ namespace join /** * @brief build a connect operation. * @param fd socket file descriptor. + * @param addr remote address. + * @param addrlen remote address length. * @param handler handler to notify on completion. * @return initialized IoOperation. */ @@ -131,7 +133,7 @@ namespace join /// buffer. void* buf; - /// number of bytes to proceed. + /// number of bytes to transfer. unsigned long len; /// registered buffer index (ignored with reactor backend). @@ -147,9 +149,11 @@ namespace join * @param buf destination buffer. * @param len number of bytes to read. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept; + static IoOperation makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler, + bool linked = false) noexcept; /** * @brief build a regular write operation. @@ -157,33 +161,37 @@ namespace join * @param buf source buffer. * @param len number of bytes to write. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept; + static IoOperation makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler, + bool linked = false) noexcept; /** - * @brief build a regular read operation. + * @brief build a fixed-buffer read operation. * @param fd file descriptor to read from. * @param buf destination buffer. * @param len number of bytes to read. * @param index registered buffer index. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept; + static IoOperation makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, CompletionHandler* handler, + bool linked = false) noexcept; /** - * @brief build a regular write operation. + * @brief build a fixed-buffer write operation. * @param fd file descriptor to write to. * @param buf source buffer. * @param len number of bytes to write. * @param index registered buffer index. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ static IoOperation makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept; + CompletionHandler* handler, bool linked = false) noexcept; /** * @brief payload for sendmsg / recvmsg. @@ -206,9 +214,11 @@ namespace join * @param msg message header. * @param flags recv flags. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeRecvmsg (int fd, msghdr* msg, int flags, CompletionHandler* handler) noexcept; + static IoOperation makeRecvmsg (int fd, msghdr* msg, int flags, CompletionHandler* handler, + bool linked = false) noexcept; /** * @brief build a send-message operation. @@ -216,9 +226,11 @@ namespace join * @param msg message header. * @param flags send flags. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeSendmsg (int fd, const msghdr* msg, int flags, CompletionHandler* handler) noexcept; + static IoOperation makeSendmsg (int fd, const msghdr* msg, int flags, CompletionHandler* handler, + bool linked = false) noexcept; /** * @brief payload for recv / send. @@ -231,7 +243,7 @@ namespace join /// buffer. void* buf; - /// number of bytes to proceed. + /// number of bytes to transfer. unsigned long len; /// send / recv flags. @@ -239,25 +251,30 @@ namespace join }; /** - * @brief build a receive-message operation. + * @brief build a receive operation. * @param fd socket file descriptor. - * @param msg message header. + * @param buf destination buffer. + * @param len number of bytes to receive. * @param flags recv flags. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeRecv (int fd, void* buf, uint32_t len, int flags, CompletionHandler* handler) noexcept; + static IoOperation makeRecv (int fd, void* buf, uint32_t len, int flags, CompletionHandler* handler, + bool linked = false) noexcept; /** - * @brief build a send-message operation. + * @brief build a send operation. * @param fd socket file descriptor. - * @param msg message header. + * @param buf source buffer. + * @param len number of bytes to send. * @param flags send flags. * @param handler handler to notify on completion. + * @param linked link this SQE to the next one (io_uring only). * @return initialized IoOperation. */ - static IoOperation makeSend (int fd, const void* buf, uint32_t len, int flags, - CompletionHandler* handler) noexcept; + static IoOperation makeSend (int fd, const void* buf, uint32_t len, int flags, CompletionHandler* handler, + bool linked = false) noexcept; union Data { @@ -280,13 +297,11 @@ namespace join /// operation state. State state = State::Idle; -#ifdef JOIN_HAS_IO_URING - /// index of this operation in the proactor pending ops vector. - uint32_t slot = 0; + /// index of this operation in the proactor pending ops (io_uring only). + uint32_t index = 0; - /// link this SQE to the next one (next executes only if this succeeds). + /// link this SQE to the next one (next executes only if this succeeds, io_uring only). bool linked = false; -#endif /// handler to dispatch to on completion. CompletionHandler* handler = nullptr; diff --git a/core/include/join/proactor.hpp b/core/include/join/proactor.hpp index 5c1f4a86..767a45d2 100644 --- a/core/include/join/proactor.hpp +++ b/core/include/join/proactor.hpp @@ -73,6 +73,14 @@ namespace join */ class join::CompletionHandler { + /// friendship with proactor. +#ifdef JOIN_HAS_IO_URING + template + friend class join::BasicProactor; +#else + friend class join::BasicProactor; +#endif + public: /** * @brief create instance. @@ -108,7 +116,7 @@ class join::CompletionHandler /** * @brief destroy instance. */ - ~CompletionHandler () = default; + virtual ~CompletionHandler () = default; protected: /** @@ -130,14 +138,6 @@ class join::CompletionHandler { // do nothing. } - - /// friendship with proactor. -#ifdef JOIN_HAS_IO_URING - template - friend class join::BasicProactor; -#else - friend class join::BasicProactor; -#endif }; /** @@ -152,7 +152,7 @@ class join::BasicProactor : public join::EventHandler { public: /** - * @brief construct with the given ring / queue size. + * @brief initialize the proactor and its I/O backend. */ explicit BasicProactor (); @@ -191,27 +191,27 @@ class join::BasicProactor : public join::EventHandler * @brief submit an asynchronous operation to the proactor. * @param op operation to submit. * @param flush call io_uring_submit after pushing the SQE if true (io_uring only). - * @param sync wait for submission acknowledgment if true. + * @param sync wait for submission acknowledgment if true (default: false). * @return 0 on success, -1 on failure. */ - int submit (IoOperation* op, bool flush = false, bool sync = true) noexcept; + int submit (IoOperation* op, bool flush = false, bool sync = false) noexcept; /** * @brief cancel an in-flight operation. * @param op operation to cancel. * @param flush call io_uring_submit after pushing the cancel SQE if true (io_uring only). - * @param sync block until the cancellation has been acknowledged. + * @param sync wait for cancellation acknowledgment if true (default: false). * @return 0 on success, -1 on failure (lastError set). */ - int cancel (IoOperation* op, bool flush = false, bool sync = true) noexcept; + int cancel (IoOperation* op, bool flush = false, bool sync = false) noexcept; #ifdef JOIN_HAS_IO_URING /** * @brief flush pending submissions to the kernel. - * @param sync block until the flush has been acknowledged if true. + * @param sync wait for flush acknowledgment if true (default: false). * @return 0 on success, -1 on failure (lastError set). */ - int flush (bool sync = true) noexcept; + int flush (bool sync = false) noexcept; #endif /** @@ -231,18 +231,18 @@ class join::BasicProactor : public join::EventHandler * @param iovecs list of buffers to register. * @return 0 on success, -1 on failure. */ - int registerBuffers (const std::vector& iovecs); + int registerBuffers (const std::vector& iovecs) noexcept; /** * @brief unregister previously registered fixed buffers from the io_uring instance. * @return 0 on success, -1 on failure. */ - int unregisterBuffers (); + int unregisterBuffers () noexcept; #endif #ifdef JOIN_HAS_NUMA /** - * @brief bind reactor command queue memory to a NUMA node. + * @brief bind proactor command queue memory to a NUMA node. * @param numa NUMA node ID. * @return 0 on success, -1 on failure. */ @@ -250,7 +250,7 @@ class join::BasicProactor : public join::EventHandler #endif /** - * @brief lock reactor command queue memory in RAM. + * @brief lock proactor command queue memory in RAM. * @return 0 on success, -1 on failure. */ int mlock () const noexcept; @@ -267,17 +267,17 @@ class join::BasicProactor : public join::EventHandler */ bool isProactorThread () const noexcept; -protected: +private: #ifdef JOIN_HAS_IO_URING /** - * @brief init wakeup eventfd descriptor. - * @return eventfd descriptor. + * @brief create the wakeup eventfd descriptor. + * @return eventfd descriptor on success, -1 on failure. */ int initWakeup (std::true_type) noexcept; /** - * @brief init wakeup eventfd descriptor. - * @return eventfd descriptor. + * @brief no-op stub; returns -1 for policies that do not use a wakeup eventfd. + * @return -1. */ int initWakeup (std::false_type) noexcept; @@ -287,7 +287,7 @@ class join::BasicProactor : public join::EventHandler void initWakeupOp (std::true_type) noexcept; /** - * @brief init wakeup eventfd descriptor. + * @brief no-op stub for policies that do not use a wakeup operation. */ void initWakeupOp (std::false_type) noexcept; @@ -348,7 +348,7 @@ class join::BasicProactor : public join::EventHandler { CommandType type; /**< command type. */ IoOperation* op; /**< target operation, or nullptr for Stop/Flush. */ - bool flush; /**< call io_uring_submit after processing. */ + bool flush; /**< if true, call io_uring_submit after processing (io_uring only). */ std::atomic* done; /**< set to true when the command is processed. */ std::error_code* errc; /**< filled with the error code on failure. */ }; @@ -362,14 +362,14 @@ class join::BasicProactor : public join::EventHandler #ifdef JOIN_HAS_IO_URING /** - * @brief write command to queue and wake dispatcher. + * @brief push command to queue and write to the wakeup eventfd to wake the dispatcher. * @param cmd command to write. * @return 0 on success, -1 on failure. */ int writeCommand (const Command& cmd, std::true_type) noexcept; /** - * @brief write command to queue and wake dispatcher. + * @brief push command to queue; the polling event loop drains it without a wakeup signal. * @param cmd command to write. * @return 0 on success, -1 on failure. */ @@ -388,18 +388,25 @@ class join::BasicProactor : public join::EventHandler void processCommand (const Command& cmd) noexcept; /** - * @brief submit op directly onto the ring. + * @brief submit an operation directly to the backend. * @param op operation to submit. + * @param flush if true, flush pending submissions to the kernel after submitting (io_uring only). * @return 0 on success, -1 on failure. */ - int submitOperation (IoOperation* op, bool flush = true) noexcept; + int submitOperation (IoOperation* op, bool flush) noexcept; /** - * @brief cancel op directly on the ring. + * @brief cancel an in-flight operation directly in the backend. * @param op operation to cancel. + * @param flush if true, flush pending submissions to the kernel after cancelling (io_uring only). * @return 0 on success, -1 on failure. */ - int cancelOperation (IoOperation* op, bool flush = true) noexcept; + int cancelOperation (IoOperation* op, bool flush) noexcept; + + /** + * @brief cancel all in-flight operations. + */ + void cancelAllOperations () noexcept; /** * @brief dispatch completion callback and reset operation state. @@ -460,7 +467,7 @@ class join::BasicProactor : public join::EventHandler void eventLoop () noexcept; /** - * @brief run the epoll backend event loop until stop() is called. + * @brief run the default io_uring event loop (blocking wait with eventfd wakeup) until stop() is called. */ void eventLoop (std::false_type, std::false_type) noexcept; @@ -477,7 +484,7 @@ class join::BasicProactor : public join::EventHandler /** * @brief return true if opcode requires EPOLLOUT. * @param code raw opcode value. - * @return true for Write, WriteFixed, SendMsg. + * @return true for Connect, Write, WriteFixed, SendMsg, Send. */ static bool isWriteOp (uint8_t code) noexcept; @@ -492,25 +499,25 @@ class join::BasicProactor : public join::EventHandler * @brief method called when data are ready to be read on handle. * @param fd file descriptor. */ - void onReadable (int fd) override; + void onReadable (int fd) noexcept override; /** * @brief method called when data are ready to be written on handle. * @param fd file descriptor. */ - void onWriteable (int fd) override; + void onWriteable (int fd) noexcept override; /** * @brief method called when handle was closed by the peer. * @param fd file descriptor. */ - void onClose (int fd) override; + void onClose (int fd) noexcept override; /** * @brief method called when an error occurred on handle. * @param fd file descriptor. */ - void onError (int fd) override; + void onError (int fd) noexcept override; #endif /// command queue size. @@ -519,8 +526,8 @@ class join::BasicProactor : public join::EventHandler /// command queue. LocalMem::Mpsc::Queue _commands; - /// stop waiter: non-null while a sync stop() is in progress. - std::atomic*> _stopDone{nullptr}; + /// set to true while a sync stop() is in progress. + std::atomic _stopping{false}; /// eventfd descriptor. int _wakeup = -1; @@ -558,6 +565,10 @@ class join::BasicProactor : public join::EventHandler #endif }; +// ========================================================================= +// CLASS : BasicProactor +// METHOD : submit +// ========================================================================= #ifdef JOIN_HAS_IO_URING template int join::BasicProactor::submit (IoOperation* op, bool flush, bool sync) noexcept @@ -573,7 +584,7 @@ inline int join::BasicProactor::submit (IoOperation* op, bool flush, bool sync) std::atomic done{false}, *pdone = nullptr; std::error_code errc, *perrc = nullptr; - if (JOIN_LIKELY (sync)) + if (JOIN_UNLIKELY (sync)) { pdone = &done; perrc = &errc; @@ -584,7 +595,7 @@ inline int join::BasicProactor::submit (IoOperation* op, bool flush, bool sync) return -1; // LCOV_EXCL_LINE } - if (JOIN_LIKELY (sync)) + if (JOIN_UNLIKELY (sync)) { Backoff backoff; while (!done.load (std::memory_order_acquire)) @@ -602,6 +613,10 @@ inline int join::BasicProactor::submit (IoOperation* op, bool flush, bool sync) return 0; } +// ========================================================================= +// CLASS : BasicProactor +// METHOD : cancel +// ========================================================================= #ifdef JOIN_HAS_IO_URING template int join::BasicProactor::cancel (IoOperation* op, bool flush, bool sync) noexcept @@ -617,7 +632,7 @@ inline int join::BasicProactor::cancel (IoOperation* op, bool flush, bool sync) std::atomic done{false}, *pdone = nullptr; std::error_code errc, *perrc = nullptr; - if (JOIN_LIKELY (sync)) + if (JOIN_UNLIKELY (sync)) { pdone = &done; perrc = &errc; @@ -628,7 +643,7 @@ inline int join::BasicProactor::cancel (IoOperation* op, bool flush, bool sync) return -1; // LCOV_EXCL_LINE } - if (JOIN_LIKELY (sync)) + if (JOIN_UNLIKELY (sync)) { Backoff backoff; while (!done.load (std::memory_order_acquire)) @@ -646,6 +661,10 @@ inline int join::BasicProactor::cancel (IoOperation* op, bool flush, bool sync) return 0; } +// ========================================================================= +// CLASS : BasicProactor +// METHOD : dispatchOperation +// ========================================================================= #ifdef JOIN_HAS_IO_URING template void join::BasicProactor::dispatchOperation (IoOperation* op, int result, bool cancelled) noexcept @@ -655,10 +674,10 @@ inline void join::BasicProactor::dispatchOperation (IoOperation* op, int result, { if (JOIN_UNLIKELY (op == nullptr)) { - return; + return; // LCOV_EXCL_LINE } - if (op->handler) + if (JOIN_LIKELY (op->handler)) { if (cancelled) { @@ -674,9 +693,9 @@ inline void join::BasicProactor::dispatchOperation (IoOperation* op, int result, } #ifdef JOIN_HAS_IO_URING -#include "proactor_uring.inl" +#include "proactor_uring_impl.hpp" #else -#include "proactor_epoll.inl" +#include "proactor_epoll_impl.hpp" #endif /** @@ -691,8 +710,8 @@ class join::BasicProactorThread { public: /** - * @brief get the global Proactor instance. - * @return reference to the singleton Proactor. + * @brief get the Proactor instance owned by the singleton ProactorThread. + * @return reference to the Proactor. */ #ifdef JOIN_HAS_IO_URING static BasicProactor& proactor () @@ -717,7 +736,7 @@ class join::BasicProactorThread * @brief get proactor thread affinity. * @return core index, or -1 if not pinned. */ - static int affinity () + static int affinity () noexcept { return instance ()._dispatcher.affinity (); } @@ -736,7 +755,7 @@ class join::BasicProactorThread * @brief get proactor thread scheduling priority. * @return current priority. */ - static int priority () + static int priority () noexcept { return instance ()._dispatcher.priority (); } @@ -745,28 +764,28 @@ class join::BasicProactorThread * @brief get the native handle of the proactor thread. * @return pthread_t handle. */ - static pthread_t handle () + static pthread_t handle () noexcept { return instance ()._dispatcher.handle (); } #ifdef JOIN_HAS_NUMA /** - * @brief bind reactor command queue memory to a NUMA node. + * @brief bind proactor command queue memory to a NUMA node. * @param numa NUMA node ID. * @return 0 on success, -1 on failure. */ - static int mbind (int numa) + static int mbind (int numa) noexcept { return instance ()._proactor.mbind (numa); } #endif /** - * @brief lock reactor command queue memory in RAM. + * @brief lock proactor command queue memory in RAM. * @return 0 on success, -1 on failure. */ - static int mlock () + static int mlock () noexcept { return instance ()._proactor.mlock (); } diff --git a/core/include/join/proactor_epoll.inl b/core/include/join/proactor_epoll.inl deleted file mode 100644 index 66725d90..00000000 --- a/core/include/join/proactor_epoll.inl +++ /dev/null @@ -1,526 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2026 Mathieu Rabine - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -namespace join -{ - inline BasicProactor::BasicProactor () - : _commands (_queueSize) - , _wakeup (eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC)) - , _readOps (256, nullptr) - , _writeOps (256, nullptr) - { - if (_wakeup == -1) - { - throw std::system_error (errno, std::system_category (), "eventfd failed"); // LCOV_EXCL_LINE - } - - _reactor.addHandler (_wakeup, this, true, false, false); - } - - inline BasicProactor::~BasicProactor () noexcept - { - stop (); - - ::close (_wakeup); - } - - inline void BasicProactor::run () - { - _reactor.run (); - } - - inline void BasicProactor::stop (bool sync) noexcept - { - if (isProactorThread ()) - { - for (size_t fd = 0; fd < _readOps.size (); ++fd) - { - IoOperation* rOp = std::exchange (_readOps[fd], nullptr); - IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); - if (rOp || wOp) - { - _reactor.delHandler (fd); - } - dispatchOperation (rOp, -ECANCELED, true); - dispatchOperation (wOp, -ECANCELED, true); - } - _reactor.stop (false); - return; - } - - if (!_reactor.isRunning ()) - { - return; - } - - std::atomic done{false}; - - if (JOIN_LIKELY (sync)) - { - std::atomic* expected = nullptr; - if (!_stopDone.compare_exchange_strong (expected, &done, std::memory_order_acq_rel)) - { - Backoff backoff; - while (isRunning ()) - { - backoff (); - } - return; - } - } - - writeCommand ({CommandType::Stop, nullptr, sync, sync ? &done : nullptr, nullptr}); - - if (JOIN_LIKELY (sync)) - { - Backoff backoff; - while (!done.load (std::memory_order_acquire)) - { - backoff (); - } - _stopDone.store (nullptr, std::memory_order_release); - } - - _reactor.stop (sync); - } - -#ifdef JOIN_HAS_NUMA - inline int BasicProactor::mbind (int numa) const noexcept - { - if (_commands.mbind (numa) == -1) - { - return -1; - } - - return _reactor.mbind (numa); - } -#endif - - inline int BasicProactor::mlock () const noexcept - { - if (_commands.mlock () == -1) - { - return -1; - } - - return _reactor.mlock (); - } - - inline bool BasicProactor::isRunning () const noexcept - { - return _reactor.isRunning (); - } - - inline bool BasicProactor::isProactorThread () const noexcept - { - return _reactor.isReactorThread (); - } - - inline int join::BasicProactor::writeCommand (const Command& cmd) noexcept - { - if (JOIN_UNLIKELY (_commands.push (cmd) == -1)) - { - return -1; // LCOV_EXCL_LINE - } - - uint64_t value = 1; - if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1)) - { - // LCOV_EXCL_START - lastError = std::error_code (errno, std::system_category ()); - return -1; - // LCOV_EXCL_STOP - } - - return 0; - } - - inline void BasicProactor::readCommands () noexcept - { - uint64_t count; - if (JOIN_UNLIKELY (::read (_wakeup, &count, sizeof (count)) == -1)) - { - return; // LCOV_EXCL_LINE - } - - Command cmd; - while (_commands.tryPop (cmd) == 0) - { - processCommand (cmd); - } - } - - inline void BasicProactor::processCommand (const Command& cmd) noexcept - { - int err = 0; - - switch (cmd.type) - { - case CommandType::Submit: - err = submitOperation (cmd.op); - break; - - case CommandType::Cancel: - err = cancelOperation (cmd.op); - break; - - case CommandType::Stop: - { - for (size_t fd = 0; fd < _readOps.size (); ++fd) - { - IoOperation* rOp = std::exchange (_readOps[fd], nullptr); - IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); - if (rOp || wOp) - { - _reactor.delHandler (fd); - } - dispatchOperation (rOp, -ECANCELED, true); - dispatchOperation (wOp, -ECANCELED, true); - } - break; - } - - default: - break; - } - - if (JOIN_UNLIKELY (cmd.done)) - { - if (cmd.errc && (err != 0)) - { - *cmd.errc = lastError; - } - cmd.done->store (true, std::memory_order_release); - } - } - - inline int BasicProactor::submitOperation (IoOperation* op, [[maybe_unused]] bool flush) noexcept - { - if (JOIN_UNLIKELY (op == nullptr)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (JOIN_UNLIKELY (op->fd () < 0)) - { - lastError = std::make_error_code (std::errc::bad_file_descriptor); - return -1; - } - - if (JOIN_UNLIKELY (op->state != IoOperation::State::Idle)) - { - lastError = make_error_code (Errc::OperationFailed); - return -1; - } - - if (static_cast (op->code) == IoOperation::Opcode::Connect) - { - if (::connect (op->data.connect.fd, op->data.connect.addr, op->data.connect.addrlen) == -1 && - errno != EINPROGRESS) - { - lastError = std::error_code (errno, std::system_category ()); - return -1; - } - } - - if (JOIN_UNLIKELY (static_cast (op->fd ()) >= _readOps.size ())) - { - size_t newSize = static_cast (op->fd ()) + 1; - _readOps.resize (newSize, nullptr); - _writeOps.resize (newSize, nullptr); - } - - bool isWrite = isWriteOp (op->code); - - if (JOIN_UNLIKELY ((isWrite && (_writeOps[op->fd ()] != nullptr)) || - (!isWrite && (_readOps[op->fd ()] != nullptr)))) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (isWrite) - { - _writeOps[op->fd ()] = op; - } - else - { - _readOps[op->fd ()] = op; - } - - op->state = IoOperation::State::Submitted; - - return _reactor.addHandler (op->fd (), this, _readOps[op->fd ()] != nullptr, _writeOps[op->fd ()] != nullptr); - } - - inline int BasicProactor::cancelOperation (IoOperation* op, [[maybe_unused]] bool flush) noexcept - { - if (JOIN_UNLIKELY (op == nullptr)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (JOIN_UNLIKELY (op->fd () < 0)) - { - lastError = std::make_error_code (std::errc::bad_file_descriptor); - return -1; - } - - if (JOIN_UNLIKELY (op->state != IoOperation::State::Submitted)) - { - lastError = make_error_code (Errc::OperationFailed); - return -1; - } - - if (JOIN_UNLIKELY (static_cast (op->fd ()) >= _readOps.size ())) - { - lastError = std::make_error_code (std::errc::bad_file_descriptor); - return -1; - } - - op->state = IoOperation::State::Cancelling; - bool isWrite = isWriteOp (op->code); - - if (JOIN_UNLIKELY ((isWrite && (_writeOps[op->fd ()] != op)) || (!isWrite && (_readOps[op->fd ()] != op)))) - { - op->state = IoOperation::State::Submitted; - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (isWrite) - { - _writeOps[op->fd ()] = nullptr; - } - else - { - _readOps[op->fd ()] = nullptr; - } - - int ret = 0; - - if (_readOps[op->fd ()] == nullptr && _writeOps[op->fd ()] == nullptr) - { - ret = _reactor.delHandler (op->fd ()); - } - else - { - ret = - _reactor.addHandler (op->fd (), this, _readOps[op->fd ()] != nullptr, _writeOps[op->fd ()] != nullptr); - } - - if (op->handler) - { - op->handler->onCancel (op, -ECANCELED); - } - - op->state = IoOperation::State::Idle; - - return ret; - } - - inline void BasicProactor::endOperation (IoOperation* op, int result, bool cancelled) noexcept - { - if (JOIN_UNLIKELY (op == nullptr)) - { - return; - } - - int fd = op->fd (); - - if (JOIN_UNLIKELY (fd < 0 || static_cast (fd) >= _readOps.size ())) - { - return; // LCOV_EXCL_LINE - } - - if (isWriteOp (op->code)) - { - _writeOps[fd] = nullptr; - } - else - { - _readOps[fd] = nullptr; - } - - if (_readOps[fd] == nullptr && _writeOps[fd] == nullptr) - { - _reactor.delHandler (fd); - } - else - { - _reactor.addHandler (fd, this, _readOps[fd] != nullptr, _writeOps[fd] != nullptr); - } - - dispatchOperation (op, result, cancelled); - } - - inline bool BasicProactor::isWriteOp (uint8_t code) noexcept - { - return code == static_cast (IoOperation::Opcode::Connect) || - code == static_cast (IoOperation::Opcode::Write) || - code == static_cast (IoOperation::Opcode::WriteFixed) || - code == static_cast (IoOperation::Opcode::SendMsg) || - code == static_cast (IoOperation::Opcode::Send); - } - - inline int BasicProactor::executeOp (IoOperation* op) noexcept - { - for (;;) - { - switch (static_cast (op->code)) - { - case IoOperation::Opcode::Accept: - { - int fd = ::accept4 (op->data.accept.fd, op->data.accept.addr, op->data.accept.addrlen, - op->data.accept.flags); - if ((fd == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (fd == -1) ? -errno : fd; - } - - case IoOperation::Opcode::Connect: - { - int err = 0; - socklen_t len = sizeof (err); - if (::getsockopt (op->data.connect.fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1) - { - return -errno; - } - return err ? -err : 0; - } - - case IoOperation::Opcode::Read: - case IoOperation::Opcode::ReadFixed: - { - ssize_t n = ::read (op->data.rw.fd, op->data.rw.buf, op->data.rw.len); - if ((n == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (n == -1) ? -errno : static_cast (n); - } - - case IoOperation::Opcode::Write: - case IoOperation::Opcode::WriteFixed: - { - ssize_t n = ::write (op->data.rw.fd, op->data.rw.buf, op->data.rw.len); - if ((n == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (n == -1) ? -errno : static_cast (n); - } - - case IoOperation::Opcode::RecvMsg: - { - ssize_t n = ::recvmsg (op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); - if ((n == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (n == -1) ? -errno : static_cast (n); - } - - case IoOperation::Opcode::SendMsg: - { - ssize_t n = ::sendmsg (op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); - if ((n == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (n == -1) ? -errno : static_cast (n); - } - - case IoOperation::Opcode::Recv: - { - ssize_t n = ::recv (op->data.stream.fd, op->data.stream.buf, op->data.stream.len, - op->data.stream.flags); - if ((n == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (n == -1) ? -errno : static_cast (n); - } - - case IoOperation::Opcode::Send: - { - ssize_t n = ::send (op->data.stream.fd, op->data.stream.buf, op->data.stream.len, - op->data.stream.flags); - if ((n == -1) && (errno == EINTR)) - { - continue; // LCOV_EXCL_LINE - } - return (n == -1) ? -errno : static_cast (n); - } - - default: - return -EINVAL; - } - } - } - - inline void BasicProactor::onReadable (int fd) - { - if (JOIN_UNLIKELY (fd == _wakeup)) - { - readCommands (); - return; - } - - endOperation (_readOps[fd], executeOp (_readOps[fd]), false); - } - - inline void BasicProactor::onWriteable (int fd) - { - endOperation (_writeOps[fd], executeOp (_writeOps[fd]), false); - } - - inline void BasicProactor::onClose (int fd) - { - IoOperation* rOp = std::exchange (_readOps[fd], nullptr); - IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); - if (rOp || wOp) - { - _reactor.delHandler (fd); - } - dispatchOperation (rOp, 0, false); - dispatchOperation (wOp, 0, false); - } - - inline void BasicProactor::onError (int fd) - { - IoOperation* rOp = std::exchange (_readOps[fd], nullptr); - IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); - if (rOp || wOp) - { - _reactor.delHandler (fd); - } - dispatchOperation (rOp, -ECONNRESET, false); - dispatchOperation (wOp, -ECONNRESET, false); - } -} diff --git a/core/include/join/proactor_epoll_impl.hpp b/core/include/join/proactor_epoll_impl.hpp new file mode 100644 index 00000000..349a2c5e --- /dev/null +++ b/core/include/join/proactor_epoll_impl.hpp @@ -0,0 +1,594 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : BasicProactor +// ========================================================================= +inline join::BasicProactor::BasicProactor () +: _commands (_queueSize) +, _wakeup (eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC)) +, _readOps (256, nullptr) +, _writeOps (256, nullptr) +{ + if (_wakeup == -1) + { + throw std::system_error (errno, std::system_category (), "eventfd failed"); // LCOV_EXCL_LINE + } + + _reactor.addHandler (_wakeup, this, true, false, false); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : ~BasicProactor +// ========================================================================= +inline join::BasicProactor::~BasicProactor () noexcept +{ + stop (true); + + ::close (_wakeup); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : run +// ========================================================================= +inline void join::BasicProactor::run () +{ + _reactor.run (); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : stop +// ========================================================================= +inline void join::BasicProactor::stop (bool sync) noexcept +{ + if (isProactorThread ()) + { + cancelAllOperations (); + _reactor.stop (false); + return; + } + + if (!_reactor.isRunning ()) + { + return; + } + + std::atomic done{false}; + + if (JOIN_LIKELY (sync)) + { + bool expected = false; + if (!_stopping.compare_exchange_strong (expected, true, std::memory_order_acq_rel)) + { + Backoff backoff; + while (isRunning ()) + { + backoff (); + } + return; + } + } + + writeCommand ({CommandType::Stop, nullptr, sync, sync ? &done : nullptr, nullptr}); + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + _stopping.store (false, std::memory_order_release); + } + + _reactor.stop (sync); +} + +#ifdef JOIN_HAS_NUMA +// ========================================================================= +// CLASS : BasicProactor +// METHOD : mbind +// ========================================================================= +inline int join::BasicProactor::mbind (int numa) const noexcept +{ + if (_commands.mbind (numa) == -1) + { + return -1; + } + + return _reactor.mbind (numa); +} +#endif + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : mlock +// ========================================================================= +inline int join::BasicProactor::mlock () const noexcept +{ + if (_commands.mlock () == -1) + { + return -1; + } + + return _reactor.mlock (); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : isRunning +// ========================================================================= +inline bool join::BasicProactor::isRunning () const noexcept +{ + return _reactor.isRunning (); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : isProactorThread +// ========================================================================= +inline bool join::BasicProactor::isProactorThread () const noexcept +{ + return _reactor.isReactorThread (); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : writeCommand +// ========================================================================= +inline int join::BasicProactor::writeCommand (const Command& cmd) noexcept +{ + if (JOIN_UNLIKELY (_commands.push (cmd) == -1)) + { + return -1; // LCOV_EXCL_LINE + } + + uint64_t value = 1; + if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1)) + { + // LCOV_EXCL_START + lastError = std::error_code (errno, std::system_category ()); + return -1; + // LCOV_EXCL_STOP + } + + return 0; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : readCommands +// ========================================================================= +inline void join::BasicProactor::readCommands () noexcept +{ + uint64_t count; + if (JOIN_UNLIKELY (::read (_wakeup, &count, sizeof (count)) == -1)) + { + return; // LCOV_EXCL_LINE + } + + Command cmd; + while (_commands.tryPop (cmd) == 0) + { + processCommand (cmd); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : processCommand +// ========================================================================= +inline void join::BasicProactor::processCommand (const Command& cmd) noexcept +{ + int err = 0; + + switch (cmd.type) + { + case CommandType::Submit: + err = submitOperation (cmd.op, cmd.flush); + break; + + case CommandType::Cancel: + err = cancelOperation (cmd.op, cmd.flush); + break; + + case CommandType::Stop: + cancelAllOperations (); + break; + + default: + break; // LCOV_EXCL_LINE + } + + if (JOIN_UNLIKELY (cmd.done)) + { + if (cmd.errc && (err != 0)) + { + *cmd.errc = lastError; + } + cmd.done->store (true, std::memory_order_release); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : submitOperation +// ========================================================================= +inline int join::BasicProactor::submitOperation (IoOperation* op, [[maybe_unused]] bool flush) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Idle)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + if (JOIN_UNLIKELY (static_cast (op->code) == IoOperation::Opcode::Connect)) + { + if (JOIN_UNLIKELY (::connect (op->data.connect.fd, op->data.connect.addr, op->data.connect.addrlen) == -1 && + errno != EINPROGRESS)) + { + lastError = std::error_code (errno, std::system_category ()); + return -1; + } + } + + if (JOIN_UNLIKELY (static_cast (op->fd ()) >= _readOps.size ())) + { + size_t newSize = static_cast (op->fd ()) + 1; + _readOps.resize (newSize, nullptr); + _writeOps.resize (newSize, nullptr); + } + + bool isWrite = isWriteOp (op->code); + + if (JOIN_UNLIKELY ((isWrite && (_writeOps[op->fd ()] != nullptr)) || + (!isWrite && (_readOps[op->fd ()] != nullptr)))) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (isWrite) + { + _writeOps[op->fd ()] = op; + } + else + { + _readOps[op->fd ()] = op; + } + + op->state = IoOperation::State::Submitted; + + return _reactor.addHandler (op->fd (), this, _readOps[op->fd ()] != nullptr, _writeOps[op->fd ()] != nullptr); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : cancelOperation +// ========================================================================= +inline int join::BasicProactor::cancelOperation (IoOperation* op, [[maybe_unused]] bool flush) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Submitted)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + if (JOIN_UNLIKELY (static_cast (op->fd ()) >= _readOps.size ())) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + op->state = IoOperation::State::Cancelling; + bool isWrite = isWriteOp (op->code); + + if (JOIN_UNLIKELY ((isWrite && (_writeOps[op->fd ()] != op)) || (!isWrite && (_readOps[op->fd ()] != op)))) + { + op->state = IoOperation::State::Submitted; + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (isWrite) + { + _writeOps[op->fd ()] = nullptr; + } + else + { + _readOps[op->fd ()] = nullptr; + } + + int ret = 0; + + if (_readOps[op->fd ()] == nullptr && _writeOps[op->fd ()] == nullptr) + { + ret = _reactor.delHandler (op->fd ()); + } + else + { + ret = _reactor.addHandler (op->fd (), this, _readOps[op->fd ()] != nullptr, _writeOps[op->fd ()] != nullptr); + } + + dispatchOperation (op, -ECANCELED, true); + + return ret; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : cancelAllOperations +// ========================================================================= +inline void join::BasicProactor::cancelAllOperations () noexcept +{ + for (size_t fd = 0; fd < _readOps.size (); ++fd) + { + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (rOp || wOp) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, -ECANCELED, true); + dispatchOperation (wOp, -ECANCELED, true); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : endOperation +// ========================================================================= +inline void join::BasicProactor::endOperation (IoOperation* op, int result, bool cancelled) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + return; // LCOV_EXCL_LINE + } + + int fd = op->fd (); + + if (JOIN_UNLIKELY (fd < 0 || static_cast (fd) >= _readOps.size ())) + { + return; // LCOV_EXCL_LINE + } + + if (isWriteOp (op->code)) + { + _writeOps[fd] = nullptr; + } + else + { + _readOps[fd] = nullptr; + } + + if (_readOps[fd] == nullptr && _writeOps[fd] == nullptr) + { + _reactor.delHandler (fd); + } + else + { + _reactor.addHandler (fd, this, _readOps[fd] != nullptr, _writeOps[fd] != nullptr); + } + + dispatchOperation (op, result, cancelled); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : isWriteOp +// ========================================================================= +inline bool join::BasicProactor::isWriteOp (uint8_t code) noexcept +{ + return code == static_cast (IoOperation::Opcode::Connect) || + code == static_cast (IoOperation::Opcode::Write) || + code == static_cast (IoOperation::Opcode::WriteFixed) || + code == static_cast (IoOperation::Opcode::SendMsg) || + code == static_cast (IoOperation::Opcode::Send); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : executeOp +// ========================================================================= +inline int join::BasicProactor::executeOp (IoOperation* op) noexcept +{ + for (;;) + { + switch (static_cast (op->code)) + { + case IoOperation::Opcode::Accept: + { + int fd = ::accept4 (op->data.accept.fd, op->data.accept.addr, op->data.accept.addrlen, + op->data.accept.flags); + if (JOIN_UNLIKELY ((fd == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (fd == -1) ? -errno : fd; + } + + case IoOperation::Opcode::Connect: + { + int err = 0; + socklen_t len = sizeof (err); + if (JOIN_UNLIKELY (::getsockopt (op->data.connect.fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1)) + { + return -errno; + } + return JOIN_UNLIKELY (err) ? -err : 0; + } + + case IoOperation::Opcode::Read: + case IoOperation::Opcode::ReadFixed: + { + ssize_t n = ::read (op->data.rw.fd, op->data.rw.buf, op->data.rw.len); + if (JOIN_UNLIKELY ((n == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::Write: + case IoOperation::Opcode::WriteFixed: + { + ssize_t n = ::write (op->data.rw.fd, op->data.rw.buf, op->data.rw.len); + if (JOIN_UNLIKELY ((n == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::RecvMsg: + { + ssize_t n = ::recvmsg (op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + if (JOIN_UNLIKELY ((n == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::SendMsg: + { + ssize_t n = ::sendmsg (op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + if (JOIN_UNLIKELY ((n == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::Recv: + { + ssize_t n = + ::recv (op->data.stream.fd, op->data.stream.buf, op->data.stream.len, op->data.stream.flags); + if (JOIN_UNLIKELY ((n == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + case IoOperation::Opcode::Send: + { + ssize_t n = + ::send (op->data.stream.fd, op->data.stream.buf, op->data.stream.len, op->data.stream.flags); + if (JOIN_UNLIKELY ((n == -1) && (errno == EINTR))) + { + continue; // LCOV_EXCL_LINE + } + return (n == -1) ? -errno : static_cast (n); + } + + default: + return -EINVAL; // LCOV_EXCL_LINE + } + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : onReadable +// ========================================================================= +inline void join::BasicProactor::onReadable (int fd) noexcept +{ + if (JOIN_UNLIKELY (fd == _wakeup)) + { + readCommands (); + return; + } + + endOperation (_readOps[fd], executeOp (_readOps[fd]), false); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : onWriteable +// ========================================================================= +inline void join::BasicProactor::onWriteable (int fd) noexcept +{ + endOperation (_writeOps[fd], executeOp (_writeOps[fd]), false); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : onClose +// ========================================================================= +inline void join::BasicProactor::onClose (int fd) noexcept +{ + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (JOIN_LIKELY (rOp || wOp)) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, 0, false); + dispatchOperation (wOp, 0, false); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : onError +// ========================================================================= +inline void join::BasicProactor::onError (int fd) noexcept +{ + IoOperation* rOp = std::exchange (_readOps[fd], nullptr); + IoOperation* wOp = std::exchange (_writeOps[fd], nullptr); + if (JOIN_LIKELY (rOp || wOp)) + { + _reactor.delHandler (fd); + } + dispatchOperation (rOp, -ECONNRESET, false); + dispatchOperation (wOp, -ECONNRESET, false); +} diff --git a/core/include/join/proactor_uring.inl b/core/include/join/proactor_uring.inl deleted file mode 100644 index 0d25804c..00000000 --- a/core/include/join/proactor_uring.inl +++ /dev/null @@ -1,748 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2026 Mathieu Rabine - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -namespace join -{ - template - BasicProactor::BasicProactor () - : _commands (_queueSize) - , _wakeup (initWakeup (is_default{})) - { - static_assert (has_spin::value || !has_sqpoll::value, "spin resuired for sq poll policy"); - - io_uring_params params{}; - params.flags = Policy::flags; - initCqEntries (params, has_cq_entries{}); - initSqThreadIdle (params, has_sq_thread_idle{}); - initSqThreadCpu (params, has_sq_thread_cpu{}); - - if (io_uring_queue_init_params (Policy::sqEntries, &_ring, ¶ms) < 0) - { - // LCOV_EXCL_START - ::close (_wakeup); - throw std::system_error (errno, std::system_category (), "io_uring_queue_init failed"); - // LCOV_EXCL_STOP - } - - initWakeupOp (is_default{}); - _pendingOps.reserve (Policy::sqEntries); - } - - template - BasicProactor::~BasicProactor () noexcept - { - stop (); - - io_uring_queue_exit (&_ring); - - if (_wakeup != -1) - { - ::close (_wakeup); - } - } - - template - int BasicProactor::flush (bool sync) noexcept - { - if (isProactorThread ()) - { - if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) - { - lastError = std::error_code (errno, std::system_category ()); - return -1; - } - return 0; - } - - std::atomic done{false}, *pdone = nullptr; - std::error_code errc, *perrc = nullptr; - - if (JOIN_LIKELY (sync)) - { - pdone = &done; - perrc = &errc; - } - - if (JOIN_UNLIKELY (writeCommand ({CommandType::Flush, nullptr, false, pdone, perrc}) == -1)) - { - return -1; // LCOV_EXCL_LINE - } - - if (JOIN_LIKELY (sync)) - { - Backoff backoff; - while (!done.load (std::memory_order_acquire)) - { - backoff (); - } - - if (JOIN_UNLIKELY (errc)) - { - lastError = errc; - return -1; - } - } - - return 0; - } - - template - void BasicProactor::run () - { - _threadId.store (pthread_self (), std::memory_order_release); - - _running.store (true, std::memory_order_release); - eventLoop (); - - _threadId.store (_invalidThreadId, std::memory_order_release); - } - - template - void BasicProactor::stop (bool sync) noexcept - { - if (isProactorThread ()) - { - _running.store (false, std::memory_order_release); - for (IoOperation* op : _pendingOps) - { - cancelOperation (op, false); - } - eventLoop (); - return; - } - - if (!isRunning ()) - { - return; - } - - std::atomic done{false}; - - if (JOIN_LIKELY (sync)) - { - std::atomic* expected = nullptr; - if (!_stopDone.compare_exchange_strong (expected, &done, std::memory_order_acq_rel)) - { - Backoff backoff; - while (isRunning ()) - { - backoff (); - } - return; - } - } - - writeCommand ({CommandType::Stop, nullptr, sync, sync ? &done : nullptr, nullptr}); - - if (JOIN_LIKELY (sync)) - { - Backoff backoff; - while (!done.load (std::memory_order_acquire)) - { - backoff (); - } - _stopDone.store (nullptr, std::memory_order_release); - } - } - - template - int BasicProactor::registerBuffers (const std::vector& iovecs) - { - int ret = io_uring_register_buffers (&_ring, iovecs.data (), iovecs.size ()); - if (ret < 0) - { - lastError = std::error_code (-ret, std::system_category ()); - return -1; - } - - return 0; - } - - template - int BasicProactor::unregisterBuffers () - { - int ret = io_uring_unregister_buffers (&_ring); - if (ret < 0) - { - lastError = std::error_code (-ret, std::system_category ()); - return -1; - } - - return 0; - } - -#ifdef JOIN_HAS_NUMA - template - int BasicProactor::mbind (int numa) const noexcept - { - return _commands.mbind (numa); - } -#endif - - template - int BasicProactor::mlock () const noexcept - { - return _commands.mlock (); - } - - template - bool BasicProactor::isRunning () const noexcept - { - return _running.load (std::memory_order_acquire); - } - - template - bool BasicProactor::isProactorThread () const noexcept - { - return _threadId.load (std::memory_order_acquire) == pthread_self (); - } - - template - int BasicProactor::initWakeup (std::true_type) noexcept - { - return eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC); - } - - template - int BasicProactor::initWakeup (std::false_type) noexcept - { - return -1; - } - - template - void BasicProactor::initWakeupOp (std::true_type) noexcept - { - _wakeupOp = IoOperation::makeRead (_wakeup, &_wakeupBuf, sizeof (_wakeupBuf), nullptr); - } - - template - void BasicProactor::initWakeupOp (std::false_type) noexcept - { - // no-op. - } - - template - void BasicProactor::initCqEntries (io_uring_params&, std::false_type) noexcept - { - // no-op. - } - - template - void BasicProactor::initCqEntries (io_uring_params& params, std::true_type) noexcept - { - params.flags |= IORING_SETUP_CQSIZE; - params.cq_entries = Policy::cqEntries; - } - - template - void BasicProactor::initSqThreadIdle (io_uring_params&, std::false_type) noexcept - { - // no-op. - } - - template - void BasicProactor::initSqThreadIdle (io_uring_params& params, std::true_type) noexcept - { - params.sq_thread_idle = Policy::sqThreadIdle; - } - - template - void BasicProactor::initSqThreadCpu (io_uring_params&, std::false_type) noexcept - { - // no-op. - } - - template - void BasicProactor::initSqThreadCpu (io_uring_params& params, std::true_type) noexcept - { - params.sq_thread_cpu = Policy::sqThreadCpu; - } - - template - int BasicProactor::writeCommand (const Command& cmd) noexcept - { - return writeCommand (cmd, is_default{}); - } - - template - int BasicProactor::writeCommand (const Command& cmd, std::true_type) noexcept - { - if (JOIN_UNLIKELY (_commands.push (cmd) == -1)) - { - return -1; - } - - uint64_t value = 1; - if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1)) - { - lastError = std::error_code (errno, std::system_category ()); - return -1; - } - - return 0; - } - - template - int BasicProactor::writeCommand (const Command& cmd, std::false_type) noexcept - { - return _commands.push (cmd); - } - - template - void BasicProactor::readCommands () noexcept - { - Command cmd; - while (_commands.tryPop (cmd) == 0) - { - processCommand (cmd); - } - } - - template - void BasicProactor::processCommand (const Command& cmd) noexcept - { - int err = 0; - - switch (cmd.type) - { - case CommandType::Submit: - err = submitOperation (cmd.op, cmd.flush); - break; - - case CommandType::Cancel: - err = cancelOperation (cmd.op, cmd.flush); - break; - - case CommandType::Stop: - { - _running.store (false, std::memory_order_release); - for (IoOperation* op : _pendingOps) - { - cancelOperation (op, false); - } - if (cmd.flush) - { - io_uring_submit (&_ring); - } - break; - } - - case CommandType::Flush: - if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) - { - lastError = std::error_code (errno, std::system_category ()); - err = -1; - } - break; - - default: - break; - } - - if (JOIN_UNLIKELY (cmd.done && (cmd.type != CommandType::Stop || _pendingOps.empty ()))) - { - if (cmd.errc && (err != 0)) - { - *cmd.errc = lastError; - } - cmd.done->store (true, std::memory_order_release); - } - } - - template - int BasicProactor::submitOperation (IoOperation* op, bool flush) noexcept - { - if (JOIN_UNLIKELY (op == nullptr)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (JOIN_UNLIKELY (op->fd () < 0)) - { - lastError = std::make_error_code (std::errc::bad_file_descriptor); - return -1; - } - - if (JOIN_UNLIKELY (op->state != IoOperation::State::Idle)) - { - lastError = make_error_code (Errc::OperationFailed); - return -1; - } - - io_uring_sqe* sqe = getSqe (); - - if (JOIN_UNLIKELY (sqe == nullptr)) - { - lastError = make_error_code (Errc::OperationFailed); - return -1; - } - - prepareSqe (sqe, op); - op->state = IoOperation::State::Submitted; - op->slot = static_cast (_pendingOps.size ()); - _pendingOps.push_back (op); - - if (flush) - { - io_uring_submit (&_ring); - } - - return 0; - } - - template - int BasicProactor::cancelOperation (IoOperation* op, bool flush) noexcept - { - if (JOIN_UNLIKELY (op == nullptr)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - if (JOIN_UNLIKELY (op->fd () < 0)) - { - lastError = std::make_error_code (std::errc::bad_file_descriptor); - return -1; - } - - if (JOIN_UNLIKELY (op->state != IoOperation::State::Submitted)) - { - lastError = make_error_code (Errc::OperationFailed); - return -1; - } - - if (JOIN_UNLIKELY (op->slot >= _pendingOps.size () || _pendingOps[op->slot] != op)) - { - lastError = make_error_code (Errc::InvalidParam); - return -1; - } - - io_uring_sqe* sqe = getSqe (); - if (JOIN_UNLIKELY (sqe == nullptr)) - { - lastError = make_error_code (Errc::OperationFailed); - return -1; - } - - op->state = IoOperation::State::Cancelling; - io_uring_prep_cancel (sqe, op, 0); - io_uring_sqe_set_data (sqe, nullptr); - - if (flush) - { - io_uring_submit (&_ring); - } - - return 0; - } - - template - void BasicProactor::endOperation (IoOperation* op, int result, bool cancelled) noexcept - { - if (JOIN_UNLIKELY (op == nullptr)) - { - return; - } - - if (op->slot < _pendingOps.size () && _pendingOps[op->slot] == op) - { - IoOperation* last = _pendingOps.back (); - _pendingOps[op->slot] = last; - last->slot = op->slot; - _pendingOps.pop_back (); - } - - dispatchOperation (op, result, cancelled); - - if (JOIN_UNLIKELY (_pendingOps.empty ())) - { - auto* p = _stopDone.load (std::memory_order_acquire); - if (p) - p->store (true, std::memory_order_release); - } - } - - template - io_uring_sqe* BasicProactor::getSqe () noexcept - { - io_uring_sqe* sqe = io_uring_get_sqe (&_ring); - - if (JOIN_UNLIKELY (sqe == nullptr)) - { - io_uring_submit (&_ring); - sqe = io_uring_get_sqe (&_ring); - } - - return sqe; - } - - template - void BasicProactor::prepareSqe (io_uring_sqe* sqe, IoOperation* op) noexcept - { - switch (static_cast (op->code)) - { - case IoOperation::Opcode::Accept: - io_uring_prep_accept (sqe, op->data.accept.fd, op->data.accept.addr, op->data.accept.addrlen, - op->data.accept.flags); - break; - - case IoOperation::Opcode::Connect: - io_uring_prep_connect (sqe, op->data.connect.fd, op->data.connect.addr, op->data.connect.addrlen); - break; - - case IoOperation::Opcode::Read: - io_uring_prep_read (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0); - break; - - case IoOperation::Opcode::Write: - io_uring_prep_write (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0); - break; - - case IoOperation::Opcode::ReadFixed: - io_uring_prep_read_fixed (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0, op->data.rw.index); - break; - - case IoOperation::Opcode::WriteFixed: - io_uring_prep_write_fixed (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0, op->data.rw.index); - break; - - case IoOperation::Opcode::RecvMsg: - io_uring_prep_recvmsg (sqe, op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); - break; - - case IoOperation::Opcode::SendMsg: - io_uring_prep_sendmsg (sqe, op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); - break; - - case IoOperation::Opcode::Recv: - io_uring_prep_recv (sqe, op->data.stream.fd, op->data.stream.buf, op->data.stream.len, - op->data.stream.flags); - break; - - case IoOperation::Opcode::Send: - io_uring_prep_send (sqe, op->data.stream.fd, op->data.stream.buf, op->data.stream.len, - op->data.stream.flags); - break; - - default: - io_uring_prep_nop (sqe); - break; - } - - io_uring_sqe_set_data (sqe, op); - - if (op->linked) - { - sqe->flags |= IOSQE_IO_LINK; - } - } - - template - void BasicProactor::rearmWakeup () noexcept - { - io_uring_sqe* sqe = getSqe (); - - if (JOIN_LIKELY (sqe != nullptr)) - { - prepareSqe (sqe, &_wakeupOp); - _wakeupOp.state = IoOperation::State::Submitted; - io_uring_submit (&_ring); - } - } - - template - void BasicProactor::dispatchCqe (io_uring_cqe* cqe) noexcept - { - dispatchCqe (cqe, is_default{}); - } - - template - void BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::true_type) noexcept - { - IoOperation* op = static_cast (io_uring_cqe_get_data (cqe)); - if (JOIN_UNLIKELY (op == nullptr)) - { - return; - } - - if (op == &_wakeupOp) - { - _wakeupOp.state = IoOperation::State::Idle; - readCommands (); - if (_running.load (std::memory_order_acquire)) - { - rearmWakeup (); - } - return; - } - - if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) - { - return; - } - - int result = cqe->res; - bool cancelled = (result < 0) && (result == -ECANCELED || op->state == IoOperation::State::Cancelling); - endOperation (op, result, cancelled); - } - - template - void BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::false_type) noexcept - { - IoOperation* op = static_cast (io_uring_cqe_get_data (cqe)); - if (JOIN_UNLIKELY (op == nullptr)) - { - return; - } - - if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) - { - return; - } - - int result = cqe->res; - bool cancelled = (result < 0) && (result == -ECANCELED || op->state == IoOperation::State::Cancelling); - endOperation (op, result, cancelled); - } - - template - void BasicProactor::eventLoop () noexcept - { - eventLoop (has_spin{}, has_sqpoll{}); - } - - template - void BasicProactor::eventLoop (std::false_type, std::false_type) noexcept - { - if (_running.load (std::memory_order_acquire)) - { - rearmWakeup (); - } - - while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) - { - io_uring_cqe* cqe = nullptr; - - if (_running.load (std::memory_order_acquire)) - { - if (JOIN_UNLIKELY (io_uring_wait_cqe (&_ring, &cqe) < 0)) - { - continue; - } - } - else - { - io_uring_submit (&_ring); - if (io_uring_peek_cqe (&_ring, &cqe) != 0) - { - continue; - } - } - - do - { - dispatchCqe (cqe); - io_uring_cqe_seen (&_ring, cqe); - } - while (io_uring_peek_cqe (&_ring, &cqe) == 0); - } - } - - template - void BasicProactor::eventLoop (std::true_type, std::false_type) noexcept - { - Backoff backoff (Policy::spin); - - while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) - { - bool running = _running.load (std::memory_order_acquire); - if (running) - { - readCommands (); - } - else - { - io_uring_submit (&_ring); - } - - io_uring_cqe* cqe = nullptr; - if (io_uring_peek_cqe (&_ring, &cqe) != 0) - { - backoff (); - continue; - } - - do - { - dispatchCqe (cqe); - io_uring_cqe_seen (&_ring, cqe); - } - while (io_uring_peek_cqe (&_ring, &cqe) == 0); - - backoff.reset (); - } - } - - template - void BasicProactor::eventLoop (std::true_type, std::true_type) noexcept - { - Backoff backoff (Policy::spin); - - while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) - { - bool running = _running.load (std::memory_order_acquire); - - if (running) - { - readCommands (); - - if (IO_URING_READ_ONCE (*_ring.sq.kflags) & IORING_SQ_NEED_WAKEUP) - { - io_uring_enter (_ring.ring_fd, 0, 0, IORING_ENTER_SQ_WAKEUP, nullptr); - } - } - else - { - io_uring_submit (&_ring); - } - - io_uring_cqe* cqe = nullptr; - if (io_uring_peek_cqe (&_ring, &cqe) != 0) - { - backoff (); - continue; - } - - do - { - dispatchCqe (cqe); - io_uring_cqe_seen (&_ring, cqe); - } - while (io_uring_peek_cqe (&_ring, &cqe) == 0); - - backoff.reset (); - } - } -} diff --git a/core/include/join/proactor_uring_impl.hpp b/core/include/join/proactor_uring_impl.hpp new file mode 100644 index 00000000..23747fa9 --- /dev/null +++ b/core/include/join/proactor_uring_impl.hpp @@ -0,0 +1,902 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : BasicProactor +// ========================================================================= +template +join::BasicProactor::BasicProactor () +: _commands (_queueSize) +, _wakeup (initWakeup (is_default{})) +{ + static_assert (has_spin::value || !has_sqpoll::value, "spin required for sq poll policy"); + + io_uring_params params{}; + params.flags = Policy::flags; + initCqEntries (params, has_cq_entries{}); + initSqThreadIdle (params, has_sq_thread_idle{}); + initSqThreadCpu (params, has_sq_thread_cpu{}); + + if (io_uring_queue_init_params (Policy::sqEntries, &_ring, ¶ms) < 0) + { + // LCOV_EXCL_START + ::close (_wakeup); + throw std::system_error (errno, std::system_category (), "io_uring_queue_init failed"); + // LCOV_EXCL_STOP + } + + initWakeupOp (is_default{}); + _pendingOps.reserve (Policy::sqEntries); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : ~BasicProactor +// ========================================================================= +template +join::BasicProactor::~BasicProactor () noexcept +{ + stop (true); + + io_uring_queue_exit (&_ring); + + if (_wakeup != -1) + { + ::close (_wakeup); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : flush +// ========================================================================= +template +int join::BasicProactor::flush (bool sync) noexcept +{ + if (isProactorThread ()) + { + if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) + { + lastError = std::error_code (errno, std::system_category ()); + return -1; + } + return 0; + } + + std::atomic done{false}, *pdone = nullptr; + std::error_code errc, *perrc = nullptr; + + if (JOIN_UNLIKELY (sync)) + { + pdone = &done; + perrc = &errc; + } + + if (JOIN_UNLIKELY (writeCommand ({CommandType::Flush, nullptr, false, pdone, perrc}) == -1)) + { + return -1; // LCOV_EXCL_LINE + } + + if (JOIN_UNLIKELY (sync)) + { + Backoff backoff; + while (!done.load (std::memory_order_acquire)) + { + backoff (); + } + + if (JOIN_UNLIKELY (errc)) + { + lastError = errc; + return -1; + } + } + + return 0; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : run +// ========================================================================= +template +void join::BasicProactor::run () +{ + _threadId.store (pthread_self (), std::memory_order_release); + + _running.store (true, std::memory_order_release); + eventLoop (); + + _threadId.store (_invalidThreadId, std::memory_order_release); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : stop +// ========================================================================= +template +void join::BasicProactor::stop (bool sync) noexcept +{ + if (isProactorThread ()) + { + _running.store (false, std::memory_order_release); + cancelAllOperations (); + eventLoop (); + return; + } + + if (!isRunning ()) + { + return; + } + + if (JOIN_LIKELY (sync)) + { + bool expected = false; + if (!_stopping.compare_exchange_strong (expected, true, std::memory_order_acq_rel)) + { + Backoff backoff; + while (_threadId.load (std::memory_order_acquire) != _invalidThreadId) + { + backoff (); + } + return; + } + } + + writeCommand ({CommandType::Stop, nullptr, sync, nullptr, nullptr}); + + if (JOIN_LIKELY (sync)) + { + Backoff backoff; + while (_threadId.load (std::memory_order_acquire) != _invalidThreadId) + { + backoff (); + } + _stopping.store (false, std::memory_order_release); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : registerBuffers +// ========================================================================= +template +int join::BasicProactor::registerBuffers (const std::vector& iovecs) noexcept +{ + int ret = io_uring_register_buffers (&_ring, iovecs.data (), iovecs.size ()); + if (JOIN_UNLIKELY (ret < 0)) + { + lastError = std::error_code (-ret, std::system_category ()); + return -1; + } + + return 0; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : unregisterBuffers +// ========================================================================= +template +int join::BasicProactor::unregisterBuffers () noexcept +{ + int ret = io_uring_unregister_buffers (&_ring); + if (JOIN_UNLIKELY (ret < 0)) + { + lastError = std::error_code (-ret, std::system_category ()); + return -1; + } + + return 0; +} + +#ifdef JOIN_HAS_NUMA +// ========================================================================= +// CLASS : BasicProactor +// METHOD : mbind +// ========================================================================= +template +int join::BasicProactor::mbind (int numa) const noexcept +{ + return _commands.mbind (numa); +} +#endif + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : mlock +// ========================================================================= +template +int join::BasicProactor::mlock () const noexcept +{ + return _commands.mlock (); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : isRunning +// ========================================================================= +template +bool join::BasicProactor::isRunning () const noexcept +{ + return _running.load (std::memory_order_acquire); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : isProactorThread +// ========================================================================= +template +bool join::BasicProactor::isProactorThread () const noexcept +{ + return _threadId.load (std::memory_order_acquire) == pthread_self (); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initWakeup +// ========================================================================= +template +int join::BasicProactor::initWakeup (std::true_type) noexcept +{ + return eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initWakeup +// ========================================================================= +template +int join::BasicProactor::initWakeup (std::false_type) noexcept +{ + return -1; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initWakeupOp +// ========================================================================= +template +void join::BasicProactor::initWakeupOp (std::true_type) noexcept +{ + _wakeupOp = IoOperation::makeRead (_wakeup, &_wakeupBuf, sizeof (_wakeupBuf), nullptr); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initWakeupOp +// ========================================================================= +template +void join::BasicProactor::initWakeupOp (std::false_type) noexcept +{ + // no-op. +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initCqEntries +// ========================================================================= +template +void join::BasicProactor::initCqEntries (io_uring_params&, std::false_type) noexcept +{ + // no-op. +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initCqEntries +// ========================================================================= +template +void join::BasicProactor::initCqEntries (io_uring_params& params, std::true_type) noexcept +{ + params.flags |= IORING_SETUP_CQSIZE; + params.cq_entries = Policy::cqEntries; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initSqThreadIdle +// ========================================================================= +template +void join::BasicProactor::initSqThreadIdle (io_uring_params&, std::false_type) noexcept +{ + // no-op. +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initSqThreadIdle +// ========================================================================= +template +void join::BasicProactor::initSqThreadIdle (io_uring_params& params, std::true_type) noexcept +{ + params.sq_thread_idle = Policy::sqThreadIdle; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initSqThreadCpu +// ========================================================================= +template +void join::BasicProactor::initSqThreadCpu (io_uring_params&, std::false_type) noexcept +{ + // no-op. +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : initSqThreadCpu +// ========================================================================= +template +void join::BasicProactor::initSqThreadCpu (io_uring_params& params, std::true_type) noexcept +{ + params.sq_thread_cpu = Policy::sqThreadCpu; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : writeCommand +// ========================================================================= +template +int join::BasicProactor::writeCommand (const Command& cmd) noexcept +{ + return writeCommand (cmd, is_default{}); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : writeCommand +// ========================================================================= +template +int join::BasicProactor::writeCommand (const Command& cmd, std::true_type) noexcept +{ + if (JOIN_UNLIKELY (_commands.push (cmd) == -1)) + { + return -1; // LCOV_EXCL_LINE + } + + uint64_t value = 1; + if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1)) + { + // LCOV_EXCL_START + lastError = std::error_code (errno, std::system_category ()); + return -1; + // LCOV_EXCL_STOP + } + + return 0; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : writeCommand +// ========================================================================= +template +int join::BasicProactor::writeCommand (const Command& cmd, std::false_type) noexcept +{ + return _commands.push (cmd); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : readCommands +// ========================================================================= +template +void join::BasicProactor::readCommands () noexcept +{ + Command cmd; + while (_commands.tryPop (cmd) == 0) + { + processCommand (cmd); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : processCommand +// ========================================================================= +template +void join::BasicProactor::processCommand (const Command& cmd) noexcept +{ + int err = 0; + + switch (cmd.type) + { + case CommandType::Submit: + err = submitOperation (cmd.op, cmd.flush); + break; + + case CommandType::Cancel: + err = cancelOperation (cmd.op, cmd.flush); + break; + + case CommandType::Stop: + _running.store (false, std::memory_order_release); + cancelAllOperations (); + if (JOIN_UNLIKELY (cmd.flush)) + { + io_uring_submit (&_ring); + } + break; + + case CommandType::Flush: + if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) + { + // LCOV_EXCL_START + lastError = std::error_code (errno, std::system_category ()); + err = -1; + // LCOV_EXCL_STOP + } + break; + + default: + break; // LCOV_EXCL_LINE + } + + if (JOIN_UNLIKELY (cmd.done)) + { + if (cmd.errc && (err != 0)) + { + *cmd.errc = lastError; + } + cmd.done->store (true, std::memory_order_release); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : submitOperation +// ========================================================================= +template +int join::BasicProactor::submitOperation (IoOperation* op, bool flush) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Idle)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + io_uring_sqe* sqe = getSqe (); + if (JOIN_UNLIKELY (sqe == nullptr)) + { + // LCOV_EXCL_START + lastError = make_error_code (Errc::OperationFailed); + return -1; + // LCOV_EXCL_STOP + } + + prepareSqe (sqe, op); + op->state = IoOperation::State::Submitted; + op->index = static_cast (_pendingOps.size ()); + _pendingOps.push_back (op); + + if (JOIN_UNLIKELY (flush)) + { + io_uring_submit (&_ring); + } + + return 0; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : cancelOperation +// ========================================================================= +template +int join::BasicProactor::cancelOperation (IoOperation* op, bool flush) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + if (JOIN_UNLIKELY (op->fd () < 0)) + { + lastError = std::make_error_code (std::errc::bad_file_descriptor); + return -1; + } + + if (JOIN_UNLIKELY (op->state != IoOperation::State::Submitted)) + { + lastError = make_error_code (Errc::OperationFailed); + return -1; + } + + if (JOIN_UNLIKELY (op->index >= _pendingOps.size () || _pendingOps[op->index] != op)) + { + lastError = make_error_code (Errc::InvalidParam); + return -1; + } + + io_uring_sqe* sqe = getSqe (); + if (JOIN_UNLIKELY (sqe == nullptr)) + { + // LCOV_EXCL_START + lastError = make_error_code (Errc::OperationFailed); + return -1; + // LCOV_EXCL_STOP + } + + op->state = IoOperation::State::Cancelling; + io_uring_prep_cancel (sqe, op, 0); + io_uring_sqe_set_data (sqe, nullptr); + + if (JOIN_UNLIKELY (flush)) + { + io_uring_submit (&_ring); + } + + return 0; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : cancelAllOperations +// ========================================================================= +template +void join::BasicProactor::cancelAllOperations () noexcept +{ + for (IoOperation* op : _pendingOps) + { + cancelOperation (op, false); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : endOperation +// ========================================================================= +template +void join::BasicProactor::endOperation (IoOperation* op, int result, bool cancelled) noexcept +{ + if (JOIN_UNLIKELY (op == nullptr)) + { + return; // LCOV_EXCL_LINE + } + + if (JOIN_LIKELY (op->index < _pendingOps.size () && _pendingOps[op->index] == op)) + { + IoOperation* last = _pendingOps.back (); + _pendingOps[op->index] = last; + last->index = op->index; + _pendingOps.pop_back (); + } + + dispatchOperation (op, result, cancelled); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : getSqe +// ========================================================================= +template +io_uring_sqe* join::BasicProactor::getSqe () noexcept +{ + io_uring_sqe* sqe = io_uring_get_sqe (&_ring); + if (JOIN_UNLIKELY (sqe == nullptr)) + { + io_uring_submit (&_ring); + sqe = io_uring_get_sqe (&_ring); + } + + return sqe; +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : prepareSqe +// ========================================================================= +template +void join::BasicProactor::prepareSqe (io_uring_sqe* sqe, IoOperation* op) noexcept +{ + switch (static_cast (op->code)) + { + case IoOperation::Opcode::Accept: + io_uring_prep_accept (sqe, op->data.accept.fd, op->data.accept.addr, op->data.accept.addrlen, + op->data.accept.flags); + break; + + case IoOperation::Opcode::Connect: + io_uring_prep_connect (sqe, op->data.connect.fd, op->data.connect.addr, op->data.connect.addrlen); + break; + + case IoOperation::Opcode::Read: + io_uring_prep_read (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0); + break; + + case IoOperation::Opcode::Write: + io_uring_prep_write (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0); + break; + + case IoOperation::Opcode::ReadFixed: + io_uring_prep_read_fixed (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0, op->data.rw.index); + break; + + case IoOperation::Opcode::WriteFixed: + io_uring_prep_write_fixed (sqe, op->data.rw.fd, op->data.rw.buf, op->data.rw.len, 0, op->data.rw.index); + break; + + case IoOperation::Opcode::RecvMsg: + io_uring_prep_recvmsg (sqe, op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + break; + + case IoOperation::Opcode::SendMsg: + io_uring_prep_sendmsg (sqe, op->data.msg.fd, op->data.msg.msg, op->data.msg.flags); + break; + + case IoOperation::Opcode::Recv: + io_uring_prep_recv (sqe, op->data.stream.fd, op->data.stream.buf, op->data.stream.len, + op->data.stream.flags); + break; + + case IoOperation::Opcode::Send: + io_uring_prep_send (sqe, op->data.stream.fd, op->data.stream.buf, op->data.stream.len, + op->data.stream.flags); + break; + + default: + io_uring_prep_nop (sqe); // LCOV_EXCL_LINE + break; + } + + io_uring_sqe_set_data (sqe, op); + + if (JOIN_UNLIKELY (op->linked)) + { + sqe->flags |= IOSQE_IO_LINK; + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : rearmWakeup +// ========================================================================= +template +void join::BasicProactor::rearmWakeup () noexcept +{ + io_uring_sqe* sqe = getSqe (); + + if (JOIN_LIKELY (sqe != nullptr)) + { + prepareSqe (sqe, &_wakeupOp); + _wakeupOp.state = IoOperation::State::Submitted; + io_uring_submit (&_ring); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : dispatchCqe +// ========================================================================= +template +void join::BasicProactor::dispatchCqe (io_uring_cqe* cqe) noexcept +{ + dispatchCqe (cqe, is_default{}); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : dispatchCqe +// ========================================================================= +template +void join::BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::true_type) noexcept +{ + IoOperation* op = static_cast (io_uring_cqe_get_data (cqe)); + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } + + if (JOIN_UNLIKELY (op == &_wakeupOp)) + { + _wakeupOp.state = IoOperation::State::Idle; + readCommands (); + if (JOIN_LIKELY (_running.load (std::memory_order_acquire))) + { + rearmWakeup (); + } + return; + } + + if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) + { + return; + } + + int result = cqe->res; + bool cancelled = (result < 0) && (result == -ECANCELED || op->state == IoOperation::State::Cancelling); + endOperation (op, result, cancelled); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : dispatchCqe +// ========================================================================= +template +void join::BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::false_type) noexcept +{ + IoOperation* op = static_cast (io_uring_cqe_get_data (cqe)); + if (JOIN_UNLIKELY (op == nullptr)) + { + return; + } + + if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) + { + return; + } + + int result = cqe->res; + bool cancelled = (result < 0) && (result == -ECANCELED || op->state == IoOperation::State::Cancelling); + endOperation (op, result, cancelled); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : eventLoop +// ========================================================================= +template +void join::BasicProactor::eventLoop () noexcept +{ + eventLoop (has_spin{}, has_sqpoll{}); +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : eventLoop +// ========================================================================= +template +void join::BasicProactor::eventLoop (std::false_type, std::false_type) noexcept +{ + if (JOIN_LIKELY (_running.load (std::memory_order_acquire))) + { + rearmWakeup (); + } + + while (_running.load (std::memory_order_acquire) || !_pendingOps.empty ()) + { + io_uring_cqe* cqe = nullptr; + + if (JOIN_LIKELY (_running.load (std::memory_order_acquire))) + { + if (JOIN_UNLIKELY (io_uring_wait_cqe (&_ring, &cqe) < 0)) + { + continue; + } + } + else + { + io_uring_submit (&_ring); + if (JOIN_UNLIKELY (io_uring_peek_cqe (&_ring, &cqe) != 0)) + { + continue; + } + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : eventLoop +// ========================================================================= +template +void join::BasicProactor::eventLoop (std::true_type, std::false_type) noexcept +{ + Backoff backoff (Policy::spin); + bool running; + + while ((running = _running.load (std::memory_order_acquire)) || !_pendingOps.empty ()) + { + if (JOIN_LIKELY (running)) + { + readCommands (); + } + else + { + io_uring_submit (&_ring); + } + + io_uring_cqe* cqe = nullptr; + if (JOIN_UNLIKELY (io_uring_peek_cqe (&_ring, &cqe) != 0)) + { + backoff (); + continue; + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + + backoff.reset (); + } +} + +// ========================================================================= +// CLASS : BasicProactor +// METHOD : eventLoop +// ========================================================================= +template +void join::BasicProactor::eventLoop (std::true_type, std::true_type) noexcept +{ + Backoff backoff (Policy::spin); + bool running; + + while ((running = _running.load (std::memory_order_acquire)) || !_pendingOps.empty ()) + { + if (JOIN_LIKELY (running)) + { + readCommands (); + + if (JOIN_UNLIKELY (IO_URING_READ_ONCE (*_ring.sq.kflags) & IORING_SQ_NEED_WAKEUP)) + { + io_uring_enter (_ring.ring_fd, 0, 0, IORING_ENTER_SQ_WAKEUP, nullptr); + } + } + else + { + io_uring_submit (&_ring); + } + + io_uring_cqe* cqe = nullptr; + if (JOIN_UNLIKELY (io_uring_peek_cqe (&_ring, &cqe) != 0)) + { + backoff (); + continue; + } + + do + { + dispatchCqe (cqe); + io_uring_cqe_seen (&_ring, cqe); + } + while (io_uring_peek_cqe (&_ring, &cqe) == 0); + + backoff.reset (); + } +} diff --git a/core/src/io_operation.cpp b/core/src/io_operation.cpp index 4deeaa8c..b12beaed 100644 --- a/core/src/io_operation.cpp +++ b/core/src/io_operation.cpp @@ -66,11 +66,12 @@ IoOperation IoOperation::makeConnect (int fd, const sockaddr* addr, socklen_t ad // CLASS : IoOperation // METHOD : makeRead // ========================================================================= -IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler) noexcept +IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHandler* handler, bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::Read); op.handler = handler; + op.linked = linked; op.data.rw.fd = fd; op.data.rw.buf = buf; op.data.rw.len = len; @@ -83,11 +84,13 @@ IoOperation IoOperation::makeRead (int fd, void* buf, uint32_t len, CompletionHa // CLASS : IoOperation // METHOD : makeWrite // ========================================================================= -IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler) noexcept +IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, CompletionHandler* handler, + bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::Write); op.handler = handler; + op.linked = linked; op.data.rw.fd = fd; op.data.rw.buf = const_cast (buf); op.data.rw.len = len; @@ -100,12 +103,13 @@ IoOperation IoOperation::makeWrite (int fd, const void* buf, uint32_t len, Compl // CLASS : IoOperation // METHOD : makeReadFixed // ========================================================================= -IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept +IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_t index, CompletionHandler* handler, + bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::ReadFixed); op.handler = handler; + op.linked = linked; op.data.rw.fd = fd; op.data.rw.buf = buf; op.data.rw.len = len; @@ -119,11 +123,12 @@ IoOperation IoOperation::makeReadFixed (int fd, void* buf, uint32_t len, uint16_ // METHOD : makeWriteFixed // ========================================================================= IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, uint16_t index, - CompletionHandler* handler) noexcept + CompletionHandler* handler, bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::WriteFixed); op.handler = handler; + op.linked = linked; op.data.rw.fd = fd; op.data.rw.buf = const_cast (buf); op.data.rw.len = len; @@ -136,11 +141,12 @@ IoOperation IoOperation::makeWriteFixed (int fd, const void* buf, uint32_t len, // CLASS : IoOperation // METHOD : makeRecvmsg // ========================================================================= -IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, int flags, CompletionHandler* handler) noexcept +IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, int flags, CompletionHandler* handler, bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::RecvMsg); op.handler = handler; + op.linked = linked; op.data.msg.fd = fd; op.data.msg.msg = msg; op.data.msg.flags = flags; @@ -151,11 +157,13 @@ IoOperation IoOperation::makeRecvmsg (int fd, msghdr* msg, int flags, Completion // CLASS : IoOperation // METHOD : makeSendmsg // ========================================================================= -IoOperation IoOperation::makeSendmsg (int fd, const msghdr* msg, int flags, CompletionHandler* handler) noexcept +IoOperation IoOperation::makeSendmsg (int fd, const msghdr* msg, int flags, CompletionHandler* handler, + bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::SendMsg); op.handler = handler; + op.linked = linked; op.data.msg.fd = fd; op.data.msg.msg = const_cast (msg); op.data.msg.flags = flags; @@ -166,11 +174,13 @@ IoOperation IoOperation::makeSendmsg (int fd, const msghdr* msg, int flags, Comp // CLASS : IoOperation // METHOD : makeRecv // ========================================================================= -IoOperation IoOperation::makeRecv (int fd, void* buf, uint32_t len, int flags, CompletionHandler* handler) noexcept +IoOperation IoOperation::makeRecv (int fd, void* buf, uint32_t len, int flags, CompletionHandler* handler, + bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::Recv); op.handler = handler; + op.linked = linked; op.data.stream.fd = fd; op.data.stream.buf = buf; op.data.stream.len = len; @@ -182,12 +192,13 @@ IoOperation IoOperation::makeRecv (int fd, void* buf, uint32_t len, int flags, C // CLASS : IoOperation // METHOD : makeSend // ========================================================================= -IoOperation IoOperation::makeSend (int fd, const void* buf, uint32_t len, int flags, - CompletionHandler* handler) noexcept +IoOperation IoOperation::makeSend (int fd, const void* buf, uint32_t len, int flags, CompletionHandler* handler, + bool linked) noexcept { IoOperation op; op.code = static_cast (IoOperation::Opcode::Send); op.handler = handler; + op.linked = linked; op.data.stream.fd = fd; op.data.stream.buf = const_cast (buf); op.data.stream.len = len; diff --git a/core/tests/hybrid_proactor_test.cpp b/core/tests/hybrid_proactor_test.cpp index b5b25c84..a20ed3d6 100644 --- a/core/tests/hybrid_proactor_test.cpp +++ b/core/tests/hybrid_proactor_test.cpp @@ -72,8 +72,9 @@ class HybridProactorTest : public CompletionHandler, public ::testing::Test */ void onComplete (IoOperation* op, int result) override { - HybridProactorThread::proactor ().submit (op, true); - HybridProactorThread::proactor ().cancel (op, true); + HybridProactorThread::proactor ().submit (op, true, true); + HybridProactorThread::proactor ().cancel (op, true, true); + HybridProactorThread::proactor ().flush (true); { ScopedLock lock (_mut); @@ -92,8 +93,9 @@ class HybridProactorTest : public CompletionHandler, public ::testing::Test */ void onCancel (IoOperation* op, int result) override { - HybridProactorThread::proactor ().submit (op, true); - HybridProactorThread::proactor ().cancel (op, true); + HybridProactorThread::proactor ().submit (op, true, true); + HybridProactorThread::proactor ().cancel (op, true, true); + HybridProactorThread::proactor ().flush (true); { ScopedLock lock (_mut); @@ -165,7 +167,7 @@ TEST_F (HybridProactorTest, stop) ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op, true, true), 0) << join::lastError.message (); proactor.stop (); th.join (); @@ -189,11 +191,11 @@ TEST_F (HybridProactorTest, submit) proactor.run (); }); - ASSERT_EQ (proactor.submit (nullptr, true), -1); + ASSERT_EQ (proactor.submit (nullptr, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (proactor.submit (&op1, true, true), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); @@ -201,19 +203,19 @@ TEST_F (HybridProactorTest, submit) op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); op1.state = IoOperation::State::Submitted; - ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (proactor.submit (&op1, true, true), -1); ASSERT_EQ (join::lastError, Errc::OperationFailed); op1.state = IoOperation::State::Idle; - ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op1, true, true), 0) << join::lastError.message (); #ifndef JOIN_HAS_IO_URING auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op2, true), -1); + ASSERT_EQ (proactor.submit (&op2, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); #endif - ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.cancel (&op1, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { @@ -237,28 +239,28 @@ TEST_F (HybridProactorTest, cancel) proactor.run (); }); - ASSERT_EQ (proactor.cancel (nullptr, true), -1); + ASSERT_EQ (proactor.cancel (nullptr, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (proactor.cancel (&op1, true, true), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (proactor.cancel (&op1, true, true), -1); ASSERT_EQ (join::lastError, Errc::OperationFailed); - ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op1, true, true), 0) << join::lastError.message (); auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); op2.state = IoOperation::State::Submitted; - ASSERT_EQ (proactor.cancel (&op2, true), -1); + ASSERT_EQ (proactor.cancel (&op2, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); - ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.cancel (&op1, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { @@ -286,8 +288,8 @@ TEST_F (HybridProactorTest, flush) ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op, false), 0) << join::lastError.message (); - ASSERT_EQ (proactor.flush (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op, false, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.flush (true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("flush", strlen ("flush"), _timeout), 0) << join::lastError.message (); @@ -318,12 +320,11 @@ TEST_F (HybridProactorTest, chain) ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this); + auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this, true); auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - writeOp.linked = true; - ASSERT_EQ (proactor.submit (&writeOp, false), 0) << join::lastError.message (); - ASSERT_EQ (proactor.submit (&readOp, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&writeOp, false, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&readOp, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); @@ -415,9 +416,37 @@ TEST_F (HybridProactorTest, registerBuffers) ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.unregisterBuffers (), -1); } #endif +/** + * @brief Test async connect. + */ +TEST_F (HybridProactorTest, asyncConnect) +{ + Tcp::Endpoint endpoint{_host, _port}; + + ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); + _client.setMode (Tcp::Socket::NonBlocking); + + auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } + + _client.setMode (Tcp::Socket::Blocking); +} + /** * @brief Test async accept. */ @@ -438,7 +467,7 @@ TEST_F (HybridProactorTest, asyncAccept) auto op = IoOperation::makeAccept (_acceptor.handle (), reinterpret_cast (&addr), &addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC, this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); { @@ -453,30 +482,41 @@ TEST_F (HybridProactorTest, asyncAccept) } /** - * @brief Test async connect. + * @brief Test async write. */ -TEST_F (HybridProactorTest, asyncConnect) +TEST_F (HybridProactorTest, asyncWrite) { - Tcp::Endpoint endpoint{_host, _port}; + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); - _client.setMode (Tcp::Socket::NonBlocking); + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); - auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + const char* msg = "asyncWrite"; + auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); - ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &op && _result == 0; + return _op == &op && _result > 0; })); + ASSERT_EQ (_result, static_cast (strlen (msg))); _op = nullptr; _result = 0; } - _client.setMode (Tcp::Socket::Blocking); + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); } /** @@ -499,7 +539,7 @@ TEST_F (HybridProactorTest, asyncRead) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("asyncRead", strlen ("asyncRead"), _timeout), 0) << join::lastError.message (); { @@ -514,27 +554,22 @@ TEST_F (HybridProactorTest, asyncRead) } /** - * @brief Test async write. + * @brief Test async write fixed. */ -TEST_F (HybridProactorTest, asyncWrite) +TEST_F (HybridProactorTest, asyncWriteFixed) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::affinity (), 0); - ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::priority (), 1); -#ifdef JOIN_HAS_NUMA - ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); -#endif - ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); - ASSERT_GT (HybridProactorThread::handle (), 0); + const char* msg = "asyncWriteFixed"; + std::vector regbuf (msg, msg + strlen (msg)); + iovec iov = {regbuf.data (), regbuf.size ()}; + std::vector iovecs = {iov}; - const char* msg = "asyncWrite"; - auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + ASSERT_EQ (HybridProactorThread::proactor ().registerBuffers (iovecs), 0) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + auto op = IoOperation::makeWriteFixed (_server.handle (), regbuf.data (), regbuf.size (), 0, this); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); @@ -549,48 +584,82 @@ TEST_F (HybridProactorTest, asyncWrite) char rbuf[256] = {}; ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); + + ASSERT_EQ (HybridProactorThread::proactor ().unregisterBuffers (), 0) << join::lastError.message (); } /** - * @brief Test async read and write. + * @brief Test async read fixed. */ -TEST_F (HybridProactorTest, asyncReadWrite) +TEST_F (HybridProactorTest, asyncReadFixed) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - const char* msg = "asyncReadWrite"; - auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - auto writeOp = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + std::vector regbuf (sizeof (_buf)); + iovec iov = {regbuf.data (), regbuf.size ()}; + std::vector iovecs = {iov}; - ASSERT_EQ (HybridProactorThread::proactor ().submit (&readOp, true), 0) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&writeOp, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().registerBuffers (iovecs), 0) << join::lastError.message (); + + auto op = IoOperation::makeReadFixed (_server.handle (), regbuf.data (), regbuf.size (), 0, this); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncReadFixed", strlen ("asyncReadFixed"), _timeout), 0) + << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &writeOp && _result > 0; + return _op == &op && _result > 0; })); - ASSERT_EQ (_result, static_cast (strlen (msg))); + ASSERT_EQ (std::string (regbuf.data (), _result), "asyncReadFixed"); _op = nullptr; _result = 0; } - ASSERT_EQ (_client.writeExactly (msg, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().unregisterBuffers (), 0) << join::lastError.message (); +} + +/** + * @brief Test async sendmsg. + */ +TEST_F (HybridProactorTest, asyncSendmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::affinity (), 0); + ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (HybridProactorThread::handle (), 0); + + const char* payload = "asyncSendmsg"; + iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &readOp && _result > 0; + return _op == &op && _result > 0; })); - ASSERT_EQ (std::string (_buf, _result), msg); + ASSERT_EQ (_result, static_cast (strlen (payload))); _op = nullptr; _result = 0; } char rbuf[256] = {}; - ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); - ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); + ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); } /** @@ -617,7 +686,7 @@ TEST_F (HybridProactorTest, asyncRecvmsg) msg.msg_iovlen = 1; auto op = IoOperation::makeRecvmsg (_server.handle (), &msg, 0, this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("asyncRecvmsg", strlen ("asyncRecvmsg"), _timeout), 0) << join::lastError.message (); @@ -633,45 +702,55 @@ TEST_F (HybridProactorTest, asyncRecvmsg) } /** - * @brief Test async sendmsg. + * @brief Test async send. */ -TEST_F (HybridProactorTest, asyncSendmsg) +TEST_F (HybridProactorTest, asyncSend) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::affinity (0), 0) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::affinity (), 0); - ASSERT_EQ (HybridProactorThread::priority (1), 0) << join::lastError.message (); - ASSERT_EQ (HybridProactorThread::priority (), 1); -#ifdef JOIN_HAS_NUMA - ASSERT_EQ (HybridProactorThread::mbind (0), 0) << join::lastError.message (); -#endif - ASSERT_EQ (HybridProactorThread::mlock (), 0) << join::lastError.message (); - ASSERT_GT (HybridProactorThread::handle (), 0); - - const char* payload = "asyncSendmsg"; - iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; - msghdr msg = {}; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + const char* msg = "asyncSend"; + auto op = IoOperation::makeSend (_server.handle (), msg, strlen (msg), 0, this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { return _op == &op && _result > 0; })); - ASSERT_EQ (_result, static_cast (strlen (payload))); + ASSERT_EQ (_result, static_cast (strlen (msg))); _op = nullptr; _result = 0; } char rbuf[256] = {}; - ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); - ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async recv. + */ +TEST_F (HybridProactorTest, asyncRecv) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRecv (_server.handle (), _buf, sizeof (_buf), 0, this); + + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRecv", strlen ("asyncRecv"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRecv"); + _op = nullptr; + _result = 0; + } } /** @@ -694,7 +773,7 @@ TEST_F (HybridProactorTest, onClose) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); _client.close (); { @@ -727,7 +806,7 @@ TEST_F (HybridProactorTest, onError) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (HybridProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); linger sl{.l_onoff = 1, .l_linger = 0}; ASSERT_EQ (setsockopt (_client.handle (), SOL_SOCKET, SO_LINGER, &sl, sizeof (sl)), 0) << strerror (errno); _client.close (); diff --git a/core/tests/proactor_test.cpp b/core/tests/proactor_test.cpp index a27c5751..d98bda53 100644 --- a/core/tests/proactor_test.cpp +++ b/core/tests/proactor_test.cpp @@ -72,8 +72,11 @@ class ProactorTest : public CompletionHandler, public ::testing::Test */ void onComplete (IoOperation* op, int result) override { - ProactorThread::proactor ().submit (op); - ProactorThread::proactor ().cancel (op); + ProactorThread::proactor ().submit (op, true, true); + ProactorThread::proactor ().cancel (op, true, true); +#ifdef JOIN_HAS_IO_URING + ProactorThread::proactor ().flush (true); +#endif { ScopedLock lock (_mut); @@ -92,8 +95,11 @@ class ProactorTest : public CompletionHandler, public ::testing::Test */ void onCancel (IoOperation* op, int result) override { - ProactorThread::proactor ().submit (op); - ProactorThread::proactor ().cancel (op); + ProactorThread::proactor ().submit (op, true, true); + ProactorThread::proactor ().cancel (op, true, true); +#ifdef JOIN_HAS_IO_URING + ProactorThread::proactor ().flush (true); +#endif { ScopedLock lock (_mut); @@ -165,7 +171,7 @@ TEST_F (ProactorTest, stop) ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op, true, true), 0) << join::lastError.message (); proactor.stop (); th.join (); @@ -189,11 +195,11 @@ TEST_F (ProactorTest, submit) proactor.run (); }); - ASSERT_EQ (proactor.submit (nullptr), -1); + ASSERT_EQ (proactor.submit (nullptr, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op1), -1); + ASSERT_EQ (proactor.submit (&op1, true, true), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); @@ -201,19 +207,19 @@ TEST_F (ProactorTest, submit) op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); op1.state = IoOperation::State::Submitted; - ASSERT_EQ (proactor.submit (&op1), -1); + ASSERT_EQ (proactor.submit (&op1, true, true), -1); ASSERT_EQ (join::lastError, Errc::OperationFailed); op1.state = IoOperation::State::Idle; - ASSERT_EQ (proactor.submit (&op1), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op1, true, true), 0) << join::lastError.message (); #ifndef JOIN_HAS_IO_URING auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op2), -1); + ASSERT_EQ (proactor.submit (&op2, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); #endif - ASSERT_EQ (proactor.cancel (&op1), 0) << join::lastError.message (); + ASSERT_EQ (proactor.cancel (&op1, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { @@ -237,28 +243,28 @@ TEST_F (ProactorTest, cancel) proactor.run (); }); - ASSERT_EQ (proactor.cancel (nullptr), -1); + ASSERT_EQ (proactor.cancel (nullptr, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.cancel (&op1), -1); + ASSERT_EQ (proactor.cancel (&op1, true, true), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.cancel (&op1), -1); + ASSERT_EQ (proactor.cancel (&op1, true, true), -1); ASSERT_EQ (join::lastError, Errc::OperationFailed); - ASSERT_EQ (proactor.submit (&op1), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op1, true, true), 0) << join::lastError.message (); auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); op2.state = IoOperation::State::Submitted; - ASSERT_EQ (proactor.cancel (&op2), -1); + ASSERT_EQ (proactor.cancel (&op2, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); - ASSERT_EQ (proactor.cancel (&op1), 0) << join::lastError.message (); + ASSERT_EQ (proactor.cancel (&op1, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { @@ -287,8 +293,8 @@ TEST_F (ProactorTest, flush) ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op, false), 0) << join::lastError.message (); - ASSERT_EQ (proactor.flush (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op, false, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.flush (true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("flush", strlen ("flush"), _timeout), 0) << join::lastError.message (); @@ -319,12 +325,11 @@ TEST_F (ProactorTest, chain) ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this); + auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this, true); auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - writeOp.linked = true; - ASSERT_EQ (proactor.submit (&writeOp, false), 0) << join::lastError.message (); - ASSERT_EQ (proactor.submit (&readOp, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&writeOp, false, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&readOp, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); @@ -417,9 +422,37 @@ TEST_F (ProactorTest, registerBuffers) ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.unregisterBuffers (), -1); } #endif +/** + * @brief Test async connect. + */ +TEST_F (ProactorTest, asyncConnect) +{ + Tcp::Endpoint endpoint{_host, _port}; + + ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); + _client.setMode (Tcp::Socket::NonBlocking); + + auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } + + _client.setMode (Tcp::Socket::Blocking); +} + /** * @brief Test async accept. */ @@ -440,7 +473,7 @@ TEST_F (ProactorTest, asyncAccept) auto op = IoOperation::makeAccept (_acceptor.handle (), reinterpret_cast (&addr), &addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC, this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); { @@ -455,30 +488,41 @@ TEST_F (ProactorTest, asyncAccept) } /** - * @brief Test async connect. + * @brief Test async write. */ -TEST_F (ProactorTest, asyncConnect) +TEST_F (ProactorTest, asyncWrite) { - Tcp::Endpoint endpoint{_host, _port}; + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); - _client.setMode (Tcp::Socket::NonBlocking); + ASSERT_EQ (ProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::affinity (), 0); + ASSERT_EQ (ProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (ProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (ProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (ProactorThread::handle (), 0); - auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + const char* msg = "asyncWrite"; + auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); - ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &op && _result == 0; + return _op == &op && _result > 0; })); + ASSERT_EQ (_result, static_cast (strlen (msg))); _op = nullptr; _result = 0; } - _client.setMode (Tcp::Socket::Blocking); + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); } /** @@ -501,7 +545,7 @@ TEST_F (ProactorTest, asyncRead) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("asyncRead", strlen ("asyncRead"), _timeout), 0) << join::lastError.message (); { @@ -515,28 +559,24 @@ TEST_F (ProactorTest, asyncRead) } } +#ifdef JOIN_HAS_IO_URING /** - * @brief Test async write. + * @brief Test async write fixed. */ -TEST_F (ProactorTest, asyncWrite) +TEST_F (ProactorTest, asyncWriteFixed) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (ProactorThread::affinity (0), 0) << join::lastError.message (); - ASSERT_EQ (ProactorThread::affinity (), 0); - ASSERT_EQ (ProactorThread::priority (1), 0) << join::lastError.message (); - ASSERT_EQ (ProactorThread::priority (), 1); -#ifdef JOIN_HAS_NUMA - ASSERT_EQ (ProactorThread::mbind (0), 0) << join::lastError.message (); -#endif - ASSERT_EQ (ProactorThread::mlock (), 0) << join::lastError.message (); - ASSERT_GT (ProactorThread::handle (), 0); + const char* msg = "asyncWriteFixed"; + std::vector regbuf (msg, msg + strlen (msg)); + iovec iov = {regbuf.data (), regbuf.size ()}; + std::vector iovecs = {iov}; - const char* msg = "asyncWrite"; - auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + ASSERT_EQ (ProactorThread::proactor ().registerBuffers (iovecs), 0) << join::lastError.message (); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + auto op = IoOperation::makeWriteFixed (_server.handle (), regbuf.data (), regbuf.size (), 0, this); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); @@ -551,48 +591,83 @@ TEST_F (ProactorTest, asyncWrite) char rbuf[256] = {}; ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); + + ASSERT_EQ (ProactorThread::proactor ().unregisterBuffers (), 0) << join::lastError.message (); } /** - * @brief Test async read and write. + * @brief Test async read fixed. */ -TEST_F (ProactorTest, asyncReadWrite) +TEST_F (ProactorTest, asyncReadFixed) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - const char* msg = "asyncReadWrite"; - auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - auto writeOp = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + std::vector regbuf (sizeof (_buf)); + iovec iov = {regbuf.data (), regbuf.size ()}; + std::vector iovecs = {iov}; - ASSERT_EQ (ProactorThread::proactor ().submit (&readOp), 0) << join::lastError.message (); - ASSERT_EQ (ProactorThread::proactor ().submit (&writeOp), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().registerBuffers (iovecs), 0) << join::lastError.message (); + + auto op = IoOperation::makeReadFixed (_server.handle (), regbuf.data (), regbuf.size (), 0, this); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncReadFixed", strlen ("asyncReadFixed"), _timeout), 0) + << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &writeOp && _result > 0; + return _op == &op && _result > 0; })); - ASSERT_EQ (_result, static_cast (strlen (msg))); + ASSERT_EQ (std::string (regbuf.data (), _result), "asyncReadFixed"); _op = nullptr; _result = 0; } - ASSERT_EQ (_client.writeExactly (msg, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().unregisterBuffers (), 0) << join::lastError.message (); +} +#endif + +/** + * @brief Test async sendmsg. + */ +TEST_F (ProactorTest, asyncSendmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (ProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::affinity (), 0); + ASSERT_EQ (ProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (ProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (ProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (ProactorThread::handle (), 0); + + const char* payload = "asyncSendmsg"; + iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &readOp && _result > 0; + return _op == &op && _result > 0; })); - ASSERT_EQ (std::string (_buf, _result), msg); + ASSERT_EQ (_result, static_cast (strlen (payload))); _op = nullptr; _result = 0; } char rbuf[256] = {}; - ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); - ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); + ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); } /** @@ -619,7 +694,7 @@ TEST_F (ProactorTest, asyncRecvmsg) msg.msg_iovlen = 1; auto op = IoOperation::makeRecvmsg (_server.handle (), &msg, 0, this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("asyncRecvmsg", strlen ("asyncRecvmsg"), _timeout), 0) << join::lastError.message (); @@ -635,45 +710,55 @@ TEST_F (ProactorTest, asyncRecvmsg) } /** - * @brief Test async sendmsg. + * @brief Test async send. */ -TEST_F (ProactorTest, asyncSendmsg) +TEST_F (ProactorTest, asyncSend) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (ProactorThread::affinity (0), 0) << join::lastError.message (); - ASSERT_EQ (ProactorThread::affinity (), 0); - ASSERT_EQ (ProactorThread::priority (1), 0) << join::lastError.message (); - ASSERT_EQ (ProactorThread::priority (), 1); -#ifdef JOIN_HAS_NUMA - ASSERT_EQ (ProactorThread::mbind (0), 0) << join::lastError.message (); -#endif - ASSERT_EQ (ProactorThread::mlock (), 0) << join::lastError.message (); - ASSERT_GT (ProactorThread::handle (), 0); - - const char* payload = "asyncSendmsg"; - iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; - msghdr msg = {}; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + const char* msg = "asyncSend"; + auto op = IoOperation::makeSend (_server.handle (), msg, strlen (msg), 0, this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { return _op == &op && _result > 0; })); - ASSERT_EQ (_result, static_cast (strlen (payload))); + ASSERT_EQ (_result, static_cast (strlen (msg))); _op = nullptr; _result = 0; } char rbuf[256] = {}; - ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); - ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async recv. + */ +TEST_F (ProactorTest, asyncRecv) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRecv (_server.handle (), _buf, sizeof (_buf), 0, this); + + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRecv", strlen ("asyncRecv"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRecv"); + _op = nullptr; + _result = 0; + } } /** @@ -696,7 +781,7 @@ TEST_F (ProactorTest, onClose) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); _client.close (); { @@ -729,7 +814,7 @@ TEST_F (ProactorTest, onError) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (ProactorThread::proactor ().submit (&op), 0) << join::lastError.message (); + ASSERT_EQ (ProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); linger sl{.l_onoff = 1, .l_linger = 0}; ASSERT_EQ (setsockopt (_client.handle (), SOL_SOCKET, SO_LINGER, &sl, sizeof (sl)), 0) << strerror (errno); _client.close (); diff --git a/core/tests/sqpoll_proactor_test.cpp b/core/tests/sqpoll_proactor_test.cpp index 2279a3d1..54b3ee95 100644 --- a/core/tests/sqpoll_proactor_test.cpp +++ b/core/tests/sqpoll_proactor_test.cpp @@ -72,8 +72,9 @@ class SqpollProactorTest : public CompletionHandler, public ::testing::Test */ void onComplete (IoOperation* op, int result) override { - SqpollProactorThread::proactor ().submit (op, true); - SqpollProactorThread::proactor ().cancel (op, true); + SqpollProactorThread::proactor ().submit (op, true, true); + SqpollProactorThread::proactor ().cancel (op, true, true); + SqpollProactorThread::proactor ().flush (true); { ScopedLock lock (_mut); @@ -92,8 +93,9 @@ class SqpollProactorTest : public CompletionHandler, public ::testing::Test */ void onCancel (IoOperation* op, int result) override { - SqpollProactorThread::proactor ().submit (op, true); - SqpollProactorThread::proactor ().cancel (op, true); + SqpollProactorThread::proactor ().submit (op, true, true); + SqpollProactorThread::proactor ().cancel (op, true, true); + SqpollProactorThread::proactor ().flush (true); { ScopedLock lock (_mut); @@ -165,7 +167,7 @@ TEST_F (SqpollProactorTest, stop) ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op, true, true), 0) << join::lastError.message (); proactor.stop (); th.join (); @@ -189,11 +191,11 @@ TEST_F (SqpollProactorTest, submit) proactor.run (); }); - ASSERT_EQ (proactor.submit (nullptr, true), -1); + ASSERT_EQ (proactor.submit (nullptr, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (proactor.submit (&op1, true, true), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); @@ -201,19 +203,19 @@ TEST_F (SqpollProactorTest, submit) op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); op1.state = IoOperation::State::Submitted; - ASSERT_EQ (proactor.submit (&op1, true), -1); + ASSERT_EQ (proactor.submit (&op1, true, true), -1); ASSERT_EQ (join::lastError, Errc::OperationFailed); op1.state = IoOperation::State::Idle; - ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op1, true, true), 0) << join::lastError.message (); #ifndef JOIN_HAS_IO_URING auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op2, true), -1); + ASSERT_EQ (proactor.submit (&op2, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); #endif - ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.cancel (&op1, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { @@ -237,28 +239,28 @@ TEST_F (SqpollProactorTest, cancel) proactor.run (); }); - ASSERT_EQ (proactor.cancel (nullptr, true), -1); + ASSERT_EQ (proactor.cancel (nullptr, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); auto op1 = IoOperation::makeRead (-1, _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (proactor.cancel (&op1, true, true), -1); ASSERT_EQ (join::lastError, std::errc::bad_file_descriptor); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); op1 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.cancel (&op1, true), -1); + ASSERT_EQ (proactor.cancel (&op1, true, true), -1); ASSERT_EQ (join::lastError, Errc::OperationFailed); - ASSERT_EQ (proactor.submit (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op1, true, true), 0) << join::lastError.message (); auto op2 = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); op2.state = IoOperation::State::Submitted; - ASSERT_EQ (proactor.cancel (&op2, true), -1); + ASSERT_EQ (proactor.cancel (&op2, true, true), -1); ASSERT_EQ (join::lastError, Errc::InvalidParam); - ASSERT_EQ (proactor.cancel (&op1, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.cancel (&op1, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { @@ -286,8 +288,8 @@ TEST_F (SqpollProactorTest, flush) ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (proactor.submit (&op, false), 0) << join::lastError.message (); - ASSERT_EQ (proactor.flush (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&op, false, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.flush (true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("flush", strlen ("flush"), _timeout), 0) << join::lastError.message (); @@ -318,12 +320,11 @@ TEST_F (SqpollProactorTest, chain) ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this); + auto writeOp = IoOperation::makeWrite (_server.handle (), "ping", 4, this, true); auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - writeOp.linked = true; - ASSERT_EQ (proactor.submit (&writeOp, false), 0) << join::lastError.message (); - ASSERT_EQ (proactor.submit (&readOp, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&writeOp, false, true), 0) << join::lastError.message (); + ASSERT_EQ (proactor.submit (&readOp, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); @@ -415,9 +416,37 @@ TEST_F (SqpollProactorTest, registerBuffers) ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); ASSERT_EQ (proactor.registerBuffers (iovecs), 0) << join::lastError.message (); ASSERT_EQ (proactor.unregisterBuffers (), 0) << join::lastError.message (); + ASSERT_EQ (proactor.unregisterBuffers (), -1); } #endif +/** + * @brief Test async connect. + */ +TEST_F (SqpollProactorTest, asyncConnect) +{ + Tcp::Endpoint endpoint{_host, _port}; + + ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); + _client.setMode (Tcp::Socket::NonBlocking); + + auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result == 0; + })); + _op = nullptr; + _result = 0; + } + + _client.setMode (Tcp::Socket::Blocking); +} + /** * @brief Test async accept. */ @@ -438,7 +467,7 @@ TEST_F (SqpollProactorTest, asyncAccept) auto op = IoOperation::makeAccept (_acceptor.handle (), reinterpret_cast (&addr), &addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC, this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); { @@ -453,30 +482,41 @@ TEST_F (SqpollProactorTest, asyncAccept) } /** - * @brief Test async connect. + * @brief Test async write. */ -TEST_F (SqpollProactorTest, asyncConnect) +TEST_F (SqpollProactorTest, asyncWrite) { - Tcp::Endpoint endpoint{_host, _port}; + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (_client.open (Tcp::v4 ()), 0) << join::lastError.message (); - _client.setMode (Tcp::Socket::NonBlocking); + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); - auto op = IoOperation::makeConnect (_client.handle (), endpoint.addr (), endpoint.length (), this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + const char* msg = "asyncWrite"; + auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); - ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &op && _result == 0; + return _op == &op && _result > 0; })); + ASSERT_EQ (_result, static_cast (strlen (msg))); _op = nullptr; _result = 0; } - _client.setMode (Tcp::Socket::Blocking); + char rbuf[256] = {}; + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); } /** @@ -499,7 +539,7 @@ TEST_F (SqpollProactorTest, asyncRead) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("asyncRead", strlen ("asyncRead"), _timeout), 0) << join::lastError.message (); { @@ -514,27 +554,22 @@ TEST_F (SqpollProactorTest, asyncRead) } /** - * @brief Test async write. + * @brief Test async write fixed. */ -TEST_F (SqpollProactorTest, asyncWrite) +TEST_F (SqpollProactorTest, asyncWriteFixed) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::affinity (), 0); - ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::priority (), 1); -#ifdef JOIN_HAS_NUMA - ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); -#endif - ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); - ASSERT_GT (SqpollProactorThread::handle (), 0); + const char* msg = "asyncWriteFixed"; + std::vector regbuf (msg, msg + strlen (msg)); + iovec iov = {regbuf.data (), regbuf.size ()}; + std::vector iovecs = {iov}; - const char* msg = "asyncWrite"; - auto op = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + ASSERT_EQ (SqpollProactorThread::proactor ().registerBuffers (iovecs), 0) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + auto op = IoOperation::makeWriteFixed (_server.handle (), regbuf.data (), regbuf.size (), 0, this); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); @@ -549,48 +584,82 @@ TEST_F (SqpollProactorTest, asyncWrite) char rbuf[256] = {}; ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); + + ASSERT_EQ (SqpollProactorThread::proactor ().unregisterBuffers (), 0) << join::lastError.message (); } /** - * @brief Test async read and write. + * @brief Test async read fixed. */ -TEST_F (SqpollProactorTest, asyncReadWrite) +TEST_F (SqpollProactorTest, asyncReadFixed) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - const char* msg = "asyncReadWrite"; - auto readOp = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - auto writeOp = IoOperation::makeWrite (_server.handle (), msg, strlen (msg), this); + std::vector regbuf (sizeof (_buf)); + iovec iov = {regbuf.data (), regbuf.size ()}; + std::vector iovecs = {iov}; - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&readOp, true), 0) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&writeOp, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().registerBuffers (iovecs), 0) << join::lastError.message (); + + auto op = IoOperation::makeReadFixed (_server.handle (), regbuf.data (), regbuf.size (), 0, this); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncReadFixed", strlen ("asyncReadFixed"), _timeout), 0) + << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &writeOp && _result > 0; + return _op == &op && _result > 0; })); - ASSERT_EQ (_result, static_cast (strlen (msg))); + ASSERT_EQ (std::string (regbuf.data (), _result), "asyncReadFixed"); _op = nullptr; _result = 0; } - ASSERT_EQ (_client.writeExactly (msg, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().unregisterBuffers (), 0) << join::lastError.message (); +} + +/** + * @brief Test async sendmsg. + */ +TEST_F (SqpollProactorTest, asyncSendmsg) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::affinity (), 0); + ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::priority (), 1); +#ifdef JOIN_HAS_NUMA + ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); +#endif + ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); + ASSERT_GT (SqpollProactorThread::handle (), 0); + + const char* payload = "asyncSendmsg"; + iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; + msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { - return _op == &readOp && _result > 0; + return _op == &op && _result > 0; })); - ASSERT_EQ (std::string (_buf, _result), msg); + ASSERT_EQ (_result, static_cast (strlen (payload))); _op = nullptr; _result = 0; } char rbuf[256] = {}; - ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); - ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); + ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); } /** @@ -617,7 +686,7 @@ TEST_F (SqpollProactorTest, asyncRecvmsg) msg.msg_iovlen = 1; auto op = IoOperation::makeRecvmsg (_server.handle (), &msg, 0, this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); ASSERT_EQ (_client.writeExactly ("asyncRecvmsg", strlen ("asyncRecvmsg"), _timeout), 0) << join::lastError.message (); @@ -633,45 +702,55 @@ TEST_F (SqpollProactorTest, asyncRecvmsg) } /** - * @brief Test async sendmsg. + * @brief Test async send. */ -TEST_F (SqpollProactorTest, asyncSendmsg) +TEST_F (SqpollProactorTest, asyncSend) { ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::affinity (0), 0) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::affinity (), 0); - ASSERT_EQ (SqpollProactorThread::priority (1), 0) << join::lastError.message (); - ASSERT_EQ (SqpollProactorThread::priority (), 1); -#ifdef JOIN_HAS_NUMA - ASSERT_EQ (SqpollProactorThread::mbind (0), 0) << join::lastError.message (); -#endif - ASSERT_EQ (SqpollProactorThread::mlock (), 0) << join::lastError.message (); - ASSERT_GT (SqpollProactorThread::handle (), 0); - - const char* payload = "asyncSendmsg"; - iovec iov = {.iov_base = const_cast (payload), .iov_len = strlen (payload)}; - msghdr msg = {}; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - auto op = IoOperation::makeSendmsg (_server.handle (), &msg, 0, this); + const char* msg = "asyncSend"; + auto op = IoOperation::makeSend (_server.handle (), msg, strlen (msg), 0, this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); { ScopedLock lock (_mut); ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { return _op == &op && _result > 0; })); - ASSERT_EQ (_result, static_cast (strlen (payload))); + ASSERT_EQ (_result, static_cast (strlen (msg))); _op = nullptr; _result = 0; } char rbuf[256] = {}; - ASSERT_EQ (_client.readExactly (rbuf, strlen (payload), _timeout), 0) << join::lastError.message (); - ASSERT_EQ (std::string (rbuf, strlen (payload)), payload); + ASSERT_EQ (_client.readExactly (rbuf, strlen (msg), _timeout), 0) << join::lastError.message (); + ASSERT_EQ (std::string (rbuf, strlen (msg)), msg); +} + +/** + * @brief Test async recv. + */ +TEST_F (SqpollProactorTest, asyncRecv) +{ + ASSERT_EQ (_client.connect ({_host, _port}), 0) << join::lastError.message (); + ASSERT_TRUE ((_server = _acceptor.accept ()).connected ()) << join::lastError.message (); + + auto op = IoOperation::makeRecv (_server.handle (), _buf, sizeof (_buf), 0, this); + + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); + ASSERT_EQ (_client.writeExactly ("asyncRecv", strlen ("asyncRecv"), _timeout), 0) << join::lastError.message (); + + { + ScopedLock lock (_mut); + ASSERT_TRUE (_cond.timedWait (lock, std::chrono::milliseconds (_timeout), [&] () { + return _op == &op && _result > 0; + })); + ASSERT_EQ (std::string (_buf, _result), "asyncRecv"); + _op = nullptr; + _result = 0; + } } /** @@ -694,7 +773,7 @@ TEST_F (SqpollProactorTest, onClose) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); _client.close (); { @@ -727,7 +806,7 @@ TEST_F (SqpollProactorTest, onError) auto op = IoOperation::makeRead (_server.handle (), _buf, sizeof (_buf), this); - ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true), 0) << join::lastError.message (); + ASSERT_EQ (SqpollProactorThread::proactor ().submit (&op, true, true), 0) << join::lastError.message (); linger sl{.l_onoff = 1, .l_linger = 0}; ASSERT_EQ (setsockopt (_client.handle (), SOL_SOCKET, SO_LINGER, &sl, sizeof (sl)), 0) << strerror (errno); _client.close (); From 26b47464b7a08c4b8bccce9a63d159aca326e5dc Mon Sep 17 00:00:00 2001 From: mrabine Date: Fri, 29 May 2026 20:26:45 +0200 Subject: [PATCH 6/7] coverage --- core/include/join/proactor_epoll_impl.hpp | 8 ++++++-- core/include/join/proactor_uring_impl.hpp | 17 +++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/core/include/join/proactor_epoll_impl.hpp b/core/include/join/proactor_epoll_impl.hpp index 349a2c5e..94199221 100644 --- a/core/include/join/proactor_epoll_impl.hpp +++ b/core/include/join/proactor_epoll_impl.hpp @@ -222,7 +222,9 @@ inline void join::BasicProactor::processCommand (const Command& cmd) noexcept break; default: - break; // LCOV_EXCL_LINE + // LCOV_EXCL_START + break; + // LCOV_EXCL_STOP } if (JOIN_UNLIKELY (cmd.done)) @@ -532,7 +534,9 @@ inline int join::BasicProactor::executeOp (IoOperation* op) noexcept } default: - return -EINVAL; // LCOV_EXCL_LINE + // LCOV_EXCL_START + return -EINVAL; + // LCOV_EXCL_STOP } } } diff --git a/core/include/join/proactor_uring_impl.hpp b/core/include/join/proactor_uring_impl.hpp index 23747fa9..8c64ba6b 100644 --- a/core/include/join/proactor_uring_impl.hpp +++ b/core/include/join/proactor_uring_impl.hpp @@ -79,8 +79,10 @@ int join::BasicProactor::flush (bool sync) noexcept { if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0)) { + // LCOV_EXCL_START lastError = std::error_code (errno, std::system_category ()); return -1; + // LCOV_EXCL_STOP } return 0; } @@ -453,7 +455,9 @@ void join::BasicProactor::processCommand (const Command& cmd) noexcept break; default: - break; // LCOV_EXCL_LINE + // LCOV_EXCL_START + break; + // LCOV_EXCL_STOP } if (JOIN_UNLIKELY (cmd.done)) @@ -671,8 +675,9 @@ void join::BasicProactor::prepareSqe (io_uring_sqe* sqe, IoOperation* op break; default: - io_uring_prep_nop (sqe); // LCOV_EXCL_LINE - break; + // LCOV_EXCL_START + io_uring_prep_nop (sqe); + // LCOV_EXCL_STOP } io_uring_sqe_set_data (sqe, op); @@ -736,7 +741,7 @@ void join::BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::true_type if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) { - return; + return; // LCOV_EXCL_LINE } int result = cqe->res; @@ -759,7 +764,7 @@ void join::BasicProactor::dispatchCqe (io_uring_cqe* cqe, std::false_typ if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle)) { - return; + return; // LCOV_EXCL_LINE } int result = cqe->res; @@ -797,7 +802,7 @@ void join::BasicProactor::eventLoop (std::false_type, std::false_type) n { if (JOIN_UNLIKELY (io_uring_wait_cqe (&_ring, &cqe) < 0)) { - continue; + continue; // LCOV_EXCL_LINE } } else From a92c61bf5edac771b27f66b2babfa95daef50b4f Mon Sep 17 00:00:00 2001 From: mrabine Date: Fri, 29 May 2026 20:43:40 +0200 Subject: [PATCH 7/7] coverage --- CMakePresets.json | 9 +++++---- core/include/join/proactor_epoll_impl.hpp | 4 ---- core/include/join/proactor_uring_impl.hpp | 10 ++++------ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index b3c4596a..93c29a04 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -19,8 +19,7 @@ "inherits": "base", "cacheVariables": { "CMAKE_C_COMPILER": "gcc", - "CMAKE_CXX_COMPILER": "g++", - "JOIN_ENABLE_IO_URING": "ON" + "CMAKE_CXX_COMPILER": "g++" } }, { @@ -39,7 +38,8 @@ "binaryDir": "${sourceDir}/build/gcc/debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", - "JOIN_ENABLE_COVERAGE": "ON" + "JOIN_ENABLE_COVERAGE": "ON", + "JOIN_ENABLE_IO_URING": "ON" } }, { @@ -59,7 +59,8 @@ "binaryDir": "${sourceDir}/build/clang/debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", - "JOIN_ENABLE_COVERAGE": "ON" + "JOIN_ENABLE_COVERAGE": "ON", + "JOIN_ENABLE_IO_URING": "ON" } }, { diff --git a/core/include/join/proactor_epoll_impl.hpp b/core/include/join/proactor_epoll_impl.hpp index 94199221..9460faab 100644 --- a/core/include/join/proactor_epoll_impl.hpp +++ b/core/include/join/proactor_epoll_impl.hpp @@ -222,9 +222,7 @@ inline void join::BasicProactor::processCommand (const Command& cmd) noexcept break; default: - // LCOV_EXCL_START break; - // LCOV_EXCL_STOP } if (JOIN_UNLIKELY (cmd.done)) @@ -534,9 +532,7 @@ inline int join::BasicProactor::executeOp (IoOperation* op) noexcept } default: - // LCOV_EXCL_START return -EINVAL; - // LCOV_EXCL_STOP } } } diff --git a/core/include/join/proactor_uring_impl.hpp b/core/include/join/proactor_uring_impl.hpp index 8c64ba6b..fc953637 100644 --- a/core/include/join/proactor_uring_impl.hpp +++ b/core/include/join/proactor_uring_impl.hpp @@ -455,9 +455,7 @@ void join::BasicProactor::processCommand (const Command& cmd) noexcept break; default: - // LCOV_EXCL_START break; - // LCOV_EXCL_STOP } if (JOIN_UNLIKELY (cmd.done)) @@ -675,9 +673,7 @@ void join::BasicProactor::prepareSqe (io_uring_sqe* sqe, IoOperation* op break; default: - // LCOV_EXCL_START io_uring_prep_nop (sqe); - // LCOV_EXCL_STOP } io_uring_sqe_set_data (sqe, op); @@ -807,11 +803,13 @@ void join::BasicProactor::eventLoop (std::false_type, std::false_type) n } else { + // LCOV_EXCL_START io_uring_submit (&_ring); if (JOIN_UNLIKELY (io_uring_peek_cqe (&_ring, &cqe) != 0)) { continue; } + // LCOV_EXCL_STOP } do @@ -841,7 +839,7 @@ void join::BasicProactor::eventLoop (std::true_type, std::false_type) no } else { - io_uring_submit (&_ring); + io_uring_submit (&_ring); // LCOV_EXCL_LINE } io_uring_cqe* cqe = nullptr; @@ -885,7 +883,7 @@ void join::BasicProactor::eventLoop (std::true_type, std::true_type) noe } else { - io_uring_submit (&_ring); + io_uring_submit (&_ring); // LCOV_EXCL_LINE } io_uring_cqe* cqe = nullptr;