diff --git a/include/tmc/atomic_condvar.hpp b/include/tmc/atomic_condvar.hpp index f1724787..29123520 100644 --- a/include/tmc/atomic_condvar.hpp +++ b/include/tmc/atomic_condvar.hpp @@ -13,6 +13,7 @@ #include "tmc/detail/waiter_list.hpp" #include +#include #include #include #include @@ -24,22 +25,24 @@ class waiter_count_accessor; namespace tmc { template class atomic_condvar; +template class aw_atomic_condvar; template class aw_atomic_condvar_co_notify; -/// The awaitable type returned by `atomic_condvar.await()` +/// The awaiter type produced by co_awaiting aw_atomic_condvar. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. template -class [[nodiscard( - "You must co_await aw_atomic_condvar for it to have any effect." -)]] aw_atomic_condvar : tmc::detail::AwaitTagNoGroupAsIs { +class aw_atomic_condvar_impl { T expected; atomic_condvar& parent; tmc::detail::waiter_list_waiter waiter; + friend class aw_atomic_condvar; friend class atomic_condvar; friend class aw_atomic_condvar_co_notify; - inline aw_atomic_condvar( - atomic_condvar& Parent TMC_LIFETIMEBOUND, T Expected + inline aw_atomic_condvar_impl( + atomic_condvar& Parent, T Expected ) noexcept : expected(Expected), parent(Parent) {} @@ -70,9 +73,40 @@ class [[nodiscard( inline void await_resume() noexcept {} // Cannot be moved or copied due to holding intrusive list pointer + aw_atomic_condvar_impl(aw_atomic_condvar_impl const&) = delete; + aw_atomic_condvar_impl& operator=(aw_atomic_condvar_impl const&) = delete; + aw_atomic_condvar_impl(aw_atomic_condvar_impl&&) = delete; + aw_atomic_condvar_impl& operator=(aw_atomic_condvar_impl&&) = delete; +}; + +/// The awaitable type returned by `atomic_condvar.await()` +template +class [[nodiscard( + "You must co_await aw_atomic_condvar for it to have any effect." +)]] aw_atomic_condvar : tmc::detail::AwaitTagNoGroupCoAwait { + atomic_condvar* parent; + T expected; + + friend class atomic_condvar; + + inline aw_atomic_condvar( + atomic_condvar& Parent TMC_LIFETIMEBOUND, T Expected + ) noexcept + : parent(&Parent), expected(Expected) {} + +public: + inline aw_atomic_condvar_impl operator co_await() && noexcept { + assert(parent != nullptr && "aw_atomic_condvar may only be awaited once"); + return aw_atomic_condvar_impl(*parent, expected); + } + + // Movable but not copyable aw_atomic_condvar(aw_atomic_condvar const&) = delete; aw_atomic_condvar& operator=(aw_atomic_condvar const&) = delete; - aw_atomic_condvar(aw_atomic_condvar&&) = delete; + inline aw_atomic_condvar(aw_atomic_condvar&& Other) noexcept + : parent(Other.parent), expected(Other.expected) { + Other.parent = nullptr; + } aw_atomic_condvar& operator=(aw_atomic_condvar&&) = delete; }; @@ -97,7 +131,7 @@ class [[nodiscard( inline std::coroutine_handle<> await_suspend(std::coroutine_handle<> Outer) noexcept { - std::vector*> wakeList = + std::vector*> wakeList = parent.get_n_waiters(notify_count); size_t sz = wakeList.size(); if (sz == 0) { @@ -112,11 +146,12 @@ class [[nodiscard( inline void await_resume() noexcept {} - // Copy/move constructors *could* be implemented, but why? + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_atomic_condvar_co_notify(aw_atomic_condvar_co_notify const&) = delete; aw_atomic_condvar_co_notify& operator=(aw_atomic_condvar_co_notify const&) = delete; - aw_atomic_condvar_co_notify(atomic_condvar&&) = delete; + aw_atomic_condvar_co_notify(aw_atomic_condvar_co_notify&&) = default; aw_atomic_condvar_co_notify& operator=(aw_atomic_condvar_co_notify&&) = delete; }; @@ -125,10 +160,10 @@ class [[nodiscard( /// `std::atomic::wait()` and `std::atomic::notify_*()`. template class atomic_condvar { std::mutex waiters_lock; - std::vector*> waiters; + std::vector*> waiters; std::atomic value; - friend class aw_atomic_condvar; + friend class aw_atomic_condvar_impl; friend class aw_atomic_condvar_co_notify; friend class ::tmc::tests::waiter_count_accessor; @@ -141,8 +176,8 @@ template class atomic_condvar { // Returns the most recently added waiter that matches the expected // condition. - inline aw_atomic_condvar* get_one_waiter() { - aw_atomic_condvar* toWake = nullptr; + inline aw_atomic_condvar_impl* get_one_waiter() { + aw_atomic_condvar_impl* toWake = nullptr; { std::scoped_lock l{waiters_lock}; auto v = value.load(std::memory_order_seq_cst); @@ -161,8 +196,8 @@ template class atomic_condvar { // Returns up to N waiters that match the expected condition. The most // recently added waiters are at the front of the returned list. - inline std::vector*> get_n_waiters(size_t N) { - std::vector*> wakeList; + inline std::vector*> get_n_waiters(size_t N) { + std::vector*> wakeList; { std::scoped_lock l{waiters_lock}; auto v = value.load(std::memory_order_seq_cst); @@ -223,7 +258,7 @@ template class atomic_condvar { /// Does not symmetric transfer; the awaiter will be posted to its executor. /// Awaiters are woken in LIFO order. inline void notify_one() { - aw_atomic_condvar* toWake = get_one_waiter(); + aw_atomic_condvar_impl* toWake = get_one_waiter(); if (toWake != nullptr) { toWake->waiter.resume(); } @@ -237,7 +272,7 @@ template class atomic_condvar { if (NotifyCount == 0) { return; } - std::vector*> wakeList = get_n_waiters(NotifyCount); + std::vector*> wakeList = get_n_waiters(NotifyCount); for (size_t i = 0; i < wakeList.size(); ++i) { wakeList[i]->waiter.resume(); } @@ -246,7 +281,7 @@ template class atomic_condvar { /// Wakes all awaiters that meet the criteria (expected != current value). /// Does not symmetric transfer; awaiters will be posted to their executors. inline void notify_all() { - std::vector*> wakeList = get_n_waiters(TMC_ALL_ONES); + std::vector*> wakeList = get_n_waiters(TMC_ALL_ONES); for (size_t i = 0; i < wakeList.size(); ++i) { wakeList[i]->waiter.resume(); } @@ -263,7 +298,7 @@ template class atomic_condvar { std::scoped_lock l{waiters_lock}; // No need to unlock before resuming here - it's not valid for resumers to // access the destroyed mutex anyway. - std::vector*> wakeList; + std::vector*> wakeList; for (size_t i = 0; i < waiters.size(); ++i) { waiters[i]->waiter.resume(); } diff --git a/include/tmc/auto_reset_event.hpp b/include/tmc/auto_reset_event.hpp index 02410435..3818c5f9 100644 --- a/include/tmc/auto_reset_event.hpp +++ b/include/tmc/auto_reset_event.hpp @@ -38,11 +38,12 @@ class [[nodiscard( inline void await_resume() noexcept {} - // Copy/move constructors *could* be implemented, but why? + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_auto_reset_event_co_set(aw_auto_reset_event_co_set const&) = delete; aw_auto_reset_event_co_set& operator=(aw_auto_reset_event_co_set const&) = delete; - aw_auto_reset_event_co_set(aw_auto_reset_event_co_set&&) = delete; + aw_auto_reset_event_co_set(aw_auto_reset_event_co_set&&) = default; aw_auto_reset_event_co_set& operator=(aw_auto_reset_event_co_set&&) = delete; }; diff --git a/include/tmc/detail/mutex.ipp b/include/tmc/detail/mutex.ipp index 027f8349..3aed46d2 100644 --- a/include/tmc/detail/mutex.ipp +++ b/include/tmc/detail/mutex.ipp @@ -22,13 +22,13 @@ mutex_scope::~mutex_scope() { } } -bool aw_mutex_lock_scope::await_ready() noexcept { +bool aw_mutex_lock_scope_impl::await_ready() noexcept { return tmc::detail::try_acquire( parent.load(std::memory_order_relaxed)->value ); } -void aw_mutex_lock_scope::await_suspend( +void aw_mutex_lock_scope_impl::await_suspend( std::coroutine_handle<> Outer ) noexcept { // This may be resumed immediately after we call add_waiter(). Access to diff --git a/include/tmc/detail/semaphore.ipp b/include/tmc/detail/semaphore.ipp index 03574a55..7985be45 100644 --- a/include/tmc/detail/semaphore.ipp +++ b/include/tmc/detail/semaphore.ipp @@ -21,13 +21,13 @@ semaphore_scope::~semaphore_scope() { } } -bool aw_semaphore_acquire_scope::await_ready() noexcept { +bool aw_semaphore_acquire_scope_impl::await_ready() noexcept { return tmc::detail::try_acquire( parent.load(std::memory_order_relaxed)->value ); } -void aw_semaphore_acquire_scope::await_suspend( +void aw_semaphore_acquire_scope_impl::await_suspend( std::coroutine_handle<> Outer ) noexcept { // This may be resumed immediately after we call add_waiter(). Access to diff --git a/include/tmc/detail/task_wrapper.hpp b/include/tmc/detail/task_wrapper.hpp index cc1534c0..2f0bf042 100644 --- a/include/tmc/detail/task_wrapper.hpp +++ b/include/tmc/detail/task_wrapper.hpp @@ -119,7 +119,16 @@ template struct task_wrapper_promise { std::is_nothrow_move_constructible_v && std::is_nothrow_move_assignable_v ) { - *customizer.result_ptr = static_cast(Value); + if constexpr (requires { + *customizer.result_ptr = static_cast(Value); + }) { + *customizer.result_ptr = static_cast(Value); + } else { + // Results that are move-constructible but not move-assignable (such as + // mutex_scope) are stored in a std::optional; construct the value in + // place instead. + customizer.result_ptr->emplace(static_cast(Value)); + } } #ifdef TMC_DEBUG_TASK_ALLOC_COUNT diff --git a/include/tmc/manual_reset_event.hpp b/include/tmc/manual_reset_event.hpp index 55b2a4d9..e40a05ed 100644 --- a/include/tmc/manual_reset_event.hpp +++ b/include/tmc/manual_reset_event.hpp @@ -65,10 +65,11 @@ class [[nodiscard( /// Returns the number of awaiters that were woken. inline size_t await_resume() noexcept { return wakeCount; } - // Copy/move constructors *could* be implemented, but why? + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_manual_reset_event_co_set(aw_manual_reset_event_co_set const&) = delete; aw_manual_reset_event_co_set& operator=(aw_manual_reset_event_co_set const&) = delete; - aw_manual_reset_event_co_set(aw_manual_reset_event_co_set&&) = delete; + aw_manual_reset_event_co_set(aw_manual_reset_event_co_set&&) = default; aw_manual_reset_event_co_set& operator=(aw_manual_reset_event_co_set&&) = delete; }; diff --git a/include/tmc/mutex.hpp b/include/tmc/mutex.hpp index c9105a5a..2788c987 100644 --- a/include/tmc/mutex.hpp +++ b/include/tmc/mutex.hpp @@ -27,7 +27,7 @@ class [[nodiscard("The mutex will be unlocked when this goes out of scope.")]] mutex_scope { mutex* parent; - friend class aw_mutex_lock_scope; + friend class aw_mutex_lock_scope_impl; inline mutex_scope(mutex* Parent TMC_LIFETIMEBOUND) noexcept : parent(Parent) {} @@ -45,17 +45,16 @@ mutex_scope { TMC_DECL ~mutex_scope(); }; -/// Same as aw_acquire but returns a nodiscard mutex_scope that unlocks the -/// mutex on destruction. -class [[nodiscard( - "You must co_await aw_mutex_lock_scope for it to have any effect." -)]] aw_mutex_lock_scope : tmc::detail::AwaitTagNoGroupAsIs { +/// The awaiter type produced by co_awaiting aw_mutex_lock_scope. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. +class aw_mutex_lock_scope_impl { tmc::detail::waiter_list_node me; std::atomic parent; - friend class mutex; + friend class aw_mutex_lock_scope; - inline aw_mutex_lock_scope(mutex& Parent) noexcept : parent(&Parent) {} + inline aw_mutex_lock_scope_impl(mutex& Parent) noexcept : parent(&Parent) {} public: TMC_DECL bool await_ready() noexcept; @@ -67,9 +66,37 @@ class [[nodiscard( } // Cannot be moved or copied due to holding intrusive list pointer + aw_mutex_lock_scope_impl(aw_mutex_lock_scope_impl const&) = delete; + aw_mutex_lock_scope_impl& operator=(aw_mutex_lock_scope_impl const&) = delete; + aw_mutex_lock_scope_impl(aw_mutex_lock_scope_impl&&) = delete; + aw_mutex_lock_scope_impl& operator=(aw_mutex_lock_scope_impl&&) = delete; +}; + +/// Same as aw_acquire but returns a nodiscard mutex_scope that unlocks the +/// mutex on destruction. +class [[nodiscard( + "You must co_await aw_mutex_lock_scope for it to have any effect." +)]] aw_mutex_lock_scope : tmc::detail::AwaitTagNoGroupCoAwait { + mutex* parent; + + friend class mutex; + + inline aw_mutex_lock_scope(mutex& Parent TMC_LIFETIMEBOUND) noexcept + : parent(&Parent) {} + +public: + inline aw_mutex_lock_scope_impl operator co_await() && noexcept { + assert(parent != nullptr && "aw_mutex_lock_scope may only be awaited once"); + return aw_mutex_lock_scope_impl(*parent); + } + + // Movable but not copyable aw_mutex_lock_scope(aw_mutex_lock_scope const&) = delete; aw_mutex_lock_scope& operator=(aw_mutex_lock_scope const&) = delete; - aw_mutex_lock_scope(aw_mutex_lock_scope&&) = delete; + inline aw_mutex_lock_scope(aw_mutex_lock_scope&& Other) noexcept + : parent(Other.parent) { + Other.parent = nullptr; + } aw_mutex_lock_scope& operator=(aw_mutex_lock_scope&&) = delete; }; @@ -90,10 +117,11 @@ class [[nodiscard( inline void await_resume() noexcept {} - // Copy/move constructors *could* be implemented, but why? + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_mutex_co_unlock(aw_mutex_co_unlock const&) = delete; aw_mutex_co_unlock& operator=(aw_mutex_co_unlock const&) = delete; - aw_mutex_co_unlock(aw_mutex_co_unlock&&) = delete; + aw_mutex_co_unlock(aw_mutex_co_unlock&&) = default; aw_mutex_co_unlock& operator=(aw_mutex_co_unlock&&) = delete; }; @@ -135,17 +163,19 @@ class [[nodiscard( [[maybe_unused]] inline void await_resume() noexcept {} + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_mutex_co_unlock_return(aw_mutex_co_unlock_return const&) = delete; aw_mutex_co_unlock_return& operator=(aw_mutex_co_unlock_return const&) = delete; - aw_mutex_co_unlock_return(aw_mutex_co_unlock_return&&) = delete; + aw_mutex_co_unlock_return(aw_mutex_co_unlock_return&&) = default; aw_mutex_co_unlock_return& operator=(aw_mutex_co_unlock_return&&) = delete; }; /// An async version of std::mutex. class mutex : protected tmc::detail::waiter_data_base { friend class aw_acquire; - friend class aw_mutex_lock_scope; + friend class aw_mutex_lock_scope_impl; friend class aw_mutex_co_unlock; template friend class aw_mutex_co_unlock_return; friend class ::tmc::tests::waiter_count_accessor; @@ -257,7 +287,7 @@ class mutex : protected tmc::detail::waiter_data_base { /// ownership to this task. Not re-entrant. /// Returns an object that will unlock the mutex (and resume an awaiter) when /// it goes out of scope. - inline aw_mutex_lock_scope lock_scope() noexcept { + inline aw_mutex_lock_scope lock_scope() noexcept TMC_LIFETIMEBOUND { return aw_mutex_lock_scope(*this); } diff --git a/include/tmc/rw_lock.hpp b/include/tmc/rw_lock.hpp index d49dfdd2..fb01ab97 100644 --- a/include/tmc/rw_lock.hpp +++ b/include/tmc/rw_lock.hpp @@ -11,6 +11,7 @@ #include "tmc/detail/waiter_list.hpp" #include +#include #include #include #include @@ -28,7 +29,7 @@ class [[nodiscard("The read lock will be released when this goes out of scope.") rw_lock_read_scope { rw_lock* parent; - friend class aw_rw_lock_read_scope; + friend class aw_rw_lock_read_scope_impl; inline rw_lock_read_scope(rw_lock* Parent TMC_LIFETIMEBOUND) noexcept : parent(Parent) {} @@ -52,7 +53,7 @@ class [[nodiscard("The write lock will be released when this goes out of scope." rw_lock_write_scope { rw_lock* parent; - friend class aw_rw_lock_write_scope; + friend class aw_rw_lock_write_scope_impl; inline rw_lock_write_scope(rw_lock* Parent TMC_LIFETIMEBOUND) noexcept : parent(Parent) {} @@ -73,7 +74,7 @@ rw_lock_write_scope { // Common state shared by all four rw_lock awaiters. Same structure as // aw_acquire but `value` has different semantics. -class aw_rw_lock_base : tmc::detail::AwaitTagNoGroupAsIs { +class aw_rw_lock_base { protected: tmc::detail::waiter_list_node me; // parent is atomic to prevent use-after-resume. See waiter_list.ipp's @@ -112,53 +113,168 @@ class aw_rw_lock_write_base : public aw_rw_lock_base { TMC_DECL void await_suspend(std::coroutine_handle<> Outer) noexcept; }; +/// The awaiter type produced by co_awaiting aw_rw_lock_read. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. +class aw_rw_lock_read_impl : public aw_rw_lock_read_base { + friend class aw_rw_lock_read; + + inline aw_rw_lock_read_impl(rw_lock& Parent) noexcept + : aw_rw_lock_read_base(Parent) {} + +public: + inline void await_resume() noexcept {} +}; + +/// The awaiter type produced by co_awaiting aw_rw_lock_write. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. +class aw_rw_lock_write_impl : public aw_rw_lock_write_base { + friend class aw_rw_lock_write; + + inline aw_rw_lock_write_impl(rw_lock& Parent) noexcept + : aw_rw_lock_write_base(Parent) {} + +public: + inline void await_resume() noexcept {} +}; + +/// The awaiter type produced by co_awaiting aw_rw_lock_read_scope. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. +class aw_rw_lock_read_scope_impl : public aw_rw_lock_read_base { + friend class aw_rw_lock_read_scope; + + inline aw_rw_lock_read_scope_impl(rw_lock& Parent) noexcept + : aw_rw_lock_read_base(Parent) {} + +public: + [[nodiscard]] inline rw_lock_read_scope await_resume() noexcept { + return rw_lock_read_scope(parent.load(std::memory_order_relaxed)); + } +}; + +/// The awaiter type produced by co_awaiting aw_rw_lock_write_scope. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. +class aw_rw_lock_write_scope_impl : public aw_rw_lock_write_base { + friend class aw_rw_lock_write_scope; + + inline aw_rw_lock_write_scope_impl(rw_lock& Parent) noexcept + : aw_rw_lock_write_base(Parent) {} + +public: + [[nodiscard]] inline rw_lock_write_scope await_resume() noexcept { + return rw_lock_write_scope(parent.load(std::memory_order_relaxed)); + } +}; + class [[nodiscard("You must co_await aw_rw_lock_read for it to have any effect.")]] -aw_rw_lock_read : public aw_rw_lock_read_base { +aw_rw_lock_read : tmc::detail::AwaitTagNoGroupCoAwait { + rw_lock* parent; + friend class rw_lock; - inline aw_rw_lock_read(rw_lock& Parent) noexcept : aw_rw_lock_read_base(Parent) {} + inline aw_rw_lock_read(rw_lock& Parent TMC_LIFETIMEBOUND) noexcept + : parent(&Parent) {} public: - inline void await_resume() noexcept {} + inline aw_rw_lock_read_impl operator co_await() && noexcept { + assert(parent != nullptr && "aw_rw_lock_read may only be awaited once"); + return aw_rw_lock_read_impl(*parent); + } + + // Movable but not copyable + aw_rw_lock_read(aw_rw_lock_read const&) = delete; + aw_rw_lock_read& operator=(aw_rw_lock_read const&) = delete; + inline aw_rw_lock_read(aw_rw_lock_read&& Other) noexcept + : parent(Other.parent) { + Other.parent = nullptr; + } + aw_rw_lock_read& operator=(aw_rw_lock_read&&) = delete; }; class [[nodiscard("You must co_await aw_rw_lock_write for it to have any effect.")]] -aw_rw_lock_write : public aw_rw_lock_write_base { +aw_rw_lock_write : tmc::detail::AwaitTagNoGroupCoAwait { + rw_lock* parent; + friend class rw_lock; - inline aw_rw_lock_write(rw_lock& Parent) noexcept : aw_rw_lock_write_base(Parent) {} + inline aw_rw_lock_write(rw_lock& Parent TMC_LIFETIMEBOUND) noexcept + : parent(&Parent) {} public: - inline void await_resume() noexcept {} + inline aw_rw_lock_write_impl operator co_await() && noexcept { + assert(parent != nullptr && "aw_rw_lock_write may only be awaited once"); + return aw_rw_lock_write_impl(*parent); + } + + // Movable but not copyable + aw_rw_lock_write(aw_rw_lock_write const&) = delete; + aw_rw_lock_write& operator=(aw_rw_lock_write const&) = delete; + inline aw_rw_lock_write(aw_rw_lock_write&& Other) noexcept + : parent(Other.parent) { + Other.parent = nullptr; + } + aw_rw_lock_write& operator=(aw_rw_lock_write&&) = delete; }; /// Same as aw_rw_lock_read but returns a nodiscard rw_lock_read_scope that /// releases the read lock on destruction. class [[nodiscard("You must co_await aw_rw_lock_read_scope for it to have any effect.")]] -aw_rw_lock_read_scope : public aw_rw_lock_read_base { +aw_rw_lock_read_scope : tmc::detail::AwaitTagNoGroupCoAwait { + rw_lock* parent; + friend class rw_lock; - inline aw_rw_lock_read_scope(rw_lock& Parent) noexcept : aw_rw_lock_read_base(Parent) {} + inline aw_rw_lock_read_scope(rw_lock& Parent TMC_LIFETIMEBOUND) noexcept + : parent(&Parent) {} public: - [[nodiscard]] inline rw_lock_read_scope await_resume() noexcept { - return rw_lock_read_scope(parent.load(std::memory_order_relaxed)); + inline aw_rw_lock_read_scope_impl operator co_await() && noexcept { + assert( + parent != nullptr && "aw_rw_lock_read_scope may only be awaited once" + ); + return aw_rw_lock_read_scope_impl(*parent); } + + // Movable but not copyable + aw_rw_lock_read_scope(aw_rw_lock_read_scope const&) = delete; + aw_rw_lock_read_scope& operator=(aw_rw_lock_read_scope const&) = delete; + inline aw_rw_lock_read_scope(aw_rw_lock_read_scope&& Other) noexcept + : parent(Other.parent) { + Other.parent = nullptr; + } + aw_rw_lock_read_scope& operator=(aw_rw_lock_read_scope&&) = delete; }; /// Same as aw_rw_lock_write but returns a nodiscard rw_lock_write_scope that /// releases the write lock on destruction. class [[nodiscard("You must co_await aw_rw_lock_write_scope for it to have any effect.")]] -aw_rw_lock_write_scope : public aw_rw_lock_write_base { +aw_rw_lock_write_scope : tmc::detail::AwaitTagNoGroupCoAwait { + rw_lock* parent; + friend class rw_lock; - inline aw_rw_lock_write_scope(rw_lock& Parent) noexcept - : aw_rw_lock_write_base(Parent) {} + inline aw_rw_lock_write_scope(rw_lock& Parent TMC_LIFETIMEBOUND) noexcept + : parent(&Parent) {} public: - [[nodiscard]] inline rw_lock_write_scope await_resume() noexcept { - return rw_lock_write_scope(parent.load(std::memory_order_relaxed)); + inline aw_rw_lock_write_scope_impl operator co_await() && noexcept { + assert( + parent != nullptr && "aw_rw_lock_write_scope may only be awaited once" + ); + return aw_rw_lock_write_scope_impl(*parent); + } + + // Movable but not copyable + aw_rw_lock_write_scope(aw_rw_lock_write_scope const&) = delete; + aw_rw_lock_write_scope& operator=(aw_rw_lock_write_scope const&) = delete; + inline aw_rw_lock_write_scope(aw_rw_lock_write_scope&& Other) noexcept + : parent(Other.parent) { + Other.parent = nullptr; } + aw_rw_lock_write_scope& operator=(aw_rw_lock_write_scope&&) = delete; }; /// An async reader-writer lock (a.k.a. std::shared_mutex). Any number of @@ -182,10 +298,10 @@ aw_rw_lock_write_scope : public aw_rw_lock_write_base { /// falls back to a size_t word; on a 32-bit platform that reduces each field to 10 bits /// (max 1023). class rw_lock { - friend class aw_rw_lock_read; - friend class aw_rw_lock_write; - friend class aw_rw_lock_read_scope; - friend class aw_rw_lock_write_scope; + friend class aw_rw_lock_read_impl; + friend class aw_rw_lock_write_impl; + friend class aw_rw_lock_read_scope_impl; + friend class aw_rw_lock_write_scope_impl; friend class aw_rw_lock_read_base; friend class aw_rw_lock_write_base; friend class ::tmc::tests::waiter_count_accessor; @@ -296,22 +412,26 @@ class rw_lock { /// Tries to acquire the read lock. If a writer is holding or waiting for /// the lock, will suspend until the read lock can be acquired by this task. /// Multiple tasks may hold the read lock simultaneously. Not re-entrant. - inline aw_rw_lock_read lock_read() noexcept { return aw_rw_lock_read(*this); } + inline aw_rw_lock_read lock_read() noexcept TMC_LIFETIMEBOUND { + return aw_rw_lock_read(*this); + } /// Tries to acquire the write lock. If the lock is held by any other task, /// will suspend until the write lock can be acquired exclusively by this /// task. Not re-entrant. - inline aw_rw_lock_write lock_write() noexcept { return aw_rw_lock_write(*this); } + inline aw_rw_lock_write lock_write() noexcept TMC_LIFETIMEBOUND { + return aw_rw_lock_write(*this); + } /// Same as lock_read(), but returns an object that will release the read /// lock (and resume awaiters) when it goes out of scope. Not re-entrant. - inline aw_rw_lock_read_scope lock_read_scope() noexcept { + inline aw_rw_lock_read_scope lock_read_scope() noexcept TMC_LIFETIMEBOUND { return aw_rw_lock_read_scope(*this); } /// Same as lock_write(), but returns an object that will release the write /// lock (and resume awaiters) when it goes out of scope. Not re-entrant. - inline aw_rw_lock_write_scope lock_write_scope() noexcept { + inline aw_rw_lock_write_scope lock_write_scope() noexcept TMC_LIFETIMEBOUND { return aw_rw_lock_write_scope(*this); } diff --git a/include/tmc/semaphore.hpp b/include/tmc/semaphore.hpp index 1dc1894f..90c59cca 100644 --- a/include/tmc/semaphore.hpp +++ b/include/tmc/semaphore.hpp @@ -11,6 +11,7 @@ #include "tmc/detail/waiter_list.hpp" #include +#include #include #include #include @@ -28,7 +29,7 @@ class [[nodiscard( )]] semaphore_scope { semaphore* parent; - friend class aw_semaphore_acquire_scope; + friend class aw_semaphore_acquire_scope_impl; inline semaphore_scope(semaphore* Parent TMC_LIFETIMEBOUND) noexcept : parent(Parent) {} @@ -46,17 +47,16 @@ class [[nodiscard( TMC_DECL ~semaphore_scope(); }; -/// Same as aw_acquire but returns a nodiscard semaphore_scope that releases -/// the semaphore on destruction. -class [[nodiscard( - "You must co_await aw_semaphore_acquire_scope for it to have any effect." -)]] aw_semaphore_acquire_scope : tmc::detail::AwaitTagNoGroupAsIs { +/// The awaiter type produced by co_awaiting aw_semaphore_acquire_scope. It is +/// constructed in place in the awaiting coroutine's frame, where it lives +/// across the suspension. +class aw_semaphore_acquire_scope_impl { tmc::detail::waiter_list_node me; std::atomic parent; - friend class semaphore; + friend class aw_semaphore_acquire_scope; - inline aw_semaphore_acquire_scope(semaphore& Parent) noexcept + inline aw_semaphore_acquire_scope_impl(semaphore& Parent) noexcept : parent(&Parent) {} public: @@ -69,10 +69,44 @@ class [[nodiscard( } // Cannot be moved or copied due to holding intrusive list pointer + aw_semaphore_acquire_scope_impl(aw_semaphore_acquire_scope_impl const&) = + delete; + aw_semaphore_acquire_scope_impl& + operator=(aw_semaphore_acquire_scope_impl const&) = delete; + aw_semaphore_acquire_scope_impl(aw_semaphore_acquire_scope_impl&&) = delete; + aw_semaphore_acquire_scope_impl& + operator=(aw_semaphore_acquire_scope_impl&&) = delete; +}; + +/// Same as aw_acquire but returns a nodiscard semaphore_scope that releases +/// the semaphore on destruction. +class [[nodiscard( + "You must co_await aw_semaphore_acquire_scope for it to have any effect." +)]] aw_semaphore_acquire_scope : tmc::detail::AwaitTagNoGroupCoAwait { + semaphore* parent; + + friend class semaphore; + + inline aw_semaphore_acquire_scope(semaphore& Parent TMC_LIFETIMEBOUND) noexcept + : parent(&Parent) {} + +public: + inline aw_semaphore_acquire_scope_impl operator co_await() && noexcept { + assert( + parent != nullptr && + "aw_semaphore_acquire_scope may only be awaited once" + ); + return aw_semaphore_acquire_scope_impl(*parent); + } + + // Movable but not copyable aw_semaphore_acquire_scope(aw_semaphore_acquire_scope const&) = delete; aw_semaphore_acquire_scope& operator=(aw_semaphore_acquire_scope const&) = delete; - aw_semaphore_acquire_scope(aw_semaphore_acquire_scope&&) = delete; + inline aw_semaphore_acquire_scope(aw_semaphore_acquire_scope&& Other) noexcept + : parent(Other.parent) { + Other.parent = nullptr; + } aw_semaphore_acquire_scope& operator=(aw_semaphore_acquire_scope&&) = delete; }; @@ -94,10 +128,11 @@ class [[nodiscard( inline void await_resume() noexcept {} - // Copy/move constructors *could* be implemented, but why? + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_semaphore_co_release(aw_semaphore_co_release const&) = delete; aw_semaphore_co_release& operator=(aw_semaphore_co_release const&) = delete; - aw_semaphore_co_release(aw_semaphore_co_release&&) = delete; + aw_semaphore_co_release(aw_semaphore_co_release&&) = default; aw_semaphore_co_release& operator=(aw_semaphore_co_release&&) = delete; }; @@ -139,11 +174,13 @@ class [[nodiscard( [[maybe_unused]] inline void await_resume() noexcept {} + // Movable so that it can be captured by value into a wrapper task when + // passed to spawn() / fork(), but not copyable. aw_semaphore_co_release_return(aw_semaphore_co_release_return const&) = delete; aw_semaphore_co_release_return& operator=(aw_semaphore_co_release_return const&) = delete; - aw_semaphore_co_release_return(aw_semaphore_co_release_return&&) = delete; + aw_semaphore_co_release_return(aw_semaphore_co_release_return&&) = default; aw_semaphore_co_release_return& operator=(aw_semaphore_co_release_return&&) = delete; }; @@ -151,7 +188,7 @@ class [[nodiscard( /// An async version of std::counting_semaphore. class semaphore : protected tmc::detail::waiter_data_base { friend class aw_acquire; - friend class aw_semaphore_acquire_scope; + friend class aw_semaphore_acquire_scope_impl; friend class aw_semaphore_co_release; template friend class aw_semaphore_co_release_return; friend class ::tmc::tests::waiter_count_accessor; @@ -268,7 +305,7 @@ class semaphore : protected tmc::detail::waiter_data_base { /// will suspend until a resource becomes ready, then transfer the ownership /// to this task. Returns an object that will release the resource (and resume /// an awaiter) when it goes out of scope. - inline aw_semaphore_acquire_scope acquire_scope() noexcept { + inline aw_semaphore_acquire_scope acquire_scope() noexcept TMC_LIFETIMEBOUND { return aw_semaphore_acquire_scope(*this); } diff --git a/include/tmc/task.hpp b/include/tmc/task.hpp index 561c90ea..d53423ee 100644 --- a/include/tmc/task.hpp +++ b/include/tmc/task.hpp @@ -268,11 +268,25 @@ template struct task_promise { template void return_value(RV&& Value) noexcept(std::is_nothrow_move_constructible_v) - requires(requires() { - { *customizer.result_ptr = static_cast(Value) }; - }) + requires( + requires() { + { *customizer.result_ptr = static_cast(Value) }; + } || + requires() { + { customizer.result_ptr->emplace(static_cast(Value)) }; + } + ) { - *customizer.result_ptr = static_cast(Value); + if constexpr (requires { + *customizer.result_ptr = static_cast(Value); + }) { + *customizer.result_ptr = static_cast(Value); + } else { + // Results that are move-constructible but not move-assignable (such as + // mutex_scope) are stored in a std::optional; construct the value in + // place instead. + customizer.result_ptr->emplace(static_cast(Value)); + } } // Whether the returned awaiter is lifetime-bound to the awaitable parameter