| Message ID | 20200316004336.1531416-1-rodgert@appliantology.com |
|---|---|
| State | New |
| Headers | show |
| Series | Add C++2a wait/notify_one/notify_all support to std::atomic<> | expand |
Updated patch attached, addresses some minor issues I found after sending the original version. Thomas Rodgers writes: > This patch adds support for wait/notify_one/notify_all to std::atomic<>. > Support for the volatile overloads will be added in a subsequent patch. > > * include/Makefile.am (bits_headers): Add new header. > * include/Makefile.in: Regenerate. > * include/bits/atomic_base.h (__atomic_base<_Itp>:wait): Define. > (__atomic_base<_Itp>::notify_one): Likewise. > (__atomic_base<_Itp>::notify_all): Likewise. > (__atomic_base<_Ptp*>::wait): Likewise. > (__atomic_base<_Ptp*>::notify_one): Likewise. > (__atomic_base<_Ptp*>::notify_all): Likewise. > (__atomic_impl::wait): Likewise. > (__atomic_impl::notify_one): Likewise. > (__atomic_impl::notify_all): Likewise. > (__atomic_float<_Fp>::wait): Likewise. > (__atomic_float<_Fp>::notify_one): Likewise. > (__atomic_float<_Fp>::notify_all): Likewise. > (__atomic_ref<_Tp>::wait): Likewise. > (__atomic_ref<_Tp>::notify_one): Likewise. > (__atomic_ref<_Tp>::notify_all): Likewise. > (atomic_wait<_Tp>): Likewise. > (atomic_wait_explicit<_Tp>): Likewise. > (atomic_notify_one<_Tp>): Likewise. > (atomic_notify_all<_Tp>): Likewise. > * include/bits/atomic_wait.h: New file. > * include/std/atomic (atomic<bool>::wait): Define. > (atomic<bool>::wait_one): Likewise. > (atomic<bool>::wait_all): Likewise. > (atomic<_Tp>::wait): Likewise. > (atomic<_Tp>::wait_one): Likewise. > (atomic<_Tp>::wait_all): Likewise. > (atomic<_Tp*>::wait): Likewise. > (atomic<_Tp*>::wait_one): Likewise. > (atomic<_Tp*>::wait_all): Likewise. > * testsuite/29_atomic/atomic/wait_notify/atomic_refs.cc: New test. > * testsuite/29_atomic/atomic/wait_notify/bool.cc: Likewise. > * testsuite/29_atomic/atomic/wait_notify/integrals.cc: Likewise. > * testsuite/29_atomic/atomic/wait_notify/floats.cc: Likewise. > * testsuite/29_atomic/atomic/wait_notify/pointers.cc: Likewise. > * testsuite/29_atomic/atomic/wait_notify/generic.h: New File. > --- > libstdc++-v3/include/Makefile.am | 1 + > libstdc++-v3/include/Makefile.in | 2 + > libstdc++-v3/include/bits/atomic_base.h | 178 ++++++++++- > libstdc++-v3/include/bits/atomic_wait.h | 295 ++++++++++++++++++ > libstdc++-v3/include/std/atomic | 61 ++++ > .../atomic/wait_notify/atomic_refs.cc | 103 ++++++ > .../29_atomics/atomic/wait_notify/bool.cc | 57 ++++ > .../29_atomics/atomic/wait_notify/floats.cc | 32 ++ > .../29_atomics/atomic/wait_notify/generic.h | 88 ++++++ > .../atomic/wait_notify/integrals.cc | 56 ++++ > .../29_atomics/atomic/wait_notify/pointers.cc | 59 ++++ > 11 files changed, 931 insertions(+), 1 deletion(-) > create mode 100644 libstdc++-v3/include/bits/atomic_wait.h > create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc > create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc > create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc > create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h > create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc > create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc > > diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am > index 80aeb3f8959..d195a721fd5 100644 > --- a/libstdc++-v3/include/Makefile.am > +++ b/libstdc++-v3/include/Makefile.am > @@ -100,6 +100,7 @@ bits_headers = \ > ${bits_srcdir}/allocated_ptr.h \ > ${bits_srcdir}/allocator.h \ > ${bits_srcdir}/atomic_base.h \ > + ${bits_srcdir}/atomic_wait.h \ > ${bits_srcdir}/atomic_futex.h \ > ${bits_srcdir}/basic_ios.h \ > ${bits_srcdir}/basic_ios.tcc \ > diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in > index eb437ad8d8d..4faaac5fb8d 100644 > --- a/libstdc++-v3/include/Makefile.in > +++ b/libstdc++-v3/include/Makefile.in > @@ -445,6 +445,7 @@ bits_headers = \ > ${bits_srcdir}/allocated_ptr.h \ > ${bits_srcdir}/allocator.h \ > ${bits_srcdir}/atomic_base.h \ > + ${bits_srcdir}/atomic_wait.h \ > ${bits_srcdir}/atomic_futex.h \ > ${bits_srcdir}/basic_ios.h \ > ${bits_srcdir}/basic_ios.tcc \ > @@ -526,6 +527,7 @@ bits_headers = \ > ${bits_srcdir}/specfun.h \ > ${bits_srcdir}/sstream.tcc \ > ${bits_srcdir}/std_abs.h \ > + ${bits_srcdir}/std_condvar.h \ > ${bits_srcdir}/std_function.h \ > ${bits_srcdir}/std_mutex.h \ > ${bits_srcdir}/stl_algo.h \ > diff --git a/libstdc++-v3/include/bits/atomic_base.h b/libstdc++-v3/include/bits/atomic_base.h > index 87fe0bd6000..b4fbe2c6eb3 100644 > --- a/libstdc++-v3/include/bits/atomic_base.h > +++ b/libstdc++-v3/include/bits/atomic_base.h > @@ -37,6 +37,11 @@ > #include <bits/atomic_lockfree_defines.h> > #include <bits/move.h> > > +#if __cplusplus > 201703L > +#include <bits/atomic_wait.h> > +#include <iostream> > +#endif > + > #ifndef _GLIBCXX_ALWAYS_INLINE > #define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__)) > #endif > @@ -134,7 +139,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > return __ret; > } > > - > // Base types for atomics. > template<typename _IntTp> > struct __atomic_base; > @@ -542,6 +546,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__m)); > } > > +#if __cplusplus > 201703L > + _GLIBCXX_ALWAYS_INLINE void > + wait(__int_type __old, memory_order __m = memory_order_seq_cst) const noexcept > + { > + __detail::__atomic_wait(&_M_i, __old, [__m, this](__int_type __o) > + { > + return this->load(__m) == __o; > + }); > + } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { > + __detail::__atomic_notify(&_M_i, false); > + } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { > + __detail::__atomic_notify(&_M_i, true); > + } > + > + // TODO add const volatile overload > +#endif // C++2a > + > _GLIBCXX_ALWAYS_INLINE __int_type > fetch_add(__int_type __i, > memory_order __m = memory_order_seq_cst) noexcept > @@ -803,6 +836,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > int(__m1), int(__m2)); > } > > +#if __cplusplus > 201703L > + _GLIBCXX_ALWAYS_INLINE void > + wait(__pointer_type __old, memory_order __m = memory_order_seq_cst) noexcept > + { > + __detail::__atomic_wait(&_M_p, __old, [__m, this](__pointer_type __o) > + { > + return this->load(__m) == __o; > + }); > + } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { > + __detail::__atomic_notify(&_M_p, false); > + } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { > + __detail::__atomic_notify(&_M_p, true); > + } > + > + // TODO add const volatile overload > +#endif // C++2a > + > _GLIBCXX_ALWAYS_INLINE __pointer_type > fetch_add(ptrdiff_t __d, > memory_order __m = memory_order_seq_cst) noexcept > @@ -891,6 +953,39 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > int(__success), int(__failure)); > } > > +#if __cplusplus > 201703L > + template<typename _Tp> > + _GLIBCXX_ALWAYS_INLINE void > + wait(const _Tp* __ptr, _Val<_Tp> __old, memory_order __m = memory_order_seq_cst) noexcept > + { > + __detail::__atomic_wait_ptr(__ptr, std::__addressof(__old), > + [=](_Tp* __o) > + { > + return load(__ptr, __m) == *__o; > + }); > + } > + > + // TODO add const volatile overload > + > + template<typename _Tp> > + _GLIBCXX_ALWAYS_INLINE void > + notify_one(const _Tp* __ptr) noexcept > + { > + __detail::__atomic_notify(__ptr, false); > + } > + > + // TODO add const volatile overload > + > + template<typename _Tp> > + _GLIBCXX_ALWAYS_INLINE void > + notify_all(const _Tp* __ptr) noexcept > + { > + __detail::__atomic_notify(__ptr, true); > + } > + > + // TODO add const volatile overload > +#endif // C++2a > + > template<typename _Tp> > _GLIBCXX_ALWAYS_INLINE _Tp > fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept > @@ -1144,6 +1239,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__order)); > } > > + _GLIBCXX_ALWAYS_INLINE void > + wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept > + { __atomic_impl::wait(&_M_fp, __old, __m); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { __atomic_impl::notify_one(&_M_fp); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { __atomic_impl::notify_all(&_M_fp); } > + > + // TODO add const volatile overload > value_type > fetch_add(value_type __i, > memory_order __m = memory_order_seq_cst) noexcept > @@ -1281,6 +1393,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__order)); > } > > + _GLIBCXX_ALWAYS_INLINE void > + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept > + { __atomic_impl::wait(_M_ptr, __old, __m); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { __atomic_impl::notify_one(_M_ptr); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { __atomic_impl::notify_all(_M_ptr); } > + > private: > _Tp* _M_ptr; > }; > @@ -1376,6 +1504,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__order)); > } > > + _GLIBCXX_ALWAYS_INLINE void > + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept > + { __atomic_impl::wait(_M_ptr, __old, __m); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { __atomic_impl::notify_one(_M_ptr); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { __atomic_impl::notify_all(_M_ptr); } > + > value_type > fetch_add(value_type __i, > memory_order __m = memory_order_seq_cst) const noexcept > @@ -1531,6 +1675,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__order)); > } > > + _GLIBCXX_ALWAYS_INLINE void > + wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept > + { __atomic_impl::wait(_M_ptr, __old, __m); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { __atomic_impl::notify_one(_M_ptr); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { __atomic_impl::notify_all(_M_ptr); } > + > value_type > fetch_add(value_type __i, > memory_order __m = memory_order_seq_cst) const noexcept > @@ -1640,6 +1800,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__order)); > } > > + _GLIBCXX_ALWAYS_INLINE void > + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept > + { __atomic_impl::wait(_M_ptr, __old, __m); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_one() const noexcept > + { __atomic_impl::notify_one(_M_ptr); } > + > + // TODO add const volatile overload > + > + _GLIBCXX_ALWAYS_INLINE void > + notify_all() const noexcept > + { __atomic_impl::notify_all(_M_ptr); } > + > _GLIBCXX_ALWAYS_INLINE value_type > fetch_add(difference_type __d, > memory_order __m = memory_order_seq_cst) const noexcept > diff --git a/libstdc++-v3/include/bits/atomic_wait.h b/libstdc++-v3/include/bits/atomic_wait.h > new file mode 100644 > index 00000000000..fa2650afc39 > --- /dev/null > +++ b/libstdc++-v3/include/bits/atomic_wait.h > @@ -0,0 +1,295 @@ > +// -*- C++ -*- header. > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// Under Section 7 of GPL version 3, you are granted additional > +// permissions described in the GCC Runtime Library Exception, version > +// 3.1, as published by the Free Software Foundation. > + > +// You should have received a copy of the GNU General Public License and > +// a copy of the GCC Runtime Library Exception along with this program; > +// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > +// <http://www.gnu.org/licenses/>. > + > +/** @file bits/atomic_wait.h > + * This is an internal header file, included by other library headers. > + * Do not attempt to use it directly. @headername{atomic} > + */ > + > +#ifndef _GLIBCXX_ATOMIC_WAIT_H > +#define _GLIBCXX_ATOMIC_WAIT_H 1 > + > +#pragma GCC system_header > + > +#include <bits/c++config.h> > +#include <bits/functional_hash.h> > +#include <bits/gthr.h> > +#include <bits/std_mutex.h> > +#include <bits/unique_lock.h> > +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX > +#include <climits> > +#include <unistd.h> > +#include <syscall.h> > +#endif > + > +#define _GLIBCXX_SPIN_COUNT_1 16 > +#define _GLIBCXX_SPIN_COUNT_2 12 > + > +// TODO get this from Autoconf > +#define _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE 1 > + > +namespace std _GLIBCXX_VISIBILITY(default) > +{ > +_GLIBCXX_BEGIN_NAMESPACE_VERSION > + namespace __detail > + { > + using __platform_wait_t = int; > + > + template<class _Tp> > + struct __platform_wait_uses_type > + { > +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX > + enum { __value = std::is_same<typename std::remove_cv<_Tp>::type, > + __platform_wait_t>::value }; > +#else > + enum { __value = std::false_type::value }; > +#endif > + }; > + > +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX > + enum > + { > +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE > + __futex_private_flag = 128, > +#else > + __futex_private_flag = 0, > +#endif > + __futex_wait = 0, > + __futex_wake = 1, > + __futex_wait_private = __futex_wait | __futex_private_flag, > + __futex_wake_private = __futex_wake | __futex_private_flag, > + }; > + > + void > + __platform_wait(__platform_wait_t* __addr, __platform_wait_t __val) noexcept > + { > + auto __e = syscall (SYS_futex, __addr, __futex_wait_private, __val, nullptr); > + if (__e && (errno != EINTR || errno != EAGAIN)) > + std::terminate(); > + } > + > + void > + __platform_notify(__platform_wait_t* __addr, bool __all) noexcept > + { > + syscall (SYS_futex, __addr, __futex_wake_private, __all ? INT_MAX : 1); > + } > +#endif > + > + struct alignas(64) __waiter > + { > + bool _M_signaled = false; > + void* _M_t; > + __waiter* _M_prev = nullptr; > + __waiter* _M_next = nullptr; > + > + __waiter(void* __t) noexcept > + : _M_t(__t) > + { } > + }; > + > + struct alignas(64) __waiters > + { > + int32_t alignas(64) _M_ver = 0; > + int32_t alignas(64) _M_wait = 0; > + > + // TODO make this used only where we don't have futexes > + using __lock_t = std::unique_lock<std::mutex>; > + mutable __lock_t::mutex_type _M_mtx; > + > +#ifdef __GTHREAD_COND_INIT > + mutable __gthread_cond_t _M_cv = __GTHREAD_COND_INIT; > + __waiters() noexcept = default; > +#else > + mutable __gthread_cond_t _M_cv; > + __waiters() noexcept > + { > + __GTHREAD_COND_INIT_FUNCTION(&_M_cond); > + } > +#endif > + > + int32_t > + _M_enter_wait() noexcept > + { > + int32_t __res; > + __atomic_load(&_M_ver, &__res, __ATOMIC_ACQUIRE); > + __atomic_fetch_add(&_M_wait, 1, __ATOMIC_ACQ_REL); > + return __res; > + } > + > + void > + _M_leave_wait() noexcept > + { > + __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_ACQ_REL); > + } > + > + void > + _M_do_wait(int32_t __version) const noexcept > + { > + int32_t __cur = 0; > + while (__cur <= __version) > + { > + __waiters::__lock_t __l(_M_mtx); > + auto __e = __gthread_cond_wait(&_M_cv, __l.mutex()->native_handle()); > + if (__e) > + std::terminate(); > + int32_t __last = __cur; > + __atomic_load(&_M_ver, &__cur, __ATOMIC_ACQUIRE); > + // protect if version overflows > + if (__cur < __last) > + break; // break the loop if version overflows > + } > + } > + > + int32_t > + _M_waiting() const noexcept > + { > + int32_t __res; > + __atomic_load(&_M_wait, &__res, __ATOMIC_ACQUIRE); > + return __res; > + } > + > + void > + _M_notify(bool __all) noexcept > + { > + __atomic_fetch_add(&_M_ver, 1, __ATOMIC_ACQ_REL); > + auto __e = __gthread_cond_broadcast(&_M_cv); > + if (__e) > + __throw_system_error(__e); > + } > + > + static __waiters& _S_for(void* __t) { > + const unsigned char __mask = 0xf; > + static __waiters __w[__mask + 1]; > + > + auto __key = _Hash_impl::hash(__t) & __mask; > + return __w[__key]; > + } > + }; > + > + void > + __thread_relax() noexcept > + { > +#if defined __i386__ || defined __x86_64__ > + __builtin_ia32_pause(); > +#elif defined _GLIBCXX_USE_SCHED_YIELD > + __gthread_yield(); > +#endif > + } > + > + void > + __thread_yield() noexcept > + { > +#if defined _GLIBCXX_USE_SCHED_YIELD > + __gthread_yield(); > +#endif > + } > + > + template<class _Tp, class _Pred> > + void > + __atomic_wait(const _Tp* __addr, _Tp __old, _Pred __pred) noexcept > + { > + for (auto __i = 0; __i < _GLIBCXX_SPIN_COUNT_1; ++__i) > + { > + if (!__pred(__old)) > + return; > + > + if (__i < _GLIBCXX_SPIN_COUNT_2) > + __thread_relax(); > + else > + __thread_yield(); > + } > + > + auto& __w = __waiters::_S_for((void*)__addr); > + auto __version = __w._M_enter_wait(); > + while (auto __res = __pred(__old)) > + { > + if constexpr (__platform_wait_uses_type<_Tp>::__value) > + { > + __platform_wait((__platform_wait_t*)(void*) __addr, __old); > + } > + else > + { > + // TODO support timed backoff when this can be moved into the lib > + __w._M_do_wait(__version); > + } > + } > + __w._M_leave_wait(); > + } > + > + template<class _Tp, class _Pred> > + void > + __atomic_wait_ptr(const _Tp* __addr, _Tp* __old, _Pred __pred) noexcept > + { > + for (auto __i = 0; __i < _GLIBCXX_SPIN_COUNT_1; ++__i) > + { > + if (!__pred(__old)) > + return; > + > + if (__i < _GLIBCXX_SPIN_COUNT_2) > + __thread_relax(); > + else > + __thread_yield(); > + } > + > + auto& __w = __waiters::_S_for((void*)__addr); > + auto __version = __w._M_enter_wait(); > + while (auto __res = __pred(__old)) > + { > + if constexpr (__platform_wait_uses_type<_Tp>::__value) > + { > + __platform_wait((__platform_wait_t*)(void*) __addr, __old); > + } > + else > + { > + // TODO support timed backoff when this can be moved into the lib > + __w._M_do_wait(__version); > + } > + } > + __w._M_leave_wait(); > + } > + > + template<class _Tp> > + void > + __atomic_notify(const _Tp* __addr, bool __all) noexcept > + { > + auto& __w = __waiters::_S_for((void*)__addr); > + if (!__w._M_waiting()) > + return; > + > + if constexpr (__platform_wait_uses_type<_Tp>::__value) > + { > + __platform_notify((__platform_wait_t*)(void*) __addr, __all); > + } > + else > + { > + __w._M_notify(__all); > + } > + } > + } // namespace __detail > + > +_GLIBCXX_END_NAMESPACE_VERSION > +} // namespace std > + > +#endif > + > diff --git a/libstdc++-v3/include/std/atomic b/libstdc++-v3/include/std/atomic > index 40f23bdfc96..cb2308b0cdf 100644 > --- a/libstdc++-v3/include/std/atomic > +++ b/libstdc++-v3/include/std/atomic > @@ -163,6 +163,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > compare_exchange_strong(bool& __i1, bool __i2, > memory_order __m = memory_order_seq_cst) volatile noexcept > { return _M_base.compare_exchange_strong(__i1, __i2, __m); } > + > +#if __cplusplus > 201703L > + void wait(bool __old, memory_order __m = memory_order_seq_cst) const noexcept > + { _M_base.wait(__old, __m); } > + > + // TODO add const volatile overload > + > + void notify_one() const noexcept > + { _M_base.notify_one(); } > + > + void notify_all() const noexcept > + { _M_base.notify_all(); } > +#endif > }; > > #if __cplusplus <= 201703L > @@ -352,6 +365,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > memory_order __m = memory_order_seq_cst) volatile noexcept > { return compare_exchange_strong(__e, __i, __m, > __cmpexch_failure_order(__m)); } > +#if __cplusplus > 201703L > + void wait(_Tp __old, memory_order __m = memory_order_seq_cst) noexcept > + { _M_i.wait(__old, __m); } > + > + // TODO add const volatile overload > + > + void notify_one() const noexcept > + { _M_i.notify_one(); } > + > + void notify_all() const noexcept > + { _M_i.notify_all(); } > +#endif > + > }; > #undef _GLIBCXX20_INIT > > @@ -590,6 +616,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > __cmpexch_failure_order(__m)); > } > > +#if __cplusplus > 201703L > + void wait(__pointer_type __old, memory_order __m = memory_order_seq_cst) noexcept > + { _M_b.wait(__old, __m); } > + > + // TODO add const volatile overload > + > + void notify_one() const noexcept > + { _M_b.notify_one(); } > + > + void notify_all() const noexcept > + { _M_b.notify_all(); } > +#endif > __pointer_type > fetch_add(ptrdiff_t __d, > memory_order __m = memory_order_seq_cst) noexcept > @@ -1342,6 +1380,29 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION > memory_order_seq_cst); > } > > + > +#if __cplusplus > 201703L > + template<typename _Tp> > + inline void atomic_wait(const atomic<_Tp>* __a, > + typename std::atomic<_Tp>::value_type __old) noexcept > + { __a->wait(__old); } > + > + template<typename _Tp> > + inline void atomic_wait_explicit(const atomic<_Tp>* __a, > + typename std::atomic<_Tp>::value_type __old, > + std::memory_order __m) noexcept > + { __a->wait(__old, __m); } > + > + template<typename _Tp> > + inline void atomic_notify_one(atomic<_Tp>* __a) noexcept > + { __a->notify_one(); } > + > + template<typename _Tp> > + inline void atomic_notify_all(atomic<_Tp>* __a) noexcept > + { __a->notify_all(); } > + > +#endif // C++2a > + > // Function templates for atomic_integral and atomic_pointer operations only. > // Some operations (and, or, xor) are only available for atomic integrals, > // which is implemented by taking a parameter of type __atomic_base<_ITp>*. > diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc > new file mode 100644 > index 00000000000..1d8d13621cc > --- /dev/null > +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc > @@ -0,0 +1,103 @@ > +// { dg-options "-std=gnu++2a -pthread -latomic -L../../libatomic/.libs" } > +// { dg-do run { target c++2a } } > +// { dg-require-effective-target pthread } > +// { dg-require-gthreads "" } > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// You should have received a copy of the GNU General Public License along > +// with this library; see the file COPYING3. If not see > +// <http://www.gnu.org/licenses/>. > + > +#include <atomic> > +#include <thread> > +#include <mutex> > +#include <condition_variable> > +#include <chrono> > +#include <type_traits> > + > +#include <testsuite_hooks.h> > + > +template<typename Tp> > +Tp check_wait_notify(Tp val1, Tp val2) > +{ > + using namespace std::literals::chrono_literals; > + > + std::mutex m; > + std::condition_variable cv; > + > + Tp aa = val1; > + std::atomic_ref<Tp> a(aa); > + std::thread t([&] > + { > + cv.notify_one(); > + a.wait(val1); > + if (a.load() != val2) > + a = val1; > + }); > + std::unique_lock<std::mutex> l(m); > + cv.wait(l); > + std::this_thread::sleep_for(100ms); > + a.store(val2); > + a.notify_one(); > + t.join(); > + return a.load(); > +} > + > +template<typename Tp, > + bool = std::is_integral_v<Tp> > + || std::is_floating_point_v<Tp>> > +struct check; > + > +template<typename Tp> > +struct check<Tp, true> > +{ > + check() > + { > + Tp a = 0; > + Tp b = 42; > + VERIFY(check_wait_notify(a, b) == b); > + } > +}; > + > +template<typename Tp> > +struct check<Tp, false> > +{ > + check(Tp b) > + { > + Tp a; > + VERIFY(check_wait_notify(a, b) == b); > + } > +}; > + > +struct foo > +{ > + long a = 0; > + long b = 0; > + > + foo& operator=(foo const&) = default; > + > + friend bool > + operator==(foo const& rhs, foo const& lhs) > + { return rhs.a == lhs.a && rhs.b == lhs.b; } > +}; > + > +int > +main () > +{ > + check<long>(); > + check<double>(); > + check<foo>({42, 48}); > + return 0; > +} > diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc > new file mode 100644 > index 00000000000..98258686945 > --- /dev/null > +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc > @@ -0,0 +1,57 @@ > +// { dg-options "-std=gnu++2a -pthread" } > +// { dg-do run { target c++2a } } > +// { dg-require-effective-target pthread } > +// { dg-require-gthreads "" } > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// You should have received a copy of the GNU General Public License along > +// with this library; see the file COPYING3. If not see > +// <http://www.gnu.org/licenses/>. > + > +#include <atomic> > +#include <thread> > +#include <mutex> > +#include <condition_variable> > +#include <type_traits> > +#include <chrono> > + > +#include <testsuite_hooks.h> > + > +int > +main () > +{ > + using namespace std::literals::chrono_literals; > + > + std::mutex m; > + std::condition_variable cv; > + > + std::atomic<bool> a(false); > + std::atomic<bool> b(false); > + std::thread t([&] > + { > + cv.notify_one(); > + a.wait(false); > + if (a.load()) > + b.store(true); > + }); > + std::unique_lock<std::mutex> l(m); > + cv.wait(l); > + std::this_thread::sleep_for(100ms); > + a.store(true); > + a.notify_one(); > + t.join(); > + VERIFY( b.load() ); > + return 0; > +} > diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc > new file mode 100644 > index 00000000000..1d032085752 > --- /dev/null > +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc > @@ -0,0 +1,32 @@ > +// { dg-options "-std=gnu++2a -pthread -latomic -L../../libatomic/.libs" } > +// { dg-do run { target c++2a } } > +// { dg-require-effective-target pthread } > +// { dg-require-gthreads "" } > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// You should have received a copy of the GNU General Public License along > +// with this library; see the file COPYING3. If not see > +// <http://www.gnu.org/licenses/>. > + > +#include "generic.h" > + > +int > +main () > +{ > + check<float> f; > + check<double> d; > + check<long double> l; > + return 0; > +} > diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h > new file mode 100644 > index 00000000000..e6eeebf4ab2 > --- /dev/null > +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h > @@ -0,0 +1,88 @@ > +// -*- C++ -*- header. > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// You should have received a copy of the GNU General Public License along > +// with this library; see the file COPYING3. If not see > +// <http://www.gnu.org/licenses/>. > + > +#include <atomic> > +#include <thread> > +#include <mutex> > +#include <condition_variable> > +#include <chrono> > + > +#include <testsuite_hooks.h> > + > +template<typename Tp> > +Tp check_wait_notify(Tp val1, Tp val2) > +{ > + using namespace std::literals::chrono_literals; > + > + std::mutex m; > + std::condition_variable cv; > + > + std::atomic<Tp> a(val1); > + std::thread t([&] > + { > + cv.notify_one(); > + a.wait(val1); > + if (a.load() != val2) > + a = val1; > + }); > + std::unique_lock<std::mutex> l(m); > + cv.wait(l); > + std::this_thread::sleep_for(100ms); > + a.store(val2); > + a.notify_one(); > + t.join(); > + return a.load(); > +} > + > +template<typename Tp> > +Tp check_atomic_wait_notify(Tp val1, Tp val2) > +{ > + using namespace std::literals::chrono_literals; > + > + std::mutex m; > + std::condition_variable cv; > + > + std::atomic<Tp> a(val1); > + std::thread t([&] > + { > + cv.notify_one(); > + std::atomic_wait(&a, val1); > + if (a.load() != val2) > + a = val1; > + }); > + std::unique_lock<std::mutex> l(m); > + cv.wait(l); > + std::this_thread::sleep_for(100ms); > + a.store(val2); > + std::atomic_notify_one(&a); > + t.join(); > + return a.load(); > +} > + > +template<typename Tp> > +struct check > +{ > + check() > + { > + Tp a = 0; > + Tp b = 42; > + VERIFY(check_wait_notify(a, b) == b); > + VERIFY(check_atomic_wait_notify(a, b) == b); > + } > +}; > diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc > new file mode 100644 > index 00000000000..2afd19a7d14 > --- /dev/null > +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc > @@ -0,0 +1,56 @@ > +// { dg-options "-std=gnu++2a -pthread" } > +// { dg-do run { target c++2a } } > +// { dg-require-effective-target pthread } > +// { dg-require-gthreads "" } > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// You should have received a copy of the GNU General Public License along > +// with this library; see the file COPYING3. If not see > +// <http://www.gnu.org/licenses/>. > + > +#include "generic.h" > + > +int > +main () > +{ > + // check<bool> bb; > + check<char> ch; > + check<signed char> sch; > + check<unsigned char> uch; > + check<short> s; > + check<unsigned short> us; > + check<int> i; > + check<unsigned int> ui; > + check<long> l; > + check<unsigned long> ul; > + check<long long> ll; > + check<unsigned long long> ull; > + > + check<wchar_t> wch; > + check<char8_t> ch8; > + check<char16_t> ch16; > + check<char32_t> ch32; > + > + check<int8_t> i8; > + check<int16_t> i16; > + check<int32_t> i32; > + check<int64_t> i64; > + > + check<uint8_t> u8; > + check<uint16_t> u16; > + check<uint32_t> u32; > + check<uint64_t> u64; > + return 0; > +} > diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc > new file mode 100644 > index 00000000000..fe0b8de9f3f > --- /dev/null > +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc > @@ -0,0 +1,59 @@ > +// { dg-options "-std=gnu++2a -pthread" } > +// { dg-do run { target c++2a } } > +// { dg-require-effective-target pthread } > +// { dg-require-gthreads "" } > + > +// Copyright (C) 2020 Free Software Foundation, Inc. > +// > +// This file is part of the GNU ISO C++ Library. This library is free > +// software; you can redistribute it and/or modify it under the > +// terms of the GNU General Public License as published by the > +// Free Software Foundation; either version 3, or (at your option) > +// any later version. > + > +// This library is distributed in the hope that it will be useful, > +// but WITHOUT ANY WARRANTY; without even the implied warranty of > +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +// GNU General Public License for more details. > + > +// You should have received a copy of the GNU General Public License along > +// with this library; see the file COPYING3. If not see > +// <http://www.gnu.org/licenses/>. > + > +#include <atomic> > +#include <thread> > +#include <mutex> > +#include <condition_variable> > +#include <type_traits> > +#include <chrono> > + > +#include <testsuite_hooks.h> > + > +int > +main () > +{ > + using namespace std::literals::chrono_literals; > + > + std::mutex m; > + std::condition_variable cv; > + > + long aa; > + long bb; > + > + std::atomic<long*> a(nullptr); > + std::thread t([&] > + { > + cv.notify_one(); > + a.wait(nullptr); > + if (a.load() == &aa) > + a.store(&bb); > + }); > + std::unique_lock<std::mutex> l(m); > + cv.wait(l); > + std::this_thread::sleep_for(100ms); > + a.store(&aa); > + a.notify_one(); > + t.join(); > + VERIFY( a.load() == &bb); > + return 0; > +}
Updated patch, fixes some whitespace issues along with ensuring that libstdc++-v3/include/Makefile.in is regenerated. Thomas Rodgers via Libstdc++ writes: > Updated patch attached, addresses some minor issues I found after > sending the original version. > > Thomas Rodgers writes: > >> This patch adds support for wait/notify_one/notify_all to std::atomic<>. >> Support for the volatile overloads will be added in a subsequent patch. >> >> * include/Makefile.am (bits_headers): Add new header. >> * include/Makefile.in: Regenerate. >> * include/bits/atomic_base.h (__atomic_base<_Itp>:wait): Define. >> (__atomic_base<_Itp>::notify_one): Likewise. >> (__atomic_base<_Itp>::notify_all): Likewise. >> (__atomic_base<_Ptp*>::wait): Likewise. >> (__atomic_base<_Ptp*>::notify_one): Likewise. >> (__atomic_base<_Ptp*>::notify_all): Likewise. >> (__atomic_impl::wait): Likewise. >> (__atomic_impl::notify_one): Likewise. >> (__atomic_impl::notify_all): Likewise. >> (__atomic_float<_Fp>::wait): Likewise. >> (__atomic_float<_Fp>::notify_one): Likewise. >> (__atomic_float<_Fp>::notify_all): Likewise. >> (__atomic_ref<_Tp>::wait): Likewise. >> (__atomic_ref<_Tp>::notify_one): Likewise. >> (__atomic_ref<_Tp>::notify_all): Likewise. >> (atomic_wait<_Tp>): Likewise. >> (atomic_wait_explicit<_Tp>): Likewise. >> (atomic_notify_one<_Tp>): Likewise. >> (atomic_notify_all<_Tp>): Likewise. >> * include/bits/atomic_wait.h: New file. >> * include/std/atomic (atomic<bool>::wait): Define. >> (atomic<bool>::wait_one): Likewise. >> (atomic<bool>::wait_all): Likewise. >> (atomic<_Tp>::wait): Likewise. >> (atomic<_Tp>::wait_one): Likewise. >> (atomic<_Tp>::wait_all): Likewise. >> (atomic<_Tp*>::wait): Likewise. >> (atomic<_Tp*>::wait_one): Likewise. >> (atomic<_Tp*>::wait_all): Likewise. >> * testsuite/29_atomic/atomic/wait_notify/atomic_refs.cc: New test. >> * testsuite/29_atomic/atomic/wait_notify/bool.cc: Likewise. >> * testsuite/29_atomic/atomic/wait_notify/integrals.cc: Likewise. >> * testsuite/29_atomic/atomic/wait_notify/floats.cc: Likewise. >> * testsuite/29_atomic/atomic/wait_notify/pointers.cc: Likewise. >> * testsuite/29_atomic/atomic/wait_notify/generic.h: New File. >> --- >> libstdc++-v3/include/Makefile.am | 1 + >> libstdc++-v3/include/Makefile.in | 2 + >> libstdc++-v3/include/bits/atomic_base.h | 178 ++++++++++- >> libstdc++-v3/include/bits/atomic_wait.h | 295 ++++++++++++++++++ >> libstdc++-v3/include/std/atomic | 61 ++++ >> .../atomic/wait_notify/atomic_refs.cc | 103 ++++++ >> .../29_atomics/atomic/wait_notify/bool.cc | 57 ++++ >> .../29_atomics/atomic/wait_notify/floats.cc | 32 ++ >> .../29_atomics/atomic/wait_notify/generic.h | 88 ++++++ >> .../atomic/wait_notify/integrals.cc | 56 ++++ >> .../29_atomics/atomic/wait_notify/pointers.cc | 59 ++++ >> 11 files changed, 931 insertions(+), 1 deletion(-) >> create mode 100644 libstdc++-v3/include/bits/atomic_wait.h >> create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc >> create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc >> create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc >> create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h >> create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc >> create mode 100644 libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc >> >> diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am >> index 80aeb3f8959..d195a721fd5 100644 >> --- a/libstdc++-v3/include/Makefile.am >> +++ b/libstdc++-v3/include/Makefile.am >> @@ -100,6 +100,7 @@ bits_headers = \ >> ${bits_srcdir}/allocated_ptr.h \ >> ${bits_srcdir}/allocator.h \ >> ${bits_srcdir}/atomic_base.h \ >> + ${bits_srcdir}/atomic_wait.h \ >> ${bits_srcdir}/atomic_futex.h \ >> ${bits_srcdir}/basic_ios.h \ >> ${bits_srcdir}/basic_ios.tcc \ >> diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in >> index eb437ad8d8d..4faaac5fb8d 100644 >> --- a/libstdc++-v3/include/Makefile.in >> +++ b/libstdc++-v3/include/Makefile.in >> @@ -445,6 +445,7 @@ bits_headers = \ >> ${bits_srcdir}/allocated_ptr.h \ >> ${bits_srcdir}/allocator.h \ >> ${bits_srcdir}/atomic_base.h \ >> + ${bits_srcdir}/atomic_wait.h \ >> ${bits_srcdir}/atomic_futex.h \ >> ${bits_srcdir}/basic_ios.h \ >> ${bits_srcdir}/basic_ios.tcc \ >> @@ -526,6 +527,7 @@ bits_headers = \ >> ${bits_srcdir}/specfun.h \ >> ${bits_srcdir}/sstream.tcc \ >> ${bits_srcdir}/std_abs.h \ >> + ${bits_srcdir}/std_condvar.h \ >> ${bits_srcdir}/std_function.h \ >> ${bits_srcdir}/std_mutex.h \ >> ${bits_srcdir}/stl_algo.h \ >> diff --git a/libstdc++-v3/include/bits/atomic_base.h b/libstdc++-v3/include/bits/atomic_base.h >> index 87fe0bd6000..b4fbe2c6eb3 100644 >> --- a/libstdc++-v3/include/bits/atomic_base.h >> +++ b/libstdc++-v3/include/bits/atomic_base.h >> @@ -37,6 +37,11 @@ >> #include <bits/atomic_lockfree_defines.h> >> #include <bits/move.h> >> >> +#if __cplusplus > 201703L >> +#include <bits/atomic_wait.h> >> +#include <iostream> >> +#endif >> + >> #ifndef _GLIBCXX_ALWAYS_INLINE >> #define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__)) >> #endif >> @@ -134,7 +139,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> return __ret; >> } >> >> - >> // Base types for atomics. >> template<typename _IntTp> >> struct __atomic_base; >> @@ -542,6 +546,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__m)); >> } >> >> +#if __cplusplus > 201703L >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(__int_type __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { >> + __detail::__atomic_wait(&_M_i, __old, [__m, this](__int_type __o) >> + { >> + return this->load(__m) == __o; >> + }); >> + } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { >> + __detail::__atomic_notify(&_M_i, false); >> + } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { >> + __detail::__atomic_notify(&_M_i, true); >> + } >> + >> + // TODO add const volatile overload >> +#endif // C++2a >> + >> _GLIBCXX_ALWAYS_INLINE __int_type >> fetch_add(__int_type __i, >> memory_order __m = memory_order_seq_cst) noexcept >> @@ -803,6 +836,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> int(__m1), int(__m2)); >> } >> >> +#if __cplusplus > 201703L >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(__pointer_type __old, memory_order __m = memory_order_seq_cst) noexcept >> + { >> + __detail::__atomic_wait(&_M_p, __old, [__m, this](__pointer_type __o) >> + { >> + return this->load(__m) == __o; >> + }); >> + } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { >> + __detail::__atomic_notify(&_M_p, false); >> + } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { >> + __detail::__atomic_notify(&_M_p, true); >> + } >> + >> + // TODO add const volatile overload >> +#endif // C++2a >> + >> _GLIBCXX_ALWAYS_INLINE __pointer_type >> fetch_add(ptrdiff_t __d, >> memory_order __m = memory_order_seq_cst) noexcept >> @@ -891,6 +953,39 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> int(__success), int(__failure)); >> } >> >> +#if __cplusplus > 201703L >> + template<typename _Tp> >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(const _Tp* __ptr, _Val<_Tp> __old, memory_order __m = memory_order_seq_cst) noexcept >> + { >> + __detail::__atomic_wait_ptr(__ptr, std::__addressof(__old), >> + [=](_Tp* __o) >> + { >> + return load(__ptr, __m) == *__o; >> + }); >> + } >> + >> + // TODO add const volatile overload >> + >> + template<typename _Tp> >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one(const _Tp* __ptr) noexcept >> + { >> + __detail::__atomic_notify(__ptr, false); >> + } >> + >> + // TODO add const volatile overload >> + >> + template<typename _Tp> >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all(const _Tp* __ptr) noexcept >> + { >> + __detail::__atomic_notify(__ptr, true); >> + } >> + >> + // TODO add const volatile overload >> +#endif // C++2a >> + >> template<typename _Tp> >> _GLIBCXX_ALWAYS_INLINE _Tp >> fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept >> @@ -1144,6 +1239,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__order)); >> } >> >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { __atomic_impl::wait(&_M_fp, __old, __m); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { __atomic_impl::notify_one(&_M_fp); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { __atomic_impl::notify_all(&_M_fp); } >> + >> + // TODO add const volatile overload >> value_type >> fetch_add(value_type __i, >> memory_order __m = memory_order_seq_cst) noexcept >> @@ -1281,6 +1393,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__order)); >> } >> >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { __atomic_impl::wait(_M_ptr, __old, __m); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { __atomic_impl::notify_one(_M_ptr); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { __atomic_impl::notify_all(_M_ptr); } >> + >> private: >> _Tp* _M_ptr; >> }; >> @@ -1376,6 +1504,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__order)); >> } >> >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { __atomic_impl::wait(_M_ptr, __old, __m); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { __atomic_impl::notify_one(_M_ptr); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { __atomic_impl::notify_all(_M_ptr); } >> + >> value_type >> fetch_add(value_type __i, >> memory_order __m = memory_order_seq_cst) const noexcept >> @@ -1531,6 +1675,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__order)); >> } >> >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { __atomic_impl::wait(_M_ptr, __old, __m); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { __atomic_impl::notify_one(_M_ptr); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { __atomic_impl::notify_all(_M_ptr); } >> + >> value_type >> fetch_add(value_type __i, >> memory_order __m = memory_order_seq_cst) const noexcept >> @@ -1640,6 +1800,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__order)); >> } >> >> + _GLIBCXX_ALWAYS_INLINE void >> + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { __atomic_impl::wait(_M_ptr, __old, __m); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_one() const noexcept >> + { __atomic_impl::notify_one(_M_ptr); } >> + >> + // TODO add const volatile overload >> + >> + _GLIBCXX_ALWAYS_INLINE void >> + notify_all() const noexcept >> + { __atomic_impl::notify_all(_M_ptr); } >> + >> _GLIBCXX_ALWAYS_INLINE value_type >> fetch_add(difference_type __d, >> memory_order __m = memory_order_seq_cst) const noexcept >> diff --git a/libstdc++-v3/include/bits/atomic_wait.h b/libstdc++-v3/include/bits/atomic_wait.h >> new file mode 100644 >> index 00000000000..fa2650afc39 >> --- /dev/null >> +++ b/libstdc++-v3/include/bits/atomic_wait.h >> @@ -0,0 +1,295 @@ >> +// -*- C++ -*- header. >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// Under Section 7 of GPL version 3, you are granted additional >> +// permissions described in the GCC Runtime Library Exception, version >> +// 3.1, as published by the Free Software Foundation. >> + >> +// You should have received a copy of the GNU General Public License and >> +// a copy of the GCC Runtime Library Exception along with this program; >> +// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see >> +// <http://www.gnu.org/licenses/>. >> + >> +/** @file bits/atomic_wait.h >> + * This is an internal header file, included by other library headers. >> + * Do not attempt to use it directly. @headername{atomic} >> + */ >> + >> +#ifndef _GLIBCXX_ATOMIC_WAIT_H >> +#define _GLIBCXX_ATOMIC_WAIT_H 1 >> + >> +#pragma GCC system_header >> + >> +#include <bits/c++config.h> >> +#include <bits/functional_hash.h> >> +#include <bits/gthr.h> >> +#include <bits/std_mutex.h> >> +#include <bits/unique_lock.h> >> +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX >> +#include <climits> >> +#include <unistd.h> >> +#include <syscall.h> >> +#endif >> + >> +#define _GLIBCXX_SPIN_COUNT_1 16 >> +#define _GLIBCXX_SPIN_COUNT_2 12 >> + >> +// TODO get this from Autoconf >> +#define _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE 1 >> + >> +namespace std _GLIBCXX_VISIBILITY(default) >> +{ >> +_GLIBCXX_BEGIN_NAMESPACE_VERSION >> + namespace __detail >> + { >> + using __platform_wait_t = int; >> + >> + template<class _Tp> >> + struct __platform_wait_uses_type >> + { >> +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX >> + enum { __value = std::is_same<typename std::remove_cv<_Tp>::type, >> + __platform_wait_t>::value }; >> +#else >> + enum { __value = std::false_type::value }; >> +#endif >> + }; >> + >> +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX >> + enum >> + { >> +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE >> + __futex_private_flag = 128, >> +#else >> + __futex_private_flag = 0, >> +#endif >> + __futex_wait = 0, >> + __futex_wake = 1, >> + __futex_wait_private = __futex_wait | __futex_private_flag, >> + __futex_wake_private = __futex_wake | __futex_private_flag, >> + }; >> + >> + void >> + __platform_wait(__platform_wait_t* __addr, __platform_wait_t __val) noexcept >> + { >> + auto __e = syscall (SYS_futex, __addr, __futex_wait_private, __val, nullptr); >> + if (__e && (errno != EINTR || errno != EAGAIN)) >> + std::terminate(); >> + } >> + >> + void >> + __platform_notify(__platform_wait_t* __addr, bool __all) noexcept >> + { >> + syscall (SYS_futex, __addr, __futex_wake_private, __all ? INT_MAX : 1); >> + } >> +#endif >> + >> + struct alignas(64) __waiter >> + { >> + bool _M_signaled = false; >> + void* _M_t; >> + __waiter* _M_prev = nullptr; >> + __waiter* _M_next = nullptr; >> + >> + __waiter(void* __t) noexcept >> + : _M_t(__t) >> + { } >> + }; >> + >> + struct alignas(64) __waiters >> + { >> + int32_t alignas(64) _M_ver = 0; >> + int32_t alignas(64) _M_wait = 0; >> + >> + // TODO make this used only where we don't have futexes >> + using __lock_t = std::unique_lock<std::mutex>; >> + mutable __lock_t::mutex_type _M_mtx; >> + >> +#ifdef __GTHREAD_COND_INIT >> + mutable __gthread_cond_t _M_cv = __GTHREAD_COND_INIT; >> + __waiters() noexcept = default; >> +#else >> + mutable __gthread_cond_t _M_cv; >> + __waiters() noexcept >> + { >> + __GTHREAD_COND_INIT_FUNCTION(&_M_cond); >> + } >> +#endif >> + >> + int32_t >> + _M_enter_wait() noexcept >> + { >> + int32_t __res; >> + __atomic_load(&_M_ver, &__res, __ATOMIC_ACQUIRE); >> + __atomic_fetch_add(&_M_wait, 1, __ATOMIC_ACQ_REL); >> + return __res; >> + } >> + >> + void >> + _M_leave_wait() noexcept >> + { >> + __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_ACQ_REL); >> + } >> + >> + void >> + _M_do_wait(int32_t __version) const noexcept >> + { >> + int32_t __cur = 0; >> + while (__cur <= __version) >> + { >> + __waiters::__lock_t __l(_M_mtx); >> + auto __e = __gthread_cond_wait(&_M_cv, __l.mutex()->native_handle()); >> + if (__e) >> + std::terminate(); >> + int32_t __last = __cur; >> + __atomic_load(&_M_ver, &__cur, __ATOMIC_ACQUIRE); >> + // protect if version overflows >> + if (__cur < __last) >> + break; // break the loop if version overflows >> + } >> + } >> + >> + int32_t >> + _M_waiting() const noexcept >> + { >> + int32_t __res; >> + __atomic_load(&_M_wait, &__res, __ATOMIC_ACQUIRE); >> + return __res; >> + } >> + >> + void >> + _M_notify(bool __all) noexcept >> + { >> + __atomic_fetch_add(&_M_ver, 1, __ATOMIC_ACQ_REL); >> + auto __e = __gthread_cond_broadcast(&_M_cv); >> + if (__e) >> + __throw_system_error(__e); >> + } >> + >> + static __waiters& _S_for(void* __t) { >> + const unsigned char __mask = 0xf; >> + static __waiters __w[__mask + 1]; >> + >> + auto __key = _Hash_impl::hash(__t) & __mask; >> + return __w[__key]; >> + } >> + }; >> + >> + void >> + __thread_relax() noexcept >> + { >> +#if defined __i386__ || defined __x86_64__ >> + __builtin_ia32_pause(); >> +#elif defined _GLIBCXX_USE_SCHED_YIELD >> + __gthread_yield(); >> +#endif >> + } >> + >> + void >> + __thread_yield() noexcept >> + { >> +#if defined _GLIBCXX_USE_SCHED_YIELD >> + __gthread_yield(); >> +#endif >> + } >> + >> + template<class _Tp, class _Pred> >> + void >> + __atomic_wait(const _Tp* __addr, _Tp __old, _Pred __pred) noexcept >> + { >> + for (auto __i = 0; __i < _GLIBCXX_SPIN_COUNT_1; ++__i) >> + { >> + if (!__pred(__old)) >> + return; >> + >> + if (__i < _GLIBCXX_SPIN_COUNT_2) >> + __thread_relax(); >> + else >> + __thread_yield(); >> + } >> + >> + auto& __w = __waiters::_S_for((void*)__addr); >> + auto __version = __w._M_enter_wait(); >> + while (auto __res = __pred(__old)) >> + { >> + if constexpr (__platform_wait_uses_type<_Tp>::__value) >> + { >> + __platform_wait((__platform_wait_t*)(void*) __addr, __old); >> + } >> + else >> + { >> + // TODO support timed backoff when this can be moved into the lib >> + __w._M_do_wait(__version); >> + } >> + } >> + __w._M_leave_wait(); >> + } >> + >> + template<class _Tp, class _Pred> >> + void >> + __atomic_wait_ptr(const _Tp* __addr, _Tp* __old, _Pred __pred) noexcept >> + { >> + for (auto __i = 0; __i < _GLIBCXX_SPIN_COUNT_1; ++__i) >> + { >> + if (!__pred(__old)) >> + return; >> + >> + if (__i < _GLIBCXX_SPIN_COUNT_2) >> + __thread_relax(); >> + else >> + __thread_yield(); >> + } >> + >> + auto& __w = __waiters::_S_for((void*)__addr); >> + auto __version = __w._M_enter_wait(); >> + while (auto __res = __pred(__old)) >> + { >> + if constexpr (__platform_wait_uses_type<_Tp>::__value) >> + { >> + __platform_wait((__platform_wait_t*)(void*) __addr, __old); >> + } >> + else >> + { >> + // TODO support timed backoff when this can be moved into the lib >> + __w._M_do_wait(__version); >> + } >> + } >> + __w._M_leave_wait(); >> + } >> + >> + template<class _Tp> >> + void >> + __atomic_notify(const _Tp* __addr, bool __all) noexcept >> + { >> + auto& __w = __waiters::_S_for((void*)__addr); >> + if (!__w._M_waiting()) >> + return; >> + >> + if constexpr (__platform_wait_uses_type<_Tp>::__value) >> + { >> + __platform_notify((__platform_wait_t*)(void*) __addr, __all); >> + } >> + else >> + { >> + __w._M_notify(__all); >> + } >> + } >> + } // namespace __detail >> + >> +_GLIBCXX_END_NAMESPACE_VERSION >> +} // namespace std >> + >> +#endif >> + >> diff --git a/libstdc++-v3/include/std/atomic b/libstdc++-v3/include/std/atomic >> index 40f23bdfc96..cb2308b0cdf 100644 >> --- a/libstdc++-v3/include/std/atomic >> +++ b/libstdc++-v3/include/std/atomic >> @@ -163,6 +163,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> compare_exchange_strong(bool& __i1, bool __i2, >> memory_order __m = memory_order_seq_cst) volatile noexcept >> { return _M_base.compare_exchange_strong(__i1, __i2, __m); } >> + >> +#if __cplusplus > 201703L >> + void wait(bool __old, memory_order __m = memory_order_seq_cst) const noexcept >> + { _M_base.wait(__old, __m); } >> + >> + // TODO add const volatile overload >> + >> + void notify_one() const noexcept >> + { _M_base.notify_one(); } >> + >> + void notify_all() const noexcept >> + { _M_base.notify_all(); } >> +#endif >> }; >> >> #if __cplusplus <= 201703L >> @@ -352,6 +365,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> memory_order __m = memory_order_seq_cst) volatile noexcept >> { return compare_exchange_strong(__e, __i, __m, >> __cmpexch_failure_order(__m)); } >> +#if __cplusplus > 201703L >> + void wait(_Tp __old, memory_order __m = memory_order_seq_cst) noexcept >> + { _M_i.wait(__old, __m); } >> + >> + // TODO add const volatile overload >> + >> + void notify_one() const noexcept >> + { _M_i.notify_one(); } >> + >> + void notify_all() const noexcept >> + { _M_i.notify_all(); } >> +#endif >> + >> }; >> #undef _GLIBCXX20_INIT >> >> @@ -590,6 +616,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> __cmpexch_failure_order(__m)); >> } >> >> +#if __cplusplus > 201703L >> + void wait(__pointer_type __old, memory_order __m = memory_order_seq_cst) noexcept >> + { _M_b.wait(__old, __m); } >> + >> + // TODO add const volatile overload >> + >> + void notify_one() const noexcept >> + { _M_b.notify_one(); } >> + >> + void notify_all() const noexcept >> + { _M_b.notify_all(); } >> +#endif >> __pointer_type >> fetch_add(ptrdiff_t __d, >> memory_order __m = memory_order_seq_cst) noexcept >> @@ -1342,6 +1380,29 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION >> memory_order_seq_cst); >> } >> >> + >> +#if __cplusplus > 201703L >> + template<typename _Tp> >> + inline void atomic_wait(const atomic<_Tp>* __a, >> + typename std::atomic<_Tp>::value_type __old) noexcept >> + { __a->wait(__old); } >> + >> + template<typename _Tp> >> + inline void atomic_wait_explicit(const atomic<_Tp>* __a, >> + typename std::atomic<_Tp>::value_type __old, >> + std::memory_order __m) noexcept >> + { __a->wait(__old, __m); } >> + >> + template<typename _Tp> >> + inline void atomic_notify_one(atomic<_Tp>* __a) noexcept >> + { __a->notify_one(); } >> + >> + template<typename _Tp> >> + inline void atomic_notify_all(atomic<_Tp>* __a) noexcept >> + { __a->notify_all(); } >> + >> +#endif // C++2a >> + >> // Function templates for atomic_integral and atomic_pointer operations only. >> // Some operations (and, or, xor) are only available for atomic integrals, >> // which is implemented by taking a parameter of type __atomic_base<_ITp>*. >> diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc >> new file mode 100644 >> index 00000000000..1d8d13621cc >> --- /dev/null >> +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc >> @@ -0,0 +1,103 @@ >> +// { dg-options "-std=gnu++2a -pthread -latomic -L../../libatomic/.libs" } >> +// { dg-do run { target c++2a } } >> +// { dg-require-effective-target pthread } >> +// { dg-require-gthreads "" } >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// You should have received a copy of the GNU General Public License along >> +// with this library; see the file COPYING3. If not see >> +// <http://www.gnu.org/licenses/>. >> + >> +#include <atomic> >> +#include <thread> >> +#include <mutex> >> +#include <condition_variable> >> +#include <chrono> >> +#include <type_traits> >> + >> +#include <testsuite_hooks.h> >> + >> +template<typename Tp> >> +Tp check_wait_notify(Tp val1, Tp val2) >> +{ >> + using namespace std::literals::chrono_literals; >> + >> + std::mutex m; >> + std::condition_variable cv; >> + >> + Tp aa = val1; >> + std::atomic_ref<Tp> a(aa); >> + std::thread t([&] >> + { >> + cv.notify_one(); >> + a.wait(val1); >> + if (a.load() != val2) >> + a = val1; >> + }); >> + std::unique_lock<std::mutex> l(m); >> + cv.wait(l); >> + std::this_thread::sleep_for(100ms); >> + a.store(val2); >> + a.notify_one(); >> + t.join(); >> + return a.load(); >> +} >> + >> +template<typename Tp, >> + bool = std::is_integral_v<Tp> >> + || std::is_floating_point_v<Tp>> >> +struct check; >> + >> +template<typename Tp> >> +struct check<Tp, true> >> +{ >> + check() >> + { >> + Tp a = 0; >> + Tp b = 42; >> + VERIFY(check_wait_notify(a, b) == b); >> + } >> +}; >> + >> +template<typename Tp> >> +struct check<Tp, false> >> +{ >> + check(Tp b) >> + { >> + Tp a; >> + VERIFY(check_wait_notify(a, b) == b); >> + } >> +}; >> + >> +struct foo >> +{ >> + long a = 0; >> + long b = 0; >> + >> + foo& operator=(foo const&) = default; >> + >> + friend bool >> + operator==(foo const& rhs, foo const& lhs) >> + { return rhs.a == lhs.a && rhs.b == lhs.b; } >> +}; >> + >> +int >> +main () >> +{ >> + check<long>(); >> + check<double>(); >> + check<foo>({42, 48}); >> + return 0; >> +} >> diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc >> new file mode 100644 >> index 00000000000..98258686945 >> --- /dev/null >> +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc >> @@ -0,0 +1,57 @@ >> +// { dg-options "-std=gnu++2a -pthread" } >> +// { dg-do run { target c++2a } } >> +// { dg-require-effective-target pthread } >> +// { dg-require-gthreads "" } >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// You should have received a copy of the GNU General Public License along >> +// with this library; see the file COPYING3. If not see >> +// <http://www.gnu.org/licenses/>. >> + >> +#include <atomic> >> +#include <thread> >> +#include <mutex> >> +#include <condition_variable> >> +#include <type_traits> >> +#include <chrono> >> + >> +#include <testsuite_hooks.h> >> + >> +int >> +main () >> +{ >> + using namespace std::literals::chrono_literals; >> + >> + std::mutex m; >> + std::condition_variable cv; >> + >> + std::atomic<bool> a(false); >> + std::atomic<bool> b(false); >> + std::thread t([&] >> + { >> + cv.notify_one(); >> + a.wait(false); >> + if (a.load()) >> + b.store(true); >> + }); >> + std::unique_lock<std::mutex> l(m); >> + cv.wait(l); >> + std::this_thread::sleep_for(100ms); >> + a.store(true); >> + a.notify_one(); >> + t.join(); >> + VERIFY( b.load() ); >> + return 0; >> +} >> diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc >> new file mode 100644 >> index 00000000000..1d032085752 >> --- /dev/null >> +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc >> @@ -0,0 +1,32 @@ >> +// { dg-options "-std=gnu++2a -pthread -latomic -L../../libatomic/.libs" } >> +// { dg-do run { target c++2a } } >> +// { dg-require-effective-target pthread } >> +// { dg-require-gthreads "" } >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// You should have received a copy of the GNU General Public License along >> +// with this library; see the file COPYING3. If not see >> +// <http://www.gnu.org/licenses/>. >> + >> +#include "generic.h" >> + >> +int >> +main () >> +{ >> + check<float> f; >> + check<double> d; >> + check<long double> l; >> + return 0; >> +} >> diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h >> new file mode 100644 >> index 00000000000..e6eeebf4ab2 >> --- /dev/null >> +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h >> @@ -0,0 +1,88 @@ >> +// -*- C++ -*- header. >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// You should have received a copy of the GNU General Public License along >> +// with this library; see the file COPYING3. If not see >> +// <http://www.gnu.org/licenses/>. >> + >> +#include <atomic> >> +#include <thread> >> +#include <mutex> >> +#include <condition_variable> >> +#include <chrono> >> + >> +#include <testsuite_hooks.h> >> + >> +template<typename Tp> >> +Tp check_wait_notify(Tp val1, Tp val2) >> +{ >> + using namespace std::literals::chrono_literals; >> + >> + std::mutex m; >> + std::condition_variable cv; >> + >> + std::atomic<Tp> a(val1); >> + std::thread t([&] >> + { >> + cv.notify_one(); >> + a.wait(val1); >> + if (a.load() != val2) >> + a = val1; >> + }); >> + std::unique_lock<std::mutex> l(m); >> + cv.wait(l); >> + std::this_thread::sleep_for(100ms); >> + a.store(val2); >> + a.notify_one(); >> + t.join(); >> + return a.load(); >> +} >> + >> +template<typename Tp> >> +Tp check_atomic_wait_notify(Tp val1, Tp val2) >> +{ >> + using namespace std::literals::chrono_literals; >> + >> + std::mutex m; >> + std::condition_variable cv; >> + >> + std::atomic<Tp> a(val1); >> + std::thread t([&] >> + { >> + cv.notify_one(); >> + std::atomic_wait(&a, val1); >> + if (a.load() != val2) >> + a = val1; >> + }); >> + std::unique_lock<std::mutex> l(m); >> + cv.wait(l); >> + std::this_thread::sleep_for(100ms); >> + a.store(val2); >> + std::atomic_notify_one(&a); >> + t.join(); >> + return a.load(); >> +} >> + >> +template<typename Tp> >> +struct check >> +{ >> + check() >> + { >> + Tp a = 0; >> + Tp b = 42; >> + VERIFY(check_wait_notify(a, b) == b); >> + VERIFY(check_atomic_wait_notify(a, b) == b); >> + } >> +}; >> diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc >> new file mode 100644 >> index 00000000000..2afd19a7d14 >> --- /dev/null >> +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc >> @@ -0,0 +1,56 @@ >> +// { dg-options "-std=gnu++2a -pthread" } >> +// { dg-do run { target c++2a } } >> +// { dg-require-effective-target pthread } >> +// { dg-require-gthreads "" } >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// You should have received a copy of the GNU General Public License along >> +// with this library; see the file COPYING3. If not see >> +// <http://www.gnu.org/licenses/>. >> + >> +#include "generic.h" >> + >> +int >> +main () >> +{ >> + // check<bool> bb; >> + check<char> ch; >> + check<signed char> sch; >> + check<unsigned char> uch; >> + check<short> s; >> + check<unsigned short> us; >> + check<int> i; >> + check<unsigned int> ui; >> + check<long> l; >> + check<unsigned long> ul; >> + check<long long> ll; >> + check<unsigned long long> ull; >> + >> + check<wchar_t> wch; >> + check<char8_t> ch8; >> + check<char16_t> ch16; >> + check<char32_t> ch32; >> + >> + check<int8_t> i8; >> + check<int16_t> i16; >> + check<int32_t> i32; >> + check<int64_t> i64; >> + >> + check<uint8_t> u8; >> + check<uint16_t> u16; >> + check<uint32_t> u32; >> + check<uint64_t> u64; >> + return 0; >> +} >> diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc >> new file mode 100644 >> index 00000000000..fe0b8de9f3f >> --- /dev/null >> +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc >> @@ -0,0 +1,59 @@ >> +// { dg-options "-std=gnu++2a -pthread" } >> +// { dg-do run { target c++2a } } >> +// { dg-require-effective-target pthread } >> +// { dg-require-gthreads "" } >> + >> +// Copyright (C) 2020 Free Software Foundation, Inc. >> +// >> +// This file is part of the GNU ISO C++ Library. This library is free >> +// software; you can redistribute it and/or modify it under the >> +// terms of the GNU General Public License as published by the >> +// Free Software Foundation; either version 3, or (at your option) >> +// any later version. >> + >> +// This library is distributed in the hope that it will be useful, >> +// but WITHOUT ANY WARRANTY; without even the implied warranty of >> +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> +// GNU General Public License for more details. >> + >> +// You should have received a copy of the GNU General Public License along >> +// with this library; see the file COPYING3. If not see >> +// <http://www.gnu.org/licenses/>. >> + >> +#include <atomic> >> +#include <thread> >> +#include <mutex> >> +#include <condition_variable> >> +#include <type_traits> >> +#include <chrono> >> + >> +#include <testsuite_hooks.h> >> + >> +int >> +main () >> +{ >> + using namespace std::literals::chrono_literals; >> + >> + std::mutex m; >> + std::condition_variable cv; >> + >> + long aa; >> + long bb; >> + >> + std::atomic<long*> a(nullptr); >> + std::thread t([&] >> + { >> + cv.notify_one(); >> + a.wait(nullptr); >> + if (a.load() == &aa) >> + a.store(&bb); >> + }); >> + std::unique_lock<std::mutex> l(m); >> + cv.wait(l); >> + std::this_thread::sleep_for(100ms); >> + a.store(&aa); >> + a.notify_one(); >> + t.join(); >> + VERIFY( a.load() == &bb); >> + return 0; >> +}
diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am index 80aeb3f8959..d195a721fd5 100644 --- a/libstdc++-v3/include/Makefile.am +++ b/libstdc++-v3/include/Makefile.am @@ -100,6 +100,7 @@ bits_headers = \ ${bits_srcdir}/allocated_ptr.h \ ${bits_srcdir}/allocator.h \ ${bits_srcdir}/atomic_base.h \ + ${bits_srcdir}/atomic_wait.h \ ${bits_srcdir}/atomic_futex.h \ ${bits_srcdir}/basic_ios.h \ ${bits_srcdir}/basic_ios.tcc \ diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in index eb437ad8d8d..4faaac5fb8d 100644 --- a/libstdc++-v3/include/Makefile.in +++ b/libstdc++-v3/include/Makefile.in @@ -445,6 +445,7 @@ bits_headers = \ ${bits_srcdir}/allocated_ptr.h \ ${bits_srcdir}/allocator.h \ ${bits_srcdir}/atomic_base.h \ + ${bits_srcdir}/atomic_wait.h \ ${bits_srcdir}/atomic_futex.h \ ${bits_srcdir}/basic_ios.h \ ${bits_srcdir}/basic_ios.tcc \ @@ -526,6 +527,7 @@ bits_headers = \ ${bits_srcdir}/specfun.h \ ${bits_srcdir}/sstream.tcc \ ${bits_srcdir}/std_abs.h \ + ${bits_srcdir}/std_condvar.h \ ${bits_srcdir}/std_function.h \ ${bits_srcdir}/std_mutex.h \ ${bits_srcdir}/stl_algo.h \ diff --git a/libstdc++-v3/include/bits/atomic_base.h b/libstdc++-v3/include/bits/atomic_base.h index 87fe0bd6000..b4fbe2c6eb3 100644 --- a/libstdc++-v3/include/bits/atomic_base.h +++ b/libstdc++-v3/include/bits/atomic_base.h @@ -37,6 +37,11 @@ #include <bits/atomic_lockfree_defines.h> #include <bits/move.h> +#if __cplusplus > 201703L +#include <bits/atomic_wait.h> +#include <iostream> +#endif + #ifndef _GLIBCXX_ALWAYS_INLINE #define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__)) #endif @@ -134,7 +139,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION return __ret; } - // Base types for atomics. template<typename _IntTp> struct __atomic_base; @@ -542,6 +546,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__m)); } +#if __cplusplus > 201703L + _GLIBCXX_ALWAYS_INLINE void + wait(__int_type __old, memory_order __m = memory_order_seq_cst) const noexcept + { + __detail::__atomic_wait(&_M_i, __old, [__m, this](__int_type __o) + { + return this->load(__m) == __o; + }); + } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { + __detail::__atomic_notify(&_M_i, false); + } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { + __detail::__atomic_notify(&_M_i, true); + } + + // TODO add const volatile overload +#endif // C++2a + _GLIBCXX_ALWAYS_INLINE __int_type fetch_add(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept @@ -803,6 +836,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION int(__m1), int(__m2)); } +#if __cplusplus > 201703L + _GLIBCXX_ALWAYS_INLINE void + wait(__pointer_type __old, memory_order __m = memory_order_seq_cst) noexcept + { + __detail::__atomic_wait(&_M_p, __old, [__m, this](__pointer_type __o) + { + return this->load(__m) == __o; + }); + } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { + __detail::__atomic_notify(&_M_p, false); + } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { + __detail::__atomic_notify(&_M_p, true); + } + + // TODO add const volatile overload +#endif // C++2a + _GLIBCXX_ALWAYS_INLINE __pointer_type fetch_add(ptrdiff_t __d, memory_order __m = memory_order_seq_cst) noexcept @@ -891,6 +953,39 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION int(__success), int(__failure)); } +#if __cplusplus > 201703L + template<typename _Tp> + _GLIBCXX_ALWAYS_INLINE void + wait(const _Tp* __ptr, _Val<_Tp> __old, memory_order __m = memory_order_seq_cst) noexcept + { + __detail::__atomic_wait_ptr(__ptr, std::__addressof(__old), + [=](_Tp* __o) + { + return load(__ptr, __m) == *__o; + }); + } + + // TODO add const volatile overload + + template<typename _Tp> + _GLIBCXX_ALWAYS_INLINE void + notify_one(const _Tp* __ptr) noexcept + { + __detail::__atomic_notify(__ptr, false); + } + + // TODO add const volatile overload + + template<typename _Tp> + _GLIBCXX_ALWAYS_INLINE void + notify_all(const _Tp* __ptr) noexcept + { + __detail::__atomic_notify(__ptr, true); + } + + // TODO add const volatile overload +#endif // C++2a + template<typename _Tp> _GLIBCXX_ALWAYS_INLINE _Tp fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept @@ -1144,6 +1239,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__order)); } + _GLIBCXX_ALWAYS_INLINE void + wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept + { __atomic_impl::wait(&_M_fp, __old, __m); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { __atomic_impl::notify_one(&_M_fp); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { __atomic_impl::notify_all(&_M_fp); } + + // TODO add const volatile overload value_type fetch_add(value_type __i, memory_order __m = memory_order_seq_cst) noexcept @@ -1281,6 +1393,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__order)); } + _GLIBCXX_ALWAYS_INLINE void + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept + { __atomic_impl::wait(_M_ptr, __old, __m); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { __atomic_impl::notify_one(_M_ptr); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { __atomic_impl::notify_all(_M_ptr); } + private: _Tp* _M_ptr; }; @@ -1376,6 +1504,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__order)); } + _GLIBCXX_ALWAYS_INLINE void + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept + { __atomic_impl::wait(_M_ptr, __old, __m); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { __atomic_impl::notify_one(_M_ptr); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { __atomic_impl::notify_all(_M_ptr); } + value_type fetch_add(value_type __i, memory_order __m = memory_order_seq_cst) const noexcept @@ -1531,6 +1675,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__order)); } + _GLIBCXX_ALWAYS_INLINE void + wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept + { __atomic_impl::wait(_M_ptr, __old, __m); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { __atomic_impl::notify_one(_M_ptr); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { __atomic_impl::notify_all(_M_ptr); } + value_type fetch_add(value_type __i, memory_order __m = memory_order_seq_cst) const noexcept @@ -1640,6 +1800,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__order)); } + _GLIBCXX_ALWAYS_INLINE void + wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept + { __atomic_impl::wait(_M_ptr, __old, __m); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_one() const noexcept + { __atomic_impl::notify_one(_M_ptr); } + + // TODO add const volatile overload + + _GLIBCXX_ALWAYS_INLINE void + notify_all() const noexcept + { __atomic_impl::notify_all(_M_ptr); } + _GLIBCXX_ALWAYS_INLINE value_type fetch_add(difference_type __d, memory_order __m = memory_order_seq_cst) const noexcept diff --git a/libstdc++-v3/include/bits/atomic_wait.h b/libstdc++-v3/include/bits/atomic_wait.h new file mode 100644 index 00000000000..fa2650afc39 --- /dev/null +++ b/libstdc++-v3/include/bits/atomic_wait.h @@ -0,0 +1,295 @@ +// -*- C++ -*- header. + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// Under Section 7 of GPL version 3, you are granted additional +// permissions described in the GCC Runtime Library Exception, version +// 3.1, as published by the Free Software Foundation. + +// You should have received a copy of the GNU General Public License and +// a copy of the GCC Runtime Library Exception along with this program; +// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +// <http://www.gnu.org/licenses/>. + +/** @file bits/atomic_wait.h + * This is an internal header file, included by other library headers. + * Do not attempt to use it directly. @headername{atomic} + */ + +#ifndef _GLIBCXX_ATOMIC_WAIT_H +#define _GLIBCXX_ATOMIC_WAIT_H 1 + +#pragma GCC system_header + +#include <bits/c++config.h> +#include <bits/functional_hash.h> +#include <bits/gthr.h> +#include <bits/std_mutex.h> +#include <bits/unique_lock.h> +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX +#include <climits> +#include <unistd.h> +#include <syscall.h> +#endif + +#define _GLIBCXX_SPIN_COUNT_1 16 +#define _GLIBCXX_SPIN_COUNT_2 12 + +// TODO get this from Autoconf +#define _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE 1 + +namespace std _GLIBCXX_VISIBILITY(default) +{ +_GLIBCXX_BEGIN_NAMESPACE_VERSION + namespace __detail + { + using __platform_wait_t = int; + + template<class _Tp> + struct __platform_wait_uses_type + { +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX + enum { __value = std::is_same<typename std::remove_cv<_Tp>::type, + __platform_wait_t>::value }; +#else + enum { __value = std::false_type::value }; +#endif + }; + +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX + enum + { +#ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE + __futex_private_flag = 128, +#else + __futex_private_flag = 0, +#endif + __futex_wait = 0, + __futex_wake = 1, + __futex_wait_private = __futex_wait | __futex_private_flag, + __futex_wake_private = __futex_wake | __futex_private_flag, + }; + + void + __platform_wait(__platform_wait_t* __addr, __platform_wait_t __val) noexcept + { + auto __e = syscall (SYS_futex, __addr, __futex_wait_private, __val, nullptr); + if (__e && (errno != EINTR || errno != EAGAIN)) + std::terminate(); + } + + void + __platform_notify(__platform_wait_t* __addr, bool __all) noexcept + { + syscall (SYS_futex, __addr, __futex_wake_private, __all ? INT_MAX : 1); + } +#endif + + struct alignas(64) __waiter + { + bool _M_signaled = false; + void* _M_t; + __waiter* _M_prev = nullptr; + __waiter* _M_next = nullptr; + + __waiter(void* __t) noexcept + : _M_t(__t) + { } + }; + + struct alignas(64) __waiters + { + int32_t alignas(64) _M_ver = 0; + int32_t alignas(64) _M_wait = 0; + + // TODO make this used only where we don't have futexes + using __lock_t = std::unique_lock<std::mutex>; + mutable __lock_t::mutex_type _M_mtx; + +#ifdef __GTHREAD_COND_INIT + mutable __gthread_cond_t _M_cv = __GTHREAD_COND_INIT; + __waiters() noexcept = default; +#else + mutable __gthread_cond_t _M_cv; + __waiters() noexcept + { + __GTHREAD_COND_INIT_FUNCTION(&_M_cond); + } +#endif + + int32_t + _M_enter_wait() noexcept + { + int32_t __res; + __atomic_load(&_M_ver, &__res, __ATOMIC_ACQUIRE); + __atomic_fetch_add(&_M_wait, 1, __ATOMIC_ACQ_REL); + return __res; + } + + void + _M_leave_wait() noexcept + { + __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_ACQ_REL); + } + + void + _M_do_wait(int32_t __version) const noexcept + { + int32_t __cur = 0; + while (__cur <= __version) + { + __waiters::__lock_t __l(_M_mtx); + auto __e = __gthread_cond_wait(&_M_cv, __l.mutex()->native_handle()); + if (__e) + std::terminate(); + int32_t __last = __cur; + __atomic_load(&_M_ver, &__cur, __ATOMIC_ACQUIRE); + // protect if version overflows + if (__cur < __last) + break; // break the loop if version overflows + } + } + + int32_t + _M_waiting() const noexcept + { + int32_t __res; + __atomic_load(&_M_wait, &__res, __ATOMIC_ACQUIRE); + return __res; + } + + void + _M_notify(bool __all) noexcept + { + __atomic_fetch_add(&_M_ver, 1, __ATOMIC_ACQ_REL); + auto __e = __gthread_cond_broadcast(&_M_cv); + if (__e) + __throw_system_error(__e); + } + + static __waiters& _S_for(void* __t) { + const unsigned char __mask = 0xf; + static __waiters __w[__mask + 1]; + + auto __key = _Hash_impl::hash(__t) & __mask; + return __w[__key]; + } + }; + + void + __thread_relax() noexcept + { +#if defined __i386__ || defined __x86_64__ + __builtin_ia32_pause(); +#elif defined _GLIBCXX_USE_SCHED_YIELD + __gthread_yield(); +#endif + } + + void + __thread_yield() noexcept + { +#if defined _GLIBCXX_USE_SCHED_YIELD + __gthread_yield(); +#endif + } + + template<class _Tp, class _Pred> + void + __atomic_wait(const _Tp* __addr, _Tp __old, _Pred __pred) noexcept + { + for (auto __i = 0; __i < _GLIBCXX_SPIN_COUNT_1; ++__i) + { + if (!__pred(__old)) + return; + + if (__i < _GLIBCXX_SPIN_COUNT_2) + __thread_relax(); + else + __thread_yield(); + } + + auto& __w = __waiters::_S_for((void*)__addr); + auto __version = __w._M_enter_wait(); + while (auto __res = __pred(__old)) + { + if constexpr (__platform_wait_uses_type<_Tp>::__value) + { + __platform_wait((__platform_wait_t*)(void*) __addr, __old); + } + else + { + // TODO support timed backoff when this can be moved into the lib + __w._M_do_wait(__version); + } + } + __w._M_leave_wait(); + } + + template<class _Tp, class _Pred> + void + __atomic_wait_ptr(const _Tp* __addr, _Tp* __old, _Pred __pred) noexcept + { + for (auto __i = 0; __i < _GLIBCXX_SPIN_COUNT_1; ++__i) + { + if (!__pred(__old)) + return; + + if (__i < _GLIBCXX_SPIN_COUNT_2) + __thread_relax(); + else + __thread_yield(); + } + + auto& __w = __waiters::_S_for((void*)__addr); + auto __version = __w._M_enter_wait(); + while (auto __res = __pred(__old)) + { + if constexpr (__platform_wait_uses_type<_Tp>::__value) + { + __platform_wait((__platform_wait_t*)(void*) __addr, __old); + } + else + { + // TODO support timed backoff when this can be moved into the lib + __w._M_do_wait(__version); + } + } + __w._M_leave_wait(); + } + + template<class _Tp> + void + __atomic_notify(const _Tp* __addr, bool __all) noexcept + { + auto& __w = __waiters::_S_for((void*)__addr); + if (!__w._M_waiting()) + return; + + if constexpr (__platform_wait_uses_type<_Tp>::__value) + { + __platform_notify((__platform_wait_t*)(void*) __addr, __all); + } + else + { + __w._M_notify(__all); + } + } + } // namespace __detail + +_GLIBCXX_END_NAMESPACE_VERSION +} // namespace std + +#endif + diff --git a/libstdc++-v3/include/std/atomic b/libstdc++-v3/include/std/atomic index 40f23bdfc96..cb2308b0cdf 100644 --- a/libstdc++-v3/include/std/atomic +++ b/libstdc++-v3/include/std/atomic @@ -163,6 +163,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION compare_exchange_strong(bool& __i1, bool __i2, memory_order __m = memory_order_seq_cst) volatile noexcept { return _M_base.compare_exchange_strong(__i1, __i2, __m); } + +#if __cplusplus > 201703L + void wait(bool __old, memory_order __m = memory_order_seq_cst) const noexcept + { _M_base.wait(__old, __m); } + + // TODO add const volatile overload + + void notify_one() const noexcept + { _M_base.notify_one(); } + + void notify_all() const noexcept + { _M_base.notify_all(); } +#endif }; #if __cplusplus <= 201703L @@ -352,6 +365,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION memory_order __m = memory_order_seq_cst) volatile noexcept { return compare_exchange_strong(__e, __i, __m, __cmpexch_failure_order(__m)); } +#if __cplusplus > 201703L + void wait(_Tp __old, memory_order __m = memory_order_seq_cst) noexcept + { _M_i.wait(__old, __m); } + + // TODO add const volatile overload + + void notify_one() const noexcept + { _M_i.notify_one(); } + + void notify_all() const noexcept + { _M_i.notify_all(); } +#endif + }; #undef _GLIBCXX20_INIT @@ -590,6 +616,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __cmpexch_failure_order(__m)); } +#if __cplusplus > 201703L + void wait(__pointer_type __old, memory_order __m = memory_order_seq_cst) noexcept + { _M_b.wait(__old, __m); } + + // TODO add const volatile overload + + void notify_one() const noexcept + { _M_b.notify_one(); } + + void notify_all() const noexcept + { _M_b.notify_all(); } +#endif __pointer_type fetch_add(ptrdiff_t __d, memory_order __m = memory_order_seq_cst) noexcept @@ -1342,6 +1380,29 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION memory_order_seq_cst); } + +#if __cplusplus > 201703L + template<typename _Tp> + inline void atomic_wait(const atomic<_Tp>* __a, + typename std::atomic<_Tp>::value_type __old) noexcept + { __a->wait(__old); } + + template<typename _Tp> + inline void atomic_wait_explicit(const atomic<_Tp>* __a, + typename std::atomic<_Tp>::value_type __old, + std::memory_order __m) noexcept + { __a->wait(__old, __m); } + + template<typename _Tp> + inline void atomic_notify_one(atomic<_Tp>* __a) noexcept + { __a->notify_one(); } + + template<typename _Tp> + inline void atomic_notify_all(atomic<_Tp>* __a) noexcept + { __a->notify_all(); } + +#endif // C++2a + // Function templates for atomic_integral and atomic_pointer operations only. // Some operations (and, or, xor) are only available for atomic integrals, // which is implemented by taking a parameter of type __atomic_base<_ITp>*. diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc new file mode 100644 index 00000000000..1d8d13621cc --- /dev/null +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/atomic_refs.cc @@ -0,0 +1,103 @@ +// { dg-options "-std=gnu++2a -pthread -latomic -L../../libatomic/.libs" } +// { dg-do run { target c++2a } } +// { dg-require-effective-target pthread } +// { dg-require-gthreads "" } + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License along +// with this library; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include <atomic> +#include <thread> +#include <mutex> +#include <condition_variable> +#include <chrono> +#include <type_traits> + +#include <testsuite_hooks.h> + +template<typename Tp> +Tp check_wait_notify(Tp val1, Tp val2) +{ + using namespace std::literals::chrono_literals; + + std::mutex m; + std::condition_variable cv; + + Tp aa = val1; + std::atomic_ref<Tp> a(aa); + std::thread t([&] + { + cv.notify_one(); + a.wait(val1); + if (a.load() != val2) + a = val1; + }); + std::unique_lock<std::mutex> l(m); + cv.wait(l); + std::this_thread::sleep_for(100ms); + a.store(val2); + a.notify_one(); + t.join(); + return a.load(); +} + +template<typename Tp, + bool = std::is_integral_v<Tp> + || std::is_floating_point_v<Tp>> +struct check; + +template<typename Tp> +struct check<Tp, true> +{ + check() + { + Tp a = 0; + Tp b = 42; + VERIFY(check_wait_notify(a, b) == b); + } +}; + +template<typename Tp> +struct check<Tp, false> +{ + check(Tp b) + { + Tp a; + VERIFY(check_wait_notify(a, b) == b); + } +}; + +struct foo +{ + long a = 0; + long b = 0; + + foo& operator=(foo const&) = default; + + friend bool + operator==(foo const& rhs, foo const& lhs) + { return rhs.a == lhs.a && rhs.b == lhs.b; } +}; + +int +main () +{ + check<long>(); + check<double>(); + check<foo>({42, 48}); + return 0; +} diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc new file mode 100644 index 00000000000..98258686945 --- /dev/null +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/bool.cc @@ -0,0 +1,57 @@ +// { dg-options "-std=gnu++2a -pthread" } +// { dg-do run { target c++2a } } +// { dg-require-effective-target pthread } +// { dg-require-gthreads "" } + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License along +// with this library; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include <atomic> +#include <thread> +#include <mutex> +#include <condition_variable> +#include <type_traits> +#include <chrono> + +#include <testsuite_hooks.h> + +int +main () +{ + using namespace std::literals::chrono_literals; + + std::mutex m; + std::condition_variable cv; + + std::atomic<bool> a(false); + std::atomic<bool> b(false); + std::thread t([&] + { + cv.notify_one(); + a.wait(false); + if (a.load()) + b.store(true); + }); + std::unique_lock<std::mutex> l(m); + cv.wait(l); + std::this_thread::sleep_for(100ms); + a.store(true); + a.notify_one(); + t.join(); + VERIFY( b.load() ); + return 0; +} diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc new file mode 100644 index 00000000000..1d032085752 --- /dev/null +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/floats.cc @@ -0,0 +1,32 @@ +// { dg-options "-std=gnu++2a -pthread -latomic -L../../libatomic/.libs" } +// { dg-do run { target c++2a } } +// { dg-require-effective-target pthread } +// { dg-require-gthreads "" } + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License along +// with this library; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include "generic.h" + +int +main () +{ + check<float> f; + check<double> d; + check<long double> l; + return 0; +} diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h new file mode 100644 index 00000000000..e6eeebf4ab2 --- /dev/null +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/generic.h @@ -0,0 +1,88 @@ +// -*- C++ -*- header. + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License along +// with this library; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include <atomic> +#include <thread> +#include <mutex> +#include <condition_variable> +#include <chrono> + +#include <testsuite_hooks.h> + +template<typename Tp> +Tp check_wait_notify(Tp val1, Tp val2) +{ + using namespace std::literals::chrono_literals; + + std::mutex m; + std::condition_variable cv; + + std::atomic<Tp> a(val1); + std::thread t([&] + { + cv.notify_one(); + a.wait(val1); + if (a.load() != val2) + a = val1; + }); + std::unique_lock<std::mutex> l(m); + cv.wait(l); + std::this_thread::sleep_for(100ms); + a.store(val2); + a.notify_one(); + t.join(); + return a.load(); +} + +template<typename Tp> +Tp check_atomic_wait_notify(Tp val1, Tp val2) +{ + using namespace std::literals::chrono_literals; + + std::mutex m; + std::condition_variable cv; + + std::atomic<Tp> a(val1); + std::thread t([&] + { + cv.notify_one(); + std::atomic_wait(&a, val1); + if (a.load() != val2) + a = val1; + }); + std::unique_lock<std::mutex> l(m); + cv.wait(l); + std::this_thread::sleep_for(100ms); + a.store(val2); + std::atomic_notify_one(&a); + t.join(); + return a.load(); +} + +template<typename Tp> +struct check +{ + check() + { + Tp a = 0; + Tp b = 42; + VERIFY(check_wait_notify(a, b) == b); + VERIFY(check_atomic_wait_notify(a, b) == b); + } +}; diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc new file mode 100644 index 00000000000..2afd19a7d14 --- /dev/null +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/integrals.cc @@ -0,0 +1,56 @@ +// { dg-options "-std=gnu++2a -pthread" } +// { dg-do run { target c++2a } } +// { dg-require-effective-target pthread } +// { dg-require-gthreads "" } + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License along +// with this library; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include "generic.h" + +int +main () +{ + // check<bool> bb; + check<char> ch; + check<signed char> sch; + check<unsigned char> uch; + check<short> s; + check<unsigned short> us; + check<int> i; + check<unsigned int> ui; + check<long> l; + check<unsigned long> ul; + check<long long> ll; + check<unsigned long long> ull; + + check<wchar_t> wch; + check<char8_t> ch8; + check<char16_t> ch16; + check<char32_t> ch32; + + check<int8_t> i8; + check<int16_t> i16; + check<int32_t> i32; + check<int64_t> i64; + + check<uint8_t> u8; + check<uint16_t> u16; + check<uint32_t> u32; + check<uint64_t> u64; + return 0; +} diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc new file mode 100644 index 00000000000..fe0b8de9f3f --- /dev/null +++ b/libstdc++-v3/testsuite/29_atomics/atomic/wait_notify/pointers.cc @@ -0,0 +1,59 @@ +// { dg-options "-std=gnu++2a -pthread" } +// { dg-do run { target c++2a } } +// { dg-require-effective-target pthread } +// { dg-require-gthreads "" } + +// Copyright (C) 2020 Free Software Foundation, Inc. +// +// This file is part of the GNU ISO C++ Library. This library is free +// software; you can redistribute it and/or modify it under the +// terms of the GNU General Public License as published by the +// Free Software Foundation; either version 3, or (at your option) +// any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License along +// with this library; see the file COPYING3. If not see +// <http://www.gnu.org/licenses/>. + +#include <atomic> +#include <thread> +#include <mutex> +#include <condition_variable> +#include <type_traits> +#include <chrono> + +#include <testsuite_hooks.h> + +int +main () +{ + using namespace std::literals::chrono_literals; + + std::mutex m; + std::condition_variable cv; + + long aa; + long bb; + + std::atomic<long*> a(nullptr); + std::thread t([&] + { + cv.notify_one(); + a.wait(nullptr); + if (a.load() == &aa) + a.store(&bb); + }); + std::unique_lock<std::mutex> l(m); + cv.wait(l); + std::this_thread::sleep_for(100ms); + a.store(&aa); + a.notify_one(); + t.join(); + VERIFY( a.load() == &bb); + return 0; +}