@@ -52,11 +52,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
#if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
struct __atomic_futex_unsigned_base
{
- // Returns false iff a timeout occurred.
+ // __s and __ns are measured against CLOCK_REALTIME. Returns false
+ // iff a timeout occurred.
bool
_M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout,
chrono::seconds __s, chrono::nanoseconds __ns);
+ // __s and __ns are measured against CLOCK_MONOTONIC. Returns
+ // false iff a timeout occurred.
+ bool
+ _M_futex_wait_until_steady(unsigned *__addr, unsigned __val, bool __has_timeout,
+ chrono::seconds __s, chrono::nanoseconds __ns);
+
// This can be executed after the object has been destroyed.
static void _M_futex_notify_all(unsigned* __addr);
};
@@ -86,6 +93,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
// value if equal is false.
// The assumed value is the caller's assumption about the current value
// when making the call.
+ // __s and __ns are measured against CLOCK_REALTIME.
unsigned
_M_load_and_test_until(unsigned __assumed, unsigned __operand,
bool __equal, memory_order __mo, bool __has_timeout,
@@ -110,6 +118,36 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
}
}
+ // If a timeout occurs, returns a current value after the timeout;
+ // otherwise, returns the operand's value if equal is true or a different
+ // value if equal is false.
+ // The assumed value is the caller's assumption about the current value
+ // when making the call.
+ // __s and __ns are measured against CLOCK_MONOTONIC.
+ unsigned
+ _M_load_and_test_until_steady(unsigned __assumed, unsigned __operand,
+ bool __equal, memory_order __mo, bool __has_timeout,
+ chrono::seconds __s, chrono::nanoseconds __ns)
+ {
+ for (;;)
+ {
+ // Don't bother checking the value again because we expect the caller
+ // to have done it recently.
+ // memory_order_relaxed is sufficient because we can rely on just the
+ // modification order (store_notify uses an atomic RMW operation too),
+ // and the futex syscalls synchronize between themselves.
+ _M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
+ bool __ret = _M_futex_wait_until_steady((unsigned*)(void*)&_M_data,
+ __assumed | _Waiter_bit,
+ __has_timeout, __s, __ns);
+ // Fetch the current value after waiting (clears _Waiter_bit).
+ __assumed = _M_load(__mo);
+ if (!__ret || ((__operand == __assumed) == __equal))
+ return __assumed;
+ // TODO adapt wait time
+ }
+ }
+
// Returns the operand's value if equal is true or a different value if
// equal is false.
// The assumed value is the caller's assumption about the current value
@@ -140,6 +178,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
true, __s.time_since_epoch(), __ns);
}
+ template<typename _Dur>
+ unsigned
+ _M_load_and_test_until_impl(unsigned __assumed, unsigned __operand,
+ bool __equal, memory_order __mo,
+ const chrono::time_point<std::chrono::steady_clock, _Dur>& __atime)
+ {
+ auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
+ auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
+ // XXX correct?
+ return _M_load_and_test_until_steady(__assumed, __operand, __equal, __mo,
+ true, __s.time_since_epoch(), __ns);
+ }
+
public:
_GLIBCXX_ALWAYS_INLINE unsigned
@@ -200,6 +251,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
return (__i & ~_Waiter_bit) == __val;
}
+ // Returns false iff a timeout occurred.
+ template<typename _Duration>
+ _GLIBCXX_ALWAYS_INLINE bool
+ _M_load_when_equal_until(unsigned __val, memory_order __mo,
+ const chrono::time_point<std::chrono::steady_clock, _Duration>& __atime)
+ {
+ unsigned __i = _M_load(__mo);
+ if ((__i & ~_Waiter_bit) == __val)
+ return true;
+ // TODO Spin-wait first. Ignore effect on timeout.
+ __i = _M_load_and_test_until_impl(__i, __val, true, __mo, __atime);
+ return (__i & ~_Waiter_bit) == __val;
+ }
+
_GLIBCXX_ALWAYS_INLINE void
_M_store_notify_all(unsigned __val, memory_order __mo)
{
@@ -36,6 +36,7 @@
// Constants for the wait/wake futex syscall operations
const unsigned futex_wait_op = 0;
const unsigned futex_wait_bitset_op = 9;
+const unsigned futex_clock_monotonic_flag = 0;
const unsigned futex_clock_realtime_flag = 256;
const unsigned futex_bitset_match_any = ~0;
const unsigned futex_wake_op = 1;
@@ -75,6 +76,38 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
}
}
+ bool
+ __atomic_futex_unsigned_base::_M_futex_wait_until_steady(unsigned *__addr,
+ unsigned __val,
+ bool __has_timeout, chrono::seconds __s, chrono::nanoseconds __ns)
+ {
+ if (!__has_timeout)
+ {
+ // Ignore whether we actually succeeded to block because at worst,
+ // we will fall back to spin-waiting. The only thing we could do
+ // here on errors is abort.
+ int ret __attribute__((unused));
+ ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr);
+ _GLIBCXX_DEBUG_ASSERT(ret == 0 || errno == EINTR || errno == EAGAIN);
+ return true;
+ }
+ else
+ {
+ struct timespec rt;
+ rt.tv_sec = __s.count();
+ rt.tv_nsec = __ns.count();
+
+ if (syscall (SYS_futex, __addr, futex_wait_bitset_op | futex_clock_monotonic_flag, __val, &rt, nullptr, futex_bitset_match_any) == -1)
+ {
+ _GLIBCXX_DEBUG_ASSERT(errno == EINTR || errno == EAGAIN
+ || errno == ETIMEDOUT);
+ if (errno == ETIMEDOUT)
+ return false;
+ }
+ return true;
+ }
+ }
+
void
__atomic_futex_unsigned_base::_M_futex_notify_all(unsigned* __addr)
{