@@ -1,3 +1,29 @@
+2017-10-24 Waiman Long <longman@redhat.com>
+
+ * nptl/pthreadP.h: Add pthread mutex macros to support TP futexes.
+ * nptl/pthread_mutex_init.c: Add test for the presence of TP futexes.
+ * nptl/pthread_mutex_lock.c: Add TP futexes support.
+ * nptl/pthread_mutex_timedlock.c: Add TP futexes support.
+ * nptl/pthread_mutex_trylock.c: Add TP futexes support.
+ * nptl/pthread_mutex_unlock.c: Add TP futexes support.
+ * nptl/pthread_mutexattr_setprotocol.c: Make it accept a new protocol
+ PTHREAD_THROUGHPUT_NP.
+ * nptl/pthread_rwlock_rdlock.c: Add TP futexes support.
+ * nptl/pthread_rwlock_timedrdlock.c: Add TP futexes support.
+ * nptl/pthread_rwlock_timedwrlock.c: Add TP futexes support.
+ * nptl/pthread_rwlock_tp.c: New file to make TP futex syscalls.
+ * nptl/pthread_rwlock_tryrdlock.c: Add TP futexes support.
+ * nptl/pthread_rwlock_trywrlock.c: Add TP futexes support.
+ * nptl/pthread_rwlock_unlock.c: Add TP futexes support.
+ * nptl/pthread_rwlock_wrlock.c: Add TP futexes support.
+ * nptl/pthread_rwlockattr_setkind_np.c: Make it accept a new kind
+ PTHREAD_RWLOCK_USE_TP_FUTEX_NP.
+ * sysdeps/nptl/pthread.h: Add PTHREAD_THROUGHPUT_NP &
+ PTHREAD_RWLOCK_USE_TP_FUTEX_NP.
+ * sysdeps/unix/sysv/linux/hppa/pthread.h: Add PTHREAD_THROUGHPUT_NP.
+ * sysdeps/unix/sysv/linux/lowlevellock-futex.h: Add new TP futexes
+ related macros.
+
2017-10-19 Valery Reznic <valery_reznic@yahoo.com>
H.J. Lu <hongjiu.lu@intel.com>
@@ -17,6 +17,7 @@
<http://www.gnu.org/licenses/>. */
#include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c. */
int
@@ -24,7 +25,9 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (rdlock_entry, 1, rwlock);
- int result = __pthread_rwlock_rdlock_full (rwlock, NULL);
+ int result = (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ ? __pthread_rwlock_rdlock_tp (rwlock, NULL)
+ : __pthread_rwlock_rdlock_full (rwlock, NULL);
LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
return result;
}
@@ -17,6 +17,7 @@
<http://www.gnu.org/licenses/>. */
#include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c. */
int
@@ -33,5 +34,7 @@ pthread_rwlock_timedrdlock (pthread_rwlock_t *rwlock,
|| abstime->tv_nsec < 0))
return EINVAL;
- return __pthread_rwlock_rdlock_full (rwlock, abstime);
+ return (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ ? __pthread_rwlock_rdlock_tp (rwlock, abstime)
+ : __pthread_rwlock_rdlock_full (rwlock, abstime);
}
@@ -17,6 +17,7 @@
<http://www.gnu.org/licenses/>. */
#include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c. */
int
@@ -33,5 +34,7 @@ pthread_rwlock_timedwrlock (pthread_rwlock_t *rwlock,
|| abstime->tv_nsec < 0))
return EINVAL;
- return __pthread_rwlock_wrlock_full (rwlock, abstime);
+ return (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ ? __pthread_rwlock_wrlock_tp (rwlock, abstime)
+ : __pthread_rwlock_wrlock_full (rwlock, abstime);
}
new file mode 100644
@@ -0,0 +1,235 @@
+/* POSIX reader--writer lock: TP futex specific code
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <errno.h>
+#include <sysdep.h>
+#include <pthread.h>
+#include <pthreadP.h>
+#include <sys/time.h>
+#include <stap-probe.h>
+#include <atomic.h>
+#include <futex-internal.h>
+
+/*
+ * The __writers_futex is used as the target of the TP futex syscalls.
+ */
+
+#ifdef __NR_futex
+
+#define FUTEX_FLAGS_MASK (~FUTEX_TID_MASK)
+#define lll_futex_tp_lock(futexp, val, timeout, private) \
+ lll_futex_syscall(4, futexp, \
+ __lll_private_flag(FUTEX_LOCK, private), \
+ val, timeout)
+#define lll_futex_tp_unlock(futexp, private) \
+ lll_futex_syscall(4, futexp, \
+ __lll_private_flag(FUTEX_UNLOCK, private), \
+ 0, NULL)
+#define lll_futex_tp_lock_shared(futexp, timeout, private) \
+ lll_futex_syscall(4, futexp, \
+ __lll_private_flag(FUTEX_LOCK_SHARED, private),\
+ 0, timeout)
+#define lll_futex_tp_unlock_shared(futexp, private) \
+ lll_futex_syscall(4, futexp, \
+ __lll_private_flag(FUTEX_UNLOCK_SHARED, private),\
+ 0, NULL)
+
+static __always_inline int
+__pthread_rwlock_private (pthread_rwlock_t *rwlock)
+{
+ return rwlock->__data.__shared != 0 ? FUTEX_SHARED : FUTEX_PRIVATE;
+}
+
+/* Return 1 if the lock acquired, 0 otherwise */
+static __always_inline int
+__pthread_rwlock_tryrdlock_tp(pthread_rwlock_t *rwlock)
+{
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+ unsigned int oldval = atomic_compare_and_exchange_val_acq
+ (futex, FUTEX_SHARED_FLAG + 1, 0);
+ if (oldval == 0)
+ return 1;
+
+ while (1)
+ {
+ unsigned int new, old = oldval;
+
+ /*
+ * Try to increment the reader count only if
+ * 1) the FUTEX_SHARED_FLAG bit is set; and
+ * 2) none of the flags bits or FUTEX_SHARED_UNLOCK is set.
+ */
+ if (!old)
+ new = FUTEX_SHARED_FLAG + 1;
+ else if ((old & FUTEX_SHARED_FLAG) &&
+ !(old & (FUTEX_FLAGS_MASK|FUTEX_SHARED_UNLOCK)))
+ new = old + 1;
+ else
+ return 0;
+
+ oldval = atomic_compare_and_exchange_val_acq(futex, new, old);
+ if (old == oldval)
+ return 1;
+ }
+ /* Unreachable */
+}
+
+/* Return 1 if the lock acquired, 0 otherwise */
+static __always_inline int
+__pthread_rwlock_trywrlock_tp(pthread_rwlock_t *rwlock)
+{
+ pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+
+ return atomic_compare_and_exchange_val_acq(futex, id, 0) == 0;
+}
+
+static __always_inline int
+__pthread_rwlock_rdlock_tp(pthread_rwlock_t *rwlock,
+ const struct timespec *abstime)
+{
+ int private = __pthread_rwlock_private(rwlock);
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+
+ if (__pthread_rwlock_tryrdlock_tp(rwlock))
+ return 0;
+
+ return lll_futex_tp_lock_shared(futex, abstime, private);
+}
+
+static __always_inline int
+__pthread_rwlock_wrlock_tp(pthread_rwlock_t *rwlock,
+ const struct timespec *abstime)
+{
+ pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+ int private = __pthread_rwlock_private(rwlock);
+ int uslockcnt = 4; /* Do 4 userspace locks before doing kernel lock */
+
+ if (__pthread_rwlock_trywrlock_tp(rwlock))
+ return 0;
+
+ while (1)
+ {
+ int err = lll_futex_tp_lock(futex, uslockcnt, abstime, private);
+
+ /* It is possible, though extremely unlikely, that a lock
+ handoff has happened and an error is returned. So we
+ need to check for that here. */
+ if (!err || ((*futex & FUTEX_TID_MASK) == id))
+ return 0;
+
+ if (err == ETIMEDOUT)
+ return err;
+
+ if ((err == EAGAIN) && __pthread_rwlock_trywrlock_tp(rwlock))
+ return 0;
+
+ uslockcnt--;
+ }
+ /* Unreachable */
+}
+
+static __always_inline void
+__pthread_rwlock_rdunlock_tp(pthread_rwlock_t *rwlock)
+{
+ int val = atomic_fetch_add_release(&rwlock->__data.__writers_futex, -1) - 1;
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+ int private = __pthread_rwlock_private(rwlock);
+
+ while (1)
+ {
+ int oldval;
+
+ /* Return if not the last reader, not in shared locking
+ mode or the unlock bit has been set. */
+ if ((val & (FUTEX_SCNT_MASK|FUTEX_SHARED_UNLOCK)) ||
+ !(val & FUTEX_SHARED_FLAG))
+ return;
+
+ if (val & FUTEX_FLAGS_MASK)
+ {
+ /* Only one task that can set the FUTEX_SHARED_UNLOCK
+ bit will do the unlock to wake up the waiters. */
+ oldval = atomic_compare_and_exchange_val_rel
+ (futex, val|FUTEX_SHARED_UNLOCK, val);
+ if (oldval == val)
+ break;
+ }
+ else
+ {
+ /* Try to clear the futex when there is no waiter. */
+ oldval = atomic_compare_and_exchange_val_rel(futex, 0, val);
+ if (oldval == val)
+ return;
+ }
+ val = oldval;
+ }
+
+ lll_futex_tp_unlock_shared(futex, private);
+}
+
+static __always_inline void
+__pthread_rwlock_wrunlock_tp(pthread_rwlock_t *rwlock)
+{
+ pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+ int private = __pthread_rwlock_private(rwlock);
+
+ if (atomic_compare_and_exchange_val_rel(futex, 0, id) == id)
+ return;
+
+ lll_futex_tp_unlock(futex, private);
+}
+
+#else /* __NR_futex */
+
+static __always_inline int
+__pthread_rwlock_tryrdlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+static __always_inline int
+__pthread_rwlock_trywrlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+
+static __always_inline void
+__pthread_rwlock_rdlock_tp(pthread_rwlock_t *rwlock,
+ const struct timespec *abstime)
+{
+}
+
+static __always_inline void
+__pthread_rwlock_wrlock_tp(pthread_rwlock_t *rwlock,
+ const struct timespec *abstime)
+{
+}
+
+static __always_inline void
+__pthread_rwlock_rdunlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+static __always_inline void
+__pthread_rwlock_wrunlock_tp(pthread_rwlock_t *rwlock)
+{
+}
+
+#endif /* __NR_futex */
@@ -21,6 +21,7 @@
#include <atomic.h>
#include <stdbool.h>
#include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c for an overview. */
@@ -39,6 +40,10 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
below. */
unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers);
unsigned int rnew;
+
+ if (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ return __pthread_rwlock_tryrdlock_tp(rwlock) ? 0 : EBUSY;
+
do
{
if ((r & PTHREAD_RWLOCK_WRPHASE) == 0)
@@ -19,6 +19,7 @@
#include <errno.h>
#include "pthreadP.h"
#include <atomic.h>
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c for an overview. */
int
@@ -37,6 +38,10 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers);
bool prefer_writer =
(rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP);
+
+ if (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ return __pthread_rwlock_trywrlock_tp(rwlock) ? 0 : EBUSY;
+
while (((r & PTHREAD_RWLOCK_WRLOCKED) == 0)
&& (((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0)
|| (prefer_writer && ((r & PTHREAD_RWLOCK_WRPHASE) != 0))))
@@ -24,6 +24,7 @@
#include <stap-probe.h>
#include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c for an overview. */
int
@@ -31,6 +32,19 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (rwlock_unlock, 1, rwlock);
+ if (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ {
+ unsigned int *futex = &rwlock->__data.__writers_futex;
+
+ /* For TP futex, the FUTEX_SHARED bit is used to distinguish between a
+ writer and reader-owned lock. */
+ if (atomic_load_relaxed (futex) & FUTEX_SHARED_FLAG)
+ __pthread_rwlock_rdunlock_tp (rwlock);
+ else
+ __pthread_rwlock_wrunlock_tp (rwlock);
+ return 0;
+ }
+
/* We distinguish between having acquired a read vs. a write lock by looking
at the writer TID. If it's equal to our TID, we must be the writer
because nobody else can have stored this value. Also, if we are a
@@ -17,6 +17,7 @@
<http://www.gnu.org/licenses/>. */
#include "pthread_rwlock_common.c"
+#include "pthread_rwlock_tp.c"
/* See pthread_rwlock_common.c. */
int
@@ -24,7 +25,9 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (wrlock_entry, 1, rwlock);
- int result = __pthread_rwlock_wrlock_full (rwlock, NULL);
+ int result = (rwlock->__data.__flags == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ ? __pthread_rwlock_wrlock_tp (rwlock, NULL)
+ : __pthread_rwlock_wrlock_full (rwlock, NULL);
LIBC_PROBE (wrlock_acquire_write, 1, rwlock);
return result;
}
@@ -17,6 +17,7 @@
<http://www.gnu.org/licenses/>. */
#include <errno.h>
+#include <assert.h>
#include "pthreadP.h"
@@ -25,7 +26,25 @@ pthread_rwlockattr_setkind_np (pthread_rwlockattr_t *attr, int pref)
{
struct pthread_rwlockattr *iattr;
- if (pref != PTHREAD_RWLOCK_PREFER_READER_NP
+ if (pref == PTHREAD_RWLOCK_USE_TP_FUTEX_NP)
+ {
+#ifdef __NR_futex
+ static int tp_futex_supported;
+ if (tp_futex_supported == 0)
+ {
+ int lock = 0;
+ INTERNAL_SYSCALL_DECL (err);
+ int ret = INTERNAL_SYSCALL (futex, err, 4, &lock, FUTEX_UNLOCK_SHARED, 0, 0);
+ assert (INTERNAL_SYSCALL_ERROR_P (ret, err));
+ tp_futex_supported = INTERNAL_SYSCALL_ERRNO (ret, err) == ENOSYS ? -1 : 1;
+ }
+ if (tp_futex_supported < 0)
+ return ENOTSUP;
+#else
+ return ENOTSUP;
+#endif
+ }
+ else if (pref != PTHREAD_RWLOCK_PREFER_READER_NP
&& pref != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
&& __builtin_expect (pref != PTHREAD_RWLOCK_PREFER_WRITER_NP, 0))
return EINVAL;
@@ -120,6 +120,7 @@ enum
PTHREAD_RWLOCK_PREFER_READER_NP,
PTHREAD_RWLOCK_PREFER_WRITER_NP,
PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP,
+ PTHREAD_RWLOCK_USE_TP_FUTEX_NP,
PTHREAD_RWLOCK_DEFAULT_NP = PTHREAD_RWLOCK_PREFER_READER_NP
};
@@ -40,11 +40,18 @@
#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_LOCK 13
#define FUTEX_UNLOCK 14
+#define FUTEX_LOCK_SHARED 15
+#define FUTEX_UNLOCK_SHARED 16
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
#define FUTEX_BITSET_MATCH_ANY 0xffffffff
+/* Bits used by TP futex */
+#define FUTEX_SHARED_FLAG 0x20000000
+#define FUTEX_SHARED_UNLOCK 0x10000000
+#define FUTEX_SCNT_MASK 0x00ffffff
+
/* Values for 'private' parameter of locking macros. Yes, the
definition seems to be backwards. But it is not. The bit will be
reversed before passing to the system call. */
This patch adds a new lock kind (PTHREAD_RWLOCK_USE_TP_FUTEX_NP) to the pthread_rwlockattr_setkind_np() to enable the pthread rwlock to use the TP futex, if available, for supporting rwlock. Signed-off-by: Waiman Long <longman@redhat.com> --- ChangeLog | 26 +++ nptl/pthread_rwlock_rdlock.c | 5 +- nptl/pthread_rwlock_timedrdlock.c | 5 +- nptl/pthread_rwlock_timedwrlock.c | 5 +- nptl/pthread_rwlock_tp.c | 235 +++++++++++++++++++++++++++ nptl/pthread_rwlock_tryrdlock.c | 5 + nptl/pthread_rwlock_trywrlock.c | 5 + nptl/pthread_rwlock_unlock.c | 14 ++ nptl/pthread_rwlock_wrlock.c | 5 +- nptl/pthread_rwlockattr_setkind_np.c | 21 ++- sysdeps/nptl/pthread.h | 1 + sysdeps/unix/sysv/linux/lowlevellock-futex.h | 7 + 12 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 nptl/pthread_rwlock_tp.c