===================================================================
@@ -0,0 +1,35 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+ Contributed by Alan Modra, IBM.
+
+ This file is part of the GNU OpenMP Library (libgomp).
+
+ Libgomp is free software; you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3, or (at your option)
+ any later version.
+
+ Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ Under Section 7 of GPL version 3, you are granted additional
+ permissions described in the GCC Runtime Library Exception, version
+ 3.1, as published by the Free Software Foundation.
+
+ You should have received a copy of the GNU General Public License and
+ a copy of the GCC Runtime Library Exception along with this program;
+ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* This type is private to the library. */
+
+#ifndef MUTEX_LOCK_H
+#define MUTEX_LOCK_H 1
+
+typedef struct linux_gomp_mutex
+{
+ /* 0 unlocked, 1 locked, -1 locked and possibly someone waiting. */
+ int flag;
+} gomp_mutex_t;
+#endif
===================================================================
@@ -3,10 +3,12 @@
structures without polluting the namespace.
When using the Linux futex primitive, non-recursive locks require
- only one int. Recursive locks require we identify the owning task
- and so require one int and a pointer. */
+ one int. Recursive locks require we identify the owning task
+ and so require in addition one int and a pointer. */
-typedef int omp_lock_t;
-typedef struct { int lock, count; void *owner; } omp_nest_lock_t;
-typedef int omp_lock_25_t;
+#include "mutex-lock.h"
+
+typedef gomp_mutex_t omp_lock_t;
+typedef struct { omp_lock_t lock; int count; void *owner; } omp_nest_lock_t;
+typedef gomp_mutex_t omp_lock_25_t;
typedef struct { int owner, count; } omp_nest_lock_25_t;
===================================================================
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp).
@@ -29,43 +29,39 @@
#ifndef GOMP_MUTEX_H
#define GOMP_MUTEX_H 1
-typedef int gomp_mutex_t;
+#include "mutex-lock.h"
#define GOMP_MUTEX_INIT_0 1
-static inline void gomp_mutex_init (gomp_mutex_t *mutex)
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
+extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
+
+static inline void
+gomp_mutex_init (gomp_mutex_t *mutex)
{
- *mutex = 0;
+ mutex->flag = 0;
}
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
-static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
+static inline void
+gomp_mutex_destroy (gomp_mutex_t *mutex)
{
- int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
- if (__builtin_expect (oldval, 0))
- gomp_mutex_lock_slow (mutex, oldval);
}
-extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
-static inline void gomp_mutex_unlock (gomp_mutex_t *mutex)
+static inline void
+gomp_mutex_lock (gomp_mutex_t *mutex)
{
- /* Warning: By definition __sync_lock_test_and_set() does not have
- proper memory barrier semantics for a mutex unlock operation.
- However, this default implementation is written assuming that it
- does, which is true for some targets.
-
- Targets that require additional memory barriers before
- __sync_lock_test_and_set to achieve the release semantics of
- mutex unlock, are encouraged to include
- "config/linux/ia64/mutex.h" in a target specific mutex.h instead
- of using this file. */
- int val = __sync_lock_test_and_set (mutex, 0);
- if (__builtin_expect (val > 1, 0))
- gomp_mutex_unlock_slow (mutex);
+ int oldval = 0;
+ __atomic_compare_exchange_4 (&mutex->flag, &oldval, 1, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+ if (__builtin_expect (oldval != 0, 0))
+ gomp_mutex_lock_slow (mutex, oldval);
}
-static inline void gomp_mutex_destroy (gomp_mutex_t *mutex)
+static inline void
+gomp_mutex_unlock (gomp_mutex_t *mutex)
{
+ int wait = __atomic_exchange_4 (&mutex->flag, 0, MEMMODEL_RELEASE);
+ if (__builtin_expect (wait < 0, 0))
+ gomp_mutex_unlock_slow (mutex);
}
-
#endif /* GOMP_MUTEX_H */
===================================================================
@@ -34,29 +34,37 @@ long int gomp_futex_wait = FUTEX_WAIT |
void
gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval)
{
+ /* First loop spins a while. */
while (oldval == 1)
{
- if (do_spin (mutex, 1))
+ if (do_spin (&mutex->flag, 1))
{
- oldval = __sync_lock_test_and_set (mutex, 2);
+ /* Spin timeout, nothing changed. Set waiting flag. */
+ oldval = __atomic_exchange_4 (&mutex->flag, -1, MEMMODEL_ACQUIRE);
if (oldval == 0)
return;
- futex_wait (mutex, 2);
+ futex_wait (&mutex->flag, -1);
break;
}
else
{
- oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+ /* Something changed. If now unlocked, we're good to go. */
+ oldval = 0;
+ __atomic_compare_exchange_4 (&mutex->flag, &oldval, 1, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
if (oldval == 0)
return;
}
}
- while ((oldval = __sync_lock_test_and_set (mutex, 2)))
- do_wait (mutex, 2);
+
+ /* Second loop waits until mutex is unlocked. We always exit this
+ loop with wait flag set, so next unlock will awaken a thread. */
+ while ((oldval = __atomic_exchange_4 (&mutex->flag, -1, MEMMODEL_ACQUIRE)))
+ do_wait (&mutex->flag, -1);
}
void
gomp_mutex_unlock_slow (gomp_mutex_t *mutex)
{
- futex_wake (mutex, 1);
+ futex_wake (&mutex->flag, 1);
}
===================================================================
@@ -1,5 +1,5 @@
-/* Copyright (C) 2005, 2009 Free Software Foundation, Inc.
- Contributed by Richard Henderson <rth@redhat.com>.
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+ Contributed by Alan Modra, IBM.
This file is part of the GNU OpenMP Library (libgomp).
@@ -24,34 +24,78 @@
/* This is a Linux specific implementation of a semaphore synchronization
mechanism for libgomp. This type is private to the library. This
- implementation uses atomic instructions and the futex syscall. */
+ counting semaphore implementation uses atomic instructions and the
+ futex syscall. */
#ifndef GOMP_SEM_H
#define GOMP_SEM_H 1
-typedef int gomp_sem_t;
+typedef struct linux_gomp_sem
+{
+ /* COUNT & 0x7fffffff threads can call gomp_sem_wait without
+ blocking. If COUNT & 0x80000000, some threads may be waiting. */
+ int count;
+} gomp_sem_t;
-static inline void gomp_sem_init (gomp_sem_t *sem, int value)
+enum memmodel
{
- *sem = value;
-}
+ MEMMODEL_RELAXED = 0,
+ MEMMODEL_CONSUME = 1,
+ MEMMODEL_ACQUIRE = 2,
+ MEMMODEL_RELEASE = 3,
+ MEMMODEL_ACQ_REL = 4,
+ MEMMODEL_SEQ_CST = 5,
+ MEMMODEL_LAST = 6
+};
+
+extern void gomp_sem_wait_slow (gomp_sem_t *, int);
+extern void gomp_sem_post_slow (gomp_sem_t *);
-extern void gomp_sem_wait_slow (gomp_sem_t *);
-static inline void gomp_sem_wait (gomp_sem_t *sem)
+static inline void
+gomp_sem_init (gomp_sem_t *sem, int value)
{
- if (!__sync_bool_compare_and_swap (sem, 1, 0))
- gomp_sem_wait_slow (sem);
+ sem->count = value;
}
-extern void gomp_sem_post_slow (gomp_sem_t *);
-static inline void gomp_sem_post (gomp_sem_t *sem)
+static inline void
+gomp_sem_destroy (gomp_sem_t *sem)
{
- if (!__sync_bool_compare_and_swap (sem, 0, 1))
- gomp_sem_post_slow (sem);
}
-static inline void gomp_sem_destroy (gomp_sem_t *sem)
+static inline void
+gomp_sem_wait (gomp_sem_t *sem)
{
+ int count = sem->count;
+
+ while ((count & 0x7fffffff) != 0)
+ {
+ int oldval = count;
+ __atomic_compare_exchange_4 (&sem->count, &oldval, count - 1,
+ false, MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+ if (__builtin_expect (oldval == count, 1))
+ return;
+ count = oldval;
+ }
+ gomp_sem_wait_slow (sem, count);
}
+static inline void
+gomp_sem_post (gomp_sem_t *sem)
+{
+ int count = sem->count;
+
+ while (1)
+ {
+ int oldval = count;
+ __atomic_compare_exchange_4 (&sem->count, &oldval,
+ (count + 1) & 0x7fffffff,
+ false, MEMMODEL_RELEASE, MEMMODEL_RELAXED);
+ if (__builtin_expect (oldval == count, 1))
+ break;
+ count = oldval;
+ }
+
+ if (__builtin_expect (count & 0x80000000, 0))
+ gomp_sem_post_slow (sem);
+}
#endif /* GOMP_SEM_H */
===================================================================
@@ -28,34 +28,69 @@
#include "wait.h"
-
void
-gomp_sem_wait_slow (gomp_sem_t *sem)
+gomp_sem_wait_slow (gomp_sem_t *sem, int count)
{
- while (1)
+ int oldval, newval;
+
+ /* First loop spins a while. */
+ while (count == 0)
{
- int val = __sync_val_compare_and_swap (sem, 0, -1);
- if (val > 0)
+ if (do_spin (&sem->count, 0))
+ {
+ /* Spin timeout, nothing changed. Set waiting flag. */
+ oldval = 0;
+ __atomic_compare_exchange_4 (&sem->count, &oldval, 0x80000000, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+ count = oldval;
+ if (oldval == 0)
+ {
+ futex_wait (&sem->count, 0x80000000);
+ count = sem->count;
+ }
+ break;
+ }
+ /* Something changed. If positive, we're good to go. */
+ else if ((count = sem->count) > 0)
{
- if (__sync_bool_compare_and_swap (sem, val, val - 1))
+ oldval = count;
+ __atomic_compare_exchange_4 (&sem->count, &oldval, count - 1, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+ if (oldval == count)
return;
+ count = oldval;
}
- do_wait (sem, -1);
+ }
+
+ /* Second loop waits until semaphore is posted. We always exit this
+ loop with wait flag set, so next post will awaken a thread. */
+ while (1)
+ {
+ oldval = count;
+ newval = 0x80000000;
+ if ((count & 0x7fffffff) != 0)
+ newval |= count - 1;
+ __atomic_compare_exchange_4 (&sem->count, &oldval, newval, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+ if (oldval == count)
+ {
+ if ((count & 0x7fffffff) != 0)
+ {
+ /* If we can wake more threads, do so now. */
+ if ((count & 0x7fffffff) > 1)
+ gomp_sem_post_slow (sem);
+ break;
+ }
+ do_wait (&sem->count, 0x80000000);
+ count = sem->count;
+ }
+ else
+ count = oldval;
}
}
void
gomp_sem_post_slow (gomp_sem_t *sem)
{
- int old, tmp = *sem, wake;
-
- do
- {
- old = tmp;
- wake = old > 0 ? old + 1 : 1;
- tmp = __sync_val_compare_and_swap (sem, old, wake);
- }
- while (old != tmp);
-
- futex_wake (sem, wake);
+ futex_wake (&sem->count, 1);
}
===================================================================
@@ -62,7 +62,10 @@ gomp_unset_lock_30 (omp_lock_t *lock)
int
gomp_test_lock_30 (omp_lock_t *lock)
{
- return __sync_bool_compare_and_swap (lock, 0, 1);
+ int oldval = 0;
+
+ return __atomic_compare_exchange_4 (lock, &oldval, 1, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
}
void
@@ -104,11 +107,14 @@ int
gomp_test_nest_lock_30 (omp_nest_lock_t *lock)
{
void *me = gomp_icv (true);
+ int oldval;
if (lock->owner == me)
return ++lock->count;
- if (__sync_bool_compare_and_swap (&lock->lock, 0, 1))
+ oldval = 0;
+ if (__atomic_compare_exchange_4 (&lock->lock, &oldval, 1, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED))
{
lock->owner = me;
lock->count = 1;
@@ -184,7 +190,9 @@ gomp_set_nest_lock_25 (omp_nest_lock_25_
while (1)
{
- otid = __sync_val_compare_and_swap (&lock->owner, 0, tid);
+ otid = 0;
+ __atomic_compare_exchange_4 (&lock->owner, &otid, tid, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
if (otid == 0)
{
lock->count = 1;
@@ -207,7 +215,7 @@ gomp_unset_nest_lock_25 (omp_nest_lock_2
if (--lock->count == 0)
{
- __sync_lock_release (&lock->owner);
+ __atomic_store_4 (&lock->owner, 0, MEMMODEL_RELEASE);
futex_wake (&lock->owner, 1);
}
}
@@ -217,7 +225,9 @@ gomp_test_nest_lock_25 (omp_nest_lock_25
{
int otid, tid = gomp_tid ();
- otid = __sync_val_compare_and_swap (&lock->owner, 0, tid);
+ otid = 0;
+ __atomic_compare_exchange_4 (&lock->owner, &otid, tid, false,
+ MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
if (otid == 0)
{
lock->count = 1;
===================================================================
@@ -1,28 +0,0 @@
-/* Copyright (C) 2010 Free Software Foundation, Inc.
- Contributed by ARM Ltd.
-
- This file is part of the GNU OpenMP Library (libgomp).
-
- Libgomp is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3, or (at your option)
- any later version.
-
- Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- Under Section 7 of GPL version 3, you are granted additional
- permissions described in the GCC Runtime Library Exception, version
- 3.1, as published by the Free Software Foundation.
-
- You should have received a copy of the GNU General Public License and
- a copy of the GCC Runtime Library Exception along with this program;
- see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
- <http://www.gnu.org/licenses/>. */
-
-/* ARM needs the same correct usage of __sync_synchronize and
- __sync_lock_test_and_set as ia64. So we just use its mutex.h. */
-
-#include "config/linux/ia64/mutex.h"
===================================================================
@@ -1,2 +0,0 @@
-/* On PowerPC __sync_lock_test_and_set isn't a full barrier. */
-#include "config/linux/ia64/mutex.h"
===================================================================
@@ -1,66 +0,0 @@
-/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
- Contributed by Richard Henderson <rth@redhat.com>.
-
- This file is part of the GNU OpenMP Library (libgomp).
-
- Libgomp is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3, or (at your option)
- any later version.
-
- Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- Under Section 7 of GPL version 3, you are granted additional
- permissions described in the GCC Runtime Library Exception, version
- 3.1, as published by the Free Software Foundation.
-
- You should have received a copy of the GNU General Public License and
- a copy of the GCC Runtime Library Exception along with this program;
- see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
- <http://www.gnu.org/licenses/>. */
-
-/* This is a Linux specific implementation of a mutex synchronization
- mechanism for libgomp. This type is private to the library. This
- implementation uses atomic instructions and the futex syscall. */
-
-#ifndef GOMP_MUTEX_H
-#define GOMP_MUTEX_H 1
-
-typedef int gomp_mutex_t;
-
-#define GOMP_MUTEX_INIT_0 1
-
-static inline void gomp_mutex_init (gomp_mutex_t *mutex)
-{
- *mutex = 0;
-}
-
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
-static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
-{
- int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
- if (__builtin_expect (oldval, 0))
- gomp_mutex_lock_slow (mutex, oldval);
-}
-
-extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
-
-/* IA64 needs a __sync_synchronize call before __sync_lock_test_and_set
- because __sync_lock_test_and_set is not a full memory fence. */
-static inline void gomp_mutex_unlock (gomp_mutex_t *mutex)
-{
- int val;
- __sync_synchronize ();
- val = __sync_lock_test_and_set (mutex, 0);
- if (__builtin_expect (val > 1, 0))
- gomp_mutex_unlock_slow (mutex);
-}
-
-static inline void gomp_mutex_destroy (gomp_mutex_t *mutex)
-{
-}
-
-#endif /* GOMP_MUTEX_H */
===================================================================
@@ -1,27 +0,0 @@
-/* Copyright (C) 2009 Free Software Foundation, Inc.
-
- This file is part of the GNU OpenMP Library (libgomp).
-
- Libgomp is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3, or (at your option)
- any later version.
-
- Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- Under Section 7 of GPL version 3, you are granted additional
- permissions described in the GCC Runtime Library Exception, version
- 3.1, as published by the Free Software Foundation.
-
- You should have received a copy of the GNU General Public License and
- a copy of the GCC Runtime Library Exception along with this program;
- see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
- <http://www.gnu.org/licenses/>. */
-
-/* MIPS needs the same correct usage of __sync_synchronize and
- __sync_lock_test_and_set as ia64. So we just use its mutex.h. */
-
-#include "config/linux/ia64/mutex.h"