Index: libgomp/config/linux/mutex-lock.h
===================================================================
--- libgomp/config/linux/mutex-lock.h	(revision 0)
+++ libgomp/config/linux/mutex-lock.h	(revision 0)
@@ -0,0 +1,35 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   Contributed by Alan Modra, IBM.
+
+   This file is part of the GNU OpenMP Library (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This type is private to the library.  */
+
+#ifndef MUTEX_LOCK_H
+#define MUTEX_LOCK_H 1
+
+typedef struct linux_gomp_mutex
+{
+  /* 0 unlocked, 1 locked, -1 locked and possibly someone waiting.  */
+  int flag;
+} gomp_mutex_t;
+#endif
Index: libgomp/config/linux/omp-lock.h
===================================================================
--- libgomp/config/linux/omp-lock.h	(revision 181683)
+++ libgomp/config/linux/omp-lock.h	(working copy)
@@ -3,10 +3,12 @@
    structures without polluting the namespace.
 
    When using the Linux futex primitive, non-recursive locks require
-   only one int.  Recursive locks require we identify the owning task
-   and so require one int and a pointer.  */
+   one int.  Recursive locks require we identify the owning task
+   and so require in addition one int and a pointer.  */
 
-typedef int omp_lock_t;
-typedef struct { int lock, count; void *owner; } omp_nest_lock_t;
-typedef int omp_lock_25_t;
+#include "mutex-lock.h"
+
+typedef gomp_mutex_t omp_lock_t;
+typedef struct { omp_lock_t lock; int count; void *owner; } omp_nest_lock_t;
+typedef gomp_mutex_t omp_lock_25_t;
 typedef struct { int owner, count; } omp_nest_lock_25_t;
Index: libgomp/config/linux/mutex.h
===================================================================
--- libgomp/config/linux/mutex.h	(revision 181683)
+++ libgomp/config/linux/mutex.h	(working copy)
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -29,43 +29,39 @@
 #ifndef GOMP_MUTEX_H
 #define GOMP_MUTEX_H 1
 
-typedef int gomp_mutex_t;
+#include "mutex-lock.h"
 
 #define GOMP_MUTEX_INIT_0 1
 
-static inline void gomp_mutex_init (gomp_mutex_t *mutex)
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
+extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
+
+static inline void
+gomp_mutex_init (gomp_mutex_t *mutex)
 {
-  *mutex = 0;
+  mutex->flag = 0;
 }
 
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
-static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
+static inline void
+gomp_mutex_destroy (gomp_mutex_t *mutex)
 {
-  int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
-  if (__builtin_expect (oldval, 0))
-    gomp_mutex_lock_slow (mutex, oldval);
 }
 
-extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
-static inline void gomp_mutex_unlock (gomp_mutex_t *mutex)
+static inline void
+gomp_mutex_lock (gomp_mutex_t *mutex)
 {
-  /* Warning: By definition __sync_lock_test_and_set() does not have
-     proper memory barrier semantics for a mutex unlock operation.
-     However, this default implementation is written assuming that it
-     does, which is true for some targets.
-
-     Targets that require additional memory barriers before
-     __sync_lock_test_and_set to achieve the release semantics of
-     mutex unlock, are encouraged to include
-     "config/linux/ia64/mutex.h" in a target specific mutex.h instead
-     of using this file.  */
-  int val = __sync_lock_test_and_set (mutex, 0);
-  if (__builtin_expect (val > 1, 0))
-    gomp_mutex_unlock_slow (mutex);
+  int oldval = 0;
+  __atomic_compare_exchange_4 (&mutex->flag, &oldval, 1, false,
+			       MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+  if (__builtin_expect (oldval != 0, 0))
+    gomp_mutex_lock_slow (mutex, oldval);
 }
 
-static inline void gomp_mutex_destroy (gomp_mutex_t *mutex)
+static inline void
+gomp_mutex_unlock (gomp_mutex_t *mutex)
 {
+  int wait = __atomic_exchange_4 (&mutex->flag, 0, MEMMODEL_RELEASE);
+  if (__builtin_expect (wait < 0, 0))
+    gomp_mutex_unlock_slow (mutex);
 }
-
 #endif /* GOMP_MUTEX_H */
Index: libgomp/config/linux/mutex.c
===================================================================
--- libgomp/config/linux/mutex.c	(revision 181683)
+++ libgomp/config/linux/mutex.c	(working copy)
@@ -34,29 +34,37 @@ long int gomp_futex_wait = FUTEX_WAIT | 
 void
 gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval)
 {
+  /* First loop spins a while.  */
   while (oldval == 1)
     {
-      if (do_spin (mutex, 1))
+      if (do_spin (&mutex->flag, 1))
 	{
-	  oldval = __sync_lock_test_and_set (mutex, 2);
+	  /* Spin timeout, nothing changed.  Set waiting flag.  */
+	  oldval = __atomic_exchange_4 (&mutex->flag, -1, MEMMODEL_ACQUIRE);
 	  if (oldval == 0)
 	    return;
-	  futex_wait (mutex, 2);
+	  futex_wait (&mutex->flag, -1);
 	  break;
 	}
       else
 	{
-	  oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+	  /* Something changed.  If now unlocked, we're good to go.  */
+	  oldval = 0;
+	  __atomic_compare_exchange_4 (&mutex->flag, &oldval, 1, false,
+				       MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
 	  if (oldval == 0)
 	    return;
 	}
     }
-  while ((oldval = __sync_lock_test_and_set (mutex, 2)))
-    do_wait (mutex, 2);
+
+  /* Second loop waits until mutex is unlocked.  We always exit this
+     loop with wait flag set, so next unlock will awaken a thread.  */
+  while ((oldval = __atomic_exchange_4 (&mutex->flag, -1, MEMMODEL_ACQUIRE)))
+    do_wait (&mutex->flag, -1);
 }
 
 void
 gomp_mutex_unlock_slow (gomp_mutex_t *mutex)
 {
-  futex_wake (mutex, 1);
+  futex_wake (&mutex->flag, 1);
 }
Index: libgomp/config/linux/sem.h
===================================================================
--- libgomp/config/linux/sem.h	(revision 181683)
+++ libgomp/config/linux/sem.h	(working copy)
@@ -1,5 +1,5 @@
-/* Copyright (C) 2005, 2009 Free Software Foundation, Inc.
-   Contributed by Richard Henderson <rth@redhat.com>.
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   Contributed by Alan Modra, IBM.
 
    This file is part of the GNU OpenMP Library (libgomp).
 
@@ -24,34 +24,78 @@
 
 /* This is a Linux specific implementation of a semaphore synchronization
    mechanism for libgomp.  This type is private to the library.  This 
-   implementation uses atomic instructions and the futex syscall.  */
+   counting semaphore implementation uses atomic instructions and the
+   futex syscall.  */ 
 
 #ifndef GOMP_SEM_H
 #define GOMP_SEM_H 1
 
-typedef int gomp_sem_t;
+typedef struct linux_gomp_sem
+{
+  /* COUNT & 0x7fffffff threads can call gomp_sem_wait without
+     blocking.  If COUNT & 0x80000000, some threads may be waiting.  */
+  int count;
+} gomp_sem_t;
 
-static inline void gomp_sem_init (gomp_sem_t *sem, int value)
+enum memmodel
 {
-  *sem = value;
-}
+  MEMMODEL_RELAXED = 0,
+  MEMMODEL_CONSUME = 1,
+  MEMMODEL_ACQUIRE = 2,
+  MEMMODEL_RELEASE = 3,
+  MEMMODEL_ACQ_REL = 4,
+  MEMMODEL_SEQ_CST = 5,
+  MEMMODEL_LAST = 6
+};
+
+extern void gomp_sem_wait_slow (gomp_sem_t *, int);
+extern void gomp_sem_post_slow (gomp_sem_t *);
 
-extern void gomp_sem_wait_slow (gomp_sem_t *);
-static inline void gomp_sem_wait (gomp_sem_t *sem)
+static inline void
+gomp_sem_init (gomp_sem_t *sem, int value)
 {
-  if (!__sync_bool_compare_and_swap (sem, 1, 0))
-    gomp_sem_wait_slow (sem);
+  sem->count = value;
 }
 
-extern void gomp_sem_post_slow (gomp_sem_t *);
-static inline void gomp_sem_post (gomp_sem_t *sem)
+static inline void
+gomp_sem_destroy (gomp_sem_t *sem)
 {
-  if (!__sync_bool_compare_and_swap (sem, 0, 1))
-    gomp_sem_post_slow (sem);
 }
 
-static inline void gomp_sem_destroy (gomp_sem_t *sem)
+static inline void
+gomp_sem_wait (gomp_sem_t *sem)
 {
+  int count = sem->count;
+
+  while ((count & 0x7fffffff) != 0)
+    {
+      int oldval = count;
+      __atomic_compare_exchange_4 (&sem->count, &oldval, count - 1,
+				   false, MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+      if (__builtin_expect (oldval == count, 1))
+	return;
+      count = oldval;
+    }
+  gomp_sem_wait_slow (sem, count);
 }
 
+static inline void
+gomp_sem_post (gomp_sem_t *sem)
+{
+  int count = sem->count;
+
+  while (1)
+    {
+      int oldval = count;
+      __atomic_compare_exchange_4 (&sem->count, &oldval,
+				   (count + 1) & 0x7fffffff,
+				   false, MEMMODEL_RELEASE, MEMMODEL_RELAXED);
+      if (__builtin_expect (oldval == count, 1))
+	break;
+      count = oldval;
+    }
+
+  if (__builtin_expect (count & 0x80000000, 0))
+    gomp_sem_post_slow (sem);
+}
 #endif /* GOMP_SEM_H */
Index: libgomp/config/linux/sem.c
===================================================================
--- libgomp/config/linux/sem.c	(revision 181683)
+++ libgomp/config/linux/sem.c	(working copy)
@@ -28,34 +28,69 @@
 
 #include "wait.h"
 
-
 void
-gomp_sem_wait_slow (gomp_sem_t *sem)
+gomp_sem_wait_slow (gomp_sem_t *sem, int count)
 {
-  while (1)
+  int oldval, newval;
+
+  /* First loop spins a while.  */
+  while (count == 0)
     {
-      int val = __sync_val_compare_and_swap (sem, 0, -1);
-      if (val > 0)
+      if (do_spin (&sem->count, 0))
+	{
+	  /* Spin timeout, nothing changed.  Set waiting flag.  */
+	  oldval = 0;
+	  __atomic_compare_exchange_4 (&sem->count, &oldval, 0x80000000, false,
+				       MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+	  count = oldval;
+	  if (oldval == 0)
+	    {
+	      futex_wait (&sem->count, 0x80000000);
+	      count = sem->count;
+	    }
+	  break;
+	}
+      /* Something changed.  If positive, we're good to go.  */
+      else if ((count = sem->count) > 0)
 	{
-	  if (__sync_bool_compare_and_swap (sem, val, val - 1))
+	  oldval = count;
+	  __atomic_compare_exchange_4 (&sem->count, &oldval, count - 1, false,
+				       MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+	  if (oldval == count)
 	    return;
+	  count = oldval;
 	}
-      do_wait (sem, -1);
+    }
+
+  /* Second loop waits until semaphore is posted.  We always exit this
+     loop with wait flag set, so next post will awaken a thread.  */
+  while (1)
+    {
+      oldval = count;
+      newval = 0x80000000;
+      if ((count & 0x7fffffff) != 0)
+	newval |= count - 1;
+      __atomic_compare_exchange_4 (&sem->count, &oldval, newval, false,
+				   MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
+      if (oldval == count)
+	{
+	  if ((count & 0x7fffffff) != 0)
+	    {
+	      /* If we can wake more threads, do so now.  */
+	      if ((count & 0x7fffffff) > 1)
+		gomp_sem_post_slow (sem);
+	      break;
+	    }
+	  do_wait (&sem->count, 0x80000000);
+	  count = sem->count;
+	}
+      else
+	count = oldval;
     }
 }
 
 void
 gomp_sem_post_slow (gomp_sem_t *sem)
 {
-  int old, tmp = *sem, wake;
-
-  do
-    {
-      old = tmp;
-      wake = old > 0 ? old + 1 : 1;
-      tmp = __sync_val_compare_and_swap (sem, old, wake);
-    }
-  while (old != tmp);
-
-  futex_wake (sem, wake);
+  futex_wake (&sem->count, 1);
 }
Index: libgomp/config/linux/lock.c
===================================================================
--- libgomp/config/linux/lock.c	(revision 181683)
+++ libgomp/config/linux/lock.c	(working copy)
@@ -62,7 +62,10 @@ gomp_unset_lock_30 (omp_lock_t *lock)
 int
 gomp_test_lock_30 (omp_lock_t *lock)
 {
-  return __sync_bool_compare_and_swap (lock, 0, 1);
+  int oldval = 0;
+
+  return __atomic_compare_exchange_4 (lock, &oldval, 1, false,
+				      MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
 }
 
 void
@@ -104,11 +107,14 @@ int
 gomp_test_nest_lock_30 (omp_nest_lock_t *lock)
 {
   void *me = gomp_icv (true);
+  int oldval;
 
   if (lock->owner == me)
     return ++lock->count;
 
-  if (__sync_bool_compare_and_swap (&lock->lock, 0, 1))
+  oldval = 0;
+  if (__atomic_compare_exchange_4 (&lock->lock, &oldval, 1, false,
+				   MEMMODEL_ACQUIRE, MEMMODEL_RELAXED))
     {
       lock->owner = me;
       lock->count = 1;
@@ -184,7 +190,9 @@ gomp_set_nest_lock_25 (omp_nest_lock_25_
 
   while (1)
     {
-      otid = __sync_val_compare_and_swap (&lock->owner, 0, tid);
+      otid = 0;
+      __atomic_compare_exchange_4 (&lock->owner, &otid, tid, false,
+				   MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
       if (otid == 0)
 	{
 	  lock->count = 1;
@@ -207,7 +215,7 @@ gomp_unset_nest_lock_25 (omp_nest_lock_2
 
   if (--lock->count == 0)
     {
-      __sync_lock_release (&lock->owner);
+      __atomic_store_4 (&lock->owner, 0, MEMMODEL_RELEASE);
       futex_wake (&lock->owner, 1);
     }
 }
@@ -217,7 +225,9 @@ gomp_test_nest_lock_25 (omp_nest_lock_25
 {
   int otid, tid = gomp_tid ();
 
-  otid = __sync_val_compare_and_swap (&lock->owner, 0, tid);
+  otid = 0;
+  __atomic_compare_exchange_4 (&lock->owner, &otid, tid, false,
+			       MEMMODEL_ACQUIRE, MEMMODEL_RELAXED);
   if (otid == 0)
     {
       lock->count = 1;
Index: libgomp/config/linux/arm/mutex.h
===================================================================
--- libgomp/config/linux/arm/mutex.h	(revision 181683)
+++ libgomp/config/linux/arm/mutex.h	(working copy)
@@ -1,28 +0,0 @@
-/* Copyright (C) 2010 Free Software Foundation, Inc.
-   Contributed by ARM Ltd.
-
-   This file is part of the GNU OpenMP Library (libgomp).
-
-   Libgomp is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3, or (at your option)
-   any later version.
-
-   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
-   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-   more details.
-
-   Under Section 7 of GPL version 3, you are granted additional
-   permissions described in the GCC Runtime Library Exception, version
-   3.1, as published by the Free Software Foundation.
-
-   You should have received a copy of the GNU General Public License and
-   a copy of the GCC Runtime Library Exception along with this program;
-   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
-   <http://www.gnu.org/licenses/>.  */
-
-/* ARM needs the same correct usage of __sync_synchronize and
-   __sync_lock_test_and_set as ia64.  So we just use its mutex.h.  */
-
-#include "config/linux/ia64/mutex.h"
Index: libgomp/config/linux/powerpc/mutex.h
===================================================================
--- libgomp/config/linux/powerpc/mutex.h	(revision 181683)
+++ libgomp/config/linux/powerpc/mutex.h	(working copy)
@@ -1,2 +0,0 @@
-/* On PowerPC __sync_lock_test_and_set isn't a full barrier.  */
-#include "config/linux/ia64/mutex.h"
Index: libgomp/config/linux/ia64/mutex.h
===================================================================
--- libgomp/config/linux/ia64/mutex.h	(revision 181683)
+++ libgomp/config/linux/ia64/mutex.h	(working copy)
@@ -1,66 +0,0 @@
-/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
-   Contributed by Richard Henderson <rth@redhat.com>.
-
-   This file is part of the GNU OpenMP Library (libgomp).
-
-   Libgomp is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3, or (at your option)
-   any later version.
-
-   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
-   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-   more details.
-
-   Under Section 7 of GPL version 3, you are granted additional
-   permissions described in the GCC Runtime Library Exception, version
-   3.1, as published by the Free Software Foundation.
-
-   You should have received a copy of the GNU General Public License and
-   a copy of the GCC Runtime Library Exception along with this program;
-   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
-   <http://www.gnu.org/licenses/>.  */
-
-/* This is a Linux specific implementation of a mutex synchronization
-   mechanism for libgomp.  This type is private to the library.  This
-   implementation uses atomic instructions and the futex syscall.  */
-
-#ifndef GOMP_MUTEX_H
-#define GOMP_MUTEX_H 1
-
-typedef int gomp_mutex_t;
-
-#define GOMP_MUTEX_INIT_0 1
-
-static inline void gomp_mutex_init (gomp_mutex_t *mutex)
-{
-  *mutex = 0;
-}
-
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
-static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
-{
-  int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
-  if (__builtin_expect (oldval, 0))
-    gomp_mutex_lock_slow (mutex, oldval);
-}
-
-extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
-
-/* IA64 needs a __sync_synchronize call before __sync_lock_test_and_set
-   because __sync_lock_test_and_set is not a full memory fence.  */
-static inline void gomp_mutex_unlock (gomp_mutex_t *mutex)
-{
-  int val;
-  __sync_synchronize ();
-  val = __sync_lock_test_and_set (mutex, 0);
-  if (__builtin_expect (val > 1, 0))
-    gomp_mutex_unlock_slow (mutex);
-}
-
-static inline void gomp_mutex_destroy (gomp_mutex_t *mutex)
-{
-}
-
-#endif /* GOMP_MUTEX_H */
Index: libgomp/config/linux/mips/mutex.h
===================================================================
--- libgomp/config/linux/mips/mutex.h	(revision 181683)
+++ libgomp/config/linux/mips/mutex.h	(working copy)
@@ -1,27 +0,0 @@
-/* Copyright (C) 2009 Free Software Foundation, Inc.
-
-   This file is part of the GNU OpenMP Library (libgomp).
-
-   Libgomp is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3, or (at your option)
-   any later version.
-
-   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
-   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-   more details.
-
-   Under Section 7 of GPL version 3, you are granted additional
-   permissions described in the GCC Runtime Library Exception, version
-   3.1, as published by the Free Software Foundation.
-
-   You should have received a copy of the GNU General Public License and
-   a copy of the GCC Runtime Library Exception along with this program;
-   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
-   <http://www.gnu.org/licenses/>.  */
-
-/* MIPS needs the same correct usage of __sync_synchronize and
-   __sync_lock_test_and_set as ia64.  So we just use its mutex.h.  */
-
-#include "config/linux/ia64/mutex.h"
