* config/linux/wait.h (do_spin): New inline, largely copied

	from do_wait, just don't do futex_wait here, instead return true if
	it should be done.
	(do_wait): Implement using do_spin.
	* config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
	to prototype.
	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
	__sync_bool_compare_and_swap, pass the oldval to
	gomp_mutex_lock_slow.
	* config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
	If all mutex contenders are just spinning and not sleeping, don't
	change state to 2 unnecessarily.  Optimize the loop when state has
	already become 2 to use just one atomic operation per loop instead
	of two.
	* config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
	to prototype.
	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
	__sync_bool_compare_and_swap, pass the oldval to
	gomp_mutex_lock_slow.

5 files changed, 60 insertions, 18 deletions

diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index 1d93273b919..36c716c518e 100644
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,3 +1,25 @@
+2011-07-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* config/linux/wait.h (do_spin): New inline, largely copied
+	from do_wait, just don't do futex_wait here, instead return true if
+	it should be done.
+	(do_wait): Implement using do_spin.
+	* config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
+	to prototype.
+	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
+	__sync_bool_compare_and_swap, pass the oldval to
+	gomp_mutex_lock_slow.
+	* config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
+	If all mutex contenders are just spinning and not sleeping, don't
+	change state to 2 unnecessarily.  Optimize the loop when state has
+	already become 2 to use just one atomic operation per loop instead
+	of two.
+	* config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
+	to prototype.
+	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
+	__sync_bool_compare_and_swap, pass the oldval to
+	gomp_mutex_lock_slow.
+
 2011-06-22  Jakub Jelinek  <jakub@redhat.com>
 
 	PR libgomp/49490
diff --git a/libgomp/config/linux/ia64/mutex.h b/libgomp/config/linux/ia64/mutex.h
index 6e294059b23..8a67673df40 100644
--- a/libgomp/config/linux/ia64/mutex.h
+++ b/libgomp/config/linux/ia64/mutex.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
   *mutex = 0;
 }
 
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex);
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
 static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
 {
-  if (!__sync_bool_compare_and_swap (mutex, 0, 1))
-    gomp_mutex_lock_slow (mutex);
+  int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+  if (__builtin_expect (oldval, 0))
+    gomp_mutex_lock_slow (mutex, oldval);
 }
 
 extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
diff --git a/libgomp/config/linux/mutex.c b/libgomp/config/linux/mutex.c
index 3ca37c19f5f..2574f7be42a 100644
--- a/libgomp/config/linux/mutex.c
+++ b/libgomp/config/linux/mutex.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -32,15 +32,27 @@ long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
 long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
 
 void
-gomp_mutex_lock_slow (gomp_mutex_t *mutex)
+gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval)
 {
-  do
+  while (oldval == 1)
     {
-      int oldval = __sync_val_compare_and_swap (mutex, 1, 2);
-      if (oldval != 0)
-	do_wait (mutex, 2);
+      if (do_spin (mutex, 1))
+	{
+	  oldval = __sync_lock_test_and_set (mutex, 2);
+	  if (oldval == 0)
+	    return;
+	  futex_wait (mutex, 2);
+	  break;
+	}
+      else
+	{
+	  oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+	  if (oldval == 0)
+	    return;
+	}
     }
-  while (!__sync_bool_compare_and_swap (mutex, 0, 2));
+  while ((oldval = __sync_lock_test_and_set (mutex, 2)))
+    do_wait (mutex, 2);
 }
 
 void
diff --git a/libgomp/config/linux/mutex.h b/libgomp/config/linux/mutex.h
index 1905ce012fd..eafb0e76734 100644
--- a/libgomp/config/linux/mutex.h
+++ b/libgomp/config/linux/mutex.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
   *mutex = 0;
 }
 
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex);
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
 static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
 {
-  if (!__sync_bool_compare_and_swap (mutex, 0, 1))
-    gomp_mutex_lock_slow (mutex);
+  int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+  if (__builtin_expect (oldval, 0))
+    gomp_mutex_lock_slow (mutex, oldval);
 }
 
 extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
diff --git a/libgomp/config/linux/wait.h b/libgomp/config/linux/wait.h
index 0e8abf111b0..4f659855811 100644
--- a/libgomp/config/linux/wait.h
+++ b/libgomp/config/linux/wait.h
@@ -44,7 +44,7 @@ extern long int gomp_futex_wait, gomp_futex_wake;
 
 #include <futex.h>
 
-static inline void do_wait (int *addr, int val)
+static inline int do_spin (int *addr, int val)
 {
   unsigned long long i, count = gomp_spin_count_var;
 
@@ -52,10 +52,16 @@ static inline void do_wait (int *addr, int val)
     count = gomp_throttled_spin_count_var;
   for (i = 0; i < count; i++)
     if (__builtin_expect (*addr != val, 0))
-      return;
+      return 0;
     else
       cpu_relax ();
-  futex_wait (addr, val);
+  return 1;
+}
+
+static inline void do_wait (int *addr, int val)
+{
+  if (do_spin (addr, val))
+    futex_wait (addr, val);
 }
 
 #ifdef HAVE_ATTRIBUTE_VISIBILITY