diff mbox

[trans-mem] Beginning of refactoring

Message ID 1306357828.13348.423.camel@triegel.csb
State New
Headers show

Commit Message

Torvald Riegel May 25, 2011, 9:10 p.m. UTC 
Here's the beginning of a refactoring aimed at being able to merge more
TM algorithms later on.

Patch 1: Just a straightfoward rename to make it clear that we're
dispatching on the level of ABI calls, not internals.

Patch 2: _ITM_dropReferences is not sufficiently defined in the ABI. It
seems to target some form of open nesting for txnal wrappers, but the
prose in the ABI specification is unclear. Thus, disable this for now
(aka fatal runtime error), and expect the related tests to fail. Pick it
up again once that the ABI has been improved and the use cases are
clear.

Patch 3: The actual change in how ABI calls are dispatched. Also,
removed method-readonly (broken, will in a similar form reappear in the
family of globallock-based algorithms), and disabled method-wbetl (needs
larger refactoring, will be revived/remerged later).

Ok for branch?


Coarse roadmap of planned future changes:

1) Improve serial-mode performance, which also gives a base for the
implementation of privatization.
2) The globallock family of algorithms (single versioned lock, with undo
or redo logging), and eager or lazy acquisition of the lock on writes.
3) Privatization safety for this globallock (includes user actions and
malloc/free commits).
4) The multilock family of algorithms (variants of LSA).

Other things: More cleanup in the ABI. Documentation.


Torvald
commit 9f1ff3be017e7d2e4ad57669de95a9e61c110146
Author: Torvald Riegel <triegel@redhat.com>
Date:   Mon May 23 12:40:34 2011 +0200

    rename gtm_disp to abi_disp
commit eaf0b413200a535cff13b0126ea51e0991f12ae1
Author: Torvald Riegel <triegel@redhat.com>
Date:   Tue May 24 18:30:28 2011 +0200

    Do not support _ITM_dropReferences anymore
    
    	* alloc_c.cc (_ITM_dropReferences): Don't support it anymore.
    	* testsuite/libitm.c++/dropref.C: _ITM_dropReferences is expected to fail.
    	* testsuite/libitm.c/dropref-2.c: Same.
    	* testsuite/libitm.c/dropref.c: Same.

diff --git a/libitm/alloc_c.cc b/libitm/alloc_c.cc
index 66d1109..549335c 100644
--- a/libitm/alloc_c.cc
+++ b/libitm/alloc_c.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Transactional Memory Library (libitm).
@@ -63,11 +63,15 @@ __attribute__((transaction_pure))
 void ITM_REGPARM
 _ITM_dropReferences (void *ptr, size_t len)
 {
-  gtm_transaction *tx = gtm_tx();
-  if (!abi_disp()->trydropreference (ptr, len))
-    tx->restart (RESTART_VALIDATE_READ);
-  tx->drop_references_local (ptr, len);
-  tx->drop_references_allocations (ptr);
+  // The semantics of _ITM_dropReferences are not sufficiently defined in the
+  // ABI specification, so it does not make sense to support it right now. See
+  // the libitm documentation for details.
+  GTM_fatal("_ITM_dropReferences is not supported");
+//  gtm_transaction *tx = gtm_tx();
+//  if (!abi_disp()->trydropreference (ptr, len))
+//    tx->restart (RESTART_VALIDATE_READ);
+//  tx->drop_references_local (ptr, len);
+//  tx->drop_references_allocations (ptr);
 }
 
 } // extern "C"
diff --git a/libitm/testsuite/libitm.c++/dropref.C b/libitm/testsuite/libitm.c++/dropref.C
index 92c8812..ee4f1bb 100644
--- a/libitm/testsuite/libitm.c++/dropref.C
+++ b/libitm/testsuite/libitm.c++/dropref.C
@@ -1,3 +1,4 @@
+/* { dg-xfail-run-if "unsupported" { *-*-* } } */
 #include <libitm.h>
 
 char *pp;
diff --git a/libitm/testsuite/libitm.c/dropref-2.c b/libitm/testsuite/libitm.c/dropref-2.c
index ddd4eae..2386b18 100644
--- a/libitm/testsuite/libitm.c/dropref-2.c
+++ b/libitm/testsuite/libitm.c/dropref-2.c
@@ -1,3 +1,4 @@
+/* { dg-xfail-run-if "unsupported" { *-*-* } } */
 #include <stdlib.h>
 #include <libitm.h>
 
diff --git a/libitm/testsuite/libitm.c/dropref.c b/libitm/testsuite/libitm.c/dropref.c
index 92c8812..ee4f1bb 100644
--- a/libitm/testsuite/libitm.c/dropref.c
+++ b/libitm/testsuite/libitm.c/dropref.c
@@ -1,3 +1,4 @@
+/* { dg-xfail-run-if "unsupported" { *-*-* } } */
 #include <libitm.h>
 
 char *pp;
commit e3454b7e33666b20866ea5410d791cbf79c297f0
Author: Torvald Riegel <triegel@redhat.com>
Date:   Wed May 25 17:21:53 2011 +0200

    New dispatch class/functions. Remove/disable old methods (readonly, wbetl).
    
    	* libitm_i.h: Move parts to common.h and dispatch.h.
    	* common.h: New file.
    	* dispatch.h: New file, new dispatch class.
    	Rename GTM::abi_dispatch::lock_type to ls_modifier.
    	RenameGTM::abi_dispatch::NOLOCK to NONTXNAL.
    	* beginend.cc (GTM::gtm_transaction::begin_transaction): Delegate mode
    	decision to retry.cc.
    	* retry.cc (GTM::gtm_transaction::decide_retry_strategy): Use serial mode
    	only.
    	(GTM::gtm_transaction::decide_begin_dispatch): Same.
    	* method-serial.cc: Adapt to new dispatch. Add serial mode with undo
    	logging.
    	* barrier.cc: Use new barriers definitions.
    	* config/x86/x86_sse.cc: Same.
    	* config/x86/x86_avx.cc: Same.
    	* Makefile.am: Don't build readonly and wbetl methods, memset.cc and
    	memcpy.cc.
    	* Makefile.in: Rebuild.
    	* method-readonly.cc: Remove.
    	* method-wbetl.cc: Rename GTM::abi_dispatch::lock_type to ls_modifier.
    	Rename GTM::abi_dispatch::NOLOCK to NONTXNAL.

diff --git a/libitm/Makefile.am b/libitm/Makefile.am
index 8ad4e5f..bf0180d 100644
--- a/libitm/Makefile.am
+++ b/libitm/Makefile.am
@@ -41,9 +41,9 @@ libitm_la_LDFLAGS = $(libitm_version_info) $(libitm_version_script) \
 
 libitm_la_SOURCES = \
 	aatree.cc alloc.cc alloc_c.cc alloc_cpp.cc barrier.cc beginend.cc \
-	clone.cc cacheline.cc cachepage.cc eh_cpp.cc local.cc memcpy.cc \
-	memset.cc query.cc retry.cc rwlock.cc useraction.cc util.cc \
-	sjlj.S tls.cc method-serial.cc method-readonly.cc method-wbetl.cc
+	clone.cc cacheline.cc cachepage.cc eh_cpp.cc local.cc \
+	query.cc retry.cc rwlock.cc useraction.cc util.cc \
+	sjlj.S tls.cc method-serial.cc
 
 if ARCH_X86
 libitm_la_SOURCES += x86_sse.cc x86_avx.cc
diff --git a/libitm/Makefile.in b/libitm/Makefile.in
index ad03c27..8fc3106 100644
--- a/libitm/Makefile.in
+++ b/libitm/Makefile.in
@@ -93,17 +93,15 @@ LTLIBRARIES = $(toolexeclib_LTLIBRARIES)
 libitm_la_LIBADD =
 am__libitm_la_SOURCES_DIST = aatree.cc alloc.cc alloc_c.cc \
 	alloc_cpp.cc barrier.cc beginend.cc clone.cc cacheline.cc \
-	cachepage.cc eh_cpp.cc local.cc memcpy.cc memset.cc query.cc \
+	cachepage.cc eh_cpp.cc local.cc query.cc \
 	retry.cc rwlock.cc useraction.cc util.cc sjlj.S tls.cc \
-	method-serial.cc method-readonly.cc method-wbetl.cc x86_sse.cc \
-	x86_avx.cc
+	method-serial.cc x86_sse.cc 86_avx.cc
 @ARCH_X86_TRUE@am__objects_1 = x86_sse.lo x86_avx.lo
 am_libitm_la_OBJECTS = aatree.lo alloc.lo alloc_c.lo alloc_cpp.lo \
 	barrier.lo beginend.lo clone.lo cacheline.lo cachepage.lo \
-	eh_cpp.lo local.lo memcpy.lo memset.lo query.lo retry.lo \
+	eh_cpp.lo local.lo query.lo retry.lo \
 	rwlock.lo useraction.lo util.lo sjlj.lo tls.lo \
-	method-serial.lo method-readonly.lo method-wbetl.lo \
-	$(am__objects_1)
+	method-serial.lo $(am__objects_1)
 libitm_la_OBJECTS = $(am_libitm_la_OBJECTS)
 DEFAULT_INCLUDES = -I.@am__isrc@
 depcomp = $(SHELL) $(top_srcdir)/../depcomp
@@ -342,6 +340,7 @@ AM_CPPFLAGS = $(addprefix -I, $(search_path))
 AM_CFLAGS = $(XCFLAGS)
 AM_CXXFLAGS = -std=gnu++0x -funwind-tables -fno-exceptions -fno-rtti \
 	$(XCFLAGS) $(abi_version)
+
 AM_CCASFLAGS = $(XCFLAGS)
 AM_LDFLAGS = $(XLDFLAGS) $(SECTION_LDFLAGS) $(OPT_LDFLAGS)
 toolexeclib_LTLIBRARIES = libitm.la
@@ -358,9 +357,9 @@ libitm_la_LDFLAGS = $(libitm_version_info) $(libitm_version_script) \
 
 libitm_la_SOURCES = aatree.cc alloc.cc alloc_c.cc alloc_cpp.cc \
 	barrier.cc beginend.cc clone.cc cacheline.cc cachepage.cc \
-	eh_cpp.cc local.cc memcpy.cc memset.cc query.cc retry.cc \
+	eh_cpp.cc local.cc query.cc retry.cc \
 	rwlock.cc useraction.cc util.cc sjlj.S tls.cc method-serial.cc \
-	method-readonly.cc method-wbetl.cc $(am__append_1)
+	$(am__append_1)
 all: config.h
 	$(MAKE) $(AM_MAKEFLAGS) all-recursive
 
@@ -470,11 +469,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/clone.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/eh_cpp.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/local.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/memcpy.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/memset.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/method-readonly.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/method-serial.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/method-wbetl.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/query.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/retry.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rwlock.Plo@am__quote@
diff --git a/libitm/barrier.cc b/libitm/barrier.cc
index 52e5a08..948dd39 100644
--- a/libitm/barrier.cc
+++ b/libitm/barrier.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Transactional Memory Library (libitm).
@@ -23,15 +23,8 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "libitm_i.h"
-#include "barrier.tpl"
-
-ITM_BARRIERS(U1)
-ITM_BARRIERS(U2)
-ITM_BARRIERS(U4)
-ITM_BARRIERS(U8)
-ITM_BARRIERS(F)
-ITM_BARRIERS(D)
-ITM_BARRIERS(E)
-ITM_BARRIERS(CF)
-ITM_BARRIERS(CD)
-ITM_BARRIERS(CE)
+
+using namespace GTM;
+
+CREATE_DISPATCH_FUNCTIONS(GTM::abi_disp()->_, )
+
diff --git a/libitm/beginend.cc b/libitm/beginend.cc
index 37e23e2..7ed5073 100644
--- a/libitm/beginend.cc
+++ b/libitm/beginend.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Transactional Memory Library (libitm).
@@ -128,13 +128,27 @@ GTM::gtm_transaction::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
 
   set_gtm_tx (tx);
 
+  // ??? pr_undoLogCode is not properly defined in the ABI. Are barriers
+  // omitted because they are not necessary (e.g., a transaction on thread-
+  // local data) or because the compiler thinks that some kind of global
+  // synchronization might perform better?
+  if (unlikely(prop & pr_undoLogCode))
+    GTM_fatal("pr_undoLogCode not supported");
+
   if ((prop & pr_doesGoIrrevocable) || !(prop & pr_instrumentedCode))
+    tx->state = (STATE_SERIAL | STATE_IRREVOCABLE);
+
+  else
+    disp = tx->decide_begin_dispatch ();
+
+  if (tx->state & STATE_SERIAL)
     {
       serial_lock.write_lock ();
 
-      tx->state = (STATE_SERIAL | STATE_IRREVOCABLE);
-
-      disp = dispatch_serial ();
+      if (tx->state & STATE_IRREVOCABLE)
+        disp = dispatch_serialirr ();
+      else
+        disp = dispatch_serial ();
 
       ret = a_runUninstrumentedCode;
       if ((prop & pr_multiwayCode) == pr_instrumentedCode)
@@ -143,14 +157,6 @@ GTM::gtm_transaction::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
   else
     {
       serial_lock.read_lock ();
-
-      // ??? Probably want some environment variable to choose the default
-      // STM implementation once we have more than one implemented.
-      if (prop & pr_readOnly)
-	disp = dispatch_readonly ();
-      else
-	disp = dispatch_wbetl ();
-
       ret = a_runInstrumentedCode | a_saveLiveVariables;
     }
 
diff --git a/libitm/common.h b/libitm/common.h
new file mode 100644
index 0000000..d94ca94
--- /dev/null
+++ b/libitm/common.h
@@ -0,0 +1,43 @@
+/* Copyright (C) 2008, 2009 Free Software Foundation, Inc.
+   Contributed by Richard Henderson <rth@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* The following are internal implementation functions and definitions.
+   To distinguish them from those defined by the Intel ABI, they all
+   begin with GTM/gtm.  */
+
+#ifndef COMMON_H
+#define COMMON_H 1
+
+#define UNUSED		__attribute__((unused))
+#define ALWAYS_INLINE	__attribute__((always_inline))
+#ifdef HAVE_ATTRIBUTE_VISIBILITY
+# define HIDDEN		__attribute__((visibility("hidden")))
+#else
+# define HIDDEN
+#endif
+
+#define likely(X)	__builtin_expect((X) != 0, 1)
+#define unlikely(X)	__builtin_expect((X), 0)
+
+#endif // COMMON_H
diff --git a/libitm/config/x86/x86_avx.cc b/libitm/config/x86/x86_avx.cc
index ee073d5..f81e3b8 100644
--- a/libitm/config/x86/x86_avx.cc
+++ b/libitm/config/x86/x86_avx.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Transactional Memory Library (libitm).
@@ -23,14 +23,16 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "libitm_i.h"
-#include "barrier.tpl"
+#include "dispatch.h"
 
-ITM_BARRIERS(M256)
+// ??? Use memcpy for now, until we have figured out how to best instantiate
+// these loads/stores.
+CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(M256, GTM::abi_disp()->_, )
 
 void ITM_REGPARM
 _ITM_LM256 (const _ITM_TYPE_M256 *ptr)
 {
-  GTM_LB (ptr, sizeof (*ptr));
+  GTM::GTM_LB (ptr, sizeof (*ptr));
 }
 
 // Helpers for re-aligning two 128-bit values.
diff --git a/libitm/config/x86/x86_sse.cc b/libitm/config/x86/x86_sse.cc
index 5bb9766..a8585e4 100644
--- a/libitm/config/x86/x86_sse.cc
+++ b/libitm/config/x86/x86_sse.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Transactional Memory Library (libitm).
@@ -23,21 +23,23 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "libitm_i.h"
-#include "barrier.tpl"
+#include "dispatch.h"
 
-ITM_BARRIERS(M64)
-ITM_BARRIERS(M128)
+// ??? Use memcpy for now, until we have figured out how to best instantiate
+// these loads/stores.
+CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(M64, GTM::abi_disp()->_, )
+CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(M128, GTM::abi_disp()->_, )
 
 void ITM_REGPARM
 _ITM_LM64 (const _ITM_TYPE_M64 *ptr)
 {
-  GTM_LB (ptr, sizeof (*ptr));
+  GTM::GTM_LB (ptr, sizeof (*ptr));
 }
 
 void ITM_REGPARM
 _ITM_LM128 (const _ITM_TYPE_M128 *ptr)
 {
-  GTM_LB (ptr, sizeof (*ptr));
+  GTM::GTM_LB (ptr, sizeof (*ptr));
 }
 
 // Helpers for re-aligning two 128-bit values.
diff --git a/libitm/dispatch.h b/libitm/dispatch.h
new file mode 100644
index 0000000..23a1453
--- /dev/null
+++ b/libitm/dispatch.h
@@ -0,0 +1,287 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   Contributed by Torvald Riegel <triegel@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef DISPATCH_H
+#define DISPATCH_H 1
+
+#include "libitm.h"
+#include "common.h"
+
+// Creates ABI load/store methods (can be made virtual or static using M).
+#define ITM_READ_M(T, LSMOD, M, M2)                                         \
+  M _ITM_TYPE_##T ITM_REGPARM _ITM_##LSMOD##T##M2 (const _ITM_TYPE_##T *ptr)\
+  {                                                                         \
+    return load(ptr, abi_dispatch::LSMOD);                                  \
+  }
+
+#define ITM_WRITE_M(T, LSMOD, M, M2)                         \
+  M void ITM_REGPARM _ITM_##LSMOD##T##M2 (_ITM_TYPE_##T *ptr,\
+                                         _ITM_TYPE_##T val)  \
+  {                                                          \
+    store(ptr, val, abi_dispatch::LSMOD);                    \
+  }
+
+// Creates ABI load/store methods for all load/store modifiers for a particular
+// type.
+#define CREATE_DISPATCH_METHODS_T(T, M, M2) \
+  ITM_READ_M(T, R, M, M2)                \
+  ITM_READ_M(T, RaR, M, M2)              \
+  ITM_READ_M(T, RaW, M, M2)              \
+  ITM_READ_M(T, RfW, M, M2)              \
+  ITM_WRITE_M(T, W, M, M2)               \
+  ITM_WRITE_M(T, WaR, M, M2)             \
+  ITM_WRITE_M(T, WaW, M, M2)
+
+// Creates ABI load/store methods for all types.
+// See CREATE_DISPATCH_FUNCTIONS for comments.
+#define CREATE_DISPATCH_METHODS(M, M2)  \
+  CREATE_DISPATCH_METHODS_T (U1, M, M2) \
+  CREATE_DISPATCH_METHODS_T (U2, M, M2) \
+  CREATE_DISPATCH_METHODS_T (U4, M, M2) \
+  CREATE_DISPATCH_METHODS_T (U8, M, M2) \
+  CREATE_DISPATCH_METHODS_T (F, M, M2)  \
+  CREATE_DISPATCH_METHODS_T (D, M, M2)  \
+  CREATE_DISPATCH_METHODS_T (E, M, M2)  \
+  CREATE_DISPATCH_METHODS_T (CF, M, M2) \
+  CREATE_DISPATCH_METHODS_T (CD, M, M2) \
+  CREATE_DISPATCH_METHODS_T (CE, M, M2)
+
+// Creates memcpy/memmove/memset methods.
+#define CREATE_DISPATCH_METHODS_MEM()  \
+virtual void _memtransfer(void *dst, const void* src, size_t size,     \
+bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod)           \
+{                                                                     \
+  memtransfer_static(dst, src, size, may_overlap, dst_mod, src_mod);  \
+}                                                                     \
+virtual void _memset(void *dst, int c, size_t size, ls_modifier mod)   \
+{                                                                     \
+  memset_static(dst, c, size, mod);                                   \
+}
+
+
+// Creates ABI load/store functions that can target either a class or an
+// object.
+#define ITM_READ(T, LSMOD, TARGET, M2)                                 \
+  _ITM_TYPE_##T ITM_REGPARM _ITM_##LSMOD##T (const _ITM_TYPE_##T *ptr) \
+  {                                                                    \
+    return TARGET##ITM_##LSMOD##T##M2(ptr);                            \
+  }
+
+#define ITM_WRITE(T, LSMOD, TARGET, M2)                                    \
+  void ITM_REGPARM _ITM_##LSMOD##T (_ITM_TYPE_##T *ptr, _ITM_TYPE_##T val) \
+  {                                                                        \
+    TARGET##ITM_##LSMOD##T##M2(ptr, val);                                  \
+  }
+
+// Creates ABI load/store functions for all load/store modifiers for a
+// particular type.
+#define CREATE_DISPATCH_FUNCTIONS_T(T, TARGET, M2) \
+  ITM_READ(T, R, TARGET, M2)                \
+  ITM_READ(T, RaR, TARGET, M2)              \
+  ITM_READ(T, RaW, TARGET, M2)              \
+  ITM_READ(T, RfW, TARGET, M2)              \
+  ITM_WRITE(T, W, TARGET, M2)               \
+  ITM_WRITE(T, WaR, TARGET, M2)             \
+  ITM_WRITE(T, WaW, TARGET, M2)
+
+// Creates ABI memcpy/memmove/memset functions.
+#define ITM_MEMTRANSFER_DEF(TARGET, M2, NAME, READ, WRITE) \
+void ITM_REGPARM _ITM_memcpy##NAME(void *dst, const void *src, size_t size)  \
+{                                                                            \
+  TARGET##memtransfer##M2 (dst, src, size,                                   \
+             false, GTM::abi_dispatch::WRITE, GTM::abi_dispatch::READ);      \
+}                                                                            \
+void ITM_REGPARM _ITM_memmove##NAME(void *dst, const void *src, size_t size) \
+{                                                                            \
+  if (GTM::abi_dispatch::NONTXNAL == GTM::abi_dispatch::WRITE ||             \
+      GTM::abi_dispatch::NONTXNAL == GTM::abi_dispatch::READ)                \
+    {                                                                        \
+      if (((uintptr_t)dst <= (uintptr_t)src ?                                \
+          (uintptr_t)dst + size > (uintptr_t)src :                           \
+          (uintptr_t)src + size > (uintptr_t)dst))                           \
+        GTM::GTM_fatal("_ITM_memmove overlapping and t/nt is not allowed");  \
+      else                                                                   \
+        TARGET##memtransfer##M2 (dst, src, size,                             \
+                false, GTM::abi_dispatch::WRITE, GTM::abi_dispatch::READ);   \
+    }                                                                        \
+  TARGET##memtransfer##M2 (dst, src, size,                                   \
+              true, GTM::abi_dispatch::WRITE, GTM::abi_dispatch::READ);      \
+}
+
+#define ITM_MEMSET_DEF(TARGET, M2, WRITE) \
+void ITM_REGPARM _ITM_memset##WRITE(void *dst, int c, size_t size) \
+{                                                                  \
+  TARGET##memset##M2 (dst, c, size, GTM::abi_dispatch::WRITE);     \
+}                                                                  \
+
+
+// ??? The number of virtual methods is large (7*4 for integers, 7*6 for FP,
+// 7*3 for vectors). Is the cache footprint so costly that we should go for
+// a small table instead (i.e., only have two virtual load/store methods for
+// each supported type)? Note that this doesn't affect custom code paths at
+// all because these use only direct calls.
+// A large cache footprint could especially decrease HTM performance (due
+// to HTM capacity). We could add the modifier (RaR etc.) as parameter, which
+// would give us just 4*2+6*2+3*2 functions (so we'd just need one line for
+// the integer loads/stores), but then the modifier can be checked only at
+// runtime.
+// For memcpy/memmove/memset, we just have two virtual methods (memtransfer
+// and memset).
+#define CREATE_DISPATCH_FUNCTIONS(TARGET, M2)  \
+  CREATE_DISPATCH_FUNCTIONS_T (U1, TARGET, M2) \
+  CREATE_DISPATCH_FUNCTIONS_T (U2, TARGET, M2) \
+  CREATE_DISPATCH_FUNCTIONS_T (U4, TARGET, M2) \
+  CREATE_DISPATCH_FUNCTIONS_T (U8, TARGET, M2) \
+  CREATE_DISPATCH_FUNCTIONS_T (F, TARGET, M2)  \
+  CREATE_DISPATCH_FUNCTIONS_T (D, TARGET, M2)  \
+  CREATE_DISPATCH_FUNCTIONS_T (E, TARGET, M2)  \
+  CREATE_DISPATCH_FUNCTIONS_T (CF, TARGET, M2) \
+  CREATE_DISPATCH_FUNCTIONS_T (CD, TARGET, M2) \
+  CREATE_DISPATCH_FUNCTIONS_T (CE, TARGET, M2) \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RnWt,     NONTXNAL, W)      \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RnWtaR,   NONTXNAL, WaR)    \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RnWtaW,   NONTXNAL, WaW)    \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtWn,     R,      NONTXNAL) \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtWt,     R,      W)        \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtWtaR,   R,      WaR)      \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtWtaW,   R,      WaW)      \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaRWn,   RaR,    NONTXNAL) \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaRWt,   RaR,    W)        \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaRWtaR, RaR,    WaR)      \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaRWtaW, RaR,    WaW)      \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaWWn,   RaW,    NONTXNAL) \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaWWt,   RaW,    W)        \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaWWtaR, RaW,    WaR)      \
+  ITM_MEMTRANSFER_DEF(TARGET, M2, RtaWWtaW, RaW,    WaW)      \
+  ITM_MEMSET_DEF(TARGET, M2, W)   \
+  ITM_MEMSET_DEF(TARGET, M2, WaR) \
+  ITM_MEMSET_DEF(TARGET, M2, WaW)
+
+
+// Creates ABI load/store functions that delegate to a transactional memcpy.
+#define ITM_READ_MEMCPY(T, LSMOD, TARGET, M2)                         \
+  _ITM_TYPE_##T ITM_REGPARM _ITM_##LSMOD##T (const _ITM_TYPE_##T *ptr)\
+  {                                                                   \
+    _ITM_TYPE_##T v;                                                  \
+    TARGET##memtransfer##M2(&v, ptr, sizeof(_ITM_TYPE_##T), false,    \
+        GTM::abi_dispatch::NONTXNAL, GTM::abi_dispatch::LSMOD);       \
+    return v;                                                         \
+  }
+
+#define ITM_WRITE_MEMCPY(T, LSMOD, TARGET, M2)                            \
+  void ITM_REGPARM _ITM_##LSMOD##T (_ITM_TYPE_##T *ptr, _ITM_TYPE_##T val)\
+  {                                                                       \
+    TARGET##memtransfer##M2(ptr, &val, sizeof(_ITM_TYPE_##T), false,      \
+        GTM::abi_dispatch::LSMOD, GTM::abi_dispatch::NONTXNAL);           \
+  }
+
+#define CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(T, TARGET, M2) \
+  ITM_READ_MEMCPY(T, R, TARGET, M2)                \
+  ITM_READ_MEMCPY(T, RaR, TARGET, M2)              \
+  ITM_READ_MEMCPY(T, RaW, TARGET, M2)              \
+  ITM_READ_MEMCPY(T, RfW, TARGET, M2)              \
+  ITM_WRITE_MEMCPY(T, W, TARGET, M2)               \
+  ITM_WRITE_MEMCPY(T, WaR, TARGET, M2)             \
+  ITM_WRITE_MEMCPY(T, WaW, TARGET, M2)
+
+
+namespace GTM HIDDEN {
+
+/// This pass-through method is the basis for other methods.
+/// It can be used for serial-irrevocable mode.
+struct abi_dispatch
+{
+public:
+  enum ls_modifier { NONTXNAL, R, RaR, RaW, RfW, W, WaR, WaW };
+
+private:
+  // Disallow copies
+  abi_dispatch(const abi_dispatch &) = delete;
+  abi_dispatch& operator=(const abi_dispatch &) = delete;
+
+public:
+  virtual bool trycommit() = 0;
+  virtual void rollback() = 0;
+  virtual void reinit() = 0;
+
+  // Use fini instead of dtor to support a static subclasses that uses
+  // a unique object and so we don't want to destroy it from common code.
+  virtual void fini() = 0;
+
+  bool read_only () const { return m_read_only; }
+  bool write_through() const { return m_write_through; }
+
+  static void *operator new(size_t s) { return xmalloc (s); }
+  static void operator delete(void *p) { free (p); }
+
+protected:
+  /// Transactional load. Will be called from the dispatch methods
+  /// created below.
+  template <typename V> static V load(const V* addr, ls_modifier mod)
+  {
+    return *addr;
+  }
+  /// Transactional store. Will be called from the dispatch methods
+  /// created below.
+  template <typename V> static void store(V* addr, const V value,
+      ls_modifier mod)
+  {
+    *addr = value;
+  }
+
+public:
+  /// Transactional memcpy/memmove. This method is static to avoid indirect
+  /// calls, and will be used by the virtual ABI dispatch methods.
+  static void memtransfer_static(void *dst, const void* src, size_t size,
+      bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod)
+  {
+    if (!may_overlap)
+      ::memcpy(dst, src, size);
+    else
+      ::memmove(dst, src, size);
+  }
+
+  /// Transactional memset. This method is static to avoid indirect
+  /// calls, and will be used by the virtual ABI dispatch methods.
+  static void memset_static(void *dst, int c, size_t size, ls_modifier mod)
+  {
+    ::memset(dst, c, size);
+  }
+
+  // Creates the ABI dispatch methods for loads and stores.
+  // ??? Should the dispatch table instead be embedded in the dispatch object
+  // to avoid the indirect lookup in the vtable?
+  CREATE_DISPATCH_METHODS(virtual, )
+  // Creates the ABI dispatch methods for memcpy/memmove/memset.
+  CREATE_DISPATCH_METHODS_MEM()
+
+protected:
+  const bool m_read_only;
+  const bool m_write_through;
+  abi_dispatch(bool ro, bool wt) : m_read_only(ro), m_write_through(wt) { }
+};
+
+}
+
+#endif // DISPATCH_H
diff --git a/libitm/libitm_i.h b/libitm/libitm_i.h
index 3291f1b..d323d45 100644
--- a/libitm/libitm_i.h
+++ b/libitm/libitm_i.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Transactional Memory Library (libitm).
@@ -38,16 +38,7 @@
 #include <unwind.h>
 #include <type_traits>
 
-#define UNUSED		__attribute__((unused))
-#define ALWAYS_INLINE	__attribute__((always_inline))
-#ifdef HAVE_ATTRIBUTE_VISIBILITY
-# define HIDDEN		__attribute__((visibility("hidden")))
-#else
-# define HIDDEN
-#endif
-
-#define likely(X)	__builtin_expect((X) != 0, 1)
-#define unlikely(X)	__builtin_expect((X), 0)
+#include "common.h"
 
 namespace GTM HIDDEN {
 
@@ -82,57 +73,12 @@ extern void * xrealloc (void *p, size_t s) __attribute__((malloc, nothrow));
 #include "cacheline.h"
 #include "cachepage.h"
 #include "stmlock.h"
+#include "dispatch.h"
 
 namespace GTM HIDDEN {
 
 // A dispatch table parameterizes the implementation of the STM.
-struct abi_dispatch
-{
- public:
-  enum lock_type { NOLOCK, R, RaR, RaW, RfW, W, WaR, WaW };
-
-  struct mask_pair
-  {
-    gtm_cacheline *line;
-    gtm_cacheline_mask *mask;
-
-    mask_pair() = default;
-    mask_pair(gtm_cacheline *l, gtm_cacheline_mask *m) : line(l), mask(m) { }
-  };
-
- private:
-  // Disallow copies
-  abi_dispatch(const abi_dispatch &) = delete;
-  abi_dispatch& operator=(const abi_dispatch &) = delete;
-
- public:
-  // The default version of these is pass-through.  This merely gives the
-  // a single location to instantiate the base class vtable.
-  virtual const gtm_cacheline *read_lock(const gtm_cacheline *, lock_type);
-  virtual mask_pair write_lock(gtm_cacheline *, lock_type);
-
-  virtual bool trycommit() = 0;
-  virtual void rollback() = 0;
-  virtual void reinit() = 0;
-  virtual bool trydropreference(void *, size_t) = 0;
-
-  // Use fini instead of dtor to support a static subclasses that uses
-  // a unique object and so we don't want to destroy it from common code.
-  virtual void fini() = 0;
-
-  bool read_only () const { return m_read_only; }
-  bool write_through() const { return m_write_through; }
-
-  static void *operator new(size_t s) { return xmalloc (s); }
-  static void operator delete(void *p) { free (p); }
-
- protected:
-  const bool m_read_only;
-  const bool m_write_through;
-  abi_dispatch(bool ro, bool wt) : m_read_only(ro), m_write_through(wt) { }
-
-  static gtm_cacheline_mask mask_sink;
-};
+struct abi_dispatch;
 
 // These values are given to GTM_restart_transaction and indicate the
 // reason for the restart.  The reason is used to decide what STM
@@ -256,6 +202,7 @@ struct gtm_transaction
 
   // In retry.cc
   void decide_retry_strategy (gtm_restart_reason);
+  abi_dispatch* decide_begin_dispatch ();
 
   // In method-serial.cc
   void serialirr_mode ();
@@ -287,9 +234,8 @@ extern void GTM_error (const char *fmt, ...)
 extern void GTM_fatal (const char *fmt, ...)
 	__attribute__((noreturn, format (printf, 1, 2)));
 
-extern abi_dispatch *dispatch_wbetl();
-extern abi_dispatch *dispatch_readonly();
 extern abi_dispatch *dispatch_serial();
+extern abi_dispatch *dispatch_serialirr();
 
 extern gtm_cacheline_mask gtm_mask_stack(gtm_cacheline *, gtm_cacheline_mask);
 
diff --git a/libitm/method-readonly.cc b/libitm/method-readonly.cc
deleted file mode 100644
index ed573e2..0000000
--- a/libitm/method-readonly.cc
+++ /dev/null
@@ -1,124 +0,0 @@
-/* Copyright (C) 2009 Free Software Foundation, Inc.
-   Contributed by Richard Henderson <rth@redhat.com>.
-
-   This file is part of the GNU Transactional Memory Library (libitm).
-
-   Libitm is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-
-   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
-   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-   more details.
-
-   Under Section 7 of GPL version 3, you are granted additional
-   permissions described in the GCC Runtime Library Exception, version
-   3.1, as published by the Free Software Foundation.
-
-   You should have received a copy of the GNU General Public License and
-   a copy of the GCC Runtime Library Exception along with this program;
-   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include "libitm_i.h"
-
-namespace {
-
-using namespace GTM;
-
-class readonly_dispatch : public abi_dispatch
-{
- private:
-  gtm_version m_start;
-
- public:
-  readonly_dispatch();
-
-  virtual const gtm_cacheline *read_lock(const gtm_cacheline *, lock_type);
-  virtual mask_pair write_lock(gtm_cacheline *, lock_type);
-  virtual bool trycommit();
-  virtual void rollback();
-  virtual void reinit();
-  virtual void fini();
-  virtual bool trydropreference (void *ptr, size_t size) { return true; }
-};
-
-inline
-readonly_dispatch::readonly_dispatch()
-  : abi_dispatch(true, true), m_start(gtm_get_clock ())
-{ }
-
-
-const gtm_cacheline *
-readonly_dispatch::read_lock(const gtm_cacheline *line, lock_type lock)
-{
-  switch (lock)
-    {
-    case NOLOCK:
-    case R:
-    case RaR:
-      return line;
-
-    case RfW:
-      gtm_tx()->restart (RESTART_NOT_READONLY);
-
-    case RaW:
-    default:
-      abort ();
-    }
-}
-
-abi_dispatch::mask_pair
-readonly_dispatch::write_lock(gtm_cacheline *line, lock_type lock)
-{
-  switch (lock)
-    {
-    case NOLOCK:
-      {
-	abi_dispatch::mask_pair pair;
-	pair.line = line;
-	pair.mask = &mask_sink;
-	return pair;
-      }
-
-    case WaW:
-      abort ();
-
-    default:
-      gtm_tx()->restart (RESTART_NOT_READONLY);
-    }
-}
-
-bool
-readonly_dispatch::trycommit ()
-{
-  return gtm_get_clock () == m_start;
-}
-
-void
-readonly_dispatch::rollback ()
-{
-  /* Nothing to do.  */
-}
-
-void
-readonly_dispatch::reinit ()
-{
-  m_start = gtm_get_clock ();
-}
-
-void
-readonly_dispatch::fini ()
-{
-  delete this;
-}
-
-} // anon namespace
-
-abi_dispatch *
-GTM::dispatch_readonly ()
-{
-  return new readonly_dispatch();
-}
diff --git a/libitm/method-serial.cc b/libitm/method-serial.cc
index b46de4d..8e4fcd2 100644
--- a/libitm/method-serial.cc
+++ b/libitm/method-serial.cc
@@ -24,24 +24,6 @@
 
 #include "libitm_i.h"
 
-namespace GTM HIDDEN {
-
-gtm_cacheline_mask abi_dispatch::mask_sink;
-
-const gtm_cacheline *
-abi_dispatch::read_lock(const gtm_cacheline *addr, lock_type)
-{
-  return addr;
-}
-
-abi_dispatch::mask_pair
-abi_dispatch::write_lock(gtm_cacheline *addr, lock_type)
-{
-  return mask_pair (addr, &mask_sink);
-}
-
-} // namespace GTM
-
 // Avoid a dependency on libstdc++ for the pure virtuals in abi_dispatch.
 extern "C" void HIDDEN
 __cxa_pure_virtual ()
@@ -58,25 +40,80 @@ class serial_dispatch : public abi_dispatch
  public:
   serial_dispatch() : abi_dispatch(false, true) { }
 
-  // The read_lock and write_lock methods are implented by the base class.
+  // The transactional load/store/mem methods are implemented by the base
+  // class (simple direct memory accesses).
 
   virtual bool trycommit() { return true; }
   virtual void rollback() { abort(); }
   virtual void reinit() { }
   virtual void fini() { }
-  virtual bool trydropreference (void *ptr, size_t size) { return true; }
+};
+
+class serial_dispatch_ul : public serial_dispatch
+{
+protected:
+  static void log(const void *addr, size_t len)
+  {
+    // TODO Ensure that this gets inlined: Use internal log interface and LTO.
+    GTM_LB(addr, len);
+  }
+
+  template <typename V> static V load(const V* addr, ls_modifier mod)
+  {
+    return *addr;
+  }
+  template <typename V> static void store(V* addr, const V value,
+      ls_modifier mod)
+  {
+    if (mod != WaW)
+      log(addr, sizeof(V));
+    *addr = value;
+  }
+
+public:
+  static void memtransfer_static(void *dst, const void* src, size_t size,
+      bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod)
+  {
+    if (dst_mod != WaW && dst_mod != NONTXNAL)
+      log(dst, size);
+    if (!may_overlap)
+      memcpy(dst, src, size);
+    else
+      memmove(dst, src, size);
+  }
+
+  static void memset_static(void *dst, int c, size_t size, ls_modifier mod)
+  {
+    if (mod != WaW)
+      log(dst, size);
+    memset(dst, c, size);
+  }
+
+  // Local undo will handle this.
+  // trydropreference() need not be changed either.
+  virtual void rollback() { }
+
+  CREATE_DISPATCH_METHODS(virtual, )
+  CREATE_DISPATCH_METHODS_MEM()
 };
 
 } // anon namespace
 
 static const serial_dispatch o_serial_dispatch;
+static const serial_dispatch_ul o_serial_dispatch_ul;
 
 abi_dispatch *
-GTM::dispatch_serial ()
+GTM::dispatch_serialirr ()
 {
   return const_cast<serial_dispatch *>(&o_serial_dispatch);
 }
 
+abi_dispatch *
+GTM::dispatch_serial ()
+{
+  return const_cast<serial_dispatch_ul *>(&o_serial_dispatch_ul);
+}
+
 // Put the transaction into serial-irrevocable mode.
 
 void
diff --git a/libitm/method-wbetl.cc b/libitm/method-wbetl.cc
index 7ce317e..aff6397 100644
--- a/libitm/method-wbetl.cc
+++ b/libitm/method-wbetl.cc
@@ -83,8 +83,8 @@ class wbetl_dispatch : public abi_dispatch
  public:
   wbetl_dispatch();
 
-  virtual const gtm_cacheline *read_lock(const gtm_cacheline *, lock_type);
-  virtual mask_pair write_lock(gtm_cacheline *, lock_type);
+  virtual const gtm_cacheline *read_lock(const gtm_cacheline *, ls_modifier);
+  virtual mask_pair write_lock(gtm_cacheline *, ls_modifier);
 
   virtual bool trycommit();
   virtual void rollback();
@@ -375,11 +375,11 @@ wbetl_dispatch::do_read_lock (const gtm_cacheline *addr, bool after_read)
 }
 
 const gtm_cacheline *
-wbetl_dispatch::read_lock (const gtm_cacheline *addr, lock_type ltype)
+wbetl_dispatch::read_lock (const gtm_cacheline *addr, ls_modifier ltype)
 {
   switch (ltype)
     {
-    case NOLOCK:
+    case NONTXNAL:
       return addr;
     case R:
       return do_read_lock (addr, false);
@@ -395,13 +395,13 @@ wbetl_dispatch::read_lock (const gtm_cacheline *addr, lock_type ltype)
 }
 
 abi_dispatch::mask_pair
-wbetl_dispatch::write_lock (gtm_cacheline *addr, lock_type ltype)
+wbetl_dispatch::write_lock (gtm_cacheline *addr, ls_modifier ltype)
 {
   gtm_cacheline *line;
 
   switch (ltype)
     {
-    case NOLOCK:
+    case NONTXNAL:
       return mask_pair (addr, &mask_sink);
     case W:
     case WaR:
diff --git a/libitm/retry.cc b/libitm/retry.cc
index a25b282..31757d5 100644
--- a/libitm/retry.cc
+++ b/libitm/retry.cc
@@ -42,6 +42,10 @@ GTM::gtm_transaction::decide_retry_strategy (gtm_restart_reason r)
       // write lock is not yet held, grab it.  Don't do this with
       // an upgrade, since we've no need to preserve the state we
       // acquired with the read.
+      // FIXME this might be dangerous if we use serial mode to change TM
+      // meta data (e.g., reallocate the lock array). Likewise, for
+      // privatization, we must get rid of old references (that is, abort)
+      // or let privatizers know we're still there by not releasing the lock.
       if ((this->state & STATE_SERIAL) == 0)
 	{
 	  this->state |= STATE_SERIAL;
@@ -65,17 +69,23 @@ GTM::gtm_transaction::decide_retry_strategy (gtm_restart_reason r)
     }
   else
     {
-      if (r == RESTART_NOT_READONLY)
-	{
-	  assert ((this->prop & pr_readOnly) == 0);
-	  if (disp->read_only ())
-	    {
-	      disp->fini ();
-	      disp = dispatch_wbetl ();
-	      set_abi_disp (disp);
-	      return;
-	    }
-	}
-      disp->reinit ();
+      GTM_fatal("internal error: unsupported mode");
     }
 }
+
+
+/// Decides which TM method should be used on the first attempt to run this
+/// transaction, setting this->disp accordingly.
+/// serial_lock will not have been acquired if this is the outer-most
+/// transaction. If the state is set to STATE_SERIAL, the caller will set the
+/// dispatch.
+GTM::abi_dispatch*
+GTM::gtm_transaction::decide_begin_dispatch ()
+{
+  // ??? Probably want some environment variable to choose the default
+  // STM implementation once we have more than one implemented.
+  state = STATE_SERIAL;
+  if (prop & pr_hasNoAbort)
+    state |= STATE_IRREVOCABLE;
+  return 0;
+}

Comments

Richard Henderson June 29, 2011, 11:01 p.m. UTC | #1 
On 05/25/2011 02:10 PM, Torvald Riegel wrote:
> Here's the beginning of a refactoring aimed at being able to merge more
> TM algorithms later on.
> 
> Patch 1: Just a straightfoward rename to make it clear that we're
> dispatching on the level of ABI calls, not internals.

Ok, I guess.  I don't think it matters that much.

> Patch 2: _ITM_dropReferences is not sufficiently defined in the ABI. It
> seems to target some form of open nesting for txnal wrappers, but the
> prose in the ABI specification is unclear. Thus, disable this for now
> (aka fatal runtime error), and expect the related tests to fail. Pick it
> up again once that the ABI has been improved and the use cases are
> clear.

Sure, but please actually delete the code rather than just comment it out.

> Patch 3: The actual change in how ABI calls are dispatched. Also,
> removed method-readonly (broken, will in a similar form reappear in the
> family of globallock-based algorithms), and disabled method-wbetl (needs
> larger refactoring, will be revived/remerged later).

> +CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(M256, GTM::abi_disp()->_, )
> +CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(M64, GTM::abi_disp()->_, )
> +CREATE_DISPATCH_FUNCTIONS_T_MEMCPY(M128, GTM::abi_disp()->_, )
> +CREATE_DISPATCH_FUNCTIONS(GTM::abi_disp()->_, )

What's the point of using "GTM::abi_disp()->_" as a mandatory argument?

Further, that second "M2" argument is universally empty.  What's that?

> +// Creates memcpy/memmove/memset methods.
> +#define CREATE_DISPATCH_METHODS_MEM()  \
> +virtual void _memtransfer(void *dst, const void* src, size_t size,     \
> +bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod)           \
> +{                                                                     \
> +  memtransfer_static(dst, src, size, may_overlap, dst_mod, src_mod);  \
> +}                                                                     \
> +virtual void _memset(void *dst, int c, size_t size, ls_modifier mod)   \
> +{                                                                     \
> +  memset_static(dst, c, size, mod);                                   \
> +}

Why are the memtransfer and memset virtuals distinguished from the statics?
For the patch as written it would seem to be ok to merge them.

> +  if (GTM::abi_dispatch::NONTXNAL == GTM::abi_dispatch::WRITE ||             \
> +      GTM::abi_dispatch::NONTXNAL == GTM::abi_dispatch::READ)                \

Formatting.

> +#define ITM_MEMTRANSFER_DEF(TARGET, M2, NAME, READ, WRITE) \
> +void ITM_REGPARM _ITM_memcpy##NAME(void *dst, const void *src, size_t size)  \
> +{                                                                            \
> +  TARGET##memtransfer##M2 (dst, src, size,                                   \
> +             false, GTM::abi_dispatch::WRITE, GTM::abi_dispatch::READ);      \
> +}                                                                            \
> +void ITM_REGPARM _ITM_memmove##NAME(void *dst, const void *src, size_t size) \
> +{                                                                            \
> +  if (GTM::abi_dispatch::NONTXNAL == GTM::abi_dispatch::WRITE ||             \
> +      GTM::abi_dispatch::NONTXNAL == GTM::abi_dispatch::READ)                \
> +    {                                                                        \
> +      if (((uintptr_t)dst <= (uintptr_t)src ?                                \
> +          (uintptr_t)dst + size > (uintptr_t)src :                           \
> +          (uintptr_t)src + size > (uintptr_t)dst))                           \
> +        GTM::GTM_fatal("_ITM_memmove overlapping and t/nt is not allowed");  \
> +      else                                                                   \
> +        TARGET##memtransfer##M2 (dst, src, size,                             \
> +                false, GTM::abi_dispatch::WRITE, GTM::abi_dispatch::READ);   \
> +    }                                                                        \
> +  TARGET##memtransfer##M2 (dst, src, size,                                   \
> +              true, GTM::abi_dispatch::WRITE, GTM::abi_dispatch::READ);      \
> +}

Ok, I realize we need macros to generate the ABI names both here and in
CREATE_DISPATCH_FUNCTIONS, but can we limit the code within macros to as
little as absolutely possible?

For instance,

  template<abi_dispatch::ls_modifier dst_mod, abi_dispatch::ls_modifier src_mod>
  void abi_memmove(void *dst, const void *src, size_t size)
  {
    if (dst_mod == NONTXNAL || src_mod == NONTXNAL)
      ...
  }

where the actual implementation under macro is limited to a function call.

Missing return/else in there?  Surely not two calls to memtransfer...

> +protected:
> +  /// Transactional load. Will be called from the dispatch methods
> +  /// created below.
> +  template <typename V> static V load(const V* addr, ls_modifier mod)
> +  {
> +    return *addr;
> +  }
> +  /// Transactional store. Will be called from the dispatch methods
> +  /// created below.
> +  template <typename V> static void store(V* addr, const V value,
> +      ls_modifier mod)
> +  {
> +    *addr = value;
> +  }

Why are these here in the base class?  I'm not sure I like having static
functions that are required to be overridden at each instance, wherein if
you forget to implement them the build silently succeeds, but of course
does the wrong thing.

What's with the 3 /// comments?  Can we stick with existing gnu standards?
I'm pretty sure we've got some documentation for at least one of libstdc++,
gold, and the go front end.

> -#define UNUSED		__attribute__((unused))
> -#define ALWAYS_INLINE	__attribute__((always_inline))
> -#ifdef HAVE_ATTRIBUTE_VISIBILITY
> -# define HIDDEN		__attribute__((visibility("hidden")))
> -#else
> -# define HIDDEN
> -#endif
> -
> -#define likely(X)	__builtin_expect((X) != 0, 1)
> -#define unlikely(X)	__builtin_expect((X), 0)
> +#include "common.h"

Why?  We're already in a header that's clearly marked "internal".


r~
diff mbox

Patch

diff --git a/libitm/alloc_c.cc b/libitm/alloc_c.cc
index 4160000..66d1109 100644
--- a/libitm/alloc_c.cc
+++ b/libitm/alloc_c.cc
@@ -64,7 +64,7 @@  void ITM_REGPARM
 _ITM_dropReferences (void *ptr, size_t len)
 {
   gtm_transaction *tx = gtm_tx();
-  if (!gtm_disp()->trydropreference (ptr, len))
+  if (!abi_disp()->trydropreference (ptr, len))
     tx->restart (RESTART_VALIDATE_READ);
   tx->drop_references_local (ptr, len);
   tx->drop_references_allocations (ptr);
diff --git a/libitm/barrier.tpl b/libitm/barrier.tpl
index 50bd89b..f74fcbc 100644
--- a/libitm/barrier.tpl
+++ b/libitm/barrier.tpl
@@ -30,12 +30,12 @@  namespace {
 using namespace GTM;
 
 template<typename T>
-T do_read (const T *ptr, gtm_dispatch::lock_type lock)
+T do_read (const T *ptr, abi_dispatch::lock_type lock)
 {
   //
   // Find the cacheline that holds the current value of *PTR.
   //
-  gtm_dispatch *disp = gtm_disp();
+  abi_dispatch *disp = abi_disp();
   uintptr_t iptr = reinterpret_cast<uintptr_t>(ptr);
   // Normalize PTR by chopping off the bottom bits so we can search
   // for PTR in the cacheline hash.
@@ -95,19 +95,19 @@  T do_read (const T *ptr, gtm_dispatch::lock_type lock)
 }
 
 template<typename T>
-void do_write (T *ptr, T val, gtm_dispatch::lock_type lock)
+void do_write (T *ptr, T val, abi_dispatch::lock_type lock)
 {
   // Note: See comments for do_read() above for hints on this
   // function.  Ideally we should abstract out a lot out of these two
   // functions, and avoid all this duplication.
 
-  gtm_dispatch *disp = gtm_disp();
+  abi_dispatch *disp = abi_disp();
   uintptr_t iptr = reinterpret_cast<uintptr_t>(ptr);
   uintptr_t iline = iptr & -CACHELINE_SIZE;
   uintptr_t iofs = iptr & (CACHELINE_SIZE - 1);
   gtm_cacheline *pline = reinterpret_cast<gtm_cacheline *>(iline);
   gtm_cacheline_mask m = ((gtm_cacheline_mask)2 << (sizeof(T) - 1)) - 1;
-  gtm_dispatch::mask_pair pair = disp->write_lock(pline, lock);
+  abi_dispatch::mask_pair pair = disp->write_lock(pline, lock);
 
   ptr = reinterpret_cast<T *>(&pair.line->b[iofs]);
 
@@ -129,7 +129,7 @@  void do_write (T *ptr, T val, gtm_dispatch::lock_type lock)
   else
     {
       *pair.mask |= m << iofs;
-      gtm_dispatch::mask_pair pair2 = disp->write_lock(pline + 1, lock);
+      abi_dispatch::mask_pair pair2 = disp->write_lock(pline + 1, lock);
 
       uintptr_t ileft = CACHELINE_SIZE - iofs;
       *pair2.mask |= m >> ileft;
@@ -151,13 +151,13 @@  void do_write (T *ptr, T val, gtm_dispatch::lock_type lock)
 #define ITM_READ(T, LOCK)						\
   _ITM_TYPE_##T ITM_REGPARM _ITM_##LOCK##T (const _ITM_TYPE_##T *ptr)	\
   {									\
-    return do_read (ptr, gtm_dispatch::LOCK);				\
+    return do_read (ptr, abi_dispatch::LOCK);				\
   }
 
 #define ITM_WRITE(T, LOCK)						\
   void ITM_REGPARM _ITM_##LOCK##T (_ITM_TYPE_##T *ptr, _ITM_TYPE_##T val) \
   {									\
-    do_write (ptr, val, gtm_dispatch::LOCK);				\
+    do_write (ptr, val, abi_dispatch::LOCK);				\
   }
 
 #define ITM_BARRIERS(T)		\
diff --git a/libitm/beginend.cc b/libitm/beginend.cc
index f9ad09a..37e23e2 100644
--- a/libitm/beginend.cc
+++ b/libitm/beginend.cc
@@ -94,7 +94,7 @@  GTM::gtm_transaction::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
   static const _ITM_transactionId_t tid_block_size = 1 << 16;
 
   gtm_transaction *tx;
-  gtm_dispatch *disp;
+  abi_dispatch *disp;
   uint32_t ret;
 
   gtm_thread *thr = setup_gtm_thr ();
@@ -154,7 +154,7 @@  GTM::gtm_transaction::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
       ret = a_runInstrumentedCode | a_saveLiveVariables;
     }
 
-  set_gtm_disp (disp);
+  set_abi_disp (disp);
 
   return ret;
 }
@@ -162,7 +162,7 @@  GTM::gtm_transaction::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
 void
 GTM::gtm_transaction::rollback ()
 {
-  gtm_disp()->rollback ();
+  abi_disp()->rollback ();
   rollback_local ();
 
   free_actions (&this->commit_actions);
@@ -202,7 +202,7 @@  _ITM_abortTransaction (_ITM_abortReason reason)
     abort ();
 
   tx->rollback ();
-  gtm_disp()->fini ();
+  abi_disp()->fini ();
 
   if (tx->state & gtm_transaction::STATE_SERIAL)
     gtm_transaction::serial_lock.write_unlock ();
@@ -218,7 +218,7 @@  _ITM_abortTransaction (_ITM_abortReason reason)
 bool
 GTM::gtm_transaction::trycommit ()
 {
-  if (gtm_disp()->trycommit ())
+  if (abi_disp()->trycommit ())
     {
       commit_local ();
       free_actions (&this->undo_actions);
@@ -234,7 +234,7 @@  GTM::gtm_transaction::trycommit_and_finalize ()
 {
   if ((this->state & gtm_transaction::STATE_ABORTING) || trycommit ())
     {
-      gtm_disp()->fini ();
+      abi_disp()->fini ();
       set_gtm_tx (this->prev);
       delete this;
       if (this->state & gtm_transaction::STATE_SERIAL)
diff --git a/libitm/config/generic/tls.h b/libitm/config/generic/tls.h
index 145375f..82d7c66 100644
--- a/libitm/config/generic/tls.h
+++ b/libitm/config/generic/tls.h
@@ -38,7 +38,7 @@  struct gtm_thread
 #ifndef HAVE_ARCH_GTM_THREAD_DISP
   // The dispatch table for the STM implementation currently in use.  Elided
   // if the target provides some efficient mechanism for storing this.
-  gtm_dispatch *disp;
+  abi_dispatch *disp;
 #endif
 
   // The maximum number of free gtm_transaction structs to be kept.
@@ -87,8 +87,8 @@  static inline void set_gtm_tx(gtm_transaction *x) { gtm_thr()->tx = x; }
 #ifndef HAVE_ARCH_GTM_THREAD_DISP
 // If the target does not provide optimized access to the currently
 // active dispatch table, simply access via GTM_THR.
-static inline gtm_dispatch * gtm_disp() { return gtm_thr()->disp; }
-static inline void set_gtm_disp(gtm_dispatch *x) { gtm_thr()->disp = x; }
+static inline abi_dispatch * abi_disp() { return gtm_thr()->disp; }
+static inline void set_abi_disp(abi_dispatch *x) { gtm_thr()->disp = x; }
 #endif
 
 } // namespace GTM
diff --git a/libitm/config/x86/tls.h b/libitm/config/x86/tls.h
index ef51d33..712b1c2 100644
--- a/libitm/config/x86/tls.h
+++ b/libitm/config/x86/tls.h
@@ -88,14 +88,14 @@  static inline void set_gtm_tx(struct gtm_transaction *x)
   asm volatile (SEG_WRITE(11) : : "r"(x));
 }
 
-static inline struct gtm_dispatch *gtm_disp(void)
+static inline struct abi_dispatch *abi_disp(void)
 {
-  struct gtm_dispatch *r;
+  struct abi_dispatch *r;
   asm (SEG_DECODE_READ(12) : "=r"(r));
   return r;
 }
 
-static inline void set_gtm_disp(struct gtm_dispatch *x)
+static inline void set_abi_disp(struct abi_dispatch *x)
 {
   void *scratch;
   asm volatile (SEG_ENCODE_WRITE(12) : "=r"(scratch) : "0"(x));
diff --git a/libitm/libitm_i.h b/libitm/libitm_i.h
index 57f18c0..3291f1b 100644
--- a/libitm/libitm_i.h
+++ b/libitm/libitm_i.h
@@ -86,7 +86,7 @@  extern void * xrealloc (void *p, size_t s) __attribute__((malloc, nothrow));
 namespace GTM HIDDEN {
 
 // A dispatch table parameterizes the implementation of the STM.
-struct gtm_dispatch
+struct abi_dispatch
 {
  public:
   enum lock_type { NOLOCK, R, RaR, RaW, RfW, W, WaR, WaW };
@@ -102,8 +102,8 @@  struct gtm_dispatch
 
  private:
   // Disallow copies
-  gtm_dispatch(const gtm_dispatch &) = delete;
-  gtm_dispatch& operator=(const gtm_dispatch &) = delete;
+  abi_dispatch(const abi_dispatch &) = delete;
+  abi_dispatch& operator=(const abi_dispatch &) = delete;
 
  public:
   // The default version of these is pass-through.  This merely gives the
@@ -129,7 +129,7 @@  struct gtm_dispatch
  protected:
   const bool m_read_only;
   const bool m_write_through;
-  gtm_dispatch(bool ro, bool wt) : m_read_only(ro), m_write_through(wt) { }
+  abi_dispatch(bool ro, bool wt) : m_read_only(ro), m_write_through(wt) { }
 
   static gtm_cacheline_mask mask_sink;
 };
@@ -287,9 +287,9 @@  extern void GTM_error (const char *fmt, ...)
 extern void GTM_fatal (const char *fmt, ...)
 	__attribute__((noreturn, format (printf, 1, 2)));
 
-extern gtm_dispatch *dispatch_wbetl();
-extern gtm_dispatch *dispatch_readonly();
-extern gtm_dispatch *dispatch_serial();
+extern abi_dispatch *dispatch_wbetl();
+extern abi_dispatch *dispatch_readonly();
+extern abi_dispatch *dispatch_serial();
 
 extern gtm_cacheline_mask gtm_mask_stack(gtm_cacheline *, gtm_cacheline_mask);
 
diff --git a/libitm/memcpy.cc b/libitm/memcpy.cc
index bbba938..f139df5 100644
--- a/libitm/memcpy.cc
+++ b/libitm/memcpy.cc
@@ -28,9 +28,9 @@  using namespace GTM;
 
 static void
 do_memcpy (uintptr_t idst, uintptr_t isrc, size_t size,
-	   gtm_dispatch::lock_type W, gtm_dispatch::lock_type R)
+	   abi_dispatch::lock_type W, abi_dispatch::lock_type R)
 {
-  gtm_dispatch *disp = gtm_disp();
+  abi_dispatch *disp = abi_disp();
   // The position in the destination cacheline where *IDST starts.
   uintptr_t dofs = idst & (CACHELINE_SIZE - 1);
   // The position in the source cacheline where *ISRC starts.
@@ -40,7 +40,7 @@  do_memcpy (uintptr_t idst, uintptr_t isrc, size_t size,
   gtm_cacheline *dst
     = reinterpret_cast<gtm_cacheline *>(idst & -CACHELINE_SIZE);
   const gtm_cacheline *sline;
-  gtm_dispatch::mask_pair dpair;
+  abi_dispatch::mask_pair dpair;
 
   if (size == 0)
     return;
@@ -177,12 +177,12 @@  do_memcpy (uintptr_t idst, uintptr_t isrc, size_t size,
 
 static void
 do_memmove (uintptr_t idst, uintptr_t isrc, size_t size,
-	    gtm_dispatch::lock_type W, gtm_dispatch::lock_type R)
+	    abi_dispatch::lock_type W, abi_dispatch::lock_type R)
 {
-  gtm_dispatch *disp = gtm_disp();
+  abi_dispatch *disp = abi_disp();
   uintptr_t dleft, sleft, sofs, dofs;
   const gtm_cacheline *sline;
-  gtm_dispatch::mask_pair dpair;
+  abi_dispatch::mask_pair dpair;
   
   if (size == 0)
     return;
@@ -194,13 +194,13 @@  do_memmove (uintptr_t idst, uintptr_t isrc, size_t size,
   if (__builtin_expect (idst == isrc, 0))
     {
       /* If the write lock is already acquired, nothing to do.  */
-      if (W == gtm_dispatch::WaW)
+      if (W == abi_dispatch::WaW)
 	return;
       /* If the destination is protected, acquire a write lock.  */
-      if (W != gtm_dispatch::NOLOCK)
-	R = gtm_dispatch::RfW;
+      if (W != abi_dispatch::NOLOCK)
+	R = abi_dispatch::RfW;
       /* Notice serial mode, where we don't acquire locks at all.  */
-      if (R == gtm_dispatch::NOLOCK)
+      if (R == abi_dispatch::NOLOCK)
 	return;
 
       idst = isrc + size;
@@ -337,12 +337,12 @@  do_memmove (uintptr_t idst, uintptr_t isrc, size_t size,
 void ITM_REGPARM _ITM_memcpy##NAME(void *dst, const void *src, size_t size)  \
 {									     \
   do_memcpy ((uintptr_t)dst, (uintptr_t)src, size,			     \
-	     gtm_dispatch::WRITE, gtm_dispatch::READ);			     \
+	     abi_dispatch::WRITE, abi_dispatch::READ);			     \
 }									     \
 void ITM_REGPARM _ITM_memmove##NAME(void *dst, const void *src, size_t size) \
 {									     \
   do_memmove ((uintptr_t)dst, (uintptr_t)src, size,			     \
-	      gtm_dispatch::WRITE, gtm_dispatch::READ);			     \
+	      abi_dispatch::WRITE, abi_dispatch::READ);			     \
 }
 
 ITM_MEM_DEF(RnWt,	NOLOCK,		W)
diff --git a/libitm/memset.cc b/libitm/memset.cc
index 39ee72b..40cab1a 100644
--- a/libitm/memset.cc
+++ b/libitm/memset.cc
@@ -27,11 +27,11 @@ 
 using namespace GTM;
 
 static void
-do_memset(uintptr_t idst, int c, size_t size, gtm_dispatch::lock_type W)
+do_memset(uintptr_t idst, int c, size_t size, abi_dispatch::lock_type W)
 {
-  gtm_dispatch *disp = gtm_disp();
+  abi_dispatch *disp = abi_disp();
   uintptr_t dofs = idst & (CACHELINE_SIZE - 1);
-  gtm_dispatch::mask_pair dpair;
+  abi_dispatch::mask_pair dpair;
   gtm_cacheline *dst
     = reinterpret_cast<gtm_cacheline *>(idst & -CACHELINE_SIZE);
 
@@ -70,7 +70,7 @@  do_memset(uintptr_t idst, int c, size_t size, gtm_dispatch::lock_type W)
 #define ITM_MEM_DEF(WRITE) \
 void ITM_REGPARM _ITM_memset##WRITE(void *dst, int c, size_t size)	\
 {									\
-  do_memset ((uintptr_t)dst, c, size, gtm_dispatch::WRITE);		\
+  do_memset ((uintptr_t)dst, c, size, abi_dispatch::WRITE);		\
 }
 
 ITM_MEM_DEF(W)
diff --git a/libitm/method-readonly.cc b/libitm/method-readonly.cc
index 5f33305..ed573e2 100644
--- a/libitm/method-readonly.cc
+++ b/libitm/method-readonly.cc
@@ -28,7 +28,7 @@  namespace {
 
 using namespace GTM;
 
-class readonly_dispatch : public gtm_dispatch
+class readonly_dispatch : public abi_dispatch
 {
  private:
   gtm_version m_start;
@@ -47,7 +47,7 @@  class readonly_dispatch : public gtm_dispatch
 
 inline
 readonly_dispatch::readonly_dispatch()
-  : gtm_dispatch(true, true), m_start(gtm_get_clock ())
+  : abi_dispatch(true, true), m_start(gtm_get_clock ())
 { }
 
 
@@ -70,14 +70,14 @@  readonly_dispatch::read_lock(const gtm_cacheline *line, lock_type lock)
     }
 }
 
-gtm_dispatch::mask_pair
+abi_dispatch::mask_pair
 readonly_dispatch::write_lock(gtm_cacheline *line, lock_type lock)
 {
   switch (lock)
     {
     case NOLOCK:
       {
-	gtm_dispatch::mask_pair pair;
+	abi_dispatch::mask_pair pair;
 	pair.line = line;
 	pair.mask = &mask_sink;
 	return pair;
@@ -117,7 +117,7 @@  readonly_dispatch::fini ()
 
 } // anon namespace
 
-gtm_dispatch *
+abi_dispatch *
 GTM::dispatch_readonly ()
 {
   return new readonly_dispatch();
diff --git a/libitm/method-serial.cc b/libitm/method-serial.cc
index cdeccc6..b46de4d 100644
--- a/libitm/method-serial.cc
+++ b/libitm/method-serial.cc
@@ -26,23 +26,23 @@ 
 
 namespace GTM HIDDEN {
 
-gtm_cacheline_mask gtm_dispatch::mask_sink;
+gtm_cacheline_mask abi_dispatch::mask_sink;
 
 const gtm_cacheline *
-gtm_dispatch::read_lock(const gtm_cacheline *addr, lock_type)
+abi_dispatch::read_lock(const gtm_cacheline *addr, lock_type)
 {
   return addr;
 }
 
-gtm_dispatch::mask_pair
-gtm_dispatch::write_lock(gtm_cacheline *addr, lock_type)
+abi_dispatch::mask_pair
+abi_dispatch::write_lock(gtm_cacheline *addr, lock_type)
 {
   return mask_pair (addr, &mask_sink);
 }
 
 } // namespace GTM
 
-// Avoid a dependency on libstdc++ for the pure virtuals in gtm_dispatch.
+// Avoid a dependency on libstdc++ for the pure virtuals in abi_dispatch.
 extern "C" void HIDDEN
 __cxa_pure_virtual ()
 {
@@ -53,10 +53,10 @@  using namespace GTM;
 
 namespace {
 
-class serial_dispatch : public gtm_dispatch
+class serial_dispatch : public abi_dispatch
 {
  public:
-  serial_dispatch() : gtm_dispatch(false, true) { }
+  serial_dispatch() : abi_dispatch(false, true) { }
 
   // The read_lock and write_lock methods are implented by the base class.
 
@@ -71,7 +71,7 @@  class serial_dispatch : public gtm_dispatch
 
 static const serial_dispatch o_serial_dispatch;
 
-gtm_dispatch *
+abi_dispatch *
 GTM::dispatch_serial ()
 {
   return const_cast<serial_dispatch *>(&o_serial_dispatch);
@@ -82,7 +82,7 @@  GTM::dispatch_serial ()
 void
 GTM::gtm_transaction::serialirr_mode ()
 {
-  struct gtm_dispatch *disp = gtm_disp ();
+  struct abi_dispatch *disp = abi_disp ();
   bool need_restart = true;
 
   if (this->state & STATE_SERIAL)
@@ -111,7 +111,7 @@  GTM::gtm_transaction::serialirr_mode ()
   else
     {
       this->state |= (STATE_SERIAL | STATE_IRREVOCABLE);
-      set_gtm_disp (dispatch_serial ());
+      set_abi_disp (dispatch_serial ());
     }
 }
 
diff --git a/libitm/method-wbetl.cc b/libitm/method-wbetl.cc
index a0359a3..7ce317e 100644
--- a/libitm/method-wbetl.cc
+++ b/libitm/method-wbetl.cc
@@ -28,7 +28,7 @@  namespace {
 
 using namespace GTM;
 
-class wbetl_dispatch : public gtm_dispatch
+class wbetl_dispatch : public abi_dispatch
 {
  private:
   static const size_t RW_SET_SIZE = 4096;
@@ -394,7 +394,7 @@  wbetl_dispatch::read_lock (const gtm_cacheline *addr, lock_type ltype)
     }
 }
 
-gtm_dispatch::mask_pair
+abi_dispatch::mask_pair
 wbetl_dispatch::write_lock (gtm_cacheline *addr, lock_type ltype)
 {
   gtm_cacheline *line;
@@ -547,7 +547,7 @@  wbetl_dispatch::trydropreference (void *ptr, size_t size)
   gtm_cacheline *src
     = reinterpret_cast<gtm_cacheline *>(isrc & -CACHELINE_SIZE);
   unsigned char *dst = (unsigned char *)ptr;
-  gtm_dispatch::mask_pair pair;
+  abi_dispatch::mask_pair pair;
 
   // If we're trying to drop a reference, we should already have a
   // write lock on it.  If we don't have one, there's no work to do.
@@ -602,7 +602,7 @@  wbetl_dispatch::trydropreference (void *ptr, size_t size)
 
 
 wbetl_dispatch::wbetl_dispatch ()
-  : gtm_dispatch (false, false)
+  : abi_dispatch (false, false)
 {
   m_rset_entries = (r_entry *) xmalloc (RW_SET_SIZE * sizeof(r_entry));
   m_rset_nb_entries = 0;
@@ -621,7 +621,7 @@  wbetl_dispatch::wbetl_dispatch ()
 
 } // anon namespace
 
-gtm_dispatch *
+abi_dispatch *
 GTM::dispatch_wbetl ()
 {
   return new wbetl_dispatch ();
diff --git a/libitm/retry.cc b/libitm/retry.cc
index f602f54..a25b282 100644
--- a/libitm/retry.cc
+++ b/libitm/retry.cc
@@ -27,7 +27,7 @@ 
 void
 GTM::gtm_transaction::decide_retry_strategy (gtm_restart_reason r)
 {
-  struct gtm_dispatch *disp = gtm_disp ();
+  struct abi_dispatch *disp = abi_disp ();
 
   this->restart_reason[r]++;
   this->restart_total++;
@@ -61,7 +61,7 @@  GTM::gtm_transaction::decide_retry_strategy (gtm_restart_reason r)
       this->state = (STATE_SERIAL | STATE_IRREVOCABLE);
       disp->fini ();
       disp = dispatch_serial ();
-      set_gtm_disp (disp);
+      set_abi_disp (disp);
     }
   else
     {
@@ -72,7 +72,7 @@  GTM::gtm_transaction::decide_retry_strategy (gtm_restart_reason r)
 	    {
 	      disp->fini ();
 	      disp = dispatch_wbetl ();
-	      set_gtm_disp (disp);
+	      set_abi_disp (disp);
 	      return;
 	    }
 	}