Index: libgupc/portals4/gupcr_lock.h
===================================================================
--- libgupc/portals4/gupcr_lock.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_lock.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,40 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef _GUPCR_LOCK_H_
+#define _GUPCR_LOCK_H_
+
+/**
+ * @file gupcr_lock.h
+ * GUPC Portals4 lock external definitions.
+ */
+
+/* Heap allocation locks.  */
+extern upc_lock_t *gupcr_heap_region_lock;
+extern upc_lock_t *gupcr_global_heap_lock;
+extern upc_lock_t *gupcr_local_heap_lock;
+
+#endif /* gupcr_lock.h */
Index: libgupc/portals4/gupcr_lock.upc
===================================================================
--- libgupc/portals4/gupcr_lock.upc	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_lock.upc	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,539 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#include <upc.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdio.h>
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_lock_sup.h"
+#include "gupcr_lock.h"
+#include "gupcr_barrier.h"
+
+/**
+ * @file gupcr_lock.upc
+ * GUPC Portals4 UPC lock implementation.
+ *
+ * The GUPC lock functions use MCS locks as described in the
+ * Mellor-Crummey and Scott paper: Algorithms for Scalable Synchronization
+ * on Shared-Memory Multiprocessors, ACM Transaction on Computer Systems,
+ * February 1991.
+ *
+ * A lock is a simple data structure that lives in the shared memory space.
+ * A pointer is used to point to the last thread on the waiting list.
+ * A lock is available if this pointer is NULL.  The following Portals
+ * atomic operations are used:
+ * - SWAP - determine if the lock is available
+ * - CSWAP - determine if the lock can be released
+ *
+ * The GUPC Portals implementation of UPC locks has the following
+ * characteristics:
+ * - The lock object has affinity to the thread that creates the lock.
+ *   If the collective function, upc_all_lock_alloc, is called, then
+ *   the lock object will have affinity to thread 0.
+ * - A thread's lock waiting queue link object has affinity to the
+ *   waiting thread.
+ * - Portals atomic functions (SWAP and CSWAP) are used to
+ *   guarantee fair access and FIFO ordering for all waiting threads.
+ * - A special Portals Table Entry (GUPCR_PTL_PTE_LOCK) is used to provide
+ *   for signaling threads taken off the waiting list.
+ *
+ * @addtogroup LOCK GUPCR Lock Functions
+ * @{
+ */
+
+typedef struct gupcr_lock_link_struct gupcr_lock_link_t;
+typedef shared [] gupcr_lock_link_t *gupcr_lock_link_ref;
+
+/** upc_lock_t is an opaque shared type.  The 'upc_lock_struct'
+    structure describes the internal representation of the
+    UPC lock type.  */
+struct upc_lock_struct
+{
+  gupcr_lock_link_ref last;	/* Must be first.  */
+  gupcr_lock_link_ref owner_link;
+  upc_lock_t *free_link;
+};
+
+struct gupcr_lock_link_struct
+{
+  gupcr_lock_link_ref next;	/* Must be first.  */
+  int signal;			/* Signal the lock ownership.  */
+  int free;			/* Lock has been freed.  */
+};
+
+struct gupcr_lock_link_cache_struct
+{
+  gupcr_lock_link_t lock_links[GUPCR_MAX_LOCKS];
+};
+typedef struct gupcr_lock_link_cache_struct gupcr_lock_link_cache_t;
+typedef shared gupcr_lock_link_cache_t *gupcr_lock_link_cache_ref;
+
+/** Array of lock links managed as a per-thread free list.  */
+static shared gupcr_lock_link_cache_t gupcr_lock_link_cache[THREADS];
+/** Per thread lock link free list.  */
+static gupcr_lock_link_ref gupcr_lock_links;
+
+/** UPC lock free list.  */
+static upc_lock_t *lock_free;
+
+/* Heap allocator locks.  */
+/** Heap region allocation lock.  */
+shared upc_lock_t gupcr_heap_region_lock_data;
+upc_lock_t *gupcr_heap_region_lock;
+/** Global heap lock.  */
+shared upc_lock_t gupcr_global_heap_lock_data;
+upc_lock_t *gupcr_global_heap_lock;
+/** Local heap locks.  */
+shared upc_lock_t gupcr_local_heap_lock_data[THREADS];
+upc_lock_t *gupcr_local_heap_lock;
+
+/**
+ * Initialize the heap allocator locks.
+ *
+ * All shared references must be local due to the fact
+ * this is called before Portals has been initialized.
+ */
+void
+gupcr_lock_heap_sup_init (void)
+{
+  if (!MYTHREAD)
+    {
+      upc_memset (&gupcr_heap_region_lock_data, '\0', sizeof (upc_lock_t));
+      upc_memset (&gupcr_global_heap_lock_data, '\0', sizeof (upc_lock_t));
+    }
+  gupcr_heap_region_lock = &gupcr_heap_region_lock_data;
+  gupcr_global_heap_lock = &gupcr_global_heap_lock_data;
+
+  upc_memset (&gupcr_local_heap_lock_data[MYTHREAD],
+	      '\0', sizeof (upc_lock_t));
+  gupcr_local_heap_lock = &gupcr_local_heap_lock_data[MYTHREAD];
+}
+
+/**
+ * Initialize the local lock free list.
+ */
+void
+gupcr_lock_free_init (void)
+{
+  lock_free = NULL;
+}
+
+/**
+ * Initialize the local lock link free list.
+ *
+ * gupcr_lock_link_init() is called before the UPC runtime
+ * is fully initialized.  Care is taken to make no UPC shared
+ * memory accesses.
+ */
+void
+gupcr_lock_link_init (void)
+{
+  gupcr_lock_link_ref link;
+  gupcr_lock_link_t *local_link;
+  gupcr_lock_links = gupcr_lock_link_cache[MYTHREAD].lock_links;
+  link = gupcr_lock_links;
+  local_link = (gupcr_lock_link_t *) link;
+  memset (local_link, '\0', sizeof (gupcr_lock_link_cache_t));
+  for (int i = 0; i < (GUPCR_MAX_LOCKS - 1); ++i)
+    (local_link++)->next = ++link;
+}
+
+/**
+ * Release the lock link record.
+ */
+static inline void
+gupcr_lock_link_free (gupcr_lock_link_ref link)
+{
+  link->next = gupcr_lock_links;
+  gupcr_lock_links = link;
+}
+
+/**
+ * Allocate a lock link record.
+ *
+ * A lock link is a data structure used to link together
+ * all threads waiting on a particular lock.
+ * Lock links are located in the shared space of the thread
+ * acquiring the lock.
+ *
+ * Lock links are kept in a locally managed list
+ * (used as a cache) rooted at 'gupcr_lock_links'.
+ * This locally managed free list avoids the need
+ * to call upc_alloc().
+ */
+static inline gupcr_lock_link_ref
+gupcr_lock_link_alloc (void)
+{
+  gupcr_lock_link_ref link;
+  link = gupcr_lock_links;
+  if (!link)
+    {
+      /* Try to find a link block that has been freed by
+         other threads and thus not returned to the free list.  */
+      gupcr_lock_link_ref mlink = (gupcr_lock_link_ref)
+	&gupcr_lock_link_cache[MYTHREAD].lock_links;
+      for (int i = 0; i < (GUPCR_MAX_LOCKS - 1); ++i)
+	{
+	  if (mlink->free)
+	    {
+	      mlink->free = 0;
+	      gupcr_lock_link_free (mlink);
+	    }
+	  mlink++;
+	}
+      link = gupcr_lock_links;
+      if (!link)
+	gupcr_fatal_error ("cannot allocate a UPC lock link. "
+			   "The number of allocated per thread lock links "
+			   "exceeds the configuration defined "
+			   "maximum of %d entries.",
+			   GUPCR_MAX_LOCKS);
+    }
+  gupcr_lock_links = link->next;
+  return link;
+}
+
+/**
+ * Allocate a lock and return a pointer to it.
+ *
+ * The 'upc_global_lock_alloc' function dynamically allocates a lock and
+ * returns a pointer to it.  The lock is created in an unlocked state.
+ * The 'upc_global_lock_alloc' function is not a collective function.
+ * If called by multiple threads, each thread will receive a pointer
+ * to a unique lock.
+ *
+ * @retval Pointer to a newly allocated lock
+ */
+upc_lock_t *
+upc_global_lock_alloc (void)
+{
+  upc_lock_t *lock;
+  gupcr_trace (FC_LOCK, "LOCK GLOBAL_ALLOC ENTER");
+  /* Check if there is a lock cached on the free list.  */
+  if (lock_free)
+    {
+      lock = lock_free;
+      lock_free = lock->free_link;
+    }
+  else
+    {
+      /* Allocate space for the lock from shared memory with
+         affinity to the calling thread.  */
+      lock = upc_alloc (sizeof (upc_lock_t));
+      if (lock == NULL)
+	gupcr_fatal_error ("cannot allocate memory for the lock");
+    }
+  lock->last = NULL;
+  lock->owner_link = NULL;
+  gupcr_trace (FC_LOCK, "LOCK GLOBAL_ALLOC EXIT %lu:0x%lx",
+	       (long unsigned) upc_threadof (lock),
+	       (long unsigned) upc_addrfield (lock));
+  return lock;
+}
+
+void
+upc_all_lock_free (upc_lock_t *ptr)
+{
+  upc_all_free (ptr);
+}
+
+/**
+ * Free all lock resources.
+ *
+ * The 'upc_lock_free' function frees all resources associated with the
+ * dynamically allocated 'upc_lock_t' pointed to by 'lock'. If 'lock' is a
+ * null pointer, no action occurs.  Otherwise, if the argument does not
+ * match a pointer earlier returned by the 'upc_global_lock_alloc' or
+ * 'upc_all_lock_alloc' function, or if the lock has been de-allocated by
+ * a previous call to 'upc_lock_free' the behavior is undefined.
+ *
+ * @param [in] lock Pointer to a lock
+ */
+void
+upc_lock_free (upc_lock_t *lock)
+{
+  gupcr_lock_link_ref link;
+
+  gupcr_trace (FC_LOCK, "LOCK FREE ENTER %lu:0x%lx",
+	       (long unsigned) upc_threadof (lock),
+	       (long unsigned) upc_addrfield (lock));
+  if (lock == NULL)
+    return;
+
+  link = lock->owner_link;
+  /* Release the link block if this thread owns the lock.  */
+  if (link)
+    {
+      if (MYTHREAD == (int) upc_threadof (link))
+	{
+	  gupcr_lock_link_free (link);
+	}
+      else
+	link->free = 1;
+    }
+  if (MYTHREAD == (int) upc_threadof (lock))
+    {
+      /* Release it on the local free list.  */
+      lock->free_link = lock_free;
+      lock_free = lock;
+    }
+  else
+    upc_free (lock);
+
+  gupcr_trace (FC_LOCK, "LOCK FREE EXIT");
+}
+
+/**
+ * Allocate a lock and return a pointer to it on all threads.
+ *
+ * The 'upc_all_lock_alloc' function dynamically allocates a lock
+ * and returns a pointer to it.  The lock is created in an unlocked state.
+ * 'upc_all_lock_alloc' is a collective function.
+ * The return value on every thread points to the same lock object.
+ *
+ * @retval Pointer to a newly allocated lock
+ */
+upc_lock_t *
+upc_all_lock_alloc (void)
+{
+  upc_lock_t *lock;
+  gupcr_trace (FC_LOCK, "LOCK ALL_ALLOC ENTER");
+  /* Allocate space for the lock from the shared memory of
+     thread 0 and broadcast its address.  */
+  if (MYTHREAD == 0)
+    {
+      if (lock_free)
+	{
+	  lock = lock_free;
+	  lock_free = lock->free_link;
+	}
+      else
+	{
+	  lock = upc_alloc (sizeof (upc_lock_t));
+	  if (lock == NULL)
+	    gupcr_fatal_error ("cannot allocate memory for the lock");
+	}
+      lock->last = NULL;
+      lock->owner_link = NULL;
+      gupcr_bcast_send (&lock, sizeof (lock));
+    }
+  else
+    gupcr_bcast_recv (&lock, sizeof (lock));
+  gupcr_trace (FC_LOCK, "LOCK ALL_ALLOC EXIT %lu:0x%lx",
+	       (long unsigned) upc_threadof (lock),
+	       (long unsigned) upc_addrfield (lock));
+  return lock;
+}
+
+/**
+ * Set the state of the lock to locked.
+ *
+ * If the lock is already in a 'locked' state due to the calling thread
+ * setting it to the 'locked' state, the result is undefined.
+ * If the lock is already in a 'locked' state, then the calling thread
+ * waits for some other thread to set the state to 'unlocked'.
+ * Once the lock is in the state 'unlocked', a single calling thread
+ * sets the state to 'locked' and the function returns.
+ * A null strict access is implied after a call to 'upc_lock'.
+ *
+ * @param [in] lock Pointer to a lock
+ */
+void
+upc_lock (upc_lock_t *lock)
+{
+  gupcr_lock_link_ref link, old_link;
+  shared [] gupcr_lock_link_ref *lock_last_addr;
+  size_t lock_last_thread, lock_last_offset;
+  gupcr_trace (FC_LOCK, "LOCK LOCK ENTER %lu:0x%lx",
+	       (long unsigned) upc_threadof (lock),
+	       (long unsigned) upc_addrfield (lock));
+  if (lock == NULL)
+    gupcr_fatal_error ("NULL lock pointer");
+  /* Allocate space for the lock waiting queue link.
+     It will have affinity to the calling thread.  */
+  link = gupcr_lock_link_alloc ();
+  if (link == NULL)
+    gupcr_fatal_error ("cannot allocate memory for the lock link");
+  link->next = NULL;
+  link->signal = 0;
+  /* Atomically set the lock value to point to the
+     calling thread's link queue object and
+     return the previous value of the lock link.  */
+  lock_last_addr = &lock->last;
+  lock_last_thread = upc_threadof (lock_last_addr);
+  lock_last_offset = upc_addrfield (lock_last_addr);
+  gupcr_lock_swap (lock_last_thread, lock_last_offset,
+		   &link, &old_link, sizeof (link));
+  if (old_link != NULL)
+    {
+      shared [] gupcr_lock_link_ref *old_link_next_addr;
+      size_t old_link_next_thread, old_link_next_offset;
+      /* We have to wait.  Clear the ownership signal field
+         and insert our pointer into the predecessor's link.  */
+      link->signal = 0;
+      upc_fence;
+      old_link_next_addr = &old_link->next;
+      old_link_next_thread = upc_threadof (old_link_next_addr);
+      old_link_next_offset = upc_addrfield (old_link_next_addr);
+      gupcr_lock_put (old_link_next_thread, old_link_next_offset,
+		      &link, sizeof (link));
+      /* At this point the thread has to wait until the lock is
+         is released.  Process counting events one by one until
+         the value of the signal word changes.  */
+      do
+	{
+	  gupcr_lock_wait ();
+	  upc_fence;
+	}
+      while (!link->signal);
+    }
+  lock->owner_link = link;
+  gupcr_trace (FC_LOCK, "LOCK LOCK EXIT");
+  upc_fence;
+}
+
+/**
+ * Attempt to set the state of the lock to locked.
+ *
+ * The 'upc_lock_attempt' function attempts to set the state of the lock
+ * pointed to by 'lock' to 'locked'.  If the lock is already in the 'locked'
+ * state due to the calling thread setting it to the 'locked' state, the
+ * result is undefined.  If the lock is already in the 'locked' state, the
+ * function returns 0.  If the lock is in the state 'unlocked',
+ * a single calling thread sets the state to 'locked' and the function
+ * returns 1.  A null strict access is implied after a call to
+ * 'upc_lock_attempt' that returns 1.
+ *
+ * @param [in] lock Pointer to a lock
+ * @retval Lock attempt result
+ *   - 1, lock was acquired successfully
+ *   - 0, lock was not acquired
+ */
+int
+upc_lock_attempt (upc_lock_t *lock)
+{
+  gupcr_lock_link_ref link;
+  gupcr_lock_link_ref null_link = NULL;
+  shared [] gupcr_lock_link_ref *lock_last_addr;
+  size_t lock_last_thread, lock_last_offset;
+  int compare_ok;
+  gupcr_trace (FC_LOCK, "LOCK ATTEMPT ENTER %lu:0x%lx",
+	       (long unsigned) upc_threadof (lock),
+	       (long unsigned) upc_addrfield (lock));
+  if (lock == NULL)
+    gupcr_fatal_error ("NULL lock pointer");
+  /* Allocate space for the lock waiting queue with affinity
+     to the calling thread.  */
+  link = gupcr_lock_link_alloc ();
+  if (link == NULL)
+    gupcr_fatal_error ("cannot allocate memory for the lock link");
+  link->next = NULL;
+  link->signal = 0;
+  /* Atomically set the lock value to the link entry and
+     return the previous value of the lock ONLY if the value
+     of the lock is already NULL.  */
+  lock_last_addr = &lock->last;
+  lock_last_thread = upc_threadof (lock_last_addr);
+  lock_last_offset = upc_addrfield (lock_last_addr);
+  compare_ok = gupcr_lock_cswap (lock_last_thread, lock_last_offset,
+				 &null_link, &link, sizeof (link));
+  if (compare_ok)
+    {
+      lock->owner_link = link;
+      upc_fence;
+      gupcr_trace (FC_LOCK, "LOCK ATTEMPT EXIT 1");
+    }
+  else
+    {
+      gupcr_lock_link_free (link);
+      gupcr_trace (FC_LOCK, "LOCK ATTEMPT EXIT 0");
+    }
+  return compare_ok;
+}
+
+/**
+ * Set the state of the lock to unlocked.
+ *
+ * The 'upc_unlock' function sets the state of the lock pointed
+ * to by 'lock' to 'unlocked'.  Unless the lock is in 'locked' state
+ * and the calling thread is the locking thread, the result is undefined.
+ * A null strict access is implied before a call to 'upc_unlock'.
+ *
+ * @param [in] lock Pointer to a lock
+ */
+void
+upc_unlock (upc_lock_t *lock)
+{
+  gupcr_lock_link_ref link = lock->owner_link;
+  gupcr_lock_link_ref null_link = NULL;
+  shared [] gupcr_lock_link_ref *lock_last_addr = &lock->last;
+  size_t lock_last_thread = upc_threadof (lock_last_addr);
+  size_t lock_last_offset = upc_addrfield (lock_last_addr);
+  int compare_ok;
+  gupcr_trace (FC_LOCK, "LOCK UNLOCK ENTER %lu:0x%lx",
+	       (long unsigned) upc_threadof (lock),
+	       (long unsigned) upc_addrfield (lock));
+  if (lock == NULL)
+    gupcr_fatal_error ("NULL lock pointer");
+  upc_fence;
+  /* Try to release the lock: write NULL into lock->last
+     if it contains a pointer to our own link block.  If it fails then
+     some other thread is on the waiting list.  */
+  lock->owner_link = NULL;
+  compare_ok = gupcr_lock_cswap (lock_last_thread, lock_last_offset,
+				 &link, &null_link, sizeof (link));
+  if (!compare_ok)
+    {
+      shared void *link_next_signal_addr;
+      size_t signal_addr_thread, signal_addr_offset;
+      int signal = 1;
+      /* Pass ownership to the next waiting thread.  Wait until
+         the link->next pointer is being set.  Use Portals call to
+	 avoid possibility of data tearing on pointer to shared.  */
+      for (;;)
+	{
+          size_t addr_offset;
+	  gupcr_lock_link_ref val;
+          addr_offset = upc_addrfield (&link->next);
+          gupcr_lock_get (MYTHREAD, addr_offset, &val, sizeof (val));
+	  if (val) break;
+	  gupcr_lock_wait ();
+	}
+      /* Signal the waiting thread that it now owns the lock.  */
+      link_next_signal_addr = &link->next->signal;
+      signal_addr_thread = upc_threadof (link_next_signal_addr);
+      signal_addr_offset = upc_addrfield (link_next_signal_addr);
+      gupcr_lock_put (signal_addr_thread, signal_addr_offset,
+		      &signal, sizeof (signal));
+    }
+  gupcr_lock_link_free (link);
+  gupcr_trace (FC_LOCK, "LOCK UNLOCK EXIT");
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_lock_sup.c
===================================================================
--- libgupc/portals4/gupcr_lock_sup.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_lock_sup.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,315 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_lib.h"
+#include "gupcr_lock_sup.h"
+#include "gupcr_sup.h"
+#include "gupcr_portals.h"
+#include "gupcr_gmem.h"
+#include "gupcr_utils.h"
+#include "gupcr_lock_sup.h"
+
+/**
+ * @file gupcr_lock_sup.c
+ * GUPC Portals4 locks implementation support routines.
+ *
+ * @addtogroup LOCK GUPCR Lock Functions
+ * @{
+ */
+
+/** Lock shared access LE handle */
+static ptl_handle_le_t gupcr_lock_le;
+/** Lock shared access LE counting events handle */
+static ptl_handle_ct_t gupcr_lock_le_ct;
+/** Lock shared access LE counting events counter */
+static ptl_size_t gupcr_lock_le_count;
+/** Lock shared access LE events queue handle */
+static ptl_handle_eq_t gupcr_lock_le_eq;
+
+/** Lock buffer for CSWAP operation */
+static char gupcr_lock_buf[16];
+/** Lock shared access MD handle */
+static ptl_handle_md_t gupcr_lock_md;
+/** Lock shared access MD counting events handle */
+static ptl_handle_ct_t gupcr_lock_md_ct;
+/** Lock shared access MD counting events counter */
+static ptl_size_t gupcr_lock_md_count;
+/** Lock shared access MD event queue handle */
+static ptl_handle_eq_t gupcr_lock_md_eq;
+
+/**
+ * Execute lock-related atomic fetch and store remote operation.
+ *
+ * Value "val" is written into the specified remote location and the
+ * old value is returned.
+ *
+ * A Portals 'swap atomic' operation is used when the acquiring thread must
+ * atomically determine if the lock is available.  A pointer to the thread's
+ * local lock waiting list link is atomically written into the lock's 'last'
+ * field, and the current value of the 'last' field is returned.  If NULL,
+ * the acquiring thread is the new owner, otherwise it must insert itself
+ * onto the waiting list.
+ */
+void
+gupcr_lock_swap (size_t dest_thread,
+		 size_t dest_offset, void *val, void *old, size_t size)
+{
+  ptl_process_t rpid;
+  ptl_ct_event_t ct;
+  gupcr_debug (FC_LOCK, "%lu:0x%lx",
+                        (long unsigned) dest_thread,
+                        (long unsigned) dest_offset);
+  rpid.rank = dest_thread;
+  gupcr_portals_call (PtlSwap, (gupcr_lock_md, (ptl_size_t) old,
+				gupcr_lock_md, (ptl_size_t) val, size, rpid,
+				GUPCR_PTL_PTE_GMEM, PTL_NO_MATCH_BITS,
+				dest_offset, PTL_NULL_USER_PTR, 0, NULL,
+				PTL_SWAP, gupcr_get_atomic_datatype (size)));
+  gupcr_lock_md_count += 1;
+  gupcr_portals_call (PtlCTWait,
+		      (gupcr_lock_md_ct, gupcr_lock_md_count, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_lock_md_eq);
+      gupcr_fatal_error ("received an error on lock MD");
+    }
+}
+
+/**
+ * Execute a lock-related atomic compare and swap operation.
+ *
+ * The value  pointed to by 'val' is written into the remote location
+ * given by ('dest_thread', 'dest_addr) only if value in the destination
+ * is identical to 'cmp'.
+ *
+ * A Portals compare and swap atomic operation is used during the lock
+ * release phase when the owner of the lock must atomically determine if
+ * there are threads waiting on the lock.  This is accomplished by using
+ * the Portals CSWAP atomic operation, where a NULL pointer is written
+ * into the lock's 'last' field only if this field contains the pointer
+ * to the owner's local lock link structure.
+ *
+ * @retval Return TRUE if the operation was successful.
+ */
+int
+gupcr_lock_cswap (size_t dest_thread,
+		  size_t dest_offset, void *cmp, void *val, size_t size)
+{
+  ptl_process_t rpid;
+  ptl_ct_event_t ct;
+  gupcr_debug (FC_LOCK, "%lu:0x%lx",
+                        (long unsigned) dest_thread,
+			(long unsigned) dest_offset);
+  rpid.rank = dest_thread;
+  gupcr_portals_call (PtlSwap, (gupcr_lock_md, (ptl_size_t) gupcr_lock_buf,
+				gupcr_lock_md, (ptl_size_t) val, size, rpid,
+				GUPCR_PTL_PTE_GMEM, PTL_NO_MATCH_BITS,
+				dest_offset, PTL_NULL_USER_PTR, 0, cmp,
+				PTL_CSWAP, gupcr_get_atomic_datatype (size)));
+  gupcr_lock_md_count += 1;
+  gupcr_portals_call (PtlCTWait,
+		      (gupcr_lock_md_ct, gupcr_lock_md_count, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_lock_md_eq);
+      gupcr_fatal_error ("received an error on lock MD");
+    }
+  return !memcmp (cmp, gupcr_lock_buf, size);
+}
+
+/*
+ * Execute a Portals put operation on the lock-related PTE.
+ *
+ * Execute a put operation on the PTE that is reserved for
+ * lock-related operations (PTL_PTE_UPC_LOCK).  This separate PTE is used
+ * to make it possible to count only Portals put operations on the
+ * 'signal' or 'next' words of a UPC lock wait list entry.
+ *
+ * gupcr_lock_put() is used to 'signal' the remote thread that:
+ * - ownership of the lock is passed to a remote thread if the remote
+ * thread is the next thread on the waiting list
+ * - a pointer to the calling thread's local control block has
+ * been appended to the lock's waiting list
+ */
+void
+gupcr_lock_put (size_t dest_thread, size_t dest_addr, void *val, size_t size)
+{
+  ptl_process_t rpid;
+  ptl_ct_event_t ct;
+  gupcr_debug (FC_LOCK, "%lu:0x%lx",
+                        (long unsigned) dest_thread,
+			(long unsigned) dest_addr);
+  rpid.rank = dest_thread;
+  gupcr_portals_call (PtlPut, (gupcr_lock_md, (ptl_size_t) val,
+			       size, PTL_ACK_REQ, rpid,
+			       GUPCR_PTL_PTE_LOCK, PTL_NO_MATCH_BITS,
+			       (ptl_size_t) dest_addr,
+			       PTL_NULL_USER_PTR, PTL_NULL_HDR_DATA));
+  gupcr_lock_md_count += 1;
+  gupcr_portals_call (PtlCTWait,
+		      (gupcr_lock_md_ct, gupcr_lock_md_count, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_lock_md_eq);
+      gupcr_fatal_error ("received an error on lock MD");
+    }
+}
+
+/*
+ * Execute a Portals get operation on the lock-related PTE.
+ *
+ * All operations on lock/link data structures must be performed
+ * through the Portals interface to prevent data tearing.
+ */
+void
+gupcr_lock_get (size_t dest_thread, size_t dest_addr, void *val, size_t size)
+{
+  ptl_process_t rpid;
+  ptl_ct_event_t ct;
+  gupcr_debug (FC_LOCK, "%lu:0x%lx",
+                        (long unsigned) dest_thread,
+			(long unsigned) dest_addr);
+  rpid.rank = dest_thread;
+  gupcr_portals_call (PtlGet, (gupcr_lock_md, (ptl_size_t) val,
+			       size, rpid,
+			       GUPCR_PTL_PTE_LOCK, PTL_NO_MATCH_BITS,
+			       (ptl_size_t) dest_addr,
+			       PTL_NULL_USER_PTR));
+  gupcr_lock_md_count += 1;
+  gupcr_portals_call (PtlCTWait,
+		      (gupcr_lock_md_ct, gupcr_lock_md_count, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_lock_md_eq);
+      gupcr_fatal_error ("received an error on lock MD");
+    }
+}
+
+/**
+ * Wait for the next counting event to be posted to lock LE.
+ *
+ * This function is called when it has been determined that
+ * the current thread needs to wait until the lock is is released.
+ *
+ * Wait until the next Portals counting event is posted
+ * to the LE reserved for this purpose and then return.
+ * The caller will check whether the lock was in fact released,
+ * and if not, will call this function again to wait for the
+ * next lock-related event to come in.
+ */
+void
+gupcr_lock_wait (void)
+{
+  ptl_ct_event_t ct;
+  gupcr_debug (FC_LOCK, "");
+  gupcr_lock_le_count += 1;
+  gupcr_portals_call (PtlCTWait,
+		      (gupcr_lock_le_ct, gupcr_lock_le_count, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_lock_le_eq);
+      gupcr_fatal_error ("received an error on lock LE");
+    }
+}
+
+/**
+ * Initialize lock resources.
+ * @ingroup INIT
+ */
+void
+gupcr_lock_init (void)
+{
+  ptl_md_t md;
+  ptl_pt_index_t pte;
+  ptl_le_t le;
+  gupcr_log (FC_LOCK, "lock init called");
+  /* Allocate Portals PTE for locks.  */
+  gupcr_portals_call (PtlEQAlloc, (gupcr_ptl_ni, 1, &gupcr_lock_le_eq));
+  gupcr_portals_call (PtlPTAlloc, (gupcr_ptl_ni, 0,
+				   gupcr_lock_le_eq, GUPCR_PTL_PTE_LOCK,
+				   &pte));
+  if (pte != GUPCR_PTL_PTE_LOCK)
+    gupcr_fatal_error ("cannot allocate PTE GUPCR_PTL_PTE_LOCK.");
+  gupcr_debug (FC_LOCK, "Lock PTE allocated: %d", GUPCR_PTL_PTE_LOCK);
+  /* Allocate LE for locks.  */
+  gupcr_portals_call (PtlCTAlloc, (gupcr_ptl_ni, &gupcr_lock_le_ct));
+  gupcr_lock_le_count = 0;
+  le.start = gupcr_gmem_base;
+  le.length = gupcr_gmem_size;
+  le.ct_handle = gupcr_lock_le_ct;
+  le.uid = PTL_UID_ANY;
+  le.options = PTL_LE_OP_PUT | PTL_LE_OP_GET | PTL_LE_EVENT_CT_COMM |
+    PTL_LE_EVENT_SUCCESS_DISABLE | PTL_LE_EVENT_LINK_DISABLE;
+  gupcr_portals_call (PtlLEAppend, (gupcr_ptl_ni, GUPCR_PTL_PTE_LOCK, &le,
+				    PTL_PRIORITY_LIST, NULL, &gupcr_lock_le));
+  gupcr_debug (FC_LOCK, "Lock LE created at 0x%lx with size 0x%lx)",
+	       (long unsigned) gupcr_gmem_base,
+	       (long unsigned) gupcr_gmem_size);
+  /* Setup MD for writes into lock data structures located on
+     other threads.  Map the entire user address space,
+     though the MD probably could be constrained to the area where
+     lock data structures are managed.  */
+  gupcr_portals_call (PtlCTAlloc, (gupcr_ptl_ni, &gupcr_lock_md_ct));
+  gupcr_lock_md_count = 0;
+  gupcr_portals_call (PtlEQAlloc, (gupcr_ptl_ni, 1, &gupcr_lock_md_eq));
+  md.length = (ptl_size_t) USER_PROG_MEM_SIZE;
+  md.start = (void *) USER_PROG_MEM_START;
+  md.options =
+    PTL_MD_EVENT_CT_ACK | PTL_MD_EVENT_CT_REPLY |
+    PTL_MD_EVENT_SUCCESS_DISABLE;
+  md.eq_handle = gupcr_lock_md_eq;
+  md.ct_handle = gupcr_lock_md_ct;
+  gupcr_portals_call (PtlMDBind, (gupcr_ptl_ni, &md, &gupcr_lock_md));
+  /* Initialize the lock link allocator.  */
+  gupcr_lock_link_init ();
+  /* Initialize the lock free list.  */
+  gupcr_lock_free_init ();
+  /* Initialize the heap allocator locks.  */
+  gupcr_lock_heap_sup_init ();
+}
+
+/**
+ * Release lock resources.
+ * @ingroup INIT
+ */
+void
+gupcr_lock_fini (void)
+{
+  gupcr_log (FC_LOCK, "lock fini called");
+  /* Release lock MD.  */
+  gupcr_portals_call (PtlMDRelease, (gupcr_lock_md));
+  gupcr_portals_call (PtlCTFree, (gupcr_lock_md_ct));
+  gupcr_portals_call (PtlEQFree, (gupcr_lock_md_eq));
+  /* Release lock LE and PTE.  */
+  gupcr_portals_call (PtlLEUnlink, (gupcr_lock_le));
+  gupcr_portals_call (PtlCTFree, (gupcr_lock_le_ct));
+  gupcr_portals_call (PtlEQFree, (gupcr_lock_le_eq));
+  gupcr_portals_call (PtlPTFree, (gupcr_ptl_ni, GUPCR_PTL_PTE_LOCK));
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_lock_sup.h
===================================================================
--- libgupc/portals4/gupcr_lock_sup.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_lock_sup.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,48 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef _GUPCR_LOCK_SUP_H_
+#define _GUPCR_LOCK_SUP_H_
+
+/**
+ * @file gupcr_lock_sup.h
+ * GUPC Portals4 lock implementation support routines.
+ */
+
+extern void gupcr_lock_init (void);
+extern void gupcr_lock_fini (void);
+extern int gupcr_lock_cswap (size_t, size_t, void *, void *, size_t);
+extern void gupcr_lock_swap (size_t, size_t, void *, void *, size_t);
+extern void gupcr_lock_put (size_t, size_t, void *, size_t);
+extern void gupcr_lock_get (size_t, size_t, void *, size_t);
+extern void gupcr_lock_wait (void);
+
+/* See: gupcr_alloc.upc.  */
+extern void gupcr_lock_link_init (void);
+extern void gupcr_lock_free_init (void);
+extern void gupcr_lock_heap_sup_init (void);
+
+#endif /* gupcr_lock_sup.h */
Index: libgupc/portals4/gupcr_main.c
===================================================================
--- libgupc/portals4/gupcr_main.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_main.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,298 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_main.c
+ * GUPC Portals4 runtime main program.
+ */
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_sup.h"
+#include "gupcr_lock_sup.h"
+#include "gupcr_alloc.h"
+#include "gupcr_broadcast.h"
+#include "gupcr_barrier.h"
+#include "gupcr_shutdown.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_runtime.h"
+#include "gupcr_gmem.h"
+#include "gupcr_node.h"
+#include "gupcr_coll_sup.h"
+#include "gupcr_atomic_sup.h"
+#include "gupcr_nb_sup.h"
+#include "gupcr_backtrace.h"
+
+/** User's main program */
+extern int GUPCR_MAIN (int argc, char *argv[]);
+
+/** The number of THREADS, as specified on the command line */
+int THREADS = -1;
+
+/** The current thread number (range: 0..THREADS-1) */
+int MYTHREAD = -1;
+
+/** OK to call finalize routines */
+int gupcr_finalize_ok = 0;
+
+/** Depth count used to implement the semantics of
+   nested upc_forall statements.  */
+int __upc_forall_depth;
+
+/** The filename of the location where a runtime
+   error was detected.  This is set by the various
+   debug-enabled ('g') UPC runtime library routines.  */
+const char *gupcr_err_filename;
+
+/** The line number of the location where a runtime
+   error was detected.  This is set by the various
+   debug-enabled ('g') UPC runtime library routines.  */
+unsigned int gupcr_err_linenum;
+
+/**
+ * @addtogroup INIT GUPCR Initialization
+ * @{
+ */
+
+/**
+ * Initialize UPC runtime.
+ *
+ * All hardware/software components of the Portals4 interface
+ * are initialized in this routine.
+ */
+static void
+gupcr_init (void)
+{
+  int run_threads_count;
+  upc_shared_ptr_t heap_region_base;
+  size_t heap_region_size;
+
+  /* Initialize Runtime.  */
+  if (gupcr_runtime_init ())
+    {
+      /* Report an error to stderr as the GUPC error reporting
+	 is not initialized yet.  Note: all threads report
+	 this error.  */
+      fprintf (stderr, "Unable to initialize runtime.\n");
+      abort ();
+    }
+
+  /* Get the thread number.  */
+  MYTHREAD = gupcr_runtime_get_rank ();
+
+  /* Set up debugging, tracing, statistics, and timing support.  */
+  gupcr_utils_init ();
+
+  /* Initialize Portals.  */
+  gupcr_portals_init ();
+
+  /* Validate program info.  */
+  gupcr_validate_pgm_info ();
+
+  run_threads_count = gupcr_get_threads_count ();
+
+  /* THREADS == -1, dynamic number of threads
+     THREADS != -1, static number of threads and
+     number of running and compiled threads must match.  */
+  if (THREADS == -1)
+    THREADS = run_threads_count;
+  else if (THREADS != run_threads_count)
+    gupcr_abort_with_msg ("number of running threads (%d) is "
+			  "not equal to compiled threads (%d)",
+			  run_threads_count, THREADS);
+  gupcr_assert (THREADS >= 1);
+
+#if HAVE_UPC_BACKTRACE                                                          
+  /* Initialize backtrace support. */                                           
+  gupcr_backtrace_init (gupcr_get_pgm_name () );                                        
+#endif 
+
+  /* Initialize the Portals Network Interface.  */
+  gupcr_portals_ni_init ();
+
+  /* Initialize this thread's multi-node tree position.  */
+  gupcr_nodetree_setup ();
+
+  /* Initialize various runtime components.  */
+  gupcr_node_init ();
+  gupcr_gmem_init ();
+  gupcr_lock_init ();
+  gupcr_barrier_init ();
+  gupcr_broadcast_init ();
+  gupcr_coll_init ();
+  gupcr_atomic_init ();
+  gupcr_nb_init ();
+  gupcr_shutdown_init ();
+
+  GUPCR_PTS_SET_NULL_SHARED (heap_region_base);
+  GUPCR_PTS_SET_THREAD (heap_region_base, MYTHREAD);
+  GUPCR_PTS_SET_VADDR (heap_region_base, gupcr_gmem_heap_base_offset);
+  heap_region_size = gupcr_gmem_heap_size;
+  gupcr_alloc_init (heap_region_base, heap_region_size);
+
+  /* Indicate that runtime initialization is complete.  */
+  gupcr_init_complete ();
+
+  /* It is ok to call the finalization routines.  */
+  gupcr_finalize_ok = 1;
+}
+
+/**
+ * UPC runtime finalization.
+ *
+ * All previously allocated Portals4 resources are released.
+ */
+void
+gupcr_fini (void)
+{
+  gupcr_shutdown_fini ();
+  gupcr_nb_fini ();
+  gupcr_atomic_fini ();
+  gupcr_broadcast_fini ();
+  gupcr_barrier_fini ();
+  gupcr_lock_fini ();
+  gupcr_gmem_fini ();
+  gupcr_node_fini ();
+  gupcr_coll_fini ();
+  gupcr_portals_ni_fini ();
+  gupcr_portals_fini ();
+  gupcr_runtime_fini ();
+  gupcr_utils_fini ();
+}
+
+/**
+ * Per thread initialization.
+ *
+ * The following pre-requisites must be met, before calling this routine:
+ * - the runtime system has been initialized.
+ * - the UPC heap manager has been initialized.
+ *
+ * The barrier that is executed subsequent to calling this
+ * per thread initialization procedure and prior to
+ * calling the main program ensures that the initialization
+ * completes before the main program runs.
+ */
+
+static void
+gupcr_per_thread_init (void)
+{
+  typedef void (*func_ptr_t) (void);
+  extern func_ptr_t GUPCR_INIT_ARRAY_START[];
+  extern func_ptr_t GUPCR_INIT_ARRAY_END[];
+  const int n_init = (int) (GUPCR_INIT_ARRAY_END - GUPCR_INIT_ARRAY_START);
+  int i;
+  for (i = 0; i < n_init; ++i)
+    {
+      func_ptr_t init_func = GUPCR_INIT_ARRAY_START[i];
+      /* Skip zero words, possibly introduced by a section marker,
+         or by the linker.  */
+      if (init_func)
+	(*init_func) ();
+    }
+}
+
+/** @} */
+
+/**
+ * UPC program exit.
+ *
+ * Calls to exit() are rewritten into calls to __upc_exit()
+ * by a "#define" in <gcc-upc.h>. Simply perform a upc_barrier and
+ * then exit the process.
+ * @param [in] status Status code of exit
+ * @ingroup GUPC-API GUPC Program Interface
+ */
+void
+__upc_exit (int status)
+{
+  /* Mask off the top bit; it is used to indicate a global exit.  */
+  const int exit_status = status & 0x7f;
+  __upc_barrier (GUPCR_RUNTIME_BARRIER_ID);
+  exit (exit_status);
+}
+
+/**
+ * Exit program with given status and terminate all other threads.
+ *
+ * A special "shutdown" PTE is used to send a signal to
+ * other threads that they should exit.
+ * @param [in] status Status code for return
+ * @ingroup UPC-LIBRARY UPC Library Interface
+ */
+void
+upc_global_exit (int status)
+{
+  /* Send exit signal to all other threads.  */
+  gupcr_signal_exit (status);
+  /* It is NOT ok to call the finalization routines as there might
+     be outstanding Portals transactions.  */
+  gupcr_finalize_ok = 0;
+  exit (status);
+}
+
+/**
+ * GUPC runtime start up.
+ *
+ * @param [in] argc Number of command line arguments
+ * @param [in] argv Command line arguments
+ * @retval Return (exit) value from the user's main program
+ * @ingroup INIT GUPCR Initialization
+ */
+/** @}*/
+int
+GUPCR_START (int argc, char *argv[])
+{
+  int status;
+
+  /* Install exit handler.  */
+  atexit (gupcr_exit);
+
+  /* Set program name for debug/trace diagnostics.  */
+  gupcr_set_pgm_name (argv[0]);
+
+  /* Initialize all runtime components.  */
+  gupcr_init ();
+
+  /* Initialize language specific variables.  */
+  __upc_forall_depth = 0;
+
+  /* Wait for all threads to finish initialization.  */
+  gupcr_startup_barrier ();
+
+  /* Perform per thread initialization.  */
+  gupcr_per_thread_init ();
+
+  /* Wait for all threads to complete per thread initialization.  */
+  __upc_barrier (GUPCR_RUNTIME_BARRIER_ID);
+
+  /* Call user main program.  */
+  status = GUPCR_MAIN (argc, argv);
+
+  /* Wait for all threads to complete.  */
+  __upc_barrier (GUPCR_RUNTIME_BARRIER_ID);
+
+  return status;
+}
Index: libgupc/portals4/gupcr_mem.c
===================================================================
--- libgupc/portals4/gupcr_mem.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_mem.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,211 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_sup.h"
+#include "gupcr_access.h"
+#include "gupcr_portals.h"
+#include "gupcr_node.h"
+#include "gupcr_gmem.h"
+#include "gupcr_utils.h"
+
+/**
+ * @file gupcr_mem.c
+ * GUPC Shared string handling functions.
+ */
+
+/**
+ * @addtogroup UPCSTR UPC Shared string handling functions
+ * @{
+ */
+
+/**
+ * Copy shared memory block.
+ *
+ * The upc_memcpy function copies n characters from a shared object having
+ * affinity with one thread to a shared object having affinity with the same or
+ * another thread.
+ *
+ * @param [in] dest Pointer-to-shared of the destination
+ * @param [in] src Pointer-to-shared of the source
+ * @param [in] n Number of bytes to transfer
+ */
+void
+upc_memcpy (upc_shared_ptr_t dest, upc_shared_ptr_t src, size_t n)
+{
+  int sthread = GUPCR_PTS_THREAD (src);
+  size_t soffset = GUPCR_PTS_OFFSET (src);
+  int dthread = GUPCR_PTS_THREAD (dest);
+  size_t doffset = GUPCR_PTS_OFFSET (dest);
+  int dthread_local, sthread_local;
+  gupcr_trace (FC_MEM, "MEM MEMCPY ENTER %d:0x%lx %d:0x%lx %lu",
+	       sthread, (long unsigned) soffset,
+	       dthread, (long unsigned) doffset, (long unsigned) n);
+  gupcr_assert (dthread < THREADS);
+  gupcr_assert (doffset != 0);
+  gupcr_assert (sthread < THREADS);
+  gupcr_assert (soffset != 0);
+  dthread_local = GUPCR_GMEM_IS_LOCAL (dthread);
+  sthread_local = GUPCR_GMEM_IS_LOCAL (sthread);
+  if (dthread_local && sthread_local)
+    memcpy (GUPCR_GMEM_OFF_TO_LOCAL (dthread, doffset),
+	    GUPCR_GMEM_OFF_TO_LOCAL (sthread, soffset), n);
+  else if (dthread_local)
+    {
+      if (gupcr_pending_strict_put)
+        gupcr_gmem_sync_puts ();
+      gupcr_gmem_get (GUPCR_GMEM_OFF_TO_LOCAL (dthread, doffset),
+		      sthread, soffset, n);
+      gupcr_gmem_sync_gets ();
+    }
+  else if (sthread_local)
+    {
+      if (n > (size_t) GUPCR_PORTALS_MAX_ORDERED_SIZE)
+	{
+	  gupcr_gmem_sync_puts ();
+	  gupcr_gmem_put (dthread, doffset,
+			  GUPCR_GMEM_OFF_TO_LOCAL (sthread, soffset), n);
+	  gupcr_pending_strict_put = 1;
+	}
+      else
+	gupcr_gmem_put (dthread, doffset,
+			GUPCR_GMEM_OFF_TO_LOCAL (sthread, soffset), n);
+    }
+  else
+    {
+      if (n > (size_t) GUPCR_PORTALS_MAX_ORDERED_SIZE)
+	{
+          gupcr_gmem_sync_puts ();
+          gupcr_gmem_copy (dthread, doffset, sthread, soffset, n);
+          gupcr_pending_strict_put = 1;
+	}
+      else
+        gupcr_gmem_copy (dthread, doffset, sthread, soffset, n);
+    }
+  gupcr_trace (FC_MEM, "MEM MEMCPY EXIT");
+}
+
+/**
+ * Get shared memory block.
+ *
+ * The upc_memget function copies n characters from a shared object with
+ * affinity to any single thread to an object on the calling thread.
+ *
+ * @param [in] dest Local destination address
+ * @param [in] src Pointer-to-shared of the source
+ * @param [in] n Number of bytes to transfer
+ */
+void
+upc_memget (void *dest, upc_shared_ptr_t src, size_t n)
+{
+  int sthread = GUPCR_PTS_THREAD (src);
+  size_t soffset = GUPCR_PTS_OFFSET (src);
+  gupcr_trace (FC_MEM, "MEM MEMGET ENTER %d:0x%lx 0x%lx %lu",
+	       sthread, (long unsigned) soffset,
+	       (long unsigned) dest, (long unsigned) n);
+  gupcr_assert (sthread < THREADS);
+  gupcr_assert (soffset != 0);
+  if (GUPCR_GMEM_IS_LOCAL (sthread))
+    memcpy (dest, GUPCR_GMEM_OFF_TO_LOCAL (sthread, soffset), n);
+  else
+    gupcr_gmem_get (dest, sthread, soffset, n);
+  gupcr_gmem_sync_gets ();
+  gupcr_trace (FC_MEM, "MEM MEMGET EXIT");
+}
+
+/**
+ * Put shared memory block.
+ *
+ * The upc_memput function copies n characters from an object on the calling
+ * thread to a shared object with affinity to any single thread.
+ *
+ * @param [in] dest Pointer-to-shared of the destination
+ * @param [in] src Local source address
+ * @param [in] n Number of bytes to transfer
+ */
+void
+upc_memput (upc_shared_ptr_t dest, const void *src, size_t n)
+{
+  int dthread = GUPCR_PTS_THREAD (dest);
+  size_t doffset = GUPCR_PTS_OFFSET (dest);
+  gupcr_trace (FC_MEM, "MEM MEMPUT ENTER 0x%lx %d:0x%lx %lu",
+	       (long unsigned) src, dthread, (long unsigned) doffset,
+	       (long unsigned) n);
+  gupcr_assert (dthread < THREADS);
+  gupcr_assert (doffset != 0);
+  if (GUPCR_GMEM_IS_LOCAL (dthread))
+    memcpy (GUPCR_GMEM_OFF_TO_LOCAL (dthread, doffset), src, n);
+  else
+    {
+      if (n > (size_t) GUPCR_PORTALS_MAX_ORDERED_SIZE)
+	{
+	  gupcr_gmem_sync_puts ();
+	  gupcr_gmem_put (dthread, doffset, src, n);
+	  gupcr_pending_strict_put = 1;
+	}
+      else
+	gupcr_gmem_put (dthread, doffset, src, n);
+    }
+  gupcr_trace (FC_MEM, "MEM MEMPUT EXIT");
+}
+
+/**
+ * Set shared memory block.
+ *
+ * The upc_memset function copies the value of c, converted to an unsigned
+ * char, to a shared object with affinity to any single thread.
+ *
+ * @param [in] dest Pointer-to-shared of the destination
+ * @param [in] c Value to set the remote memory
+ * @param [in] n Number of bytes to set
+ */
+void
+upc_memset (upc_shared_ptr_t dest, int c, size_t n)
+{
+  int dthread = GUPCR_PTS_THREAD (dest);
+  size_t doffset = GUPCR_PTS_OFFSET (dest);
+  gupcr_trace (FC_MEM, "MEM MEMSET ENTER 0x%x %d:0x%lx %lu",
+	       c, dthread, (long unsigned) doffset, (long unsigned) n);
+  gupcr_assert (dthread < THREADS);
+  gupcr_assert (doffset != 0);
+  if (GUPCR_GMEM_IS_LOCAL (dthread))
+    memset (GUPCR_GMEM_OFF_TO_LOCAL (dthread, doffset), c, n);
+  else
+    {
+      if (n > (size_t) GUPCR_PORTALS_MAX_ORDERED_SIZE)
+	{
+	  gupcr_gmem_sync_puts ();
+	  gupcr_gmem_set (dthread, doffset, c, n);
+	  gupcr_pending_strict_put = 1;
+	}
+      else
+	gupcr_gmem_set (dthread, doffset, c, n);
+    }
+  gupcr_trace (FC_MEM, "MEM MEMSET EXIT");
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_nb.upc
===================================================================
--- libgupc/portals4/gupcr_nb.upc	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_nb.upc	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,320 @@
+/* Copyright (C) 2013-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_nb.upc
+ * GUPC Portals4 Non-Blocking Transfers Operations.
+ *
+ * @addtogroup UPC-NON-BLOCKING UPC Non-Blocking Transfer Operations
+ * @{
+ */
+
+#include <upc.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <upc_nb.h>
+#include <portals4.h>
+#include "gupcr_gmem.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_nb_sup.h"
+
+/**
+ * Copy memory with non-blocking explicit handle transfer
+ *
+ * @param[in] dst Destination shared memory pointer
+ * @param[in] src Source shared memory pointer
+ * @param[in] n Number of bytes to transfer
+ * @retval UPC non-blocking transfer handle
+ */
+upc_handle_t
+upc_memcpy_nb (shared void *restrict dst,
+	       shared const void *restrict src, size_t n)
+{
+  size_t sthread, soffset;
+  size_t dthread, doffset;
+  upc_handle_t handle;
+  gupcr_trace (FC_NB, "NB MEMCOPY_NB ENTER");
+  sthread = upc_threadof ((shared void *) src);
+  dthread = upc_threadof ((shared void *) dst);
+  if ((int) sthread == MYTHREAD)
+    {
+      doffset = upc_addrfield (dst);
+      gupcr_nb_put (dthread, doffset, (char *) src, n, &handle);
+    }
+  else if ((int) dthread == MYTHREAD)
+    {
+      soffset = upc_addrfield ((shared void *) src);
+      gupcr_nb_get (sthread, soffset, (char *) dst, n, &handle);
+    }
+  else
+    {
+      /* Third party copying.  Just use upc_memcpy().  */
+      upc_memcpy (dst, src, n);
+      handle = UPC_COMPLETE_HANDLE; 
+    }
+  gupcr_trace (FC_NB, "NB MEMCOPY_NB EXIT");
+  return handle;
+}
+
+/**
+ * Get memory with non-blocking explicit handle transfer
+ *
+ * @param[in] dst Destination local memory pointer
+ * @param[in] src Source remote memory pointer
+ * @param[in] n Number of bytes to transfer
+ * @retval UPC non-blocking transfer handle
+ */
+upc_handle_t
+upc_memget_nb (void *restrict dst,
+	       shared const void *restrict src, size_t n)
+{
+  gupcr_trace (FC_NB, "NB MEMGET_NB ENTER");
+  upc_handle_t handle = UPC_COMPLETE_HANDLE;
+  if (n)
+    {
+      size_t sthread, soffset;
+      sthread = upc_threadof ((shared void *) src);
+      soffset = upc_addrfield ((shared void *) src);
+      gupcr_nb_get (sthread, soffset, dst, n, &handle);
+    }
+  gupcr_trace (FC_NB, "NB MEMGET_NB EXIT");
+  return handle;
+}
+
+/**
+ * Put memory with non-blocking explicit handle transfer
+ *
+ * @param[in] dst Destination remote memory pointer
+ * @param[in] src Source local memory pointer
+ * @param[in] n Number of bytes to transfer
+ * @retval UPC non-blocking transfer handle
+ */
+upc_handle_t
+upc_memput_nb (shared void *restrict dst,
+	       const void *restrict src, size_t n)
+{
+  gupcr_trace (FC_NB, "NB MEMPUT_NB ENTER");
+  upc_handle_t handle = UPC_COMPLETE_HANDLE;
+  if (n)
+    {
+      size_t dthread, doffset;
+      dthread = upc_threadof ((shared void *) dst);
+      doffset = upc_addrfield ((shared void *) dst);
+      gupcr_nb_put (dthread, doffset, src, n, &handle);
+    }
+  gupcr_trace (FC_NB, "NB MEMPUT_NB EXIT");
+  return handle;
+}
+
+/**
+ * Set memory with non-blocking implicit handle transfer
+ *
+ * @param[in] dst Shared remote pointer
+ * @param[in] c Value for set operation
+ * @param[in] n Number of bytes to set
+ * @retval UPC non-blocking transfer handle
+ *
+ * This call completes the transfer before returning.
+ */
+upc_handle_t
+upc_memset_nb (shared void *dst, int c, size_t n)
+{
+  gupcr_trace (FC_NB, "NB MEMSET_NB ENTER");
+  if (n)
+    {
+      upc_memset (dst, c, n);
+      upc_fence;
+    }
+  gupcr_trace (FC_NB, "NB MEMSET_NB EXIT");
+  return UPC_COMPLETE_HANDLE;
+}
+
+/**
+ * Explicit handle non-blocking transfer sync attempt
+ *
+ * @param[in] handle Transfer explicit handle
+ * @retval UPC_NB_COMPLETED returned if transfer completed,
+ *	   otherwise UPC_NB_NOT_COMPLETED
+ */
+int
+upc_sync_attempt (upc_handle_t handle)
+{
+  int comp;
+  gupcr_trace (FC_NB, "NB SYNC_ATTEMPT ENTER");
+  if (handle == UPC_COMPLETE_HANDLE
+    || gupcr_nb_completed (handle))
+    comp = UPC_NB_COMPLETED;
+  else
+    comp = UPC_NB_NOT_COMPLETED;
+  gupcr_trace (FC_NB, "NB SYNC_ATTEMPT EXIT");
+  return comp;
+}
+
+/**
+ * Explicit handle non-blocking transfer sync
+ *
+ * @param[in] handle Non-blocking transfer explicit handle
+ */
+void
+upc_sync (upc_handle_t handle)
+{
+  gupcr_trace (FC_NB, "NB SYNC ENTER");
+  if (handle != UPC_COMPLETE_HANDLE)
+    gupcr_sync (handle);
+  gupcr_trace (FC_NB, "NB SYNC EXIT");
+}
+
+/**
+ * Copy memory with non-blocking implicit handle transfer
+ *
+ * @param[in] dst Shared remote memory pointer
+ * @param[in] src Shared remote memory pointer
+ * @param[in] n Number of bytes to transfer
+ */
+void
+upc_memcpy_nbi (shared void *restrict dst,
+		shared const void *restrict src, size_t n)
+{
+  size_t sthread, soffset;
+  size_t dthread, doffset;
+  gupcr_trace (FC_NB, "NB MEMCOPY_NBI ENTER");
+  sthread = upc_threadof ((shared void *) src);
+  dthread = upc_threadof ((shared void *) dst);
+  if ((int) sthread == MYTHREAD)
+    {
+      doffset = upc_addrfield (dst);
+      gupcr_nb_put (dthread, doffset, (char *) src, n, NULL);
+    }
+  else if ((int) dthread == MYTHREAD)
+    {
+      soffset = upc_addrfield ((shared void *) src);
+      gupcr_nb_get (sthread, soffset, (char *) dst, n, NULL);
+    }
+  else
+    {
+      /* Third party copying.  Just use upc_memcpy().  */
+      upc_memcpy (dst, src, n);
+    }
+  gupcr_trace (FC_NB, "NB MEMCOPY_NBI EXIT");
+}
+
+/**
+ * Get memory with non-blocking implicit handle transfer
+ *
+ * @param[in] dst Local memory pointer
+ * @param[in] src Shared remote memory pointer
+ * @param[in] n Number of bytes to transfer
+ */
+void
+upc_memget_nbi (void *restrict dst,
+		shared const void *restrict src, size_t n)
+{
+  gupcr_trace (FC_NB, "NB MEMGET_NBI ENTER");
+  if (n)
+    {
+      size_t sthread, soffset;
+      sthread = upc_threadof ((shared void *) src);
+      soffset = upc_addrfield ((shared void *) src);
+      gupcr_nb_get (sthread, soffset, dst, n, NULL);
+    }
+  gupcr_trace (FC_NB, "NB MEMGET_NBI EXIT");
+}
+
+/**
+ * Put memory with non-blocking implicit handle transfer
+ *
+ * @param[in] dst Shared remote memory pointer
+ * @param[in] src Local memory pointer
+ * @param[in] n Number of bytes to transfer
+ */
+void
+upc_memput_nbi (shared void *restrict dst,
+		const void *restrict src, size_t n)
+{
+  gupcr_trace (FC_NB, "NB MEMPUT_NBI ENTER");
+  if (n)
+    {
+      size_t dthread, doffset;
+      dthread = upc_threadof ((shared void *) dst);
+      doffset = upc_addrfield ((shared void *) dst);
+      gupcr_nb_put (dthread, doffset, src, n, NULL);
+    }
+  gupcr_trace (FC_NB, "NB MEMPUT_NBI EXIT");
+}
+
+/**
+ * Set memory with non-blocking implicit handle transfer
+ *
+ * @param[in] dst Shared remote pointer
+ * @param[in] c Value for set operation
+ * @param[in] n Number of bytes to set
+ */
+void
+upc_memset_nbi (shared void *dst, int c, size_t n)
+{
+  gupcr_trace (FC_NB, "NB MEMSET_NBI ENTER");
+  if (n)
+    {
+      upc_memset (dst, c, n);
+      upc_fence;
+    }
+  gupcr_trace (FC_NB, "NB MEMSET_NBI EXIT");
+}
+
+/**
+ * Check on implicit handle non-blocking transfers
+ *
+ * @retval UPC_NB_COMPLETED if no transfers pending, otherwise
+ *         UPC_NB_NOT_COMPLETED is returned
+ */
+int
+upc_synci_attempt (void)
+{
+  int result;
+  gupcr_trace (FC_NB, "NB SYNCI ATTEMPT ENTER");
+  if (gupcr_nbi_outstanding ())
+    result = UPC_NB_NOT_COMPLETED;
+  else
+    result = UPC_NB_COMPLETED;
+  gupcr_trace (FC_NB, "NB SYNCI ATTEMPT EXIT");
+  return result;
+}
+
+/**
+ * Complete implicit handle non-blocking transfers
+ */
+void
+upc_synci (void)
+{
+  gupcr_trace (FC_NB, "NB SYNCI ENTER");
+  gupcr_synci ();
+  gupcr_trace (FC_NB, "NB SYNCI EXIT");
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_nb_sup.c
===================================================================
--- libgupc/portals4/gupcr_nb_sup.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_nb_sup.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,597 @@
+/* Copyright (C) 2013-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#include "malloc.h"
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_lib.h"
+#include "gupcr_sup.h"
+#include "gupcr_portals.h"
+#include "gupcr_gmem.h"
+#include "gupcr_utils.h"
+#include "gupcr_nb_sup.h"
+
+/**
+ * @file gupcr_nb_sup.c
+ * GUPC Portals4 non-blocking transfers support routines.
+ *
+ * @addtogroup NON-BLOCKING GUPCR Non-Blocking Transfer Support Functions
+ * @{
+ */
+
+/** Non-blocking shared access LE handle */
+static ptl_handle_le_t gupcr_nb_le;
+/** Explicit non-blocking MD handle */
+static ptl_handle_md_t gupcr_nb_md;
+/** Explicit non-blocking MD event queue handle */
+static ptl_handle_eq_t gupcr_nb_md_eq;
+/** Start of the explicit non-blocking MD */
+static char *gupcr_nb_md_start;
+
+/** Implicit non-blocking MD handle */
+static ptl_handle_md_t gupcr_nbi_md;
+/** Implicit non-blocking MD EQ handle */
+static ptl_handle_eq_t gupcr_nbi_md_eq;
+/** Implicit non-blocking MD CT handle */
+static ptl_handle_ct_t gupcr_nbi_md_ct;
+/** Implicit non-blocking number of received ACKs on local md */
+static ptl_size_t gupcr_nbi_md_count;
+/** Start of the implicit non-blocking MD */
+static char *gupcr_nbi_md_start;
+
+/* All non-blocking transfers with explicit handle
+   are managed through the 'gupcr_nbcb' structure
+   (control block).  Free control blocks are on
+   the free list, while those with active transfers
+   are on the 'active' list.  A single linked list
+   is used to link CBs on the free or active list.
+
+   Non-blocking transfer handle is an unsigned long
+   number that we increment every time a new transfer is
+   requested.  */
+
+/** Non-blocking transfers control block */
+struct gupcr_nbcb
+{
+  struct gupcr_nbcb *next; /** forward link on the free or used list */
+  unsigned long id; /** UPC handle for non-blocking transfer */
+  int status; /** non-blocking transfer status */
+};
+typedef struct gupcr_nbcb gupcr_nbcb_t;
+typedef struct gupcr_nbcb *gupcr_nbcb_p;
+
+/** nb transfer status value */
+#define	NB_STATUS_COMPLETED 1
+#define	NB_STATUS_NOT_COMPLETED 0
+
+/** NB handle values */
+unsigned long gupcr_nb_handle_next;
+
+/** NB cb free list */
+gupcr_nbcb_p gupcr_nbcb_cb_free = NULL;
+/** List of NB active transfers */
+gupcr_nbcb_p gupcr_nbcb_active = NULL;
+
+/** Number of outstanding transfers with explicit handle */
+int gupcr_nb_outstanding;
+void gupcr_nb_check_outstanding (void);
+
+/**
+ * Allocate free NB control block
+ */
+static gupcr_nbcb_p
+gupcr_nbcb_alloc (void)
+{
+  gupcr_nbcb_p cb;
+  if (gupcr_nbcb_cb_free)
+    {
+      cb = gupcr_nbcb_cb_free;
+      gupcr_nbcb_cb_free = cb->next;
+    }
+  else
+    {
+      /* Allocate memory for the new block.  */
+      cb = calloc (sizeof (struct gupcr_nbcb), 1);
+      if (cb == NULL)
+	gupcr_fatal_error ("cannot allocate local memory");
+    }
+  return cb;
+}
+
+/**
+ * Place NB control block on the free list
+ */
+static void
+gupcr_nbcb_free (gupcr_nbcb_p cb)
+{
+  cb->next = gupcr_nbcb_cb_free;
+  gupcr_nbcb_cb_free = cb;
+}
+
+/**
+ * Place NB control block on the active list
+ */
+static void
+gupcr_nbcb_active_insert (gupcr_nbcb_p cb)
+{
+  cb->next = gupcr_nbcb_active;
+  gupcr_nbcb_active = cb;
+}
+
+/**
+ * Remove NB control block from the active list
+ */
+static void
+gupcr_nbcb_active_remove (gupcr_nbcb_p cb)
+{
+  gupcr_nbcb_p acb = gupcr_nbcb_active;
+  gupcr_nbcb_p prev_acb = acb;
+  while (acb)
+    {
+      if (acb == cb)
+	{
+	  if (acb == gupcr_nbcb_active)
+	    gupcr_nbcb_active = acb->next;
+	  else
+	    prev_acb->next = acb->next;
+	  return;
+	}
+      prev_acb = acb;
+      acb = acb->next;
+    }
+}
+
+/**
+ * Find NB control block on the active list
+ */
+static gupcr_nbcb_p
+gupcr_nbcb_find (unsigned long id)
+{
+  gupcr_nbcb_p cb = gupcr_nbcb_active;
+  while (cb)
+    {
+      if (cb->id == id)
+	return cb;
+      cb = cb->next;
+    }
+  return NULL;
+}
+
+/**
+ * Non-blocking GET operation
+ *
+ * @param[in] sthread Source thread
+ * @param[in] soffset Source offset
+ * @param[in] dst_ptr Destination local pointer
+ * @param[in] size Number of bytes to transfer
+ * @param[in] handle Transfer handle (NULL for implicit)
+ */
+void
+gupcr_nb_get (size_t sthread, size_t soffset, char *dst_ptr,
+	      size_t size, unsigned long *handle)
+{
+  ptl_process_t rpid;
+  size_t n_rem = size;
+  ptl_size_t local_offset = dst_ptr - gupcr_nbi_md_start;
+
+  if (handle)
+    {
+      gupcr_nbcb_p cb = gupcr_nbcb_alloc ();
+      cb->id = gupcr_nb_handle_next++;
+      cb->status = NB_STATUS_NOT_COMPLETED;
+      gupcr_nbcb_active_insert (cb);
+      *handle = cb->id;
+      gupcr_nb_check_outstanding ();
+    }
+  gupcr_debug (FC_NB, "%s %lu:0x%lx(%ld) -> 0x%lx (%lu)",
+	       handle ? "NB" : "NBI", sthread, soffset,
+	       size, (long unsigned int) dst_ptr, handle ? *handle : 0);
+
+  /* Large transfers must be done in chunks.  Only the last chunk
+     behaves as a non-blocking transfer.  */
+  while (n_rem > 0)
+    {
+      size_t n_xfer;
+      n_xfer = GUPCR_MIN (n_rem, GUPCR_PORTALS_MAX_MSG_SIZE);
+      rpid.rank = sthread;
+      gupcr_portals_call (PtlGet, (handle ? gupcr_nb_md : gupcr_nbi_md,
+				   local_offset,
+				   n_xfer, rpid, GUPCR_PTL_PTE_NB,
+				   PTL_NO_MATCH_BITS, soffset,
+				   handle ? (void *) *handle : NULL));
+      if (handle)
+        gupcr_nb_outstanding += 1;
+      else
+        gupcr_nbi_md_count += 1;
+      n_rem -= n_xfer;
+      local_offset += n_xfer;
+      if (n_rem)
+	{
+	  /* Unfortunately, there are more data to transfer, we have to
+	     wait for all non-blocking transfers to complete.  */
+	  if (handle)
+	    gupcr_sync (*handle);
+	  else
+	    gupcr_synci ();
+	}
+    }
+}
+
+/**
+ * Non-blocking transfer PUT operation
+ *
+ * @param[in] dthread Destination thread
+ * @param[in] doffset Destination offset
+ * @param[in] src_ptr Source local pointer
+ * @param[in] size Number of bytes to transfer
+ * @param[in] handle Transfer handle (NULL for implicit)
+ */
+void
+gupcr_nb_put (size_t dthread, size_t doffset, const void *src_ptr,
+	      size_t size, unsigned long *handle)
+{
+  ptl_process_t rpid;
+  size_t n_rem = size;
+  ptl_size_t local_offset = (char *) src_ptr - gupcr_nbi_md_start;
+
+  if (handle)
+    {
+      gupcr_nbcb_p cb = gupcr_nbcb_alloc ();
+      cb->id = gupcr_nb_handle_next++;
+      cb->status = NB_STATUS_NOT_COMPLETED;
+      gupcr_nbcb_active_insert (cb);
+      *handle = cb->id;
+      gupcr_nb_check_outstanding ();
+    }
+
+  gupcr_debug (FC_NB, "%s 0x%lx(%ld) -> %lu:0x%lx (%lu)",
+	       handle ? "NB" : "NBI", (long unsigned int) src_ptr, size,
+	       dthread, doffset, handle ? *handle : 0);
+
+  /* Large transfers must be done in chunks.  Only the last chunk
+     behaves as a non-blocking transfer.  */
+  while (n_rem > 0)
+    {
+      size_t n_xfer;
+      n_xfer = GUPCR_MIN (n_rem, GUPCR_PORTALS_MAX_MSG_SIZE);
+      rpid.rank = dthread;
+      gupcr_portals_call (PtlPut, (handle ? gupcr_nb_md : gupcr_nbi_md,
+				   local_offset, n_xfer, PTL_ACK_REQ, rpid,
+				   GUPCR_PTL_PTE_NB, PTL_NO_MATCH_BITS,
+				   doffset, handle ? (void *) *handle : NULL,
+				   PTL_NULL_HDR_DATA));
+      if (handle)
+        gupcr_nb_outstanding += 1;
+      else
+        gupcr_nbi_md_count += 1;
+      n_rem -= n_xfer;
+      local_offset += n_xfer;
+      if (n_rem)
+	{
+	  /* Unfortunately, there are more data to transfer, we have to
+	     wait for all non-blocking transfers to complete.  */
+	  if (handle)
+	    gupcr_sync (*handle);
+	  else
+	    gupcr_synci ();
+	}
+    }
+}
+
+/**
+ * Check for the max number of outstanding non-blocking
+ * transfers with explicit handle
+ *
+ * We cannot allow for number of outstanding transfers
+ * to go over the event queue size.  Otherwise, some ACK/REPLY
+ * can be dropped.
+ */
+void
+gupcr_nb_check_outstanding (void)
+{
+  if (gupcr_nb_outstanding == GUPCR_NB_MAX_OUTSTANDING)
+    {
+      /* We have to wait for at least one to complete.  */
+      ptl_event_t event;
+      gupcr_portals_call (PtlEQWait, (gupcr_nb_md_eq, &event));
+
+      /* Process only ACKs and REPLYs,  */
+      if (event.type == PTL_EVENT_ACK || event.type == PTL_EVENT_REPLY)
+	{
+	  gupcr_nbcb_p cb;
+	  unsigned long id = (unsigned long) event.user_ptr;
+	  gupcr_debug (FC_NB, "received event for handle %lu", id);
+	  cb = gupcr_nbcb_find (id);
+	  if (!cb || cb->status == NB_STATUS_COMPLETED)
+	    {
+	      gupcr_fatal_error
+		("received event for non-existent or "
+		 "already completed NB handle");
+	    }
+	  cb->status = NB_STATUS_COMPLETED;
+	  gupcr_nb_outstanding--;
+	}
+      else
+	{
+	  gupcr_fatal_error ("received event of invalid type: %s",
+			      gupcr_streqtype (event.type));
+	}
+    }
+}
+
+/**
+ * Check for non-blocking transfer complete
+ *
+ * @param[in] handle Transfer handle
+ * @retval "1" if transfer completed
+ */
+int
+gupcr_nb_completed (unsigned long handle)
+{
+  ptl_event_t event;
+  int done = 0;
+  gupcr_nbcb_p cb;
+
+  /* Handle Portals completion events.  */
+  while (!done)
+    {
+      int pstatus;
+      gupcr_portals_call_with_status (PtlEQGet, pstatus,
+				      (gupcr_nb_md_eq, &event));
+      if (pstatus == PTL_OK)
+	{
+	  /* There is something to process.  */
+	  if (event.type == PTL_EVENT_ACK || event.type == PTL_EVENT_REPLY)
+	    {
+	      unsigned long id = (unsigned long) event.user_ptr;
+	      gupcr_debug (FC_NB, "received event for handle %lu", id);
+	      cb = gupcr_nbcb_find (id);
+	      if (!cb)
+		gupcr_fatal_error ("received event for invalid NB handle");
+	      cb->status = NB_STATUS_COMPLETED;
+	      gupcr_nb_outstanding--;
+	    }
+	  else
+	    {
+	      gupcr_fatal_error ("received event of invalid type: %s",
+				 gupcr_streqtype (event.type));
+	    }
+	}
+      else
+	done = 1;
+    }
+
+  /* Check if transfer is completed.  */
+  cb = gupcr_nbcb_find (handle);
+  if (cb && cb->status == NB_STATUS_COMPLETED)
+    {
+      gupcr_nbcb_active_remove (cb);
+      gupcr_nbcb_free (cb);
+      return 1;
+    }
+
+  return 0;
+}
+
+/**
+ * Complete non-blocking transfers with explicit handle
+ *
+ * Wait for outstanding request to complete.
+ *
+ * @param[in] handle Transfer handle
+ */
+void
+gupcr_sync (unsigned long handle)
+{
+  gupcr_nbcb_p cb;
+
+  gupcr_debug (FC_NB, "waiting for handle %lu", handle);
+  /* Check if transfer already completed.  */
+  cb = gupcr_nbcb_find (handle);
+  if (!cb)
+    {
+      /* Handle does not exist.  Assume it is a duplicate
+         sync request.  */
+      return;
+    }
+  if (cb->status == NB_STATUS_COMPLETED)
+    {
+      /* Already completed.  */
+      gupcr_nbcb_active_remove (cb);
+      gupcr_nbcb_free (cb);
+    }
+  else
+    {
+      int done = 0;
+      /* Must wait for portals to complete the transfer.  */
+      while (!done)
+	{
+	  ptl_event_t event;
+	  int pstatus;
+	  gupcr_portals_call_with_status (PtlEQGet, pstatus,
+					  (gupcr_nb_md_eq, &event));
+	  if (pstatus == PTL_OK)
+	    {
+	      /* Process only ACKs and REPLYs,  */
+	      gupcr_debug (FC_NB, "received event of type %s",
+			   gupcr_streqtype (event.type));
+	      if (event.type == PTL_EVENT_ACK
+		  || event.type == PTL_EVENT_REPLY)
+		{
+		  unsigned long id = (unsigned long) event.user_ptr;
+		  gupcr_debug (FC_NB, "received event for handle %lu", id);
+		  cb = gupcr_nbcb_find (id);
+		  if (!cb || cb->status == NB_STATUS_COMPLETED)
+		    {
+		      gupcr_fatal_error
+			("received event for non-existent or already "
+			 "completed NB handle");
+		    }
+		  cb->status = NB_STATUS_COMPLETED;
+		  gupcr_nb_outstanding--;
+		  if (id == handle)
+		    {
+		      gupcr_nbcb_active_remove (cb);
+		      gupcr_nbcb_free (cb);
+		      done = 1;
+		    }
+		}
+	      else
+		{
+		  gupcr_fatal_error ("received event of invalid type: %s",
+				     gupcr_streqtype (event.type));
+		}
+	    }
+	}
+    }
+}
+
+/**
+ * Check for any outstanding implicit handle non-blocking transfer
+ *
+ * @retval Number of outstanding transfers
+ */
+int
+gupcr_nbi_outstanding (void)
+{
+  ptl_ct_event_t ct;
+
+  /* Check the number of completed transfers.  */
+  gupcr_portals_call (PtlCTGet, (gupcr_nbi_md_ct, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_nbi_md_eq);
+      gupcr_fatal_error ("received an error on NBI MD");
+    }
+  return (int) (gupcr_nbi_md_count - ct.success);
+}
+
+/**
+ * Complete non-blocking transfers with implicit handle
+ *
+ * Wait for all outstanding requests to complete.
+ */
+void
+gupcr_synci (void)
+{
+  ptl_ct_event_t ct;
+  gupcr_portals_call (PtlCTWait, (gupcr_nbi_md_ct, gupcr_nbi_md_count, &ct));
+  if (ct.failure)
+    {
+      gupcr_process_fail_events (gupcr_nbi_md_eq);
+      gupcr_fatal_error ("received an error on NBI MD");
+    }
+}
+
+/**
+ * Initialize non-blocking transfer resources
+ * @ingroup INIT
+ */
+void
+gupcr_nb_init (void)
+{
+  ptl_md_t md;
+  ptl_pt_index_t pte;
+  ptl_le_t le;
+
+  gupcr_log (FC_NB, "non-blocking transfer init called");
+
+  /* Non-blocking transfers use their own PTE.  */
+  gupcr_portals_call (PtlPTAlloc, (gupcr_ptl_ni, 0,
+				   PTL_EQ_NONE, GUPCR_PTL_PTE_NB, &pte));
+  if (pte != GUPCR_PTL_PTE_NB)
+    gupcr_fatal_error ("cannot allocate PTE GUPCR_PTL_PTE_NB.");
+  gupcr_debug (FC_NB, "Non-blocking PTE allocated: %d", GUPCR_PTL_PTE_NB);
+  /* Allocate LE for non-blocking transfers.  */
+  le.start = gupcr_gmem_base;
+  le.length = gupcr_gmem_size;
+  le.ct_handle = PTL_CT_NONE;
+  le.uid = PTL_UID_ANY;
+  le.options = PTL_LE_OP_PUT | PTL_LE_OP_GET;
+  gupcr_portals_call (PtlLEAppend,
+		      (gupcr_ptl_ni, GUPCR_PTL_PTE_NB, &le,
+		       PTL_PRIORITY_LIST, NULL, &gupcr_nb_le));
+  gupcr_debug (FC_NB,
+	       "Non-blocking LE created at 0x%lx with size 0x%lx)",
+	       (long unsigned) gupcr_gmem_base,
+	       (long unsigned) gupcr_gmem_size);
+
+  /* Setup the Portals MD for local source/destination copying.
+     We need to map the whole user's space (same as gmem).  */
+  /* Non-blocking transfers with explicit handles must use full events
+     there is no need for counting events.  */
+  gupcr_portals_call (PtlEQAlloc,
+		      (gupcr_ptl_ni, GUPCR_NB_MAX_OUTSTANDING,
+		       &gupcr_nb_md_eq));
+  md.length = (ptl_size_t) USER_PROG_MEM_SIZE;
+  md.start = (void *) USER_PROG_MEM_START;
+  md.options = PTL_MD_EVENT_SEND_DISABLE;
+  md.eq_handle = gupcr_nb_md_eq;
+  md.ct_handle = PTL_CT_NONE;
+  gupcr_portals_call (PtlMDBind, (gupcr_ptl_ni, &md, &gupcr_nb_md));
+  gupcr_nb_md_start = md.start;
+  /* Non-blocking transfers with implicit handles use counting events.  */
+  gupcr_portals_call (PtlEQAlloc, (gupcr_ptl_ni, 1, &gupcr_nbi_md_eq));
+  gupcr_portals_call (PtlCTAlloc, (gupcr_ptl_ni, &gupcr_nbi_md_ct));
+  md.length = (ptl_size_t) USER_PROG_MEM_SIZE;
+  md.start = (void *) USER_PROG_MEM_START;
+  md.options = PTL_MD_EVENT_CT_ACK | PTL_MD_EVENT_CT_REPLY |
+    PTL_MD_EVENT_SUCCESS_DISABLE;
+  md.eq_handle = gupcr_nbi_md_eq;
+  md.ct_handle = gupcr_nbi_md_ct;
+  gupcr_portals_call (PtlMDBind, (gupcr_ptl_ni, &md, &gupcr_nbi_md));
+  gupcr_nbi_md_start = md.start;
+
+  /* Reset number of acknowledgments.  */
+  gupcr_nbi_md_count = 0;
+
+  /* Initialize NB handle values.  */
+  gupcr_nb_handle_next = 1;
+  /* Initialize number of outstanding transfers.  */
+  gupcr_nb_outstanding = 0;
+}
+
+/**
+ * Release non-blocking transfer resources
+ * @ingroup INIT
+ */
+void
+gupcr_nb_fini (void)
+{
+  gupcr_log (FC_NB, "non-blocking transfer fini called");
+  /* Release explicit handle NB MD and its resources.  */
+  gupcr_portals_call (PtlMDRelease, (gupcr_nb_md));
+  gupcr_portals_call (PtlEQFree, (gupcr_nb_md_eq));
+  /* Release implicit handle NB MD and its resources.  */
+  gupcr_portals_call (PtlMDRelease, (gupcr_nbi_md));
+  gupcr_portals_call (PtlEQFree, (gupcr_nbi_md_eq));
+  gupcr_portals_call (PtlCTFree, (gupcr_nbi_md_ct));
+  /* Release LE and PTE.  */
+  gupcr_portals_call (PtlLEUnlink, (gupcr_nb_le));
+  gupcr_portals_call (PtlPTFree, (gupcr_ptl_ni, GUPCR_PTL_PTE_NB));
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_nb_sup.h
===================================================================
--- libgupc/portals4/gupcr_nb_sup.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_nb_sup.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,54 @@
+/* Copyright (C) 2013-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_nb_sup.h
+ * GUPC Portals4 non-blocking transfer implementation support routines.
+ *
+ * @addtogroup NON-BLOCKING GUPCR Non-Blocking Transfer Support Functions
+ * @{
+ */
+
+#ifndef _GUPCR_NB_SUP_H_
+#define _GUPCR_NB_SUP_H_ 1
+
+/** Maximum number of outstanding non-blocking transfers */
+#define GUPCR_NB_MAX_OUTSTANDING 128
+
+extern void gupcr_nb_put (size_t, size_t, const void *,
+			  size_t, unsigned long *);
+extern void gupcr_nb_get (size_t, size_t, char *, size_t,
+			  unsigned long *);
+extern int gupcr_nb_completed (unsigned long);
+extern void gupcr_sync (unsigned long);
+extern int gupcr_nbi_outstanding (void);
+extern void gupcr_synci (void);
+extern void gupcr_nb_init (void);
+extern void gupcr_nb_fini (void);
+
+#endif /* gupcr_nb_sup.h */
+
+/** }@ */
Index: libgupc/portals4/gupcr_node.c
===================================================================
--- libgupc/portals4/gupcr_node.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_node.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,268 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_node.c
+ * GUPC Node Local Memory.
+ */
+
+/**
+ * @addtogroup NODE GUPCR Node Local Memory
+ * @{
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_runtime.h"
+#include "gupcr_gmem.h"
+#include "gupcr_node.h"
+
+/**
+ * GUPC Node Local Memory Support
+ *
+ * Multiple UPC threads can run on a single node and access
+ * each other's data via shared memory instead of
+ * the Portals library API.  This is true for all
+ * ordinary shared accesses (get/put), but not for atomic or
+ * some specialized Portals operations (e.g. triggered
+ * operations).  Future runtime improvements might remove
+ * this restriction in some cases.
+ *
+ * DISABLED
+ *   Each thread allocates its own shared memory with
+ *   a private mapping (MAP_ANONYMOUS).
+ *
+ *   This functionality is useful for testing purposes.
+ *
+ * ENABLED
+ *   Shared memory can be configured to use
+ *   an mmap backed file or POSIX shared memory.
+ *
+ *   Shared memory allocation and mapping:
+ *
+ *   (1) Each thread creates a shared object or file name in the
+ *       form of "upc-mem-MYTHREAD-MYPID" where MYTHREAD and MYPID
+ *       are 6 digit numbers with leading zeros.
+ *       This name refers to either a file name (mapped via mmap())
+ *       or a POSIX shared memory object.
+ *   (2) Each thread maps its own shared space and verifies that
+ *       it can be accessed (by writing to the first and last page).
+ *   (3) A runtime barrier is used to allow for every thread in
+ *       the system to complete its own setup.
+ *   (4) By using Portals pid-to-nid mappings, each thread searches for
+ *       other threads that reside on the same node (same nid).
+ *   (5) For each found thread, a POSIX shared object or a file
+ *       is opened (in the same manner as under the step 1), mapped,
+ *       and verified.
+ *
+ * UPC ACCESS
+ *   Each thread keeps a private array of addresses to other
+ *   threads' shared spaces.  A of NULL value indicates that
+ *   specified thread is NOT on the same node and Portals
+ *   functions should be used.  Otherwise, the mapped address
+ *   is the base address of the target thread's shared address space.
+ */
+
+/** Node local memory map (indexed by thread id).  */
+char **gupcr_node_map;
+/** Current thread for mapping verification.  */
+int gupcr_check_thread;
+
+/**
+ * SIGBUS handler for shared memory check.
+ *
+ * SIGBUS happens if the current thread is unable to
+ * access shared memory of some other thread.
+ */
+static void
+gupcr_mem_sigbus (int sig __attribute__ ((unused)))
+{
+#if GUPCR_NODE_LOCAL_MEM
+  gupcr_mem_local_unlink ();
+#endif
+  gupcr_fatal_error ("cannot access shared memory of thread %d",
+		     gupcr_check_thread);
+}
+
+/**
+ * Shared memory check.
+ *
+ * Check that memory at the specified address and with the
+ * specified size can be accessed.  As memory might not
+ * accessible, this procedure installs its own SIGBUS handler
+ * in order to provide the user with better diagnostic.
+ */
+static void
+gupcr_mem_check (char *mem, size_t size, int thread)
+{
+  char temp;
+  struct sigaction action;
+  struct sigaction old_action;
+
+  /* Set file name for better diagnostic.  */
+  gupcr_check_thread = thread;
+
+  /* Install new SIGBUS handler to catch memory faults.  */
+  action.sa_handler = gupcr_mem_sigbus;
+  sigemptyset (&action.sa_mask);
+  action.sa_flags = 0;
+  sigaction (SIGBUS, &action, &old_action);
+
+  if (gupcr_is_forcetouch_enabled ())
+    {
+      volatile char *memp = (volatile char *) mem;
+      while (memp < mem + size)
+	{
+	  temp = *memp;
+	  *memp = temp;
+	  memp += GUPCR_MEMORY_PAGE_SIZE;
+	}
+    }
+  else
+    {
+      /* Only check the first and the last page.  */
+      temp = *(volatile char *) mem;
+      *(volatile char *) mem = temp;
+      temp = *(volatile char *) (mem + size - 16);
+      *(volatile char *) (mem + size - 16) = temp;
+    }
+
+  /* Restore SIGBUS handler.  */
+  sigaction (SIGBUS, &old_action, NULL);
+}
+
+/**
+ * Allocate memory for a thread's shared space.
+ *
+ * This memory is never shared with other threads
+ * on the same node and is visible through Portals
+ * functions only.
+ *
+ * @retval Memory address of mapping
+ */
+char *
+gupcr_mem_private (size_t size)
+{
+  char *memaddr;
+  memaddr = mmap (NULL, size, PROT_READ | PROT_WRITE,
+		  MAP_PRIVATE | MAP_ANONYMOUS, -1, OFFSET_ZERO);
+  if (!memaddr || memaddr == MAP_ERROR)
+    gupcr_fatal_error
+      ("cannot mmap 0x%lx bytes of node's private shared memory (%s)",
+       (long unsigned) size, strerror (errno));
+  gupcr_log (FC_MEM, "using private mapping for shared space");
+  gupcr_mem_check (memaddr, size, MYTHREAD);
+  return memaddr;
+}
+
+/**
+ * Shared space allocation and mapping.
+ *
+ * Allocate and map each thread's node local shared space.
+ * Learn and map the shared space of other threads that reside on
+ * the same node.
+ *
+ * @in size Size of the shared space.
+ * @retval Address of the shared space.
+ */
+char *
+gupcr_node_local_alloc (size_t size)
+{
+  /* Allocate zero initialized space for node local memory map.  */
+  gupcr_node_map = calloc (THREADS, sizeof (char *));
+  if (!gupcr_node_map)
+    gupcr_fatal_error ("cannot allocate space for node local memory map");
+
+#if GUPCR_NODE_LOCAL_MEM
+  /* Node Local Memory can be disabled by env variable.  */
+  if (gupcr_is_node_local_memory_enabled ())
+    {
+      int i;
+      /* Create mapping for this thread.  */
+      gupcr_node_map[MYTHREAD] = gupcr_mem_local_map (MYTHREAD, size);
+      /* Verify that we can access memory.  */
+      gupcr_mem_check (gupcr_node_map[MYTHREAD], size, MYTHREAD);
+
+      /* Wait for all other threads to complete the same.  */
+      gupcr_runtime_barrier ();
+
+      /* Map shared memory of other threads on the same node.  */
+      {
+	int nid = gupcr_get_rank_nid (MYTHREAD);
+	for (i = 0; i < THREADS; i++)
+	  {
+	    if (i != MYTHREAD && nid == gupcr_get_rank_nid (i))
+	      {
+		gupcr_node_map[i] = gupcr_mem_local_map (i, size);
+	      }
+	  }
+      }
+      /* Make sure everybody completed their mappings.  */
+      gupcr_runtime_barrier ();
+      /* At this point it is safe to cleanup.  */
+      gupcr_mem_local_unlink ();
+    }
+  else
+    {
+      /* The node local shared memory optimization has been
+         disabled (via an environment variable).
+	 Create and map only the shared memory contribution of this node.  */
+      gupcr_node_map[MYTHREAD] = gupcr_mem_private (size);
+    }
+#else
+  /* The node local shared memory optimization has been
+     disabled (via a configuration option).
+     Create and map only the shared memory contribution of this node.  */
+  gupcr_node_map[MYTHREAD] = gupcr_mem_private (size);
+#endif /* GUPCR_NODE_LOCAL_MEM */
+  return gupcr_node_map[MYTHREAD];
+}
+
+/**
+ * Initialize node specific data.
+ */
+void
+gupcr_node_init (void)
+{
+#if GUPCR_NODE_LOCAL_MEM
+  if (gupcr_is_node_local_memory_enabled ())
+    gupcr_mem_local_init ();
+#endif
+}
+
+/**
+ * Finalize node specific data.
+ */
+void
+gupcr_node_fini (void)
+{
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_node.h
===================================================================
--- libgupc/portals4/gupcr_node.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_node.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,61 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef _GUPCR_NODE_H_
+#define _GUPCR_NODE_H_ 1
+
+/**
+ * @file gupcr_node.h
+ * GUPC Node Local Memory.
+ */
+
+/**
+ * @addtogroup NODE GUPCR Node Local Memory
+ * @{
+ */
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_runtime.h"
+
+//begin lib_node_local
+/** Memory map for threads that share node local memory.  */
+extern char **gupcr_node_map;
+//end lib_node_local
+
+/** Node local memory file/object prefix.  */
+#define GUPCR_LOCAL_NAME_PREFIX "upc-mem"
+
+char *gupcr_node_local_alloc (size_t);
+void gupcr_node_init (void);
+void gupcr_node_fini (void);
+char *gupcr_mem_local_map (int, size_t);
+void gupcr_mem_local_unlink (void);
+void gupcr_mem_local_init (void);
+/** @} */
+
+#endif /* gupcr_node.h */
Index: libgupc/portals4/gupcr_node_mem_mmap.c
===================================================================
--- libgupc/portals4/gupcr_node_mem_mmap.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_node_mem_mmap.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,102 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_node_mem_mmap.c
+ * GUPC Node Local Memory - Mmap Support.
+ */
+
+/**
+ * @addtogroup NODE GUPCR Node Local Memory
+ * @{
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_node.h"
+
+/** Thread's node local memory shared space.  */
+static char gupcr_thread_local_name[FILENAME_MAX];
+
+/**
+ * Map specified thread's shared memory into the current
+ * thread's memory.
+ *
+ * @param [in] thread Thread ID
+ * @param [in] size Shared memory size
+ * @retval Memory address of the map.
+ */
+char *
+gupcr_mem_local_map (int thread, size_t size)
+{
+  int fd;
+  char fname[FILENAME_MAX];
+  char *maddr;
+  /* Create file name for mmap.  */
+  gupcr_unique_local_name (fname, GUPCR_LOCAL_NAME_PREFIX, thread, 1);
+  if (thread == MYTHREAD)
+    {
+      strcpy (gupcr_thread_local_name, fname);
+      gupcr_log (FC_MEM, "shared memory file: %s", fname);
+    }
+  fd = open (fname, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+  if (fd < 0)
+    gupcr_fatal_error ("cannot create file for mmap %s", fname);
+  if (ftruncate (fd, size))
+    gupcr_fatal_error ("cannot resize file %s 0x%lx", fname, size);
+  maddr = mmap (NULL, size, PROT_READ | PROT_WRITE,
+		MAP_SHARED, fd, OFFSET_ZERO);
+  if (!maddr || maddr == MAP_ERROR)
+    gupcr_fatal_error ("cannot map 0x%lx bytes of %s", size, fname);
+  return maddr;
+}
+
+/**
+ * Remove the file used for mmap.
+ * On Unix systems, this removes the file name, but since
+ * the file is open, the underlying file storage will remain
+ * until the program terminates.
+ */
+void
+gupcr_mem_local_unlink (void)
+{
+  unlink (gupcr_thread_local_name);
+}
+
+/*
+ * Initialize mmap support.
+ */
+void
+gupcr_mem_local_init (void)
+{
+  gupcr_log (FC_MEM, "using node local mmap shared memory file");
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_node_mem_posix.c
===================================================================
--- libgupc/portals4/gupcr_node_mem_posix.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_node_mem_posix.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,106 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_node_mem_posix.c
+ * GUPC Node Local Memory - POSIX Shared Memory.
+ */
+
+/**
+ * @addtogroup NODE GUPCR Node Local Memory
+ * @{
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_runtime.h"
+#include "gupcr_node.h"
+
+/** Thread's node local memory shared space.  */
+static char gupcr_thread_local_name[FILENAME_MAX];
+
+/**
+ * Map specified thread's shared memory into the current
+ * thread's memory.
+ *
+ * @param [in] thread Thread ID
+ * @param [in] size Shared object size
+ * @retval Memory address.
+ */
+char *
+gupcr_mem_local_map (int thread, size_t size)
+{
+  int fd;
+  char fname[FILENAME_MAX];
+  char *maddr;
+  /* Create shared object name.  */
+  gupcr_unique_local_name (fname, GUPCR_LOCAL_NAME_PREFIX, thread, 0);
+  if (thread == MYTHREAD)
+    {
+      strcpy (gupcr_thread_local_name, fname);
+      gupcr_log (FC_MEM, "node local memory shared object: %s", fname);
+    }
+  fd = shm_open (fname, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+  if (fd < 0)
+    gupcr_fatal_error ("cannot create POSIX shared object %s", fname);
+  if (ftruncate (fd, size))
+    gupcr_fatal_error ("cannot resize POSIX shared object %s 0x%lx",
+		       fname, (long unsigned) size);
+  maddr = mmap (NULL, size, PROT_READ | PROT_WRITE,
+		MAP_SHARED, fd, OFFSET_ZERO);
+  if (!maddr || maddr == MAP_ERROR)
+    gupcr_fatal_error ("cannot map POSIX shared object: 0x%lx bytes of %s",
+		       (long unsigned) size, fname);
+  return maddr;
+}
+
+/**
+ * Remove the file used for the shared object.
+ *
+ * On Unix systems, this removes the file name, but since
+ * the file is open, the underlying file storage will remain
+ * until the program terminates.
+ */
+void
+gupcr_mem_local_unlink (void)
+{
+  shm_unlink (gupcr_thread_local_name);
+}
+
+/**
+ * Initialize the POSIX shared node local memory support.
+ */
+void
+gupcr_mem_local_init (void)
+{
+  gupcr_log (FC_MEM, "Using node local POSIX shared memory");
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_pgm_info.c
===================================================================
--- libgupc/portals4/gupcr_pgm_info.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_pgm_info.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,243 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_pgm_info.c
+ * GUPC Runtime program information routines
+ */
+
+/**
+ * @addtogroup GUPCUTILS GUPCR Utility Functions
+ * @{
+ */
+
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_sup.h"
+#include "gupcr_utils.h"
+
+
+typedef enum
+{
+  upc_threads_model_none = 0,
+  upc_threads_model_process = 1,
+} upc_threads_model_t;
+
+typedef struct upc_compiled_thread_info_struct
+{
+    /** Next on the list of files */
+  struct upc_compiled_thread_info_struct *next;
+    /** File name */
+  char *filename;
+    /** Number of compiled threads for the file */
+  int nthreads;
+    /** Thread's model (process/pthreads) */
+  upc_threads_model_t threads_model;
+} upc_compiled_thread_info_t;
+typedef upc_compiled_thread_info_t *upc_compiled_thread_info_p;
+
+/* List of compiled UPC files, and the value of THREADS
+   specified at compile-time (-1 means no value given
+   at compile-time).  */
+static upc_compiled_thread_info_p gupcr_compiled_thread_info = 0;
+
+static void
+gupcr_print_upc_compiled_thread_info (void)
+{
+  upc_compiled_thread_info_p p;
+  gupcr_error ("   THREADS   Threads Model  Filename\n");
+  for (p = gupcr_compiled_thread_info; p; p = p->next)
+    {
+      if (p->nthreads > 0)
+	gupcr_error ("%10d", p->nthreads);
+      else
+	gupcr_error (" <dynamic>");
+      if (p->threads_model == upc_threads_model_process)
+	gupcr_error ("         process");
+      gupcr_error (" %s\n", p->filename);
+    }
+}
+
+static void
+gupcr_register_pgm_info (char *filename, int nthreads,
+			 upc_threads_model_t threads_model)
+{
+  upc_compiled_thread_info_p info, *p;
+  gupcr_malloc (info, (sizeof (upc_compiled_thread_info_t)));
+  /* Insertion into list is ordered by file name.  */
+  for (p = &gupcr_compiled_thread_info;
+       *p && strcmp (filename, (*p)->filename) >= 0;
+       p = &(*p)->next) /* loop */ ;
+  info->filename = filename;
+  info->nthreads = nthreads;
+  info->threads_model = threads_model;
+  info->next = *p;
+  *p = info;
+}
+
+static void
+gupcr_skip_spaces (const char **s)
+{
+  while (**s == ' ')
+    ++(*s);
+}
+
+static int
+gupcr_match_string (const char **s, const char *string)
+{
+  int slen = strlen (string);
+  if (strncmp (*s, string, slen) != 0)
+    return 0;
+  *s += slen;
+  return 1;
+}
+
+static int
+gupcr_match_until (const char **s, const char *string)
+{
+  int slen = strlen (string);
+  while (**s && (strncmp (*s, string, slen) != 0))
+    ++(*s);
+  if (!**s)
+    return 0;
+  *s += slen;
+  return 1;
+}
+
+static int
+gupcr_match_num (const char **s, int *num)
+{
+  *num = 0;
+  while (**s >= '0' && **s <= '9')
+    {
+      *num = *num * 10 + (**s - '0');
+      ++(*s);
+    }
+  if (*num == 0)
+    return 0;
+  return 1;
+}
+
+/* Examples:
+ $GCCUPCConfig: (t.upc) dynamicthreads process$
+ $GCCUPCConfig: (t.upc) staticcthreads=4 pthreads-tls staticpthreads=4$ */
+static void
+gupcr_parse_program_info (char *info)
+{
+  char *filename;
+  int nthreads = -1;
+  upc_threads_model_t threads_model = upc_threads_model_none;
+  const char *fname;
+  int fname_len;
+  const char *s = info;
+  if (!gupcr_match_string (&s, "$GCCUPCConfig:"))
+    return;
+  gupcr_skip_spaces (&s);
+  if (!gupcr_match_string (&s, "("))
+    return;
+  fname = s;
+  if (!gupcr_match_until (&s, ")"))
+    return;
+  fname_len = (s - fname - 1);
+  gupcr_malloc (filename, (fname_len + 1));
+  strncpy (filename, fname, fname_len);
+  filename[fname_len] = '\0';
+  while (*s)
+    {
+      gupcr_skip_spaces (&s);
+      if (gupcr_match_string (&s, "$"))
+	{
+	  break;
+	}
+      else if (gupcr_match_string (&s, "dynamicthreads"))
+	{
+	  nthreads = -1;
+	}
+      else if (gupcr_match_string (&s, "staticthreads="))
+	{
+	  if (!gupcr_match_num (&s, &nthreads))
+	    return;
+	}
+      else if (gupcr_match_string (&s, "process"))
+	{
+	  threads_model = upc_threads_model_process;
+	}
+      else
+	return;
+    }
+  gupcr_register_pgm_info (filename, nthreads, threads_model);
+}
+
+void
+gupcr_validate_pgm_info (void)
+{
+  upc_compiled_thread_info_p p;
+  char *info;
+  int nthreads = -1;
+  /* Process all the strings within the program information section.
+     (Ignore intervening null bytes.)  */
+  for (info = GUPCR_PGM_INFO_SECTION_START;
+       info < GUPCR_PGM_INFO_SECTION_END; ++info)
+    {
+      if (*info)
+	{
+	  gupcr_parse_program_info (info);
+	  info += strlen (info);
+	}
+    }
+  if (!gupcr_compiled_thread_info)
+    gupcr_abort_with_msg ("there are no UPC source files "
+			  "compiled into this program, "
+			  "or perhaps <upc.h> was not included");
+  for (p = gupcr_compiled_thread_info; p; p = p->next)
+    {
+      if (p->nthreads > 0 && nthreads <= 0)
+	nthreads = p->nthreads;
+      /* Static threads compilations can be intermixed
+         with dynamic threads compilations, but the static values
+         must agree.  */
+      if (((p->nthreads != nthreads)
+	   && (p->nthreads > 0)
+	   && (nthreads > 0))
+	  || (p->threads_model != gupcr_compiled_thread_info->threads_model))
+	{
+	  gupcr_assert (MYTHREAD >= 0);
+	  if (!MYTHREAD)
+	    {
+	      gupcr_error ("the UPC source files in this "
+			   "program were not compiled with the same value "
+			   "of UPC settings;\n"
+			   "a list of each UPC source file and "
+			   "its compiled UPC settings follows");
+	      gupcr_print_upc_compiled_thread_info ();
+	    }
+	  exit (2);
+	}
+    }
+  THREADS = nthreads;
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_portals.c
===================================================================
--- libgupc/portals4/gupcr_portals.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_portals.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,521 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_portals.c
+ * GUPC Portals4 Initialization.
+ */
+
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_runtime.h"
+
+/* Portals network interface handle/limits/rank.  */
+ptl_handle_ni_t gupcr_ptl_ni;
+ptl_ni_limits_t gupcr_ptl_ni_limits;
+ptl_rank_t gupcr_ptl_rank;
+ptl_nid_t gupcr_ptl_nid;
+ptl_pid_t gupcr_ptl_pid;
+int gupcr_ptl_size;
+
+int gupcr_child[GUPCR_TREE_FANOUT];
+int gupcr_child_cnt;
+int gupcr_parent_thread;
+
+size_t gupcr_max_ordered_size;
+size_t gupcr_max_msg_size;
+size_t gupcr_max_volatile_size;
+
+/** Mapping to nid/pid for each rank */
+static ptl_process_t *gupcr_ptl_proc_map;
+
+/**
+ * @addtogroup PORTALS_RUNTIME GUPCR Portals runtime interface
+ * @{
+ */
+
+/**
+ * Return Portals error description string.
+ *
+ * @param [in] errnum Portals error number
+ * @retval Error description string
+ */
+const char *
+gupcr_strptlerror (int errnum)
+{
+  static char gupcr_strptlerror_buf[64];
+  switch (errnum)
+    {
+    case PTL_OK:
+      return "portals operation successful";
+    case PTL_ARG_INVALID:
+      return "invalid portals argument";
+    case PTL_CT_NONE_REACHED:
+      return "timeout reached";
+    case PTL_EQ_DROPPED:
+      return "portals event dropped";
+    case PTL_EQ_EMPTY:
+      return "portals event queue empty";
+    case PTL_FAIL:
+      return "portals operation failed";
+    case PTL_IGNORED:
+      return "logical map set failed";
+    case PTL_IN_USE:
+      return "portals resource already in use";
+    case PTL_INTERRUPTED:
+      return "portals operation interrupted";
+    case PTL_LIST_TOO_LONG:
+      return "portals list too long";
+    case PTL_NO_INIT:
+      return "portals not initialized";
+    case PTL_NO_SPACE:
+      return "portals out of memory";
+    case PTL_PID_IN_USE:
+      return "portals portal ID already in use";
+    case PTL_PT_FULL:
+      return "portals portal table is full";
+    case PTL_PT_EQ_NEEDED:
+      return "portals event queue needed for flow control";
+    case PTL_PT_IN_USE:
+      return "portals PTE already in use";
+    default:
+      break;
+    }
+  sprintf (gupcr_strptlerror_buf, "unknown portals status code: %d", errnum);
+  return gupcr_strptlerror_buf;
+}
+
+/**
+ * Return Event Queue type description string.
+ *
+ * @param [in] eqtype Event queue type
+ * @retval Event queue type description string
+ */
+const char *
+gupcr_streqtype (ptl_event_kind_t eqtype)
+{
+  switch (eqtype)
+    {
+    case PTL_EVENT_GET:
+      return "PTL_EVENT_GET";
+    case PTL_EVENT_GET_OVERFLOW:
+      return "PTL_EVENT_GET_OVERFLOW";
+    case PTL_EVENT_PUT:
+      return "PTL_EVENT_PUT";
+    case PTL_EVENT_PUT_OVERFLOW:
+      return "PTL_EVENT_PUT_OVERFLOW";
+    case PTL_EVENT_ATOMIC:
+      return "PTL_EVENT_ATOMIC";
+    case PTL_EVENT_ATOMIC_OVERFLOW:
+      return "PTL_EVENT_ATOMIC_OVERFLOW";
+    case PTL_EVENT_FETCH_ATOMIC:
+      return "PTL_EVENT_ATOMIC";
+    case PTL_EVENT_FETCH_ATOMIC_OVERFLOW:
+      return "PTL_EVENT_ATOMIC_OVERFLOW";
+    case PTL_EVENT_REPLY:
+      return "PTL_EVENT_REPLY";
+    case PTL_EVENT_SEND:
+      return "PTL_EVENT_SEND";
+    case PTL_EVENT_ACK:
+      return "PTL_EVENT_ACK";
+    case PTL_EVENT_PT_DISABLED:
+      return "PTL_EVENT_PT_DISABLED";
+    case PTL_EVENT_LINK:
+      return "PTL_EVENT_LINK";
+    case PTL_EVENT_AUTO_UNLINK:
+      return "PTL_EVENT_AUTO_UNLINK";
+    case PTL_EVENT_AUTO_FREE:
+      return "PTL_EVENT_AUTO_FREE";
+    case PTL_EVENT_SEARCH:
+      return "PTL_EVENT_SEARCH";
+    }
+  return "UNKNOWN EVENT TYPE";
+}
+
+/**
+ * Return Data type description string.
+ *
+ * @param [in] datatype Data type
+ * @retval Data type description string
+ */
+const char *
+gupcr_strptldatatype (ptl_datatype_t datatype)
+{
+  switch (datatype)
+    {
+    case PTL_INT8_T:
+      return "PTL_INT8_T";
+    case PTL_UINT8_T:
+      return "PTL_UINT8_T";
+    case PTL_INT16_T:
+      return "PTL_INT16_T";
+    case PTL_UINT16_T:
+      return "PTL_UINT16_T";
+    case PTL_INT32_T:
+      return "PTL_INT32_T";
+    case PTL_UINT32_T:
+      return "PTL_UINT32_T";
+    case PTL_FLOAT:
+      return "PTL_FLOAT";
+    case PTL_INT64_T:
+      return "PTL_INT64_T";
+    case PTL_UINT64_T:
+      return "PTL_UINT64_T";
+    case PTL_DOUBLE:
+      return "PTL_DOUBLE";
+    case PTL_FLOAT_COMPLEX:
+      return "PTL_FLOAT_COMPLEX";
+    case PTL_DOUBLE_COMPLEX:
+      return "PTL_DOUBLE_COMPLEX";
+    case PTL_LONG_DOUBLE:
+      return "PTL_LONG_DOUBLE";
+    case PTL_LONG_DOUBLE_COMPLEX:
+      return "PTL_LONG_DOUBLE_COMPLEX";
+    }
+  return "UNKNOWN DATA TYPE";
+}
+
+/**
+ * Return Atomic operation description string.
+ *
+ * @param [in] op Atomic operation type
+ * @retval Atomic operation description string
+ */
+const char *
+gupcr_strptlop (ptl_op_t op)
+{
+  switch (op)
+    {
+    case PTL_MIN:
+      return "PTL_MIN";
+    case PTL_MAX:
+      return "PTL_MAX";
+    case PTL_SUM:
+      return "PTL_SUM";
+    case PTL_PROD:
+      return "PTL_PROD";
+    case PTL_LOR:
+      return "PTL_LOR";
+    case PTL_LAND:
+      return "PTL_LAND";
+    case PTL_BOR:
+      return "PTL_BOR";
+    case PTL_BAND:
+      return "PTL_BAND";
+    case PTL_LXOR:
+      return "PTL_LXOR";
+    case PTL_BXOR:
+      return "PTL_BXOR";
+    case PTL_SWAP:
+      return "PTL_SWAP";
+    case PTL_CSWAP:
+      return "PTL_CSWAP";
+    case PTL_CSWAP_NE:
+      return "PTL_CSWAP_NE";
+    case PTL_CSWAP_LE:
+      return "PTL_CSWAP_LE";
+    case PTL_CSWAP_LT:
+      return "PTL_CSWAP_LT";
+    case PTL_CSWAP_GE:
+      return "PTL_CSWAP_GE";
+    case PTL_CSWAP_GT:
+      return "PTL_CSWAP_GT";
+    case PTL_MSWAP:
+      return "PTL_MSWAP";
+    }
+  return "UNKNOWN ATOMIC OPERATION TYPE";
+}
+
+/**
+ * Return Network Interface error description string.
+ *
+ * @param [in] nitype NI failure type
+ * @retval NI failure description string.
+ */
+const char *
+gupcr_nifailtype (ptl_ni_fail_t nitype)
+{
+  switch (nitype)
+    {
+    case PTL_NI_OK:
+      return "PTL_NI_OK";
+    case PTL_NI_UNDELIVERABLE:
+      return "PTL_NI_UNDELIVERABLE";
+    case PTL_NI_DROPPED:
+      return "PTL_NI_DROPPED";
+    case PTL_NI_PT_DISABLED:
+      return "PTL_NI_PT_DISABLED";
+    case PTL_NI_PERM_VIOLATION:
+      return "PTL_NI_PERM_VIOLATION";
+    case PTL_NI_OP_VIOLATION:
+      return "PTL_NI_OP_VIOLATION";
+    case PTL_NI_NO_MATCH:
+      return "PTL_NI_NO_MATCH";
+    case PTL_NI_SEGV:
+      return "PTL_NI_SEGV";
+    }
+  return "NI_FAILURE_UNKNOWN";
+}
+
+/**
+ * Return Portals atomic data type from the specified size.
+ */
+ptl_datatype_t
+gupcr_get_atomic_datatype (int size)
+{
+  switch (size)
+    {
+    case 1:
+      return PTL_UINT8_T;
+    case 2:
+      return PTL_UINT16_T;
+    case 4:
+      return PTL_UINT32_T;
+    case 8:
+      return PTL_UINT64_T;
+    case 16:
+      return PTL_DOUBLE_COMPLEX;
+    default:
+      gupcr_fatal_error
+	("Unable to convert size of %d into Portals atomic data type.", size);
+    }
+  return -1;
+}
+
+/**
+ * Return Portals data size from the specified atomic type.
+ *
+ * @param [in] type Portals atomic data type
+ * @retval Portals atomic data type size
+ */
+size_t
+gupcr_get_atomic_size (ptl_datatype_t type)
+{
+  switch (type)
+    {
+    case PTL_INT8_T:
+    case PTL_UINT8_T:
+      return 1;
+    case PTL_INT16_T:
+    case PTL_UINT16_T:
+      return 2;
+    case PTL_INT32_T:
+    case PTL_UINT32_T:
+      return 4;
+    case PTL_INT64_T:
+    case PTL_UINT64_T:
+      return 8;
+    case PTL_FLOAT:
+      return __SIZEOF_FLOAT__;
+    case PTL_FLOAT_COMPLEX:
+      return 2 * __SIZEOF_FLOAT__;
+    case PTL_DOUBLE:
+      return __SIZEOF_DOUBLE__;
+    case PTL_DOUBLE_COMPLEX:
+      return 2 * __SIZEOF_DOUBLE__;
+#ifdef __SIZEOF_LONG_DOUBLE__
+    case PTL_LONG_DOUBLE:
+      return __SIZEOF_LONG_DOUBLE__;
+    case PTL_LONG_DOUBLE_COMPLEX:
+      return 2 * __SIZEOF_LONG_DOUBLE__;
+#endif
+    default:
+      gupcr_fatal_error ("unknown atomic type %d", (int) type);
+    }
+  return -1;
+}
+
+/**
+ * @fn gupcr_process_fail_events (ptl_handle_eq_t eq)
+ * Show information on failed events.
+ *
+ * This procedure prints the contents of the event queue.  As barrier
+ * implementation does not use full events, the event queue contains
+ * only failure events.  This procedure is called only if any of the
+ * counting events reported a failure.
+ *
+ * @param [in] eq Event Queue ID
+ */
+void
+gupcr_process_fail_events (ptl_handle_eq_t eq)
+{
+  ptl_event_t ev;
+  int status;
+
+  while ((status = PtlEQGet (eq, &ev)) != PTL_EQ_EMPTY)
+    {
+      const char *eqerr = gupcr_streqtype (ev.type);
+      const char *nierr = gupcr_nifailtype (ev.ni_fail_type);
+      gupcr_error_print ("event failure %s (%s (%x))", eqerr, nierr,
+			 ev.ni_fail_type);
+    }
+}
+
+/**
+ * Get current thread rank.
+ * @retval Rank of the current thread
+ */
+int
+gupcr_get_rank (void)
+{
+  return gupcr_ptl_rank;
+}
+
+/**
+ * Get number of running threads.
+ * @retval Number of running threads
+ */
+int
+gupcr_get_threads_count (void)
+{
+  return gupcr_ptl_size;
+}
+
+/**
+ * Get process PID for specified rank.
+ * @param [in] rank Rank of the thread
+ * @retval PID of the thread
+ */
+int
+gupcr_get_rank_pid (int rank)
+{
+  return gupcr_ptl_proc_map[rank].phys.pid;
+}
+
+/**
+ * Get process NID for specified rank.
+ * @param [in] rank Rank of the thread
+ * @retval NID of the thread
+ */
+int
+gupcr_get_rank_nid (int rank)
+{
+  return gupcr_ptl_proc_map[rank].phys.nid;
+}
+
+/**
+ * Wait for all threads to complete initialization.
+ */
+void
+gupcr_startup_barrier (void)
+{
+  gupcr_runtime_barrier ();
+}
+
+/** @} */
+
+/**
+ * @addtogroup INIT GUPCR Initialization
+ * @{
+ */
+
+/**
+ * Initialize Portals Interface.
+ *
+ * Calls to get rank/number of threads can
+ * be made only after Portals is initialized.
+ * Also, initialize the necessary runtime.
+ */
+void
+gupcr_portals_init (void)
+{
+  gupcr_portals_call (PtlInit, ());
+  /* Set rank for this thread.  */
+  gupcr_ptl_rank = gupcr_runtime_get_rank ();
+  gupcr_ptl_size = gupcr_runtime_get_size ();
+}
+
+/**
+ * Close Portals.
+ */
+void
+gupcr_portals_fini (void)
+{
+  PtlFini ();
+}
+
+/**
+ * Initialize Portals Networking Interface.
+ */
+void
+gupcr_portals_ni_init (void)
+{
+  /* Initialize network interface.  */
+  gupcr_portals_call (PtlNIInit,
+		      (PTL_IFACE_DEFAULT, PTL_NI_NO_MATCHING | PTL_NI_LOGICAL,
+		       PTL_PID_ANY, NULL, &gupcr_ptl_ni_limits,
+		       &gupcr_ptl_ni));
+  /* Initialize limits used by GMEM.  */
+  gupcr_max_ordered_size = gupcr_ptl_ni_limits.max_waw_ordered_size;
+  gupcr_max_msg_size = gupcr_ptl_ni_limits.max_msg_size;
+  gupcr_max_volatile_size = gupcr_ptl_ni_limits.max_volatile_size;
+
+  /* Initialize the mapping from rank -> nid/pid.  */
+  gupcr_ptl_proc_map = gupcr_runtime_get_mapping (gupcr_ptl_ni);
+  gupcr_portals_call (PtlSetMap, (gupcr_ptl_ni, (ptl_size_t) gupcr_ptl_size,
+				  gupcr_ptl_proc_map));
+
+  /* Get this thread physical IDs.  */
+  gupcr_ptl_pid = gupcr_get_rank_pid (gupcr_ptl_rank);
+  gupcr_ptl_nid = gupcr_get_rank_nid (gupcr_ptl_rank);
+}
+
+/**
+ * Close Portals Networking Interface.
+ */
+void
+gupcr_portals_ni_fini (void)
+{
+  gupcr_portals_call (PtlNIFini, (gupcr_ptl_ni));
+}
+
+/**
+ * Find the node's parent and all its children.
+ */
+void
+gupcr_nodetree_setup (void)
+{
+  int i;
+  gupcr_log ((FC_BARRIER | FC_BROADCAST),
+	     "node tree initialized with fanout of %d", GUPCR_TREE_FANOUT);
+  for (i = 0; i < GUPCR_TREE_FANOUT; i++)
+    {
+      int child = GUPCR_TREE_FANOUT * MYTHREAD + i + 1;
+      if (child < THREADS)
+	{
+	  gupcr_child_cnt++;
+	  gupcr_child[i] = child;
+	}
+    }
+  if (MYTHREAD == 0)
+    gupcr_parent_thread = ROOT_PARENT;
+  else
+    gupcr_parent_thread = (MYTHREAD - 1) / GUPCR_TREE_FANOUT;
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_portals.h
===================================================================
--- libgupc/portals4/gupcr_portals.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_portals.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,167 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef _GUPCR_PORTALS_H_
+#define _GUPCR_PORTALS_H_
+
+#include <portals4.h>
+
+/**
+ * @file gupcr_portals.h
+ * GUPC Portals4 Global Definitions.
+ */
+
+/**
+ * @addtogroup CONFIG GUPCR Configuration
+ * @{
+ */
+
+/* GUPCR Portals Table Entries */
+/** Memory put/get functions PTE */
+#define	GUPCR_PTL_PTE_GMEM		GUPCR_PTE_BASE
+/** Barrier messages to parent node PTE */
+#define	GUPCR_PTL_PTE_BARRIER_UP	GUPCR_PTE_BASE+1
+/** Barrier messages from parent node PTE */
+#define	GUPCR_PTL_PTE_BARRIER_DOWN	GUPCR_PTE_BASE+2
+/** Lock signaling PTE */
+#define	GUPCR_PTL_PTE_LOCK		GUPCR_PTE_BASE+3
+/** Shutdown service signaling PTE */
+#define	GUPCR_PTL_PTE_SHUTDOWN		GUPCR_PTE_BASE+4
+/** Collectives service signaling PTE */
+#define	GUPCR_PTL_PTE_COLL		GUPCR_PTE_BASE+5
+/** Non-blocking transfers PTE */
+#define	GUPCR_PTL_PTE_NB		GUPCR_PTE_BASE+6
+/** @} */
+
+//begin lib_portals
+
+/** Max ordered size - per network interface */
+extern size_t gupcr_max_ordered_size;
+#define GUPCR_PORTALS_MAX_ORDERED_SIZE gupcr_max_ordered_size
+/** Max size of a message (put, get, or reply) */
+extern size_t gupcr_max_msg_size;
+#define GUPCR_PORTALS_MAX_MSG_SIZE gupcr_max_msg_size
+/** Max size of a message that can use volatile memory descriptor */
+extern size_t gupcr_max_volatile_size;
+#define GUPCR_PORTALS_MAX_VOLATILE_SIZE gupcr_max_volatile_size
+
+//end lib_portals
+
+/** NULL value for matching bits */
+#define PTL_NO_MATCH_BITS ((ptl_match_bits_t) 0)
+/** NULL value for user pointer */
+#define PTL_NULL_USER_PTR ((void *) 0)
+/** NULL value for header data */
+#define PTL_NULL_HDR_DATA ((ptl_hdr_data_t) 0)
+/** Execute portals call and abort if error */
+#define gupcr_portals_call(portals_func, args)				\
+    do									\
+      {									\
+        int pstatus;							\
+        pstatus = portals_func args;					\
+	if (pstatus != PTL_OK)						\
+	  gupcr_fatal_error ("UPC runtime Portals call "		\
+	                     "`%s' on thread %d failed: %s\n", 		\
+			     __STRING(portals_func), gupcr_get_rank (),	\
+	                     gupcr_strptlerror (pstatus));		\
+      }									\
+    while (0)
+
+/** Execute portals call and return status if there is no fatal error */
+#define gupcr_portals_call_with_status(portals_func, pstatus, args)	\
+    do									\
+      {									\
+        pstatus = portals_func args;					\
+	if ((pstatus != PTL_OK) &&					\
+	    (pstatus != PTL_EQ_EMPTY) &&				\
+	    (pstatus != PTL_CT_NONE_REACHED) &&				\
+	    (pstatus != PTL_IN_USE) &&					\
+	    (pstatus != PTL_INTERRUPTED))		        	\
+	  gupcr_fatal_error ("UPC runtime Portals call "		\
+	                     "`%s' on thread %d failed: %s\n",		\
+			     __STRING(portals_func), gupcr_get_rank (),	\
+	                     gupcr_strptlerror (pstatus));		\
+      }									\
+    while (0)
+
+/**
+ * @addtogroup GLOBAL GUPCR Global Variables
+ * @{
+ */
+
+/** Network Interface Handle */
+extern ptl_handle_ni_t gupcr_ptl_ni;
+/** Network Interface Limits */
+extern ptl_ni_limits_t gupcr_ptl_ni_limits;
+/** Thread's process info */
+extern ptl_process_t gupcr_ptl_myproc;
+/** Thread's rank */
+extern ptl_rank_t gupcr_ptl_rank;
+/** Thread's NID */
+extern ptl_nid_t gupcr_ptl_nid;
+/** Thread's PID */
+extern ptl_pid_t gupcr_ptl_pid;
+
+/* For the purposes of implementing GUPC barriers, all UPC threads
+   in a given job are organized as a tree.  Thread 0 is the
+   root node (at the top of the tree).  Other threads can represent
+   either an inner node (has at least one child), or a leaf
+   node (has no children).  */
+
+/** Parent of the root thread */
+#define ROOT_PARENT -1
+
+/** Thread IDs of all children of the current thread.  */
+extern int gupcr_child[GUPCR_TREE_FANOUT];
+/** Number of children of the current thread.  */
+extern int gupcr_child_cnt;
+/** Parent thread ID of the current thread.
+    The tree root thread has a parent ID of -1.  */
+extern int gupcr_parent_thread;
+
+/** @} */
+
+extern const char *gupcr_strptlerror (int);
+extern const char *gupcr_streqtype (ptl_event_kind_t);
+extern const char *gupcr_strptlop (ptl_op_t);
+extern const char *gupcr_strptldatatype (ptl_datatype_t);
+extern const char *gupcr_nifailtype (ptl_ni_fail_t);
+extern void gupcr_process_fail_events (ptl_handle_eq_t);
+extern ptl_datatype_t gupcr_get_atomic_datatype (int);
+extern size_t gupcr_get_atomic_size (ptl_datatype_t);
+extern int gupcr_get_rank (void);
+extern int gupcr_get_threads_count (void);
+extern int gupcr_get_rank_pid (int rank);
+extern int gupcr_get_rank_nid (int rank);
+extern void gupcr_startup_barrier (void);
+
+extern void gupcr_portals_init (void);
+extern void gupcr_portals_fini (void);
+extern void gupcr_portals_ni_init (void);
+extern void gupcr_portals_ni_fini (void);
+extern void gupcr_nodetree_setup (void);
+
+#endif /* gupcr_portals.h */
Index: libgupc/portals4/gupcr_pts.h
===================================================================
--- libgupc/portals4/gupcr_pts.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_pts.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,193 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_pts.h
+ * GUPC Runtime pointer to shared definitions.
+ */
+
+#ifndef _GUPCR_PTS_H_
+#define _GUPCR_PTS_H_ 1
+
+//begin lib_pts_defs
+
+/* UPC pointer representation.  */
+
+#if (defined (GUPCR_PTS_STRUCT_REP) + defined (GUPCR_PTS_WORD_PAIR_REP) \
+     + defined (GUPCR_PTS_PACKED_REP)) == 0
+#error Unknown PTS representation.
+#elif (defined (GUPCR_PTS_STRUCT_REP) + defined (GUPCR_PTS_WORD_PAIR_REP) \
+     + defined (GUPCR_PTS_PACKED_REP)) != 1
+#error Only one UPC shared pointer representation setting is permitted.
+#endif
+
+#ifdef GUPCR_PTS_STRUCT_REP
+
+#ifndef GUPCR_PTS_THREAD_FIELD
+#define GUPCR_PTS_THREAD_FIELD
+#endif
+#ifndef GUPCR_PTS_PHASE_FIELD
+#define GUPCR_PTS_PHASE_FIELD
+#endif
+#if GUPCR_PTS_THREAD_SIZE == 32
+#undef GUPCR_PTS_THREAD_TYPE
+#define GUPCR_PTS_THREAD_TYPE u_intSI_t
+#elif GUPCR_PTS_THREAD_SIZE == 16
+#undef GUPCR_PTS_THREAD_TYPE
+#define GUPCR_PTS_THREAD_TYPE u_intHI_t
+#endif
+#if GUPCR_PTS_PHASE_SIZE == 32
+#undef GUPCR_PTS_PHASE_TYPE
+#define GUPCR_PTS_PHASE_TYPE u_intSI_t
+#elif GUPCR_PTS_PHASE_SIZE == 16
+#undef GUPCR_PTS_PHASE_TYPE
+#define GUPCR_PTS_PHASE_TYPE u_intHI_t
+#endif
+
+#if !__GCC_UPC__
+/* The UPC compiler pre-defines upc_shared_ptr_t to be the
+   representation of a shared pointer.  Since most of the
+   runtime is written in regular "C", we need to define
+   the pointer representation here.  */
+typedef struct shared_ptr_struct
+{
+#if GUPCR_PTS_VADDR_FIRST
+  GUPCR_PTS_VADDR_TYPE vaddr;
+  GUPCR_PTS_THREAD_TYPE thread GUPCR_PTS_THREAD_FIELD;
+  GUPCR_PTS_PHASE_TYPE phase GUPCR_PTS_PHASE_FIELD;
+#else
+  GUPCR_PTS_PHASE_TYPE phase GUPCR_PTS_PHASE_FIELD;
+  GUPCR_PTS_THREAD_TYPE thread GUPCR_PTS_THREAD_FIELD;
+  GUPCR_PTS_VADDR_TYPE vaddr;
+#endif
+} upc_shared_ptr_t __attribute__ ((aligned (GUPCR_PTS_ALIGN)));
+typedef upc_shared_ptr_t *upc_shared_ptr_p;
+/* upc_dbg_shared_ptr_t is used by the debugger to figure out
+   shared pointer layout.  */
+typedef upc_shared_ptr_t upc_dbg_shared_ptr_t;
+#endif
+
+#define GUPCR_PTS_TO_REP(V) *((upc_shared_ptr_t *)&(V))
+#define GUPCR_PTS_IS_NULL(P) (!(P).vaddr && !(P).thread && !(P).phase)
+#define GUPCR_PTS_SET_NULL_SHARED(P) \
+   {(P).vaddr = 0; (P).thread = 0; (P).phase = 0;}
+
+#define GUPCR_PTS_VADDR(P) ((size_t)(P).vaddr \
+                              - (size_t)GUPCR_SHARED_SECTION_START)
+#define GUPCR_PTS_OFFSET(P) ((size_t)(P).vaddr \
+                              - (size_t)GUPCR_SHARED_SECTION_START)
+#define GUPCR_PTS_THREAD(P) (P).thread
+#define GUPCR_PTS_PHASE(P) (P).phase
+
+#define GUPCR_PTS_SET_VADDR(P,V) (P).vaddr = (void *)((size_t)(V) \
+				+ (size_t)GUPCR_SHARED_SECTION_START)
+#define GUPCR_PTS_INCR_VADDR(P,V) (P).vaddr += ((size_t)(V))
+#define GUPCR_PTS_SET_THREAD(P,V) (P).thread = (size_t)(V)
+#define GUPCR_PTS_SET_PHASE(P,V) (P).phase = (size_t)(V)
+
+#elif GUPCR_PTS_PACKED_REP
+
+#if GUPCR_PTS_VADDR_FIRST
+#define GUPCR_PTS_VADDR_SHIFT	(GUPCR_PTS_THREAD_SHIFT \
+                                 + GUPCR_PTS_THREAD_SIZE)
+#define GUPCR_PTS_THREAD_SHIFT	GUPCR_PTS_PHASE_SIZE
+#define GUPCR_PTS_PHASE_SHIFT	0
+#else
+#define GUPCR_PTS_VADDR_SHIFT   0
+#define GUPCR_PTS_THREAD_SHIFT  GUPCR_PTS_VADDR_SIZE
+#define GUPCR_PTS_PHASE_SHIFT   (GUPCR_PTS_THREAD_SHIFT \
+                                 + GUPCR_PTS_THREAD_SIZE)
+#endif
+#define GUPCR_PTS_TO_REP(V) *((upc_shared_ptr_t *)&(V))
+#if GUPCR_TARGET64
+#define GUPCR_ONE 1UL
+#define GUPCR_PTS_REP_T unsigned long
+#else
+#define GUPCR_ONE 1ULL
+#define GUPCR_PTS_REP_T unsigned long long
+#endif
+#define GUPCR_PTS_VADDR_MASK	((GUPCR_ONE << GUPCR_PTS_VADDR_SIZE) \
+                                 - GUPCR_ONE)
+#define GUPCR_PTS_THREAD_MASK	((GUPCR_ONE << GUPCR_PTS_THREAD_SIZE) \
+                                 - GUPCR_ONE)
+#define GUPCR_PTS_PHASE_MASK	((GUPCR_ONE << GUPCR_PTS_PHASE_SIZE) \
+                                 - GUPCR_ONE)
+
+#if !__GCC_UPC__
+/* upc_dbg_shared_ptr_t is used by debugger to figure out
+   shared pointer layout.  */
+typedef struct shared_ptr_struct
+{
+#if GUPCR_PTS_VADDR_FIRST
+  unsigned long long vaddr:GUPCR_PTS_VADDR_SIZE;
+  unsigned int thread:GUPCR_PTS_THREAD_SIZE;
+  unsigned int phase:GUPCR_PTS_PHASE_SIZE;
+#else
+  unsigned int phase:GUPCR_PTS_PHASE_SIZE;
+  unsigned int thread:GUPCR_PTS_THREAD_SIZE;
+  unsigned long long vaddr:GUPCR_PTS_VADDR_SIZE;
+#endif
+} upc_dbg_shared_ptr_t;
+
+typedef GUPCR_PTS_REP_T upc_shared_ptr_t;
+typedef upc_shared_ptr_t *upc_shared_ptr_p;
+#endif
+
+#define GUPCR_PTS_IS_NULL(P) !(P)
+#define GUPCR_PTS_SET_NULL_SHARED(P) { (P) = 0; }
+
+/* access functions are optimized for a representation of the
+   form (vaddr,thread,phase) and where the value is unsigned.
+   Thus, right shift is logical (not arithmetic), and masking
+   is avoided for vaddr, and shifting is avoided for phase.
+   Further, the value being inserted must fit into the field.
+   It will not be masked.  */
+#define GUPCR_PTS_VADDR(P)  \
+  (void *)((size_t)((P)>>GUPCR_PTS_VADDR_SHIFT & GUPCR_PTS_VADDR_MASK))
+#define GUPCR_PTS_THREAD(P) ((size_t)((P)>>GUPCR_PTS_THREAD_SHIFT \
+                             & GUPCR_PTS_THREAD_MASK))
+#define GUPCR_PTS_PHASE(P)  ((size_t)((P)>>GUPCR_PTS_PHASE_SHIFT \
+                             & GUPCR_PTS_PHASE_MASK))
+#define GUPCR_PTS_OFFSET(P) ((size_t)((P)>>GUPCR_PTS_VADDR_SHIFT \
+                             & GUPCR_PTS_VADDR_MASK))
+
+#define GUPCR_PTS_SET_VADDR(P,V) \
+  (P) = ((P) & ~(GUPCR_PTS_VADDR_MASK << GUPCR_PTS_VADDR_SHIFT)) \
+               | ((GUPCR_PTS_REP_T)(V) << GUPCR_PTS_VADDR_SHIFT)
+#define GUPCR_PTS_SET_THREAD(P,V) (P) = ((P) \
+                      & ~(GUPCR_PTS_THREAD_MASK << GUPCR_PTS_THREAD_SHIFT)) \
+                        | ((GUPCR_PTS_REP_T)(V) << GUPCR_PTS_THREAD_SHIFT)
+#define GUPCR_PTS_SET_PHASE(P,V) (P) = ((P) \
+             & ~(GUPCR_PTS_PHASE_MASK << GUPCR_PTS_PHASE_SHIFT)) \
+               | ((GUPCR_PTS_REP_T)(V) << GUPCR_PTS_PHASE_SHIFT)
+#define GUPCR_PTS_INCR_VADDR(P,V) \
+  ((P) += ((GUPCR_PTS_REP_T)(V) << GUPCR_PTS_VADDR_SHIFT))
+#elif GUPCR_PTS_WORD_PAIR_REP
+#error UPC word pair representation is unsupported.
+#endif /* GUPCR_PTS_*_REP__ */
+//end lib_pts_defs
+
+#endif /* gupcr_pts.h */
Index: libgupc/portals4/gupcr_runtime.c
===================================================================
--- libgupc/portals4/gupcr_runtime.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_runtime.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,368 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/* Copyright (c) 2011-2012, Sandia Corporation.
+   All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ . Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+ . Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+ . Neither the name of the Sandia Corporation nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.  */
+
+/**
+ * @file gupcr_runtime.c
+ * GUPC Portals4 Runtime.
+ */
+
+/**
+ * @addtogroup RUNTIME GUPCR PMI
+ * @{
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <portals4.h>
+#ifdef GUPCR_JOB_LAUNCHER_SLURM
+#include <slurm/pmi.h>
+#else
+#include <portals4/pmi.h>
+#endif
+
+struct map_t
+{
+  ptl_handle_ni_t handle;
+  ptl_process_t *mapping;
+};
+
+/** Process rank */
+static int rank = -1;
+/** Number of processes */
+static int size = 0;
+struct map_t maps[4] = {
+  {PTL_INVALID_HANDLE, NULL},
+  {PTL_INVALID_HANDLE, NULL},
+  {PTL_INVALID_HANDLE, NULL},
+  {PTL_INVALID_HANDLE, NULL}
+};
+
+static int max_name_len, max_key_len, max_val_len;
+static char *name, *key, *val;
+
+static int
+encode (const void *inval, int invallen, char *outval, int outvallen)
+{
+  static unsigned char encodings[] = {
+    '0', '1', '2', '3', '4', '5', '6', '7',
+    '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
+  };
+  int i;
+
+  if (invallen * 2 + 1 > outvallen)
+    {
+      return 1;
+    }
+
+  for (i = 0; i < invallen; i++)
+    {
+      outval[2 * i] = encodings[((unsigned char *) inval)[i] & 0xf];
+      outval[2 * i + 1] = encodings[((unsigned char *) inval)[i] >> 4];
+    }
+
+  outval[invallen * 2] = '\0';
+
+  return 0;
+}
+
+static int
+decode (const char *inval, void *outval, int outvallen)
+{
+  int i;
+  char *ret = (char *) outval;
+
+  if (outvallen != (int) (strlen (inval) / 2))
+    {
+      return 1;
+    }
+
+  for (i = 0; i < outvallen; ++i)
+    {
+      if (*inval >= '0' && *inval <= '9')
+	{
+	  ret[i] = *inval - '0';
+	}
+      else
+	{
+	  ret[i] = *inval - 'a' + 10;
+	}
+      inval++;
+      if (*inval >= '0' && *inval <= '9')
+	{
+	  ret[i] |= ((*inval - '0') << 4);
+	}
+      else
+	{
+	  ret[i] |= ((*inval - 'a' + 10) << 4);
+	}
+      inval++;
+    }
+
+  return 0;
+}
+
+int
+gupcr_runtime_init (void)
+{
+  int initialized;
+
+  if (PMI_SUCCESS != PMI_Initialized (&initialized))
+    {
+      return 1;
+    }
+
+  if (0 == initialized)
+    {
+      if (PMI_SUCCESS != PMI_Init (&initialized))
+	{
+	  return 2;
+	}
+    }
+
+  if (PMI_SUCCESS != PMI_Get_rank (&rank))
+    {
+      return 3;
+    }
+
+  if (PMI_SUCCESS != PMI_Get_size (&size))
+    {
+      return 4;
+    }
+
+  /* Initialize key/val work strings.  */
+
+  if (PMI_SUCCESS != PMI_KVS_Get_name_length_max (&max_name_len))
+    {
+      return 5;
+    }
+  name = (char *) malloc (max_name_len);
+  if (NULL == name)
+    return 5;
+
+  if (PMI_SUCCESS != PMI_KVS_Get_key_length_max (&max_key_len))
+    {
+      return 5;
+    }
+  key = (char *) malloc (max_key_len);
+  if (NULL == key)
+    return 5;
+
+  if (PMI_SUCCESS != PMI_KVS_Get_value_length_max (&max_val_len))
+    {
+      return 5;
+    }
+  val = (char *) malloc (max_val_len);
+  if (NULL == val)
+    return 5;
+
+  if (PMI_SUCCESS != PMI_KVS_Get_my_name (name, max_name_len))
+    {
+      return 5;
+    }
+
+  return 0;
+}
+
+int
+gupcr_runtime_fini (void)
+{
+  int i;
+
+  for (i = 0; i < 4; ++i)
+    {
+      if (NULL != maps[i].mapping)
+	{
+	  free (maps[i].mapping);
+	}
+    }
+
+  PMI_Finalize ();
+
+  return 0;
+}
+
+ptl_process_t *
+gupcr_runtime_get_mapping (ptl_handle_ni_t ni_h)
+{
+  int i, ret;
+  ptl_process_t my_id;
+  struct map_t *map = NULL;
+
+  for (i = 0; i < 4; ++i)
+    {
+      if (maps[i].handle == ni_h)
+	{
+	  return maps[i].mapping;
+	}
+    }
+
+  for (i = 0; i < 4; ++i)
+    {
+      if (PTL_INVALID_HANDLE == maps[i].handle)
+	{
+	  map = &maps[i];
+	  break;
+	}
+    }
+
+  if (NULL == map)
+    return NULL;
+
+  ret = PtlGetPhysId (ni_h, &my_id);
+  if (PTL_OK != ret)
+    return NULL;
+
+  /* Put my information.  */
+  snprintf (key, max_key_len, "libgupc-%lu-%lu-nid",
+	    (long unsigned) ni_h, (long unsigned) rank);
+  if (0 != encode (&my_id.phys.nid, sizeof (my_id.phys.nid), val,
+		   max_val_len))
+    {
+      return NULL;
+    }
+  if (PMI_SUCCESS != PMI_KVS_Put (name, key, val))
+    {
+      return NULL;
+    }
+
+  snprintf (key, max_key_len, "libgupc-%lu-%lu-pid",
+	    (long unsigned) ni_h, (long unsigned) rank);
+  if (0 != encode (&my_id.phys.pid, sizeof (my_id.phys.pid), val,
+		   max_val_len))
+    {
+      return NULL;
+    }
+  if (PMI_SUCCESS != PMI_KVS_Put (name, key, val))
+    {
+      return NULL;
+    }
+
+  if (PMI_SUCCESS != PMI_KVS_Commit (name))
+    {
+      return NULL;
+    }
+
+  if (PMI_SUCCESS != PMI_Barrier ())
+    {
+      return NULL;
+    }
+
+  /* Get everyone's information.  */
+  map->mapping = malloc (sizeof (ptl_process_t) * size);
+  if (NULL == map->mapping)
+    return NULL;
+
+  for (i = 0; i < size; ++i)
+    {
+      snprintf (key, max_key_len, "libgupc-%lu-%lu-nid",
+		(long unsigned) ni_h, (long unsigned) i);
+      if (PMI_SUCCESS != PMI_KVS_Get (name, key, val, max_val_len))
+	{
+	  return NULL;
+	}
+      if (0 != decode (val, &(map->mapping)[i].phys.nid,
+		       sizeof ((map->mapping)[i].phys.nid)))
+	{
+	  return NULL;
+	}
+
+      snprintf (key, max_key_len, "libgupc-%lu-%lu-pid",
+		(long unsigned) ni_h, (long unsigned) i);
+      if (PMI_SUCCESS != PMI_KVS_Get (name, key, val, max_val_len))
+	{
+	  return NULL;
+	}
+      if (0 != decode (val, &(map->mapping)[i].phys.pid,
+		       sizeof ((map->mapping)[i].phys.pid)))
+	{
+	  return NULL;
+	}
+    }
+
+  return map->mapping;
+}
+
+/**
+ * Return this process rank.
+ */
+int
+gupcr_runtime_get_rank (void)
+{
+  return rank;
+}
+
+/**
+ * Return number of processes in the system.
+ */
+int
+gupcr_runtime_get_size (void)
+{
+  return size;
+}
+
+/**
+ * Runtime barrier.
+ */
+void
+gupcr_runtime_barrier (void)
+{
+  PMI_Barrier ();
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_runtime.h
===================================================================
--- libgupc/portals4/gupcr_runtime.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_runtime.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,41 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef _GUPCR_RUNTIME_H_
+#define _GUPCR_RUNTIME_H_
+
+/**
+ * @file gupcr_runtime.h
+ * GUPC Portals4 Runtime.
+ */
+
+extern int gupcr_runtime_init (void);
+extern int gupcr_runtime_fini (void);
+extern int gupcr_runtime_get_rank (void);
+extern int gupcr_runtime_get_size (void);
+extern void gupcr_runtime_barrier (void);
+ptl_process_t *gupcr_runtime_get_mapping (ptl_handle_ni_t);
+#endif /* gupcr_runtime.h */
Index: libgupc/portals4/gupcr_shutdown.c
===================================================================
--- libgupc/portals4/gupcr_shutdown.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_shutdown.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,328 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_shutdown.c
+ * GUPC Portals4 shutdown support.
+ *
+ * Support for upc_global_exit ().
+ *
+ * Each UPC thread (process) creates a helper (shutdown) pthread
+ * with the sole purpose of waiting for receipt of a remote request
+ * to shutdown, as a result of the other thread issuing a call
+ * to upc_global_exit.
+ *
+ * This pthread uses a special PTE/LE (GUPCR_PTL_PTE_SHUTDOWN) to receive a
+ * global exit code from another UPC thread.  A simple PtlPut of the
+ * exit code issued to the shutdown PTE on some other UPC thread
+ * triggers exit of the receiving thread.
+ *
+ * The following steps are taken to initialize, wait, and signal
+ * the UPC global exit:
+ *
+ * - Each thread initializes a PTE/LE to receive an exit code that
+ *     was passed in as the argument to upc_global_exit().
+ * - Each thread creates a helper pthread - gupcr_shutdown_pthread()
+ *     that waits on the shutdown LE's counting event (one count only).
+ * - The main UPC thread installs a signal handler for SHUTDWON_SIGNAL
+ *     that is used by the shutdown pthread to signal a need for
+ *     global exit.
+ * - Remote shutdown takes the following steps:
+ *     -# A UPC thread executing a call to upc_global_exit() sends the
+ *        exit code to all other UPC threads by using the shutdown PTE.
+ *     -# The pthread associated with each UPC thread receives the
+ *        exit code and returns from the counting event Portals
+ *        wait call.
+ *     -# The receiving pthread sends the SHUTDOWN_SIGNAL to the main
+ *     	  UPC thread and calls pthread_exit().
+ *     -# The main UPC thread receives the signal, which invokes
+ *        the signal handler.
+ *     -# The signal handler waits for the shutdown pthread to exit,
+ *        and then calls exit() with the code received from
+ *        the thread that sent the shutdown request.  The invoking thread
+ *        also waits for ACKs from the first step with the configured timeout,
+ *
+ *  @note
+ *  -# The gupcr_exit() function is registered with atexit()
+ *     and will be executed when exit() is called.
+ *  -# Upon regular exit, the main UPC thread disables the
+ *     SHUTDOWN_SIGNAL signal, and terminates the shutdown pthread
+ *     by writing a dummy value using its own shutdown PTE.
+ *
+ * @addtogroup SHUTDOWN GUPCR Shutdown Functions
+ * @{
+ */
+
+#include <pthread.h>
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_utils.h"
+#include "gupcr_portals.h"
+#include "gupcr_shutdown.h"
+
+/** Shutdown signal to main thread */
+#define SHUTDOWN_SIGNAL SIGUSR2
+/** Shutdown check interval (100 miliseconds) */
+#define SHUTDOWN_MICROSEC_WAIT 100000L
+
+/** Shutdown LE buffer */
+static int gupcr_shutdown_status;
+/** Shutdown LE handle */
+static ptl_handle_le_t gupcr_shutdown_le;
+/** Shutdown LE event counter */
+static ptl_handle_ct_t gupcr_shutdown_le_ct;
+
+/** Shutdown memory buffer for sending data */
+static int gupcr_shutdown_send_status;
+/** Shutdown memory MD handle */
+static ptl_handle_md_t gupcr_shutdown_md;
+/** Shutdown memory MD event counter */
+static ptl_handle_ct_t gupcr_shutdown_md_ct;
+/** Shutdown memory MD event queue */
+static ptl_handle_eq_t gupcr_shutdown_md_eq;
+
+/** Shutdown pthread ID */
+static pthread_t gupcr_shutdown_pthread_id;
+/** Shutdown pthread declaration */
+static void *gupcr_shutdown_pthread (void *arg) __attribute__ ((noreturn));
+
+/**
+ * Send a remote shutdown request to all threads.
+ *
+ * Wait for the thread's pthread and ACks from sending
+ * messages to other threads (with timeout).
+ *
+ * @param [in] status exit code passed to other threads
+ */
+void
+gupcr_signal_exit (int status)
+{
+  int thread;
+  int wait_cnt = GUPCR_GLOBAL_EXIT_TIMEOUT *
+    (1000000L / SHUTDOWN_MICROSEC_WAIT);
+  int done = 0;
+  ptl_process_t pid;
+
+  /* Protect local global exit from remote shutdown request.  */
+  gupcr_signal_disable (SHUTDOWN_SIGNAL);
+
+  gupcr_shutdown_send_status = status;
+  /* Send global exit code to all threads.  */
+  for (thread = 0; thread < THREADS; thread++)
+    {
+      pid.rank = thread;
+      gupcr_portals_call (PtlPut, (gupcr_shutdown_md, 0,
+				   sizeof (gupcr_shutdown_send_status),
+				   PTL_CT_ACK_REQ, pid,
+				   GUPCR_PTL_PTE_SHUTDOWN, PTL_NO_MATCH_BITS,
+				   0, PTL_NULL_USER_PTR, PTL_NULL_HDR_DATA));
+    }
+  /* It is NOT ok to call finalize routines as there might
+     be outstanding transactions.  */
+  gupcr_finalize_ok = 0;
+  /* Wait for our own shutdown pthread to complete.  */
+  pthread_join (gupcr_shutdown_pthread_id, NULL);
+  /* Wait for ACKs from all threads.  It should happen quickly
+     if everything is ok, otherwise timeout after configured
+     number of seconds.  */
+  do
+    {
+      ptl_ct_event_t ct = { 0, 0 };
+      gupcr_portals_call (PtlCTGet, (gupcr_shutdown_md_ct, &ct));
+      if ((ct.success + ct.failure) == (ptl_size_t) THREADS)
+	done = 1;
+      else
+	gupcr_cpu_delay (SHUTDOWN_MICROSEC_WAIT);
+    }
+  while (!done && wait_cnt--);
+}
+
+/**
+ * Terminate shutdown pthread
+ *
+ * To terminate the local shutdown pthread a dummy value must
+ * be sent by this thread to its own shutdown PTE.  The main thread
+ * then waits for pthread to exit.
+ */
+static void
+gupcr_shutdown_terminate_pthread (void)
+{
+  ptl_process_t pid;
+  pid.rank = MYTHREAD;
+
+  /* Disable interrupts before sending a signal to
+     shutdown pthread.  */
+  gupcr_signal_disable (SHUTDOWN_SIGNAL);
+
+  gupcr_shutdown_send_status = 0;
+  gupcr_portals_call (PtlPut, (gupcr_shutdown_md, 0,
+			       sizeof (gupcr_shutdown_send_status),
+			       PTL_NO_ACK_REQ, pid, GUPCR_PTL_PTE_SHUTDOWN,
+			       PTL_NO_MATCH_BITS, 0, PTL_NULL_USER_PTR,
+			       PTL_NULL_HDR_DATA));
+  pthread_join (gupcr_shutdown_pthread_id, NULL);
+}
+
+/**
+ * Shutdown pthread that waits for remote shutdown requests.
+ *
+ * This pthread waits on a shutdown PTE to receive a shutdown
+ * request from any other thread that executed upc_global_exit().
+ * Then, it uses signal (SHUTDOWN_SIGNAL) to inform the main thread
+ * of a need to shutdown this UPC thread.
+ * @param [in] arg Pthread arguments (not used in this case)
+ * @retval Pthread's exit value
+ */
+static void *
+gupcr_shutdown_pthread (void *arg __attribute ((unused)))
+{
+  ptl_ct_event_t ct = { 0, 0 };
+  int pstatus;
+
+  gupcr_log (FC_MISC, "Shutdown pthread started");
+  /* Wait for the shutdown request.  Yield control of the
+     CPU frequently as this is a low priority activity
+     that should minimize competition for resources with the
+     main thread.  */
+  do
+    {
+      gupcr_cpu_delay (SHUTDOWN_MICROSEC_WAIT);
+      gupcr_portals_call_with_status (PtlCTGet, pstatus,
+				      (gupcr_shutdown_le_ct, &ct));
+    }
+  while (!(ct.success + ct.failure));
+  gupcr_debug (FC_MISC, "Shutdown pthread received exit %d",
+	       gupcr_shutdown_status);
+
+  /* Signal the main thread to exit.  */
+  kill (getpid (), SHUTDOWN_SIGNAL);
+  /* No need for this helper pthread any more.  */
+  pthread_exit (NULL);
+}
+
+/**
+ * Signal handler that performs shutdown.
+ *
+ * This signal handler will terminate the pthread
+ * that listens for shutdown requests and then
+ * exit the current process (which is associated with
+ * a UPC thread).  The UPC will exit with the
+ * the status code received from the remote thread.
+ * @param [in] signum Signal number (unused)
+ */
+void
+gupcr_shutdown_signal_handler (int signum __attribute__ ((unused)))
+{
+  gupcr_debug (FC_MISC, "Shutdown signal handler for signal %d", signum);
+  /* Wait for shutdown pthread to exit.  */
+  pthread_join (gupcr_shutdown_pthread_id, NULL);
+  /* It is NOT ok to call finalize routines as there might
+     be outstanding transactions.  */
+  gupcr_finalize_ok = 0;
+  /* Exit with global exit code.  */
+  exit (gupcr_shutdown_status);
+}
+
+/**
+ * Initialize remote shutdown resources.
+ * @ingroup INIT
+ */
+void
+gupcr_shutdown_init (void)
+{
+  ptl_md_t md;
+  ptl_pt_index_t pte;
+  ptl_le_t le;
+
+  gupcr_log (FC_MISC, "shutdown init called");
+
+  /* Create the PTE used to communicate shutdown requests.  */
+  gupcr_portals_call (PtlPTAlloc, (gupcr_ptl_ni, 0,
+				   PTL_EQ_NONE, GUPCR_PTL_PTE_SHUTDOWN,
+				   &pte));
+  gupcr_assert (pte != GUPCR_PTL_PTE_LOCK);
+
+  gupcr_debug (FC_MISC, "Shutdown PTE allocated: %d", GUPCR_PTL_PTE_SHUTDOWN);
+
+  /* Allocate the LE used for shutdown requests.  */
+  gupcr_portals_call (PtlCTAlloc, (gupcr_ptl_ni, &gupcr_shutdown_le_ct));
+  le.start = &gupcr_shutdown_status;
+  le.length = sizeof (gupcr_shutdown_status);
+  le.ct_handle = gupcr_shutdown_le_ct;
+  le.uid = PTL_UID_ANY;
+  le.options = PTL_LE_OP_PUT | PTL_LE_OP_GET | PTL_LE_EVENT_CT_COMM;
+  gupcr_portals_call (PtlLEAppend,
+		      (gupcr_ptl_ni, GUPCR_PTL_PTE_SHUTDOWN, &le,
+		       PTL_PRIORITY_LIST, NULL, &gupcr_shutdown_le));
+
+  gupcr_debug (FC_MISC, "Service LE created: %lx (%lx)",
+	       (long unsigned) &gupcr_shutdown_status,
+	       (long unsigned) sizeof (gupcr_shutdown_status));
+
+  /* Setup the MD used to send a shutdown request to other threads.  */
+  gupcr_portals_call (PtlCTAlloc, (gupcr_ptl_ni, &gupcr_shutdown_md_ct));
+  gupcr_portals_call (PtlEQAlloc, (gupcr_ptl_ni, 1, &gupcr_shutdown_md_eq));
+
+  md.start = &gupcr_shutdown_send_status;
+  md.length = sizeof (gupcr_shutdown_send_status);
+  md.options = PTL_MD_EVENT_CT_ACK | PTL_MD_EVENT_SUCCESS_DISABLE;
+  md.eq_handle = gupcr_shutdown_md_eq;
+  md.ct_handle = gupcr_shutdown_md_ct;
+  gupcr_portals_call (PtlMDBind, (gupcr_ptl_ni, &md, &gupcr_shutdown_md));
+
+  /* Start a pthread that listens for remote shutdown requests.  */
+  gupcr_syscall (pthread_create,
+		 (&gupcr_shutdown_pthread_id, (pthread_attr_t *) NULL,
+		  &gupcr_shutdown_pthread, NULL));
+
+  /* Install a signal handler that processes remote
+     shutdown global exit requests.  */
+  gupcr_signal_enable (SHUTDOWN_SIGNAL, gupcr_shutdown_signal_handler);
+}
+
+/**
+ * Release remote shutdown resources.
+ * @ingroup INIT
+ */
+void
+gupcr_shutdown_fini (void)
+{
+  gupcr_log (FC_MISC, "shutdown fini called");
+
+  /* Terminate the shutdown pthread.  */
+  gupcr_shutdown_terminate_pthread ();
+
+  /* Release the shutdown MD.  */
+  gupcr_portals_call (PtlMDRelease, (gupcr_shutdown_md));
+  gupcr_portals_call (PtlCTFree, (gupcr_shutdown_md_ct));
+
+  /* Release the shutdown LE and PTE.  */
+  gupcr_portals_call (PtlLEUnlink, (gupcr_shutdown_le));
+  gupcr_portals_call (PtlCTFree, (gupcr_shutdown_le_ct));
+  gupcr_portals_call (PtlPTFree, (gupcr_ptl_ni, GUPCR_PTL_PTE_SHUTDOWN));
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_shutdown.h
===================================================================
--- libgupc/portals4/gupcr_shutdown.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_shutdown.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,39 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef _GUPCR_SHUTDOWN_H_
+#define _GUPCR_SHUTDOWN_H_
+
+/**
+ * @file gupcr_shutdown.h
+ * GUPC Portals4 shutdown support.
+ */
+
+extern void gupcr_signal_exit (int status);
+extern void gupcr_shutdown_init (void);
+extern void gupcr_shutdown_fini (void);
+
+#endif /* gupcr_shutdown.h */
Index: libgupc/portals4/gupcr_sup.h
===================================================================
--- libgupc/portals4/gupcr_sup.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_sup.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,48 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_sup.h
+ * GUPC Runtime prototypes to UPC specific functions.
+ */
+
+#ifndef _GUPCR_SUP_H_
+#define _GUPCR_SUP_H_
+
+/* Internal runtime routines and external symbols.  */
+
+//begin lib_runtime_api
+
+extern void *__cvtaddr (upc_shared_ptr_t);
+extern void *__getaddr (upc_shared_ptr_t);
+extern void __upc_barrier (int barrier_id);
+extern void __upc_notify (int barrier_id);
+extern void __upc_wait (int barrier_id);
+extern void __upc_exit (int status) __attribute__ ((__noreturn__));
+
+//end lib_runtime_api
+
+#endif /* gupcr_sup.h */
Index: libgupc/portals4/gupcr_sync.h
===================================================================
--- libgupc/portals4/gupcr_sync.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_sync.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,87 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_sync.h
+ * GUPC Runtime memory read/write sync definitions.
+ */
+
+#ifndef _GUPCR_SYNC_H_
+#define _GUPCR_SYNC_H_
+
+//begin lib_mem_barrier
+
+/*
+
+The following table is excerpted from
+"Implementing the UPC memory consistency model for
+shared-memory architectures", Dan Bonachea et al.
+
+CPU		Write fence		Read fence
+--------------------------------------------------
+Power/PowerPC	sync			isync
+Alpha		wmb			mb
+x86		lock; addl $0,0(%%esp)  none reqd.
+Athlon/Opteron	mfence			none reqd.
+Itanium		mf			none reqd.
+SPARC		stbar			none reqd.
+MIPS		sync			none reqd.
+PA-RISC		SYNC			none reqd. */
+
+#define GUPCR_MEM_BARRIER() \
+	{ GUPCR_READ_MEM_BARRIER (); GUPCR_WRITE_MEM_BARRIER (); }
+
+#if defined (PPC)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("sync":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("isync":::"memory")
+#elif defined (alpha)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("wmb":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("mb":::"memory")
+#elif defined (__x86_64__)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("mfence":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("":::"memory")
+#elif defined (__ia64__)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("mf":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("":::"memory")
+#elif defined (i386)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ( \
+                                "lock; addl $0,0(%%esp)":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("":::"memory")
+#elif defined (sparc)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("stbar":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("":::"memory")
+#elif defined (mips)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("sync":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("":::"memory")
+#elif defined (hppa)
+#define GUPCR_WRITE_MEM_BARRIER() asm __volatile__ ("SYNC":::"memory")
+#define GUPCR_READ_MEM_BARRIER() asm __volatile__ ("":::"memory")
+#else
+#error "No memory fence  operations provided for this cpu."
+#endif
+//end lib_mem_barrier
+
+#endif /* gupcr_sync.h */
Index: libgupc/portals4/gupcr_tick.c
===================================================================
--- libgupc/portals4/gupcr_tick.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_tick.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,77 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_lib.h"
+
+#if HAVE_CLOCK_GETTIME
+#ifdef CLOCK_MONOTONIC_RAW
+/* System clock id passed to clock_gettime.  CLOCK_MONOTONIC_RAW
+   is preferred.  It has been available in the Linux kernel
+   since version 2.6.28.  */
+#define SYS_RT_CLOCK_ID CLOCK_MONOTONIC_RAW
+#else
+#define SYS_RT_CLOCK_ID CLOCK_MONOTONIC
+#endif
+
+upc_tick_t
+upc_ticks_now (void)
+{
+  struct timespec ts;
+  upc_tick_t t;
+  if (clock_gettime (SYS_RT_CLOCK_ID, &ts) != 0)
+    {
+      perror ("clock_gettime");
+      abort ();
+    }
+  t = (upc_tick_t) ts.tv_sec * 1000000000LL + (upc_tick_t) ts.tv_nsec;
+  return t;
+}
+
+#else /* not HAVE_CLOCK_GETTIME */
+
+upc_tick_t
+upc_ticks_now (void)
+{
+  struct timeval tv;
+  upc_tick_t t;
+  if (gettimeofday (&tv, NULL) != 0)
+    {
+      perror ("gettimeofday");
+      abort ();
+    }
+  t = tv.tv_sec * 1000000000 + tv.tv_usec * 1000;
+  return t;
+}
+
+#endif
+
+uint64_t
+upc_ticks_to_ns (upc_tick_t ticks)
+{
+  return ticks;
+}
Index: libgupc/portals4/gupcr_utils.c
===================================================================
--- libgupc/portals4/gupcr_utils.c	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_utils.c	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,820 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_utils.c
+ * GUPC Runtime utility routines
+ */
+
+/**
+ * @addtogroup GUPCUTILS GUPCR Utility Functions
+ * @{
+ */
+
+#include "gupcr_config.h"
+#include "gupcr_defs.h"
+#include "gupcr_portals.h"
+#include "gupcr_runtime.h"
+#include "gupcr_utils.h"
+
+#define PATH_SEP '/'
+
+typedef struct gupcr_open_file_struct *gupcr_open_file_ref;
+typedef struct gupcr_open_file_struct
+{
+  gupcr_open_file_ref next;
+  FILE *file_ptr;
+  const char *pathname;
+} gupcr_open_file_t;
+
+static void gupcr_close_all_open_files (void);
+static void gupcr_debug_fini (void);
+static void gupcr_debug_init (void);
+static FILE *gupcr_find_open_file (const char *pathname);
+static void gupcr_open_file_list_add (FILE * file, const char *pathname);
+static void gupcr_set_short_pgm_name (void);
+static void gupcr_stats_fini (void);
+static void gupcr_stats_init (void);
+static void gupcr_trace_fini (void);
+static void gupcr_trace_init (void);
+static void gupcr_write_log (FILE * file, const char *filename,
+			     const char *fmt, va_list ap);
+
+static int gupcr_is_init;
+static int gupcr_error_count;
+static const char *gupcr_pgm_name = "";
+static char gupcr_short_pgm_name[13];
+static int gupcr_pid;
+static char gupcr_pid_string[13];
+static int gupcr_inform_user = 1;
+static int gupcr_warn_user = 1;
+static size_t gupcr_shared_heap_size;
+static int gupcr_node_local_memory = 1;
+static int gupcr_forcetouch = 1;
+static int gupcr_backtrace = 0;
+
+static gupcr_open_file_ref gupcr_open_files_list;
+static int gupcr_debug_enabled;
+static const char *gupcr_debug_filename = "stderr";
+static FILE *gupcr_debug_file;
+static int gupcr_log_enabled;
+static const char *gupcr_log_filename = "stderr";
+static FILE *gupcr_log_file;
+static int gupcr_stats_enabled;
+static const char *gupcr_stats_filename = "stderr";
+static FILE *gupcr_stats_file __attribute ((unused));
+static int gupcr_trace_enabled;
+static const char *gupcr_trace_filename = "stderr";
+static FILE *gupcr_trace_file;
+
+gupcr_facility_t gupcr_debug_facility_mask;
+gupcr_facility_t gupcr_log_facility_mask;
+gupcr_facility_t gupcr_stats_facility_mask;
+gupcr_facility_t gupcr_trace_facility_mask;
+
+const char *
+gupcr_strsignal (int sig)
+{
+  return strsignal (sig);
+}
+
+const char *
+gupcr_strerror (void)
+{
+  return strerror (errno);
+}
+
+void
+gupcr_signal_enable (int signal, void (*handler) (int))
+{
+  struct sigaction action;
+
+  action.sa_handler = handler;
+  sigemptyset (&action.sa_mask);
+  action.sa_flags = 0;
+  sigaction (signal, &action, NULL);
+}
+
+void
+gupcr_signal_disable (int signal)
+{
+  struct sigaction action;
+
+  action.sa_handler = SIG_IGN;
+  sigemptyset (&action.sa_mask);
+  action.sa_flags = 0;
+  sigaction (signal, &action, NULL);
+}
+
+static void
+gupcr_write_log (FILE * file, const char *filename,
+		 const char *fmt, va_list ap)
+{
+  const double timestamp = gupcr_clock ();
+  static double prev_timestamp;
+  double delta_timestamp;
+  char buf[1024];
+  char *bp = buf;
+  va_list args;
+  delta_timestamp = timestamp - prev_timestamp;
+  prev_timestamp = timestamp;
+  bp += sprintf (bp, "%lld ", (long long) (timestamp * 1.0e6));
+  bp += sprintf (bp, "%#.3g%s ",
+		 ((delta_timestamp > 1.0) ? 1.0
+		  : ((delta_timestamp > 0.001) ? 1.0e3 : 1.0e6))
+		 * delta_timestamp,
+		 ((delta_timestamp > 1.0) ? "s"
+		  : ((delta_timestamp > 0.001) ? "m" : "u")));
+  if (MYTHREAD >= 0 && MYTHREAD < THREADS)
+    bp += sprintf (bp, "%d ", MYTHREAD);
+  else
+    bp += sprintf (bp, "??? ");
+  va_copy (args, ap);
+  bp += vsprintf (bp, fmt, args);
+  va_end (args);
+  if (fputs (buf, file) == EOF)
+    gupcr_abort_with_msg ("UPC runtime write to `%s' failed: %s",
+			  filename, gupcr_strerror ());
+}
+
+void
+gupcr_abort (void)
+{
+  gupcr_utils_fini ();
+  abort ();
+}
+
+/* NOTE: registered as an exit routine.  */
+void
+gupcr_exit ()
+{
+  if (gupcr_finalize_ok)
+    gupcr_fini ();
+  else
+    {
+      gupcr_runtime_fini ();
+      gupcr_utils_fini ();
+    }
+}
+
+void
+gupcr_shutdown (int exit_code)
+{
+  exit (exit_code);
+}
+
+void
+gupcr_abort_with_msg (const char *fmt, ...)
+{
+  va_list args;
+  va_start (args, fmt);
+  vfprintf (stderr, fmt, args);
+  va_end (args);
+  gupcr_abort ();
+}
+
+const char *
+gupcr_get_pid_as_string (void)
+{
+  return gupcr_pid_string;
+}
+
+const char *
+gupcr_get_pgm_name (void)
+{
+  return gupcr_pgm_name;
+}
+
+void
+gupcr_set_pgm_name (const char *pgm_name)
+{
+  gupcr_pgm_name = pgm_name;
+}
+
+void
+gupcr_init_complete (void)
+{
+  gupcr_is_init = 1;
+  if (gupcr_error_count)
+    gupcr_abort ();
+}
+
+void
+gupcr_be_quiet (void)
+{
+  gupcr_inform_user = 0;
+  gupcr_warn_user = 0;
+}
+
+void
+gupcr_no_warn (void)
+{
+  gupcr_warn_user = 0;
+}
+
+void
+gupcr_set_shared_heap_size (size_t heap_size)
+{
+  gupcr_shared_heap_size = heap_size;
+}
+
+size_t
+gupcr_get_shared_heap_size (void)
+{
+  return gupcr_shared_heap_size;
+}
+
+void
+gupcr_set_node_local_memory (int value)
+{
+  gupcr_node_local_memory = value;
+}
+
+int
+gupcr_is_node_local_memory_enabled (void)
+{
+  return gupcr_node_local_memory;
+}
+
+void
+gupcr_set_forcetouch (int value)
+{
+  gupcr_forcetouch = value;
+}
+
+int
+gupcr_is_forcetouch_enabled (void)
+{
+  return gupcr_forcetouch;
+}
+
+void
+gupcr_set_backtrace (int value)
+{
+  gupcr_backtrace = value;
+}
+
+int
+gupcr_is_backtrace_enabled (void)
+{
+  return gupcr_backtrace;
+}
+
+/** Node local unique name.  */
+#define GUPCR_LOCAL_NAME_FMT "-%06d-%06d"
+#define GUPCR_LOCAL_NAME_FMT_SIZE 14
+
+void
+gupcr_unique_local_name (char *fname, const char *prefix, int thread,
+			 int tmpenv)
+{
+  const char *tmpdir = NULL;
+  char *n = fname;
+  *n = 0;
+  if (tmpenv)
+    {
+      /* Honor TMDIR and/or TMP environment variables.  */
+      if ((getuid () == geteuid ()) && (getgid () == getegid ()))
+	{
+	  tmpdir = getenv ("TMPDIR");
+	  if (!tmpdir)
+	    tmpdir = getenv ("TMP");
+	}
+      if (!tmpdir)
+	tmpdir = "/tmp";
+      if (strlen (tmpdir) > FILENAME_MAX)
+	gupcr_fatal_error ("tmp path too long: %s", tmpdir);
+      strcpy (n, tmpdir);
+      n = n + strlen (n);
+      strcat (n, "/");
+      n = n + 1;
+    }
+  if (strlen (n) + strlen (prefix) + GUPCR_LOCAL_NAME_FMT_SIZE > FILENAME_MAX)
+    gupcr_fatal_error ("unique local name too long");
+  sprintf (n, "%s" GUPCR_LOCAL_NAME_FMT, prefix, thread,
+	   gupcr_get_rank_pid (thread));
+}
+
+/* Convert buffer to string (max of 16 characters) */
+const char *
+gupcr_get_buf_as_hex (char *bufstr, const void *buf, size_t size)
+{
+  size_t cnt = (size > 16) ? 16 : size;
+  bufstr[0] = 0;
+  if (cnt > 0)
+    {
+      char *tmp = bufstr;
+      size_t i;
+      for (i = 0; i < cnt - 1; ++i)
+	{
+	  sprintf (tmp, "%02x ", ((unsigned char *) buf)[i]);
+	  tmp += 3;
+	}
+      sprintf (tmp, "%02x", ((unsigned char *) buf)[cnt - 1]);
+    }
+  return bufstr;
+}
+
+void
+gupcr_error_print (const char *fmt, ...)
+{
+  va_list args;
+  gupcr_assert (MYTHREAD >= 0);
+  /* If the runtime has not initialized yet, print the
+     error message on thread 0 only.  */
+  if (gupcr_is_init || MYTHREAD == 0)
+    {
+      va_start (args, fmt);
+      vfprintf (stderr, fmt, args);
+      va_end (args);
+    }
+  /* Abort is deferred until the runtime has initialized
+     so that all non-fatal error diagnostics are printed
+     before the program aborts.  */
+  if (gupcr_is_init)
+    gupcr_abort ();
+  ++gupcr_error_count;
+}
+
+void
+gupcr_warn_print (const char *fmt, ...)
+{
+  if (gupcr_warn_user)
+    {
+      va_list args;
+      va_start (args, fmt);
+      vfprintf (stderr, fmt, args);
+      va_end (args);
+    }
+}
+
+void
+gupcr_info_print (const char *fmt, ...)
+{
+  if (gupcr_inform_user)
+    {
+      va_list args;
+      va_start (args, fmt);
+      vfprintf (stderr, fmt, args);
+      va_end (args);
+    }
+}
+
+void
+gupcr_debug_print (const char *fmt, ...)
+{
+  if (gupcr_debug_enabled)
+    {
+      va_list args;
+      if (!gupcr_debug_file)
+	{
+	  gupcr_assert (gupcr_debug_filename != NULL);
+	  gupcr_debug_file = gupcr_fopen ("debug", gupcr_debug_filename, "w");
+	}
+      va_start (args, fmt);
+      gupcr_write_log (gupcr_debug_file, gupcr_debug_filename, fmt, args);
+      va_end (args);
+    }
+}
+
+void
+gupcr_set_debug_facility (gupcr_facility_t facility_mask)
+{
+  gupcr_debug_facility_mask = facility_mask;
+  gupcr_debug_enabled = (facility_mask != 0);
+}
+
+void
+gupcr_set_debug_filename (const char *filename)
+{
+  gupcr_assert (filename != NULL);
+  gupcr_debug_filename = filename;
+}
+
+void
+gupcr_log_print (const char *fmt, ...)
+{
+  if (gupcr_log_enabled)
+    {
+      va_list args;
+      if (!gupcr_log_file)
+	{
+	  gupcr_assert (gupcr_log_filename != NULL);
+	  gupcr_log_file = gupcr_fopen ("debug", gupcr_log_filename, "w");
+	}
+      va_start (args, fmt);
+      gupcr_write_log (gupcr_log_file, gupcr_log_filename, fmt, args);
+      va_end (args);
+    }
+}
+
+void
+gupcr_set_log_facility (gupcr_facility_t facility_mask)
+{
+  gupcr_log_facility_mask = facility_mask;
+  gupcr_log_enabled = (facility_mask != 0);
+}
+
+void
+gupcr_set_log_filename (const char *filename)
+{
+  gupcr_assert (filename != NULL);
+  gupcr_log_filename = filename;
+}
+
+void
+gupcr_set_stats_facility (gupcr_facility_t facility_mask)
+{
+  gupcr_stats_facility_mask = facility_mask;
+  gupcr_stats_enabled = (facility_mask != 0);
+}
+
+void
+gupcr_set_stats_filename (const char *filename)
+{
+  gupcr_assert (filename != NULL);
+  gupcr_stats_filename = filename;
+}
+
+void
+gupcr_trace_print (const char *fmt, ...)
+{
+  if (gupcr_trace_enabled)
+    {
+      va_list args;
+      if (!gupcr_trace_file)
+	{
+	  gupcr_assert (gupcr_trace_filename != NULL);
+	  gupcr_trace_file = gupcr_fopen ("trace", gupcr_trace_filename, "w");
+	}
+      va_start (args, fmt);
+      gupcr_write_log (gupcr_trace_file, gupcr_trace_filename, fmt, args);
+      va_end (args);
+    }
+}
+
+void
+gupcr_set_trace_facility (gupcr_facility_t facility_mask)
+{
+  gupcr_trace_facility_mask = facility_mask;
+  gupcr_trace_enabled = (facility_mask != 0);
+}
+
+void
+gupcr_set_trace_filename (const char *filename)
+{
+  gupcr_assert (filename != NULL);
+  gupcr_trace_filename = filename;
+}
+
+static void
+gupcr_set_short_pgm_name (void)
+{
+  char *pgm_name, *filename, *cp;
+  gupcr_strdup (pgm_name, gupcr_pgm_name);
+  filename = basename (pgm_name);
+  gupcr_assert ((sizeof (gupcr_short_pgm_name) - 1) == 12);
+  strncpy (gupcr_short_pgm_name, filename, 12);
+  gupcr_short_pgm_name[12] = '\0';
+  for (cp = gupcr_short_pgm_name; *cp; ++cp)
+    {
+      if (!(isalnum (*cp) || *cp == '_' || *cp == '.'))
+	*cp = '_';
+    }
+  gupcr_free (pgm_name);
+}
+
+static FILE *
+gupcr_find_open_file (const char *pathname)
+{
+  gupcr_open_file_ref f;
+  for (f = gupcr_open_files_list; f; f = f->next)
+    {
+      if (!strcmp (f->pathname, pathname))
+	return f->file_ptr;
+    }
+  return NULL;
+}
+
+static void
+gupcr_open_file_list_add (FILE * file, const char *pathname)
+{
+  gupcr_open_file_ref open_file;
+  gupcr_assert (file != NULL);
+  gupcr_assert (pathname != NULL && pathname[0]);
+  gupcr_malloc (open_file, sizeof (gupcr_open_file_t));
+  open_file->next = gupcr_open_files_list;
+  gupcr_strdup (open_file->pathname, pathname);
+  open_file->file_ptr = file;
+  gupcr_open_files_list = open_file;
+}
+
+FILE *
+gupcr_fopen (const char *purpose, const char *pathname, const char *mode)
+{
+  FILE *file, *already_open;
+  already_open = gupcr_find_open_file (pathname);
+  if (already_open)
+    file = already_open;
+  else
+    {
+      if (!strcmp (mode, "w"))
+	/* Create any intervening directories.  */
+	gupcr_mkpath (pathname);
+      file = fopen (pathname, mode);
+      if (!file)
+	gupcr_error ("UPC runtime cannot open %s file `%s' for %s: %s",
+		     purpose, pathname,
+		     !strcmp (mode, "r") ? "read" : "write",
+		     errno ? gupcr_strerror () : "unknown reason");
+      gupcr_open_file_list_add (file, pathname);
+    }
+  return file;
+}
+
+static void
+gupcr_close_all_open_files (void)
+{
+  gupcr_open_file_ref f, f_next;
+  for (f = gupcr_open_files_list; f; f = f_next)
+    {
+      f_next = f->next;
+      gupcr_assert (f->file_ptr != NULL);
+      (void) fclose (f->file_ptr);
+      gupcr_free ((void *) f->pathname);
+      gupcr_free (f);
+    }
+}
+
+void
+gupcr_mkpath (const char *const path)
+{
+  char *path_dup, *dirpath;
+  gupcr_strdup (path_dup, path);
+  dirpath = dirname (path_dup);
+  if (strcmp (dirpath, ".") != 0 && strcmp (dirpath, "/") != 0)
+    {
+      char *dir, *p;
+      int nlevels;
+      for (p = dirpath, nlevels = 0;
+	   (dir = strtok (p, "/")); p = NULL, ++nlevels)
+	{
+	  gupcr_mkdir_unless_exists (dir);
+	  gupcr_syscall (chdir, (dir));
+	}
+      while (nlevels--)
+	gupcr_syscall (chdir, (".."));
+    }
+  gupcr_free (path_dup);
+}
+
+/* Parse the numeric string given by STR_ARG, and
+   multiply it by any scaling factor ('K', 'M', or 'G')
+   if present and return the result.  If the conversion
+   is successful then *STATUS will be 0.  Otherwise,
+	-1	There are no valid digits in the input string.
+	-2	There are extra characters in the multiplier suffix.
+	-3	The multiplier suffix specifies an invalid character.
+	-4	The number is out-of-range.  */
+
+long long
+gupcr_strtoll (const char *const str,
+	       long long int min_val, long long int max_val, int *status)
+{
+  long long result = 0;
+  *status = 0;
+  if (str && str[0])
+    {
+      char *suffix;
+      errno = 0;
+      result = strtoll (str, &suffix, 0);
+      if (errno == ERANGE)
+	/* Underflow/Overflow.  */
+	*status = -4;
+      else if (suffix[0])
+	{
+	  if (suffix == str)
+	    *status = -1;
+	  else if (suffix[1] && suffix[2])
+	    *status = -2;
+	  else if (suffix[1] && suffix[1] != 'B' && suffix[1] != 'b')
+	    *status = -2;
+	  else
+	    {
+	      int shift_count = 0;
+	      long long int tmp_result = result;
+	      switch (suffix[0])
+		{
+		case 'k':
+		case 'K':
+		  shift_count = 10;
+		  break;
+		case 'm':
+		case 'M':
+		  shift_count = 20;
+		  break;
+		case 'g':
+		case 'G':
+		  shift_count = 30;
+		  break;
+		case 't':
+		case 'T':
+		  shift_count = 40;
+		  break;
+		default:
+		  *status = -3;
+		  break;
+		}
+	      if (!*status && shift_count)
+		{
+		  const int is_neg = (tmp_result < 0);
+		  unsigned int i;
+		  tmp_result = is_neg ? -tmp_result : tmp_result;
+		  if (tmp_result >= 0)
+		    {
+		      for (i = shift_count; i > 0; --i)
+			{
+			  tmp_result <<= 1;
+			  if (tmp_result < 0 && !(is_neg && i == 1))
+			    {
+			      *status = -4;
+			      break;
+			    }
+			}
+		    }
+		  else
+		    *status = -4;
+		  if (*status)
+		    result = is_neg ? LLONG_MIN : LLONG_MAX;
+		  else
+		    result = is_neg ? -tmp_result : tmp_result;
+		}
+	    }
+	}
+      if (!*status && (result < min_val || result > max_val))
+	*status = -4;
+    }
+  else
+    *status = -1;
+  return result;
+}
+
+/* Given a non-zero status returned from calling gupcr_strtoll()
+   with the STR argument, print an informative error message.  */
+
+void
+gupcr_strtoll_error (const char *const str,
+		     long long int val_min, long long int val_max, int status)
+{
+  gupcr_assert (status < 0 && status >= -4);
+  switch (status)
+    {
+    case -1:
+      gupcr_error ("there are no valid digits in " "the value: `%s'", str);
+      break;
+    case -2:
+      gupcr_error ("there are extra characters in "
+		   "the multiplier suffix: `%s'", str);
+      break;
+    case -3:
+      gupcr_error ("invalid multiplier suffix; the suffix "
+		   "may be K, M, G, or T: `%s'", str);
+      break;
+    case -4:
+      gupcr_error ("the value `%s' is out of range; it must be "
+		   "in the range %lld .. %lld", str, val_min, val_max);
+      break;
+    default:
+      gupcr_fatal_error ("unreachable statement");
+      break;
+    }
+}
+
+/* Given a "tag" (a relative filename ending in XXXXXX),
+   create a temporary file using the tag.
+   Return a file descriptor associated with the newly
+   created temporary file.
+   [see: http://www.linux.com/howtos/Secure-Programs-HOWTO/avoid-race.shtml]  */
+
+int
+gupcr_create_temp_file (const char *tag, char *tmp_fname,
+                        const char **err_msg)
+{
+  const char *tmpdir = NULL;
+  mode_t old_mode;
+  int fd;
+  if ((getuid () == geteuid ()) && (getgid () == getegid ()))
+    {
+      tmpdir = getenv ("TMPDIR");
+      if (!tmpdir)
+        tmpdir = getenv ("TMP");
+    }
+  if (!tmpdir)
+    tmpdir = "/tmp";
+  sprintf (tmp_fname, "%s/%s", tmpdir, tag);
+  /* Create file with restrictive permissions */
+  old_mode = umask (077);
+  fd = mkstemp (tmp_fname);
+  (void) umask (old_mode);
+  if (fd < 0)
+    *err_msg = "Couldn't open temporary file";
+  return fd;
+}
+
+static void
+gupcr_debug_init (void)
+{
+  if (gupcr_debug_enabled)
+    {
+      gupcr_log (FC_ALL, "GUPCR debug enabled");
+    }
+}
+
+static void
+gupcr_stats_init (void)
+{
+  if (gupcr_stats_enabled)
+    {
+      gupcr_log (FC_ALL, "GUPCR stats enabled");
+    }
+}
+
+static void
+gupcr_trace_init (void)
+{
+  if (gupcr_trace_enabled)
+    {
+      gupcr_log (FC_ALL, "GUPCR trace enabled");
+    }
+}
+
+static void
+gupcr_debug_fini (void)
+{
+}
+
+static void
+gupcr_stats_fini (void)
+{
+}
+
+static void
+gupcr_trace_fini (void)
+{
+}
+
+void
+gupcr_utils_init (void)
+{
+  gupcr_pid = getpid ();
+  sprintf (gupcr_pid_string, "%d", gupcr_pid);
+  gupcr_shared_heap_size = GUPCR_DEFAULT_PER_THREAD_HEAP_SIZE;
+  gupcr_set_short_pgm_name ();
+  gupcr_inform_user = 1;
+  gupcr_warn_user = 1;
+  if (stdin)
+    gupcr_open_file_list_add (stdin, "stdin");
+  if (stdout)
+    gupcr_open_file_list_add (stdout, "stdout");
+  if (stderr)
+    gupcr_open_file_list_add (stderr, "stderr");
+  gupcr_env_init ();
+  gupcr_debug_init ();
+  gupcr_stats_init ();
+  gupcr_trace_init ();
+  gupcr_clock_init ();
+}
+
+void
+gupcr_utils_fini (void)
+{
+  gupcr_trace_fini ();
+  gupcr_stats_fini ();
+  gupcr_debug_fini ();
+  gupcr_close_all_open_files ();
+}
+
+/** @} */
Index: libgupc/portals4/gupcr_utils.h
===================================================================
--- libgupc/portals4/gupcr_utils.h	(.../trunk)	(revision 0)
+++ libgupc/portals4/gupcr_utils.h	(.../branches/gupc)	(revision 231080)
@@ -0,0 +1,327 @@
+/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
+   This file is part of the UPC runtime Library.
+   Written by Gary Funck <gary@intrepid.com>
+   and Nenad Vukicevic <nenad@intrepid.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+/**
+ * @file gupcr_utils.h
+ * GUPC Runtime utility routines
+ */
+
+/**
+ * @addtogroup GUPCUTILS GUPCR Utility Functions
+ * @{
+ */
+
+#ifndef _GUPCR_UTILS_H_
+
+//begin lib_utils_api
+#define _GUPCR_UTILS_H_ 1
+
+#ifndef __STRING
+/** Return 'x' argument as string */
+#define __STRING(x)     #x
+#endif
+
+/* Give up control of the CPU for a small time interval.  */
+#ifdef _POSIX_PRIORITY_SCHEDULING
+#define gupcr_yield_cpu() do { sched_yield (); } while (0)
+#else
+#define gupcr_yield_cpu() do { usleep (1000L); } while (0)
+#endif
+
+/* Give up control of the CPU for specified amount of time.  */
+#define gupcr_cpu_delay(MICROSECS) do { usleep(MICROSECS); } while (0)
+
+/* The definitions below use GCC's extensions to variadic macros.
+ * See: http://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html
+ * These extensions permit empty variadic arg lists.  */
+
+#define gupcr_error(fmt, args...)					\
+  gupcr_error_print ("%s: UPC error: " fmt "\n",			\
+                    gupcr_get_pgm_name (), ##args)
+
+#define gupcr_warn(fmt, args...)					\
+  gupcr_warn_print ("%s: UPC warning: " fmt "\n",			\
+                    gupcr_get_pgm_name (), ##args)
+
+#define gupcr_info(fmt, args...)					\
+  gupcr_info_print ("%s: UPC info: " fmt "\n",				\
+                    gupcr_get_pgm_name (), ##args)
+
+#define gupcr_fatal_error(fmt, args...)					\
+  gupcr_abort_with_msg ("[%d] %s: %s:%d: %s: " fmt "\n",		\
+               MYTHREAD, gupcr_get_pgm_name (),				\
+	       __FILE__, __LINE__, __func__, ##args)
+
+#ifdef GUPCR_HAVE_CHECKS
+#define gupcr_assert(expr) (expr) ? (void)(0)				\
+   : gupcr_fatal_error ("UPC runtime assertion `%s' failed",		\
+                        __STRING (expr))
+#else
+#define gupcr_assert(expr)
+#endif
+
+#define gupcr_syscall(syscall, args)					\
+    do									\
+      {									\
+        int status;							\
+        status = syscall args;						\
+	if (status < 0)					        	\
+	  gupcr_fatal_error ("UPC runtime system call `%s' failed: %s",	\
+	                     __STRING (syscall),				\
+	                     gupcr_strerror ());			\
+      }									\
+    while (0)
+
+#define gupcr_getcwd(path)						\
+    do									\
+      {									\
+        path = getcwd (NULL, 0);					\
+	if (!path)					        	\
+	  gupcr_fatal_error ("UPC runtime `getcwd' failed: %s",		\
+	                     gupcr_strerror ());			\
+      }									\
+    while (0)
+
+#define gupcr_mkdir_unless_exists(dir)					\
+    do									\
+      {									\
+        int status;							\
+        status = mkdir (dir, 0755);					\
+	if (status < 0 && errno != EEXIST)		        	\
+	  gupcr_fatal_error ("UPC runtime `mkdir' of `%s' failed: %s",	\
+	                     dir, __STRING (syscall));			\
+      }									\
+    while (0)
+
+#define gupcr_malloc(ptr, size)						\
+    do									\
+      {									\
+        ptr = malloc (size);						\
+	if (!ptr)					        	\
+	  gupcr_fatal_error ("UPC runtime malloc of %lld bytes failed",	\
+	                     (long long int)size);			\
+      }									\
+    while (0)
+
+#define gupcr_strdup(str, orig)						\
+    do									\
+      {									\
+        if (!orig)							\
+	  gupcr_fatal_error ("UPC runtime attempt to "			\
+	                     "duplicate a null string");		\
+        str = strdup (orig);						\
+	if (!str)					        	\
+	  gupcr_fatal_error ("UPC runtime strdup of %ld bytes failed",	\
+	                     (long int)strlen (orig));			\
+      }									\
+    while (0)
+
+
+#define gupcr_free(ptr)							\
+    do									\
+      {									\
+        if (!ptr)							\
+	  gupcr_fatal_error ("UPC runtime attempt to "			\
+	                     "free a null pointer");			\
+	free (ptr);							\
+      }									\
+    while (0)
+
+#ifdef GUPCR_HAVE_DEBUG
+#define gupcr_debug_enabled(facility)					\
+          (((facility) & gupcr_debug_facility_mask) != 0)
+#define gupcr_debug(facility, fmt, args...)				\
+          if (gupcr_debug_enabled (facility))				\
+            gupcr_debug_print ("%s: " fmt "\n", __func__ , ##args)
+#define gupcr_log(facility, fmt, args...)				\
+          if (facility & gupcr_log_facility_mask)			\
+            gupcr_log_print (fmt "\n", ##args)
+#define gupcr_stats(facility, fmt, args...)				\
+          if (facility & gupcr_stats_facility_mask)			\
+            gupcr_stats_print (fmt "\n", ##args)
+#define gupcr_trace(facility, fmt, args...)				\
+          if (facility & gupcr_trace_facility_mask)			\
+            gupcr_trace_print (fmt "\n", ##args)
+#else
+#define gupcr_debug_enabled(facility) (0)
+#define gupcr_debug(facility, fmt, args...)
+#define gupcr_log(facility, fmt, args...)
+#define gupcr_stats(facility, fmt, args...)
+#define gupcr_trace(facility, fmt, args...)
+#endif
+
+typedef enum
+{
+  FC_NONE      = 0b0000000000000,
+  FC_ADDR      = 0b0000000000001,
+  FC_ALLOC     = 0b0000000000010,
+  FC_ATOMIC    = 0b0000000000100,
+  FC_BARRIER   = 0b0000000001000,
+  FC_BROADCAST = 0b0000000010000,
+  FC_COLL      = 0b0000000100000,
+  FC_INFO      = 0b0000001000000,
+  FC_LOCK      = 0b0000010000000,
+  FC_MEM       = 0b0000100000000,
+  FC_MISC      = 0b0001000000000,
+  FC_NB        = 0b0010000000000,
+  FC_PORTALS   = 0b0100000000000,
+  FC_SYSTEM    = 0b1000000000000,
+  FC_ALL       = 0b1111111111111
+} gupcr_facility_t;
+
+#ifndef LONG_LONG_BITS
+#define LONG_LONG_BITS (__SIZEOF_LONG_LONG__ * 8)
+#endif /* LONG_LONG_BITS */
+
+#ifndef SIZE_T_BITS
+#define SIZE_T_BITS (__SIZEOF_SIZE_T__ * 8)
+#endif /* SIZE_T_BITS */
+
+/** Return TRUE, if 'v' is an exact power of 2.  */
+static inline unsigned int
+gupcr_is_pow_2 (unsigned long long v)
+{
+  return ((v & ~-v) == 0);
+}
+
+/** Return the value 'v' with the 1 bit at position 'bit' cleared.  */
+static inline unsigned long long int
+gupcr_clear_bit (unsigned long long v, unsigned int bit)
+{
+  return v & ~((unsigned long long) 1 << ((LONG_LONG_BITS - 1) - bit));
+}
+
+/** Return the value 'v' with the 1 bit at position 'bit' set.  */
+static inline unsigned long long int
+gupcr_set_bit (unsigned long long v, unsigned int bit)
+{
+  return v | ((unsigned long long) 1 << ((LONG_LONG_BITS - 1) - bit));
+}
+
+/** Return the position of the next non-zero bit in 'v'.
+    If 'v' is zero, then the value LONG_LONG_BITS is returned.  */
+static inline unsigned int
+gupcr_find_first_one (unsigned long long v)
+{
+  return __builtin_clzll (v);
+}
+
+/** Return the largest power of 2 that is <= 'v'.  */
+static inline unsigned int
+gupcr_floor_log2 (unsigned long long v)
+{
+  return (LONG_LONG_BITS - 1) - __builtin_clzll (v);
+}
+
+/** Return the smallest power of 2 that is >= 'v'.  */
+static inline unsigned int
+gupcr_log2 (unsigned long long v)
+{
+  return gupcr_floor_log2 (v) + !gupcr_is_pow_2 (v);
+}
+
+extern gupcr_facility_t gupcr_debug_facility_mask;
+extern gupcr_facility_t gupcr_log_facility_mask;
+extern gupcr_facility_t gupcr_stats_facility_mask;
+extern gupcr_facility_t gupcr_trace_facility_mask;
+
+extern void gupcr_exit (void);
+extern void gupcr_abort (void)
+  __attribute__ ((__nothrow__)) __attribute__ ((__noreturn__));
+extern void gupcr_abort_with_msg (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)))
+  __attribute__ ((__nothrow__)) __attribute__ ((__noreturn__));
+extern void gupcr_shutdown (int exit_code);
+extern const char *gupcr_strerror (void);
+extern const char *gupcr_get_pgm_name (void);
+extern void gupcr_set_pgm_name (const char *pgm_name);
+extern void gupcr_init_complete (void);
+extern size_t gupcr_get_shared_heap_size (void);
+extern void gupcr_signal_enable (int signal, void (*handler) (int));
+extern void gupcr_signal_disable (int signal);
+extern FILE *gupcr_fopen (const char *, const char *, const char *);
+extern void gupcr_mkpath (const char *const);
+extern long long gupcr_strtoll (const char *const,
+				long long int, long long int, int *);
+extern void gupcr_strtoll_error (const char *const str,
+				 long long int, long long int val_max, int);
+extern void gupcr_size_cvt_error (const char *const str, int);
+extern const char *gupcr_get_buf_as_hex (char *bufstr,
+					 const void *buf, size_t size);
+extern const char *gupcr_get_pid_as_string (void);
+extern size_t gupcr_get_shared_heap_size (void);
+extern int gupcr_is_node_local_memory_enabled (void);
+extern int gupcr_is_forcetouch_enabled (void);
+extern int gupcr_is_backtrace_enabled (void);
+extern void gupcr_unique_local_name (char *, const char *, int, int);
+extern void gupcr_log_print (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+extern void gupcr_debug_print (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+extern void gupcr_error_print (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+extern void gupcr_info_print (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+extern void gupcr_trace_print (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+extern void gupcr_warn_print (const char *fmt, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+extern int gupcr_create_temp_file (const char *, char *,
+				   const char **);
+extern void gupcr_utils_init (void);
+extern void gupcr_utils_fini (void);
+
+/* Called from: gupcr_env.c.  */
+extern void gupcr_be_quiet (void);
+extern void gupcr_no_warn (void);
+extern void gupcr_set_shared_heap_size (size_t heap_size);
+extern void gupcr_set_node_local_memory (int value);
+extern void gupcr_set_forcetouch (int value);
+extern void gupcr_set_backtrace (int value);
+extern void gupcr_set_debug_facility (gupcr_facility_t);
+extern void gupcr_set_debug_filename (const char *);
+extern void gupcr_set_log_facility (gupcr_facility_t);
+extern void gupcr_set_log_filename (const char *);
+extern void gupcr_set_stats_facility (gupcr_facility_t);
+extern void gupcr_set_stats_filename (const char *);
+extern void gupcr_set_trace_facility (gupcr_facility_t);
+extern void gupcr_set_trace_filename (const char *);
+
+/* See: gupcr_clock.c.  */
+extern double gupcr_clock (void);
+extern double gupcr_clock_resolution (void);
+extern void gupcr_clock_init (void);
+
+/* See: gupcr_env.c.  */
+extern void gupcr_env_init (void);
+
+/* See: gupcr_pgm_info.c.  */
+extern void gupcr_validate_pgm_info (void);
+
+//end lib_utils_api
+
+/** @} */
+#endif /* gupcr_utils.h */
