Implement std::pmr::synchronized_pool_resource

Message ID 20181108180229.GA31340@redhat.com
State New
Headers show
Series
  • Implement std::pmr::synchronized_pool_resource
Related show

Commit Message

Jonathan Wakely Nov. 8, 2018, 6:02 p.m.
Define the thread-safe pool resource, using a shared_mutex to allow
multiple threads to concurrently allocate from thread-specific pools.

Define new weak symbols for the pthread_rwlock_t functions, to avoid
making libstdc++.so depend on libpthread.so

	* config/abi/pre/gnu.ver: Add new symbols.
	* include/std/memory_resource [_GLIBCXX_HAS_GTHREADS]
	(synchronized_pool_resource): New class.
	* include/std/shared_mutex (__glibcxx_rwlock_rdlock)
	(__glibcxx_rwlock_tryrdlock, __glibcxx_rwlock_wrlock)
	(__glibcxx_rwlock_trywrlock, __glibcxx_rwlock_unlock)
	(__glibcxx_rwlock_destroy, __glibcxx_rwlock_init)
	(__glibcxx_rwlock_timedrdlock, __glibcxx_rwlock_timedwrlock): Define
	weak symbols for POSIX rwlock functions.
	(__shared_mutex_pthread): Use weak symbols.
	* src/c++17/memory_resource.cc [_GLIBCXX_HAS_GTHREADS]
	(synchronized_pool_resource::_TPools): New class.
	(destroy_TPools): New function for pthread_key_create destructor.
	(synchronized_pool_resource::synchronized_pool_resource)
	(synchronized_pool_resource::~synchronized_pool_resource)
	(synchronized_pool_resource::release)
	(synchronized_pool_resource::do_allocate)
	(synchronized_pool_resource::do_deallocate): Define public members.
	(synchronized_pool_resource::_M_thread_specific_pools)
	(synchronized_pool_resource::_M_alloc_tpools)
	(synchronized_pool_resource::_M_alloc_shared_tpools): Define private
	members.

The performance of this implementation is ... not great. But it's
better than simply adding a mutex to unsynchronized_pool_resource and
locking that around every allocation and deallocation.

I am not committing this yet, because I still need to test on non-GNU
targets, where the new weak wrappers around the pthread_rwlock_t
functions might not work properly.
commit adf32f3dd71a8312f212aadb7df2b6d32d8fe59b
Author: Jonathan Wakely <jwakely@redhat.com>
Date:   Sat Nov 3 23:13:26 2018 +0000

    Implement std::pmr::synchronized_pool_resource
    
    Define the thread-safe pool resource, using a shared_mutex to allow
    multiple threads to concurrently allocate from thread-specific pools.
    
    Define new weak symbols for the pthread_rwlock_t functions, to avoid
    making libstdc++.so depend on libpthread.so
    
            * config/abi/pre/gnu.ver: Add new symbols.
            * include/std/memory_resource [_GLIBCXX_HAS_GTHREADS]
            (synchronized_pool_resource): New class.
            * include/std/shared_mutex (__glibcxx_rwlock_rdlock)
            (__glibcxx_rwlock_tryrdlock, __glibcxx_rwlock_wrlock)
            (__glibcxx_rwlock_trywrlock, __glibcxx_rwlock_unlock)
            (__glibcxx_rwlock_destroy, __glibcxx_rwlock_init)
            (__glibcxx_rwlock_timedrdlock, __glibcxx_rwlock_timedwrlock): Define
            weak symbols for POSIX rwlock functions.
            (__shared_mutex_pthread): Use weak symbols.
            * src/c++17/memory_resource.cc [_GLIBCXX_HAS_GTHREADS]
            (synchronized_pool_resource::_TPools): New class.
            (destroy_TPools): New function for pthread_key_create destructor.
            (synchronized_pool_resource::synchronized_pool_resource)
            (synchronized_pool_resource::~synchronized_pool_resource)
            (synchronized_pool_resource::release)
            (synchronized_pool_resource::do_allocate)
            (synchronized_pool_resource::do_deallocate): Define public members.
            (synchronized_pool_resource::_M_thread_specific_pools)
            (synchronized_pool_resource::_M_alloc_tpools)
            (synchronized_pool_resource::_M_alloc_shared_tpools): Define private
            members.

Patch

diff --git a/libstdc++-v3/config/abi/pre/gnu.ver b/libstdc++-v3/config/abi/pre/gnu.ver
index 9d66f908e1a..c301fc31afd 100644
--- a/libstdc++-v3/config/abi/pre/gnu.ver
+++ b/libstdc++-v3/config/abi/pre/gnu.ver
@@ -2039,13 +2039,6 @@  GLIBCXX_3.4.26 {
     _ZNSt7__cxx1118basic_stringstreamI[cw]St11char_traitsI[cw]ESaI[cw]EEC[12]Ev;
     _ZNSt7__cxx1119basic_[io]stringstreamI[cw]St11char_traitsI[cw]ESaI[cw]EEC[12]Ev;
 
-    _ZNSt3pmr19new_delete_resourceEv;
-    _ZNSt3pmr20null_memory_resourceEv;
-    _ZNSt3pmr20get_default_resourceEv;
-    _ZNSt3pmr20set_default_resourceEPNS_15memory_resourceE;
-    _ZNSt3pmr25monotonic_buffer_resource13_M_new_bufferE[jmy][jmy];
-    _ZNSt3pmr25monotonic_buffer_resource18_M_release_buffersEv;
-
     # std::__throw_ios_failure(const char*, int);
     _ZSt19__throw_ios_failurePKci;
 
@@ -2057,6 +2050,18 @@  GLIBCXX_3.4.26 {
     _ZN11__gnu_debug25_Safe_local_iterator_base16_M_attach_singleEPNS_19_Safe_sequence_baseEb;
 
     # <memory_resource> members
+    _ZNSt3pmr19new_delete_resourceEv;
+    _ZNSt3pmr20null_memory_resourceEv;
+    _ZNSt3pmr20get_default_resourceEv;
+    _ZNSt3pmr20set_default_resourceEPNS_15memory_resourceE;
+    _ZNSt3pmr25monotonic_buffer_resource13_M_new_bufferE[jmy][jmy];
+    _ZNSt3pmr25monotonic_buffer_resource18_M_release_buffersEv;
+    _ZTINSt3pmr26synchronized_pool_resourceE;
+    _ZNSt3pmr26synchronized_pool_resourceC1ERKNS_12pool_optionsEPNS_15memory_resourceE;
+    _ZNSt3pmr26synchronized_pool_resourceD[12]Ev;
+    _ZNSt3pmr26synchronized_pool_resource7releaseEv;
+    _ZNSt3pmr26synchronized_pool_resource11do_allocateE[jmy][jmy];
+    _ZNSt3pmr26synchronized_pool_resource13do_deallocateEPv[jmy][jmy];
     _ZTINSt3pmr28unsynchronized_pool_resourceE;
     _ZNSt3pmr28unsynchronized_pool_resourceC[12]ERKNS_12pool_optionsEPNS_15memory_resourceE;
     _ZNSt3pmr28unsynchronized_pool_resourceD[12]Ev;
diff --git a/libstdc++-v3/include/std/memory_resource b/libstdc++-v3/include/std/memory_resource
index 40486af82fe..86866f6c4d5 100644
--- a/libstdc++-v3/include/std/memory_resource
+++ b/libstdc++-v3/include/std/memory_resource
@@ -37,6 +37,7 @@ 
 #include <utility>			// pair, index_sequence
 #include <vector>			// vector
 #include <cstddef>			// size_t, max_align_t
+#include <shared_mutex>			// shared_mutex
 #include <debug/assertions.h>
 
 namespace std _GLIBCXX_VISIBILITY(default)
@@ -338,7 +339,72 @@  namespace pmr
     const int _M_npools;
   };
 
-  // TODO class synchronized_pool_resource
+#ifdef _GLIBCXX_HAS_GTHREADS
+  /// A thread-safe memory resource that manages pools of fixed-size blocks.
+  class synchronized_pool_resource : public memory_resource
+  {
+  public:
+    synchronized_pool_resource(const pool_options& __opts,
+				 memory_resource* __upstream)
+    __attribute__((__nonnull__));
+
+    synchronized_pool_resource()
+    : synchronized_pool_resource(pool_options(), get_default_resource())
+    { }
+
+    explicit
+    synchronized_pool_resource(memory_resource* __upstream)
+    __attribute__((__nonnull__))
+    : synchronized_pool_resource(pool_options(), __upstream)
+    { }
+
+    explicit
+    synchronized_pool_resource(const pool_options& __opts)
+    : synchronized_pool_resource(__opts, get_default_resource()) { }
+
+    synchronized_pool_resource(const synchronized_pool_resource&) = delete;
+
+    virtual ~synchronized_pool_resource();
+
+    synchronized_pool_resource&
+    operator=(const synchronized_pool_resource&) = delete;
+
+    void release();
+
+    memory_resource*
+    upstream_resource() const noexcept
+    __attribute__((__returns_nonnull__))
+    { return _M_impl.resource(); }
+
+    pool_options options() const noexcept { return _M_impl._M_opts; }
+
+  protected:
+    void*
+    do_allocate(size_t __bytes, size_t __alignment) override;
+
+    void
+    do_deallocate(void* __p, size_t __bytes, size_t __alignment) override;
+
+    bool
+    do_is_equal(const memory_resource& __other) const noexcept override
+    { return this == &__other; }
+
+  public:
+    // Thread-specific pools (only public for access by implementation details)
+    struct _TPools;
+
+  private:
+    _TPools* _M_alloc_tpools(lock_guard<shared_mutex>&);
+    _TPools* _M_alloc_shared_tpools(lock_guard<shared_mutex>&);
+    auto _M_thread_specific_pools() noexcept;
+
+    __pool_resource _M_impl;
+    __gthread_key_t _M_key;
+    // Linked list of thread-specific pools. All threads share _M_tpools[0].
+    _TPools* _M_tpools = nullptr;
+    mutable shared_mutex _M_mx;
+  };
+#endif
 
   /// A non-thread-safe memory resource that manages pools of fixed-size blocks.
   class unsynchronized_pool_resource : public memory_resource
diff --git a/libstdc++-v3/include/std/shared_mutex b/libstdc++-v3/include/std/shared_mutex
index dce97f48a3f..8aa6e9f0d4f 100644
--- a/libstdc++-v3/include/std/shared_mutex
+++ b/libstdc++-v3/include/std/shared_mutex
@@ -57,6 +57,90 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
   class shared_timed_mutex;
 
 #if _GLIBCXX_USE_PTHREAD_RWLOCK_T
+#ifdef __gthrw
+#define _GLIBCXX_GTHRW(name) \
+  __gthrw(pthread_ ## name); \
+  static inline int \
+  __glibcxx_ ## name (pthread_rwlock_t *__rwlock) \
+  { \
+    if (__gthread_active_p ()) \
+      return __gthrw_(pthread_ ## name) (__rwlock); \
+    else \
+      return 0; \
+  }
+  _GLIBCXX_GTHRW(rwlock_rdlock)
+  _GLIBCXX_GTHRW(rwlock_tryrdlock)
+  _GLIBCXX_GTHRW(rwlock_wrlock)
+  _GLIBCXX_GTHRW(rwlock_trywrlock)
+  _GLIBCXX_GTHRW(rwlock_unlock)
+# ifndef PTHREAD_RWLOCK_INITIALIZER
+  _GLIBCXX_GTHRW(rwlock_destroy)
+  __gthrw(pthread_rwlock_init);
+  static inline int
+  __glibcxx_rwlock_init (pthread_rwlock_t *__rwlock)
+  {
+    if (__gthread_active_p ())
+      return __gthrw_(pthread_rwlock_init) (__rwlock, NULL);
+    else
+      return 0;
+  }
+# endif
+# if _GTHREAD_USE_MUTEX_TIMEDLOCK
+   __gthrw(pthread_rwlock_timedrdlock);
+  static inline int
+  __glibcxx_rwlock_timedrdlock (pthread_rwlock_t *__rwlock,
+				const timespec *__ts)
+  {
+    if (__gthread_active_p ())
+      return __gthrw_(pthread_rwlock_timedrdlock) (__rwlock, __ts);
+    else
+      return 0;
+  }
+   __gthrw(pthread_rwlock_timedwrlock);
+  static inline int
+  __glibcxx_rwlock_timedwrlock (pthread_rwlock_t *__rwlock,
+				const timespec *__ts)
+  {
+    if (__gthread_active_p ())
+      return __gthrw_(pthread_rwlock_timedwrlock) (__rwlock, __ts);
+    else
+      return 0;
+  }
+# endif
+#else
+  static inline int
+  __glibcxx_rwlock_rdlock (pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_rdlock (__rwlock); }
+  static inline int
+  __glibcxx_rwlock_tryrdlock (pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_tryrdlock (__rwlock); }
+  static inline int
+  __glibcxx_rwlock_wrlock (pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_wrlock (__rwlock); }
+  static inline int
+  __glibcxx_rwlock_trywrlock (pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_trywrlock (__rwlock); }
+  static inline int
+  __glibcxx_rwlock_unlock (pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_unlock (__rwlock); }
+  static inline int
+  __glibcxx_rwlock_destroy(pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_destroy (__rwlock); }
+  static inline int
+  __glibcxx_rwlock_init(pthread_rwlock_t *__rwlock)
+  { return pthread_rwlock_init (__rwlock, NULL); }
+# if _GTHREAD_USE_MUTEX_TIMEDLOCK
+  static inline int
+  __glibcxx_rwlock_timedrdlock (pthread_rwlock_t *__rwlock,
+				const timespec *__ts)
+  { return pthread_rwlock_timedrdlock (__rwlock, __ts); }
+  static inline int
+  __glibcxx_rwlock_timedwrlock (pthread_rwlock_t *__rwlock,
+				const timespec *__ts)
+  { return pthread_rwlock_timedwrlock (__rwlock, __ts); }
+# endif
+#endif
+
   /// A shared mutex type implemented using pthread_rwlock_t.
   class __shared_mutex_pthread
   {
@@ -74,7 +158,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
   public:
     __shared_mutex_pthread()
     {
-      int __ret = pthread_rwlock_init(&_M_rwlock, NULL);
+      int __ret = __glibcxx_rwlock_init(&_M_rwlock, NULL);
       if (__ret == ENOMEM)
 	__throw_bad_alloc();
       else if (__ret == EAGAIN)
@@ -87,7 +171,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
     ~__shared_mutex_pthread()
     {
-      int __ret __attribute((__unused__)) = pthread_rwlock_destroy(&_M_rwlock);
+      int __ret __attribute((__unused__)) = __glibcxx_rwlock_destroy(&_M_rwlock);
       // Errors not handled: EBUSY, EINVAL
       __glibcxx_assert(__ret == 0);
     }
@@ -99,7 +183,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
     void
     lock()
     {
-      int __ret = pthread_rwlock_wrlock(&_M_rwlock);
+      int __ret = __glibcxx_rwlock_wrlock(&_M_rwlock);
       if (__ret == EDEADLK)
 	__throw_system_error(int(errc::resource_deadlock_would_occur));
       // Errors not handled: EINVAL
@@ -109,7 +193,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
     bool
     try_lock()
     {
-      int __ret = pthread_rwlock_trywrlock(&_M_rwlock);
+      int __ret = __glibcxx_rwlock_trywrlock(&_M_rwlock);
       if (__ret == EBUSY) return false;
       // Errors not handled: EINVAL
       __glibcxx_assert(__ret == 0);
@@ -119,7 +203,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
     void
     unlock()
     {
-      int __ret __attribute((__unused__)) = pthread_rwlock_unlock(&_M_rwlock);
+      int __ret __attribute((__unused__)) = __glibcxx_rwlock_unlock(&_M_rwlock);
       // Errors not handled: EPERM, EBUSY, EINVAL
       __glibcxx_assert(__ret == 0);
     }
@@ -135,7 +219,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
       // is okay based on the current specification of forward progress
       // guarantees by the standard.
       do
-	__ret = pthread_rwlock_rdlock(&_M_rwlock);
+	__ret = __glibcxx_rwlock_rdlock(&_M_rwlock);
       while (__ret == EAGAIN);
       if (__ret == EDEADLK)
 	__throw_system_error(int(errc::resource_deadlock_would_occur));
@@ -146,7 +230,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
     bool
     try_lock_shared()
     {
-      int __ret = pthread_rwlock_tryrdlock(&_M_rwlock);
+      int __ret = __glibcxx_rwlock_tryrdlock(&_M_rwlock);
       // If the maximum number of read locks has been exceeded, we just fail
       // to acquire the lock.  Unlike for lock(), we are not allowed to throw
       // an exception.
@@ -413,7 +497,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	    static_cast<long>(__ns.count())
 	  };
 
-	int __ret = pthread_rwlock_timedwrlock(&_M_rwlock, &__ts);
+	int __ret = __glibcxx_rwlock_timedwrlock(&_M_rwlock, &__ts);
 	// On self-deadlock, we just fail to acquire the lock.  Technically,
 	// the program violated the precondition.
 	if (__ret == ETIMEDOUT || __ret == EDEADLK)
@@ -466,7 +550,7 @@  _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	// mistaken for a spurious failure, which might help users realise
 	// there is a deadlock.
 	do
-	  __ret = pthread_rwlock_timedrdlock(&_M_rwlock, &__ts);
+	  __ret = __glibcxx_rwlock_timedrdlock(&_M_rwlock, &__ts);
 	while (__ret == EAGAIN || __ret == EDEADLK);
 	if (__ret == ETIMEDOUT)
 	  return false;
diff --git a/libstdc++-v3/src/c++17/memory_resource.cc b/libstdc++-v3/src/c++17/memory_resource.cc
index 781bdada381..c34b817abe0 100644
--- a/libstdc++-v3/src/c++17/memory_resource.cc
+++ b/libstdc++-v3/src/c++17/memory_resource.cc
@@ -863,6 +863,11 @@  namespace pmr
     return 1;
   }
 
+#ifdef _GLIBCXX_HAS_GTHREADS
+  using shared_lock = std::shared_lock<shared_mutex>;
+  using exclusive_lock = lock_guard<shared_mutex>;
+#endif
+
   } // namespace
 
   __pool_resource::
@@ -948,6 +953,292 @@  namespace pmr
     return p;
   }
 
+#ifdef _GLIBCXX_HAS_GTHREADS
+  // synchronized_pool_resource members.
+
+  /* Notes on implementation and thread safety:
+   *
+   * Each synchronized_pool_resource manages an linked list of N+1 _TPools
+   * objects, where N is the number of threads using the pool resource.
+   * Each _TPools object has its own set of pools, with their own chunks.
+   * The first element of the list, _M_tpools[0], can be used by any thread.
+   * The rest of the list contains a _TPools object for each thread,
+   * accessed via the thread-specific key _M_key (and referred to for
+   * exposition as _M_tpools[_M_key]).
+   * The first element, _M_tpools[0], contains "orphaned chunks" which were
+   * allocated by a thread which has since exited, and so there is no
+   * _M_tpools[_M_key] for that thread.
+   * A thread can access its own thread-specific set of pools via _M_key
+   * while holding a shared lock on _M_mx. Accessing _M_impl._M_unpooled
+   * or _M_tpools[0] or any other thread's _M_tpools[_M_key] requires an
+   * exclusive lock.
+   * The upstream_resource() pointer can be obtained without a lock, but
+   * any dereference of that pointer requires an exclusive lock.
+   * The _M_impl._M_opts and _M_impl._M_npools members are immutable,
+   * and can safely be accessed concurrently.
+   */
+
+  extern "C" {
+    static void destroy_TPools(void*);
+  }
+
+  struct synchronized_pool_resource::_TPools
+  {
+    // Exclusive lock must be held in the thread where this constructor runs.
+    explicit
+    _TPools(synchronized_pool_resource& owner, exclusive_lock&)
+    : owner(owner), pools(owner._M_impl._M_alloc_pools())
+    {
+      // __builtin_printf("%p constructing\n", this);
+      __glibcxx_assert(pools);
+    }
+
+    // Exclusive lock must be held in the thread where this destructor runs.
+    ~_TPools()
+    {
+      __glibcxx_assert(pools);
+      if (pools)
+	{
+	  memory_resource* r = owner.upstream_resource();
+	  for (int i = 0; i < owner._M_impl._M_npools; ++i)
+	    pools[i].release(r);
+	  std::destroy_n(pools, owner._M_impl._M_npools);
+	  polymorphic_allocator<__pool_resource::_Pool> a(r);
+	  a.deallocate(pools, owner._M_impl._M_npools);
+	}
+      if (prev)
+	prev->next = next;
+      if (next)
+	next->prev = prev;
+    }
+
+    // Exclusive lock must be held in the thread where this function runs.
+    void move_nonempty_chunks()
+    {
+      __glibcxx_assert(pools);
+      if (!pools)
+	return;
+      memory_resource* r = owner.upstream_resource();
+      // move all non-empty chunks to the shared _TPools
+      for (int i = 0; i < owner._M_impl._M_npools; ++i)
+	for (auto& c : pools[i]._M_chunks)
+	  if (!c.empty())
+	    owner._M_tpools->pools[i]._M_chunks.insert(std::move(c), r);
+    }
+
+    synchronized_pool_resource& owner;
+    __pool_resource::_Pool* pools = nullptr;
+    _TPools* prev = nullptr;
+    _TPools* next = nullptr;
+
+    static void destroy(_TPools* p)
+    {
+      exclusive_lock l(p->owner._M_mx);
+      // __glibcxx_assert(p != p->owner._M_tpools);
+      p->move_nonempty_chunks();
+      polymorphic_allocator<_TPools> a(p->owner.upstream_resource());
+      p->~_TPools();
+      a.deallocate(p, 1);
+    }
+  };
+
+  // Called when a thread exits
+  extern "C" {
+    static void destroy_TPools(void* p)
+    {
+      using _TPools = synchronized_pool_resource::_TPools;
+      _TPools::destroy(static_cast<_TPools*>(p));
+    }
+  }
+
+  // Constructor
+  synchronized_pool_resource::
+  synchronized_pool_resource(const pool_options& opts,
+			     memory_resource* upstream)
+  : _M_impl(opts, upstream)
+  {
+    if (int err = __gthread_key_create(&_M_key, destroy_TPools))
+      __throw_system_error(err);
+    exclusive_lock l(_M_mx);
+    _M_tpools = _M_alloc_shared_tpools(l);
+  }
+
+  // Destructor
+  synchronized_pool_resource::~synchronized_pool_resource()
+  {
+    release();
+    __gthread_key_delete(_M_key); // does not run destroy_TPools
+  }
+
+  void
+  synchronized_pool_resource::release()
+  {
+    exclusive_lock l(_M_mx);
+    if (_M_tpools)
+      {
+	__gthread_key_delete(_M_key); // does not run destroy_TPools
+	__gthread_key_create(&_M_key, destroy_TPools);
+	polymorphic_allocator<_TPools> a(upstream_resource());
+	// destroy+deallocate each _TPools
+	do
+	  {
+	    _TPools* p = _M_tpools;
+	    _M_tpools = _M_tpools->next;
+	    p->~_TPools();
+	    a.deallocate(p, 1);
+	  }
+	while (_M_tpools);
+      }
+    // release unpooled memory
+    _M_impl.release();
+  }
+
+  // Caller must hold shared or exclusive lock to ensure the pointer
+  // isn't invalidated before it can be used.
+  auto
+  synchronized_pool_resource::_M_thread_specific_pools() noexcept
+  {
+    __pool_resource::_Pool* pools = nullptr;
+    if (auto tp = static_cast<_TPools*>(__gthread_getspecific(_M_key)))
+      {
+      pools = tp->pools;
+      __glibcxx_assert(tp->pools);
+      }
+    return pools;
+  }
+
+  // Override for memory_resource::do_allocate
+  void*
+  synchronized_pool_resource::
+  do_allocate(size_t bytes, size_t alignment)
+  {
+    const auto block_size = std::max(bytes, alignment);
+    if (block_size <= _M_impl._M_opts.largest_required_pool_block)
+      {
+	const ptrdiff_t index = pool_index(block_size, _M_impl._M_npools);
+	memory_resource* const r = upstream_resource();
+	const pool_options opts = _M_impl._M_opts;
+	{
+	  // Try to allocate from the thread-specific pool
+	  shared_lock l(_M_mx);
+	  if (auto pools = _M_thread_specific_pools()) // [[likely]]
+	    {
+	      // Need exclusive lock to replenish so use try_allocate:
+	      if (void* p = pools[index].try_allocate())
+		return p;
+	      // Need to take exclusive lock and replenish pool.
+	    }
+	  // Need to allocate or replenish thread-specific pools using
+	  // upstream resource, so need to hold exclusive lock.
+	}
+	// N.B. Another thread could call release() now lock is not held.
+	exclusive_lock excl(_M_mx);
+	if (!_M_tpools) // [[unlikely]]
+	  _M_tpools = _M_alloc_shared_tpools(excl);
+	auto pools = _M_thread_specific_pools();
+	if (!pools)
+	  pools = _M_alloc_tpools(excl)->pools;
+	return pools[index].allocate(r, opts);
+      }
+    exclusive_lock l(_M_mx);
+    return _M_impl.allocate(bytes, alignment); // unpooled allocation
+  }
+
+  // Override for memory_resource::do_deallocate
+  void
+  synchronized_pool_resource::
+  do_deallocate(void* p, size_t bytes, size_t alignment)
+  {
+    size_t block_size = std::max(bytes, alignment);
+    if (block_size <= _M_impl._M_opts.largest_required_pool_block)
+      {
+	const ptrdiff_t index = pool_index(block_size, _M_impl._M_npools);
+	__glibcxx_assert(index != -1);
+	{
+	  shared_lock l(_M_mx);
+	  auto pools = _M_thread_specific_pools();
+	  if (pools)
+	    {
+	      // No need to lock here, no other thread is accessing this pool.
+	      if (pools[index].deallocate(upstream_resource(), p))
+		return;
+	    }
+	  // Block might have come from a different thread's pool,
+	  // take exclusive lock and check every pool.
+	}
+	// TODO store {p, bytes, alignment} somewhere and defer returning
+	// the block to the correct thread-specific pool until we next
+	// take the exclusive lock.
+	exclusive_lock excl(_M_mx);
+	for (_TPools* t = _M_tpools; t != nullptr; t = t->next)
+	  {
+	    if (t->pools) // [[likely]]
+	      {
+		if (t->pools[index].deallocate(upstream_resource(), p))
+		  return;
+	      }
+	  }
+      }
+    exclusive_lock l(_M_mx);
+    _M_impl.deallocate(p, bytes, alignment);
+  }
+
+  // Allocate a thread-specific _TPools object and add it to the linked list.
+  auto
+  synchronized_pool_resource::_M_alloc_tpools(exclusive_lock& l)
+  -> _TPools*
+  {
+    __glibcxx_assert(_M_tpools != nullptr);
+    // dump_list(_M_tpools);
+    polymorphic_allocator<_TPools> a(upstream_resource());
+    _TPools* p = a.allocate(1);
+    bool constructed = false;
+    __try
+      {
+	a.construct(p, *this, l);
+	constructed = true;
+	// __glibcxx_assert(__gthread_getspecific(_M_key) == nullptr);
+	if (int err = __gthread_setspecific(_M_key, p))
+	  __throw_system_error(err);
+      }
+    __catch(...)
+      {
+	if (constructed)
+	  a.destroy(p);
+	a.deallocate(p, 1);
+	__throw_exception_again;
+      }
+    p->prev = _M_tpools;
+    p->next = _M_tpools->next;
+    _M_tpools->next = p;
+    if (p->next)
+      p->next->prev = p;
+    return p;
+  }
+
+  // Allocate the shared _TPools object, _M_tpools[0]
+  auto
+  synchronized_pool_resource::_M_alloc_shared_tpools(exclusive_lock& l)
+  -> _TPools*
+  {
+    __glibcxx_assert(_M_tpools == nullptr);
+    polymorphic_allocator<_TPools> a(upstream_resource());
+    _TPools* p = a.allocate(1);
+    __try
+      {
+	a.construct(p, *this, l);
+      }
+    __catch(...)
+      {
+	a.deallocate(p, 1);
+	__throw_exception_again;
+      }
+    // __glibcxx_assert(p->next == nullptr);
+    // __glibcxx_assert(p->prev == nullptr);
+    return p;
+  }
+#endif // _GLIBCXX_HAS_GTHREADS
+
   // unsynchronized_pool_resource member functions
 
   // Constructor