From 0e544bd8adbea9ab369d2c6796a6b41e054c37fa Mon Sep 17 00:00:00 2001
From: Szabolcs Nagy <szabolcs.nagy@arm.com>
Date: Tue, 31 Dec 2019 16:01:41 +0000
Subject: [PATCH] Avoid using up static TLS surplus for optimizations [BZ
#25051]
v2:
- Add dl.nns tunable.
- Add dl.optional_static_tls tunable.
- New surplus TLS usage contract that works reliably up to dl.nns
namespaces.
- TODO: document the tunables
- TODO: static linking vs opportunistic tls?
On some targets static TLS surplus area can be used opportunistically
for dynamically loaded modules such that the TLS access then becomes
faster (TLSDESC and powerpc TLS optimization). However we don't want
all surplus TLS to be used for this optimization because dynamically
loaded modules with initial-exec model TLS can only use surplus TLS.
Currently TLS_STATIC_SURPLUS is 1664 which is not even enough to cover
all the IE TLS need by libc.so and other libraries when DL_NNS (== 16)
namespaces are created (let alone opportunistic usage).
So the contract for static TLS surplus use is changed as follows:
- libc.so can have up to 192 bytes of IE TLS,
- other system libraries together can have up to 144 bytes of IE TLS,
- and 512 bytes of surplus TLS is available for opportunistic use.
By default at most 4 namespaces are supported so the surplus TLS
requirement is 3*192 + 4*144 + 512 = 1664 bytes. (i.e. the same as
before so the externally visible behaviour is not changed by default,
other than limiting opportunistic TLS use on affected targets.)
The surplus TLS available for opportunistic use is tunable
(dl.optional_static_tls), so users can directly affect the allocated
static TLS size. Note that this portion can easily run out when used
for dynamic TLS optimization, in particular module unloading does not
reclaim the opportunistically allocated static TLS.
Since users may need more dlmopen namespaces (5 .. 16) the maximum
supported number is now a tunable (dl.nns), when it is increased the
static TLS allocation increases according to the contract. If users
use more namespaces than the tunable limit, static TLS may run out
(just like before). If users dynamically load libraries with IE TLS
beyond what's allowed by the contract, static TLS may run out (just
like before). These conditions are not checked or enforced, but the
user's responsibility for now.
Tested on aarch64-linux-gnu and x86_64-linux-gnu.
---
csu/libc-start.c | 2 +
csu/libc-tls.c | 4 ++
elf/Makefile | 17 ++++++-
elf/dl-reloc.c | 38 +++++++++++----
elf/dl-support.c | 3 ++
elf/dl-sysdep.c | 2 +
elf/dl-tls.c | 54 ++++++++++++++++++++-
elf/dl-tunables.h | 4 ++
elf/dl-tunables.list | 13 +++++
elf/dynamic-link.h | 5 +-
elf/tst-tls-ie-mod.h | 40 ++++++++++++++++
elf/tst-tls-ie-mod0.c | 4 ++
elf/tst-tls-ie-mod1.c | 4 ++
elf/tst-tls-ie-mod2.c | 4 ++
elf/tst-tls-ie-mod3.c | 4 ++
elf/tst-tls-ie-mod4.c | 4 ++
elf/tst-tls-ie-mod5.c | 4 ++
elf/tst-tls-ie.c | 98 ++++++++++++++++++++++++++++++++++++++
sysdeps/generic/ldsodefs.h | 7 +++
19 files changed, 296 insertions(+), 15 deletions(-)
create mode 100644 elf/tst-tls-ie-mod.h
create mode 100644 elf/tst-tls-ie-mod0.c
create mode 100644 elf/tst-tls-ie-mod1.c
create mode 100644 elf/tst-tls-ie-mod2.c
create mode 100644 elf/tst-tls-ie-mod3.c
create mode 100644 elf/tst-tls-ie-mod4.c
create mode 100644 elf/tst-tls-ie-mod5.c
create mode 100644 elf/tst-tls-ie.c
@@ -188,6 +188,8 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
__tunables_init (__environ);
+ _dl_static_tls_tunables_init ();
+
ARCH_INIT_CPU_FEATURES ();
/* Perform IREL{,A} relocations. */
@@ -53,6 +53,10 @@ size_t _dl_tls_static_size = 2048;
size_t _dl_tls_static_used;
/* Alignment requirement of the static TLS block. */
size_t _dl_tls_static_align;
+/* Remaining limit for optional use of the static TLS block. */
+size_t _dl_tls_static_optional;
+/* Limit for optional use of the static TLS block. */
+size_t _dl_tls_static_optional_size;
/* Generation counter for the dtv. */
size_t _dl_tls_generation;
@@ -202,7 +202,8 @@ tests += restest1 preloadtest loadfail multiload origtest resolvfail \
tst-sonamemove-link tst-sonamemove-dlopen tst-dlopen-tlsmodid \
tst-dlopen-self tst-auditmany tst-initfinilazyfail tst-dlopenfail \
tst-dlopenfail-2 \
- tst-filterobj tst-filterobj-dlopen tst-auxobj tst-auxobj-dlopen
+ tst-filterobj tst-filterobj-dlopen tst-auxobj tst-auxobj-dlopen \
+ tst-tls-ie
# reldep9
tests-internal += loadtest unload unload2 circleload1 \
neededtest neededtest2 neededtest3 neededtest4 \
@@ -314,7 +315,10 @@ modules-names = testobj1 testobj2 testobj3 testobj4 testobj5 testobj6 \
tst-initlazyfailmod tst-finilazyfailmod \
tst-dlopenfailmod1 tst-dlopenfaillinkmod tst-dlopenfailmod2 \
tst-dlopenfailmod3 tst-ldconfig-ld-mod \
- tst-filterobj-flt tst-filterobj-aux tst-filterobj-filtee
+ tst-filterobj-flt tst-filterobj-aux tst-filterobj-filtee \
+ tst-tls-ie-mod0 tst-tls-ie-mod1 tst-tls-ie-mod2 \
+ tst-tls-ie-mod3 tst-tls-ie-mod4 tst-tls-ie-mod5
+
# Most modules build with _ISOMAC defined, but those filtered out
# depend on internal headers.
modules-names-tests = $(filter-out ifuncmod% tst-libc_dlvsym-dso tst-tlsmod%,\
@@ -1728,3 +1732,12 @@ $(objpfx)tst-auxobj: $(objpfx)tst-filterobj-aux.so
$(objpfx)tst-auxobj-dlopen: $(libdl)
$(objpfx)tst-auxobj.out: $(objpfx)tst-filterobj-filtee.so
$(objpfx)tst-auxobj-dlopen.out: $(objpfx)tst-filterobj-filtee.so
+
+$(objpfx)tst-tls-ie: $(libdl) $(shared-thread-library)
+$(objpfx)tst-tls-ie.out: \
+ $(objpfx)tst-tls-ie-mod0.so \
+ $(objpfx)tst-tls-ie-mod1.so \
+ $(objpfx)tst-tls-ie-mod2.so \
+ $(objpfx)tst-tls-ie-mod3.so \
+ $(objpfx)tst-tls-ie-mod4.so \
+ $(objpfx)tst-tls-ie-mod5.so
@@ -39,13 +39,17 @@
/* We are trying to perform a static TLS relocation in MAP, but it was
dynamically loaded. This can only work if there is enough surplus in
the static TLS area already allocated for each running thread. If this
- object's TLS segment is too big to fit, we fail. If it fits,
- we set MAP->l_tls_offset and return.
- This function intentionally does not return any value but signals error
- directly, as static TLS should be rare and code handling it should
- not be inlined as much as possible. */
+ object's TLS segment is too big to fit, we fail with -1. If it fits,
+ we set MAP->l_tls_offset and return 0.
+ A small portion of static TLS surplus is reserved for opportunistic use
+ to optimize TLS access when both static and dynamically allocated TLS
+ works, but static TLS results in faster access (e.g. TLSDESC).
+ If OPTIONAL is true then only this reserved portion of static TLS
+ is used and if it runs out, then the caller is expected to have a fallback
+ that does not rely on static TLS. OPTIONAL should be false if static TLS
+ is required and the caller has no fallback. */
int
-_dl_try_allocate_static_tls (struct link_map *map)
+_dl_try_allocate_static_tls (struct link_map *map, bool optional)
{
/* If we've already used the variable with dynamic access, or if the
alignment requirements are too high, fail. */
@@ -68,8 +72,14 @@ _dl_try_allocate_static_tls (struct link_map *map)
size_t n = (freebytes - blsize) / map->l_tls_align;
- size_t offset = GL(dl_tls_static_used) + (freebytes - n * map->l_tls_align
- - map->l_tls_firstbyte_offset);
+ /* Account optional static TLS surplus usage. */
+ size_t use = freebytes - n * map->l_tls_align - map->l_tls_firstbyte_offset;
+ if (optional && use > GL(dl_tls_static_optional))
+ goto fail;
+ else if (optional)
+ GL(dl_tls_static_optional) -= use;
+
+ size_t offset = GL(dl_tls_static_used) + use;
map->l_tls_offset = GL(dl_tls_static_used) = offset;
#elif TLS_DTV_AT_TP
@@ -83,6 +93,13 @@ _dl_try_allocate_static_tls (struct link_map *map)
if (used > GL(dl_tls_static_size))
goto fail;
+ /* Account optional static TLS surplus usage. */
+ size_t use = used - GL(dl_tls_static_used);
+ if (optional && use > GL(dl_tls_static_optional))
+ goto fail;
+ else if (optional)
+ GL(dl_tls_static_optional) -= use;
+
map->l_tls_offset = offset;
map->l_tls_firstbyte_offset = GL(dl_tls_static_used);
GL(dl_tls_static_used) = used;
@@ -110,12 +127,15 @@ _dl_try_allocate_static_tls (struct link_map *map)
return 0;
}
+/* This function intentionally does not return any value but signals error
+ directly, as static TLS should be rare and code handling it should
+ not be inlined as much as possible. */
void
__attribute_noinline__
_dl_allocate_static_tls (struct link_map *map)
{
if (map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET
- || _dl_try_allocate_static_tls (map))
+ || _dl_try_allocate_static_tls (map, false))
{
_dl_signal_error (0, map->l_name, NULL, N_("\
cannot allocate memory in static TLS block"));
@@ -110,6 +110,9 @@ struct link_namespaces _dl_ns[DL_NNS] =
};
size_t _dl_nns = 1;
+/* Number of namespaces for which enough static TLS is allocated. */
+size_t _dl_nns_supported = 1;
+
/* Incremented whenever something may have been added to dl_loaded. */
unsigned long long _dl_load_adds = 1;
@@ -222,6 +222,8 @@ _dl_sysdep_start (void **start_argptr,
__tunables_init (_environ);
+ _dl_static_tls_tunables_init ();
+
#ifdef DL_SYSDEP_INIT
DL_SYSDEP_INIT;
#endif
@@ -29,10 +29,60 @@
#include <dl-tls.h>
#include <ldsodefs.h>
+#define TUNABLE_NAMESPACE dl
+#include <dl-tunables.h>
+
+/* Default value for GLRO(dl_nns_supported). */
+#define DL_NNS_SUPPORTED_DEFAULT 4
+/* Size of initial-exec TLS in libc.so itself. */
+#define LIBC_IE_TLS 192
+/* Size of initial-exec TLS in libraries other than libc.so.
+ This should be large enough to cover runtime libraries of the
+ compiler such as libgomp and libraries in libc other than libc.so. */
+#define OTHER_IE_TLS 144
+/* Default size of static TLS that may be opportunistically used for
+ dynamic TLS, e.g. to optimize TLSDESC, not required for correctness.
+ Actual value is GLRO(dl_tls_static_optional_size). */
+#define OPTIONAL_STATIC_TLS_DEFAULT 512
+
/* Amount of excess space to allocate in the static TLS area
- to allow dynamic loading of modules defining IE-model TLS data. */
-#define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
+ to allow dynamic loading of modules defining IE-model TLS data.
+ Static TLS surplus is used for
+ - IE-TLS in libc.so in all dynamically created namespaces, in the initial
+ namespace libc.so is always expected to be loaded at program startup
+ time, not dynamically,
+ - IE-TLS in other libraries which may be dynamically loaded even in the
+ initial namespace,
+ - and for opportunistic dynamic TLS optimization.
+
+ The maximum number of namespaces is DL_NNS, but to support that many
+ namespaces correctly the static TLS allocation should be significantly
+ increased, which may run into problems because of bug 11787.
+ So there is a separate GLRO(dl_nns_supported) limit that can be tuned
+ between 1 and DL_NNS and by default it's small enough so static TLS
+ is not increased while correctly supporting namespaces below the limit.
+ It's the user's responsibility to increase the limit if more namespaces
+ are used, there is no check in dlmopen to enforce the limit. */
+#define TLS_STATIC_SURPLUS \
+ (GLRO(dl_nns_supported) - 1) * LIBC_IE_TLS \
+ + GLRO(dl_nns_supported) * OTHER_IE_TLS \
+ + GLRO(dl_tls_static_optional_size)
+void
+_dl_static_tls_tunables_init (void)
+{
+#if HAVE_TUNABLES
+ GLRO(dl_nns_supported) = TUNABLE_GET (nns, size_t, NULL);
+ GLRO(dl_tls_static_optional_size)
+ = TUNABLE_GET (optional_static_tls, size_t, NULL);
+#else
+ GLRO(dl_nns_supported) = DL_NNS_SUPPORTED_DEFAULT;
+ GLRO(dl_tls_static_optional_size) = OPTIONAL_STATIC_TLS_DEFAULT;
+#endif
+ if (GLRO(dl_nns_supported) > DL_NNS)
+ GLRO(dl_nns_supported) = DL_NNS;
+ GL(dl_tls_static_optional) = GLRO(dl_tls_static_optional_size);
+}
/* Out-of-memory handler. */
static void
@@ -128,4 +128,8 @@ tunable_is_name (const char *orig, const char *envname)
}
#endif
+
+/* Initializers of tunables in the dl tunable namespace. */
+void _dl_static_tls_tunables_init (void);
+
#endif
@@ -126,4 +126,17 @@ glibc {
default: 3
}
}
+
+ dl {
+ nns {
+ type: SIZE_T
+ minval: 1
+ maxval: 16
+ default: 4
+ }
+ optional_static_tls {
+ type: SIZE_T
+ default: 512
+ }
+ }
}
@@ -40,9 +40,10 @@
(__builtin_expect ((sym_map)->l_tls_offset \
!= FORCED_DYNAMIC_TLS_OFFSET, 1) \
&& (__builtin_expect ((sym_map)->l_tls_offset != NO_TLS_OFFSET, 1) \
- || _dl_try_allocate_static_tls (sym_map) == 0))
+ || _dl_try_allocate_static_tls (sym_map, true) == 0))
-int _dl_try_allocate_static_tls (struct link_map *map) attribute_hidden;
+int _dl_try_allocate_static_tls (struct link_map *map, bool optional)
+ attribute_hidden;
#include <elf.h>
new file mode 100644
@@ -0,0 +1,40 @@
+/* Module with specified TLS size and model.
+ Copyright (C) 2020 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* This file is parameterized by macros N, SIZE and MODEL. */
+
+#include <stdio.h>
+#include <string.h>
+
+#define CONCATX(x, y) x ## y
+#define CONCAT(x, y) CONCATX (x, y)
+#define STRX(x) #x
+#define STR(x) STRX (x)
+
+#define VAR CONCAT (var, N)
+
+__attribute__ ((aligned (8), tls_model (MODEL)))
+__thread char VAR[SIZE];
+
+void
+CONCAT (access, N) (void)
+{
+ printf (STR (VAR) "[%d]:\t %p .. %p " MODEL "\n", SIZE, VAR, VAR + SIZE);
+ fflush (stdout);
+ memset (VAR, 1, SIZE);
+}
new file mode 100644
@@ -0,0 +1,4 @@
+#define N 0
+#define SIZE 1520
+#define MODEL "global-dynamic"
+#include "tst-tls-ie-mod.h"
new file mode 100644
@@ -0,0 +1,4 @@
+#define N 1
+#define SIZE 120
+#define MODEL "global-dynamic"
+#include "tst-tls-ie-mod.h"
new file mode 100644
@@ -0,0 +1,4 @@
+#define N 2
+#define SIZE 48
+#define MODEL "global-dynamic"
+#include "tst-tls-ie-mod.h"
new file mode 100644
@@ -0,0 +1,4 @@
+#define N 3
+#define SIZE 16
+#define MODEL "global-dynamic"
+#include "tst-tls-ie-mod.h"
new file mode 100644
@@ -0,0 +1,4 @@
+#define N 4
+#define SIZE 120
+#define MODEL "initial-exec"
+#include "tst-tls-ie-mod.h"
new file mode 100644
@@ -0,0 +1,4 @@
+#define N 5
+#define SIZE 8
+#define MODEL "initial-exec"
+#include "tst-tls-ie-mod.h"
new file mode 100644
@@ -0,0 +1,98 @@
+/* Test dlopen of modules with initial-exec TLS.
+ Copyright (C) 2016-2020 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* This test tries to ensure that at least 128 byte surplus TLS is
+ available for dlopening modules with initial-exec TLS. */
+
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int do_test (void);
+#include <support/xthread.h>
+#include <support/xdlfcn.h>
+#include <support/test-driver.c>
+
+/* Have some big TLS in the main exe: should not use surplus TLS. */
+__thread char maintls[1000];
+
+static pthread_barrier_t barrier;
+
+/* Forces multi-threaded behaviour. */
+static void *
+blocked_thread_func (void *closure)
+{
+ xpthread_barrier_wait (&barrier);
+ /* TLS load and access tests run here in the main thread. */
+ xpthread_barrier_wait (&barrier);
+ return NULL;
+}
+
+static void *
+load_and_access (const char *mod, const char *func)
+{
+ /* Load module with TLS. */
+ void *p = xdlopen (mod, RTLD_NOW);
+ /* Access the TLS variable to ensure it is allocated. */
+ void (*f) (void) = (void (*) (void))xdlsym (p, func);
+ f ();
+ return p;
+}
+
+static int
+do_test (void)
+{
+ void *mods[6];
+
+ {
+ int ret = pthread_barrier_init (&barrier, NULL, 2);
+ if (ret != 0)
+ {
+ errno = ret;
+ printf ("error: pthread_barrier_init: %m\n");
+ exit (1);
+ }
+ }
+
+ pthread_t blocked_thread = xpthread_create (NULL, blocked_thread_func, NULL);
+ xpthread_barrier_wait (&barrier);
+
+ printf ("maintls[%zu]:\t %p .. %p\n",
+ sizeof maintls, maintls, maintls + sizeof maintls);
+ memset (maintls, 1, sizeof maintls);
+
+ /* Load modules with dynamic TLS (may use surplus TLS opportunistically). */
+ mods[0] = load_and_access ("tst-tls-ie-mod0.so", "access0");
+ mods[1] = load_and_access ("tst-tls-ie-mod1.so", "access1");
+ mods[2] = load_and_access ("tst-tls-ie-mod2.so", "access2");
+ mods[3] = load_and_access ("tst-tls-ie-mod3.so", "access3");
+ /* Load modules with initial-exec TLS (can only use surplus TLS). */
+ mods[4] = load_and_access ("tst-tls-ie-mod4.so", "access4");
+ mods[5] = load_and_access ("tst-tls-ie-mod5.so", "access5");
+
+ xpthread_barrier_wait (&barrier);
+ xpthread_join (blocked_thread);
+
+ /* Close the modules. */
+ for (int i = 0; i < 6; ++i)
+ xdlclose (mods[i]);
+
+ return 0;
+}
@@ -438,6 +438,8 @@ struct rtld_global
EXTERN size_t _dl_tls_static_used;
/* Alignment requirement of the static TLS block. */
EXTERN size_t _dl_tls_static_align;
+ /* Remaining limit for optional use of the static TLS block. */
+ EXTERN size_t _dl_tls_static_optional;
/* Number of additional entries in the slotinfo array of each slotinfo
list element. A large number makes it almost certain take we never
@@ -579,6 +581,11 @@ struct rtld_global_ro
binaries, don't honor for PIEs). */
EXTERN ElfW(Addr) _dl_use_load_bias;
+ /* Number of namespaces for which enough static TLS is allocated. */
+ EXTERN size_t _dl_nns_supported;
+ /* Limit for optional use of the static TLS block. */
+ EXTERN size_t _dl_tls_static_optional_size;
+
/* Name of the shared object to be profiled (if any). */
EXTERN const char *_dl_profile;
/* Filename of the output file. */
--
2.17.1