Message ID | 87czu6ihok.fsf@oldenburg.str.redhat.com |
---|---|
State | New |
Headers | show |
Series | elf, nptl: Initialize static TLS directly in ld.so | expand |
On Tue, May 4, 2021 at 12:01 PM Florian Weimer via Libc-alpha <libc-alpha@sourceware.org> wrote: > > The stack list is available in ld.so since commit > 1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list > variables into _rtld_global"), so it's possible to walk the stack > list directly in ld.so and perform the initialization there. > > This eliminates an unprotected function pointer from _rtld_global > and reduces the libpthread initialization code. > > --- > elf/dl-open.c | 2 +- > elf/dl-reloc.c | 5 +++-- > elf/dl-support.c | 3 +-- > elf/dl-tls.c | 39 +++++++++++++++++++++++++++++++++++++++ > elf/rtld.c | 2 ++ > nptl/allocatestack.c | 35 ----------------------------------- > nptl/nptl-init.c | 2 -- > nptl/pthreadP.h | 2 -- > sysdeps/generic/ldsodefs.h | 19 +++++++++++++++++++ > 9 files changed, 65 insertions(+), 44 deletions(-) > > diff --git a/elf/dl-open.c b/elf/dl-open.c > index ab7aaa345e..09f0df7d38 100644 > --- a/elf/dl-open.c > +++ b/elf/dl-open.c > @@ -426,7 +426,7 @@ TLS generation counter wrapped! Please report this.")); > _dl_update_slotinfo (imap->l_tls_modid); > #endif > > - GL(dl_init_static_tls) (imap); > + dl_init_static_tls (imap); > assert (imap->l_need_tls_init == 0); > } > } > diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c > index c2df26deea..bb9ca1a101 100644 > --- a/elf/dl-reloc.c > +++ b/elf/dl-reloc.c > @@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional) > (void) _dl_update_slotinfo (map->l_tls_modid); > #endif > > - GL(dl_init_static_tls) (map); > + dl_init_static_tls (map); > } > else > map->l_need_tls_init = 1; > @@ -141,6 +141,7 @@ cannot allocate memory in static TLS block")); > } > } > > +#if !THREAD_GSCOPE_IN_TCB > /* Initialize static TLS area and DTV for current (only) thread. > libpthread implementations should provide their own hook > to handle all threads. */ > @@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map) > memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), > '\0', map->l_tls_blocksize - map->l_tls_initimage_size); > } > - > +#endif /* !THREAD_GSCOPE_IN_TCB */ > > void > _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[], > diff --git a/elf/dl-support.c b/elf/dl-support.c > index 7fc2ee78e2..f966a2e7cd 100644 > --- a/elf/dl-support.c > +++ b/elf/dl-support.c > @@ -138,8 +138,6 @@ void *_dl_random; > #include <dl-procruntime.c> > #include <dl-procinfo.c> > > -void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls; > - > size_t _dl_pagesize = EXEC_PAGESIZE; > > size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ; > @@ -197,6 +195,7 @@ list_t _dl_stack_user; > int _dl_stack_cache_lock; > #else > int _dl_thread_gscope_count; > +void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls; > #endif > struct dl_scope_free_list *_dl_scope_free_list; > > diff --git a/elf/dl-tls.c b/elf/dl-tls.c > index f8b32b3ecb..6baff0c1ea 100644 > --- a/elf/dl-tls.c > +++ b/elf/dl-tls.c > @@ -29,6 +29,10 @@ > #include <dl-tls.h> > #include <ldsodefs.h> > > +#if THREAD_GSCOPE_IN_TCB > +# include <list.h> > +#endif > + > #define TUNABLE_NAMESPACE rtld > #include <dl-tunables.h> > > @@ -1005,3 +1009,38 @@ cannot create TLS data structures")); > listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; > } > } > + > +#if THREAD_GSCOPE_IN_TCB > +static inline void __attribute__((always_inline)) > +init_one_static_tls (struct pthread *curp, struct link_map *map) > +{ > +# if TLS_TCB_AT_TP > + void *dest = (char *) curp - map->l_tls_offset; > +# elif TLS_DTV_AT_TP > + void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; > +# else > +# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" > +# endif > + > + /* Initialize the memory. */ > + memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), > + '\0', map->l_tls_blocksize - map->l_tls_initimage_size); > +} > + > +void > +_dl_init_static_tls (struct link_map *map) > +{ > + lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE); > + > + /* Iterate over the list with system-allocated threads first. */ > + list_t *runp; > + list_for_each (runp, &GL (dl_stack_used)) > + init_one_static_tls (list_entry (runp, struct pthread, list), map); > + > + /* Now the list with threads using user-allocated stacks. */ > + list_for_each (runp, &GL (dl_stack_user)) > + init_one_static_tls (list_entry (runp, struct pthread, list), map); > + > + lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE); > +} > +#endif /* THREAD_GSCOPE_IN_TCB */ > diff --git a/elf/rtld.c b/elf/rtld.c > index 34879016ad..ad325d4c10 100644 > --- a/elf/rtld.c > +++ b/elf/rtld.c > @@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr, > struct dl_main_state state; > dl_main_state_init (&state); > > +#if !THREAD_GSCOPE_IN_TCB > GL(dl_init_static_tls) = &_dl_nothread_init_static_tls; > +#endif > > #if defined SHARED && defined _LIBC_REENTRANT \ > && defined __rtld_lock_default_lock_recursive > diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c > index f1270c3109..8aaba088b1 100644 > --- a/nptl/allocatestack.c > +++ b/nptl/allocatestack.c > @@ -951,38 +951,3 @@ __reclaim_stacks (void) > GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER; > __default_pthread_attr_lock = LLL_LOCK_INITIALIZER; > } > - > - > -static inline void __attribute__((always_inline)) > -init_one_static_tls (struct pthread *curp, struct link_map *map) > -{ > -# if TLS_TCB_AT_TP > - void *dest = (char *) curp - map->l_tls_offset; > -# elif TLS_DTV_AT_TP > - void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; > -# else > -# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" > -# endif > - > - /* Initialize the memory. */ > - memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), > - '\0', map->l_tls_blocksize - map->l_tls_initimage_size); > -} > - > -void > -attribute_hidden > -__pthread_init_static_tls (struct link_map *map) > -{ > - lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE); > - > - /* Iterate over the list with system-allocated threads first. */ > - list_t *runp; > - list_for_each (runp, &GL (dl_stack_used)) > - init_one_static_tls (list_entry (runp, struct pthread, list), map); > - > - /* Now the list with threads using user-allocated stacks. */ > - list_for_each (runp, &GL (dl_stack_user)) > - init_one_static_tls (list_entry (runp, struct pthread, list), map); > - > - lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE); > -} > diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c > index b0879bd87e..fcab5a0904 100644 > --- a/nptl/nptl-init.c > +++ b/nptl/nptl-init.c > @@ -191,8 +191,6 @@ __pthread_initialize_minimal_internal (void) > GL(dl_make_stack_executable_hook) = &__make_stacks_executable; > #endif > > - GL(dl_init_static_tls) = &__pthread_init_static_tls; > - > /* Register the fork generation counter with the libc. */ > #ifndef TLS_MULTIPLE_THREADS_IN_TCB > __libc_multiple_threads_ptr = > diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h > index 00d2cfe764..0ce63672c4 100644 > --- a/nptl/pthreadP.h > +++ b/nptl/pthreadP.h > @@ -381,8 +381,6 @@ extern int __pthread_multiple_threads attribute_hidden; > extern int *__libc_multiple_threads_ptr attribute_hidden; > #endif > > -extern void __pthread_init_static_tls (struct link_map *) attribute_hidden; > - > extern size_t __pthread_get_minstack (const pthread_attr_t *attr); > > /* Namespace save aliases. */ > diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h > index 67c6686015..1b064c5894 100644 > --- a/sysdeps/generic/ldsodefs.h > +++ b/sysdeps/generic/ldsodefs.h > @@ -464,7 +464,9 @@ struct rtld_global > /* Generation counter for the dtv. */ > EXTERN size_t _dl_tls_generation; > > +#if !THREAD_GSCOPE_IN_TCB > EXTERN void (*_dl_init_static_tls) (struct link_map *); > +#endif > > /* Scopes to free after next THREAD_GSCOPE_WAIT (). */ > EXTERN struct dl_scope_free_list > @@ -1270,6 +1272,23 @@ extern void _dl_non_dynamic_init (void) > extern void _dl_aux_init (ElfW(auxv_t) *av) > attribute_hidden; > > +/* Initialize the static TLS space for the link map in all existing > + threads. */ > +#if THREAD_GSCOPE_IN_TCB > +void _dl_init_static_tls (struct link_map *map) attribute_hidden; > +#endif > +static inline void > +dl_init_static_tls (struct link_map *map) > +{ > +#if THREAD_GSCOPE_IN_TCB > + /* The stack list is available to ld.so, so the initialization can > + be handled within ld.so directly. */ > + _dl_init_static_tls (map); > +#else > + GL (dl_init_static_tls) (map); > +#endif > +} > + > /* Return true if the ld.so copy in this namespace is actually active > and working. If false, the dl_open/dlfcn hooks have to be used to > call into the outer dynamic linker (which happens after static > LGTM. Thanks.
diff --git a/elf/dl-open.c b/elf/dl-open.c index ab7aaa345e..09f0df7d38 100644 --- a/elf/dl-open.c +++ b/elf/dl-open.c @@ -426,7 +426,7 @@ TLS generation counter wrapped! Please report this.")); _dl_update_slotinfo (imap->l_tls_modid); #endif - GL(dl_init_static_tls) (imap); + dl_init_static_tls (imap); assert (imap->l_need_tls_init == 0); } } diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c index c2df26deea..bb9ca1a101 100644 --- a/elf/dl-reloc.c +++ b/elf/dl-reloc.c @@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional) (void) _dl_update_slotinfo (map->l_tls_modid); #endif - GL(dl_init_static_tls) (map); + dl_init_static_tls (map); } else map->l_need_tls_init = 1; @@ -141,6 +141,7 @@ cannot allocate memory in static TLS block")); } } +#if !THREAD_GSCOPE_IN_TCB /* Initialize static TLS area and DTV for current (only) thread. libpthread implementations should provide their own hook to handle all threads. */ @@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map) memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), '\0', map->l_tls_blocksize - map->l_tls_initimage_size); } - +#endif /* !THREAD_GSCOPE_IN_TCB */ void _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[], diff --git a/elf/dl-support.c b/elf/dl-support.c index 7fc2ee78e2..f966a2e7cd 100644 --- a/elf/dl-support.c +++ b/elf/dl-support.c @@ -138,8 +138,6 @@ void *_dl_random; #include <dl-procruntime.c> #include <dl-procinfo.c> -void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls; - size_t _dl_pagesize = EXEC_PAGESIZE; size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ; @@ -197,6 +195,7 @@ list_t _dl_stack_user; int _dl_stack_cache_lock; #else int _dl_thread_gscope_count; +void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls; #endif struct dl_scope_free_list *_dl_scope_free_list; diff --git a/elf/dl-tls.c b/elf/dl-tls.c index f8b32b3ecb..6baff0c1ea 100644 --- a/elf/dl-tls.c +++ b/elf/dl-tls.c @@ -29,6 +29,10 @@ #include <dl-tls.h> #include <ldsodefs.h> +#if THREAD_GSCOPE_IN_TCB +# include <list.h> +#endif + #define TUNABLE_NAMESPACE rtld #include <dl-tunables.h> @@ -1005,3 +1009,38 @@ cannot create TLS data structures")); listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; } } + +#if THREAD_GSCOPE_IN_TCB +static inline void __attribute__((always_inline)) +init_one_static_tls (struct pthread *curp, struct link_map *map) +{ +# if TLS_TCB_AT_TP + void *dest = (char *) curp - map->l_tls_offset; +# elif TLS_DTV_AT_TP + void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; +# else +# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" +# endif + + /* Initialize the memory. */ + memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), + '\0', map->l_tls_blocksize - map->l_tls_initimage_size); +} + +void +_dl_init_static_tls (struct link_map *map) +{ + lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE); + + /* Iterate over the list with system-allocated threads first. */ + list_t *runp; + list_for_each (runp, &GL (dl_stack_used)) + init_one_static_tls (list_entry (runp, struct pthread, list), map); + + /* Now the list with threads using user-allocated stacks. */ + list_for_each (runp, &GL (dl_stack_user)) + init_one_static_tls (list_entry (runp, struct pthread, list), map); + + lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE); +} +#endif /* THREAD_GSCOPE_IN_TCB */ diff --git a/elf/rtld.c b/elf/rtld.c index 34879016ad..ad325d4c10 100644 --- a/elf/rtld.c +++ b/elf/rtld.c @@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr, struct dl_main_state state; dl_main_state_init (&state); +#if !THREAD_GSCOPE_IN_TCB GL(dl_init_static_tls) = &_dl_nothread_init_static_tls; +#endif #if defined SHARED && defined _LIBC_REENTRANT \ && defined __rtld_lock_default_lock_recursive diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index f1270c3109..8aaba088b1 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -951,38 +951,3 @@ __reclaim_stacks (void) GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER; __default_pthread_attr_lock = LLL_LOCK_INITIALIZER; } - - -static inline void __attribute__((always_inline)) -init_one_static_tls (struct pthread *curp, struct link_map *map) -{ -# if TLS_TCB_AT_TP - void *dest = (char *) curp - map->l_tls_offset; -# elif TLS_DTV_AT_TP - void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; -# else -# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" -# endif - - /* Initialize the memory. */ - memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), - '\0', map->l_tls_blocksize - map->l_tls_initimage_size); -} - -void -attribute_hidden -__pthread_init_static_tls (struct link_map *map) -{ - lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE); - - /* Iterate over the list with system-allocated threads first. */ - list_t *runp; - list_for_each (runp, &GL (dl_stack_used)) - init_one_static_tls (list_entry (runp, struct pthread, list), map); - - /* Now the list with threads using user-allocated stacks. */ - list_for_each (runp, &GL (dl_stack_user)) - init_one_static_tls (list_entry (runp, struct pthread, list), map); - - lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE); -} diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c index b0879bd87e..fcab5a0904 100644 --- a/nptl/nptl-init.c +++ b/nptl/nptl-init.c @@ -191,8 +191,6 @@ __pthread_initialize_minimal_internal (void) GL(dl_make_stack_executable_hook) = &__make_stacks_executable; #endif - GL(dl_init_static_tls) = &__pthread_init_static_tls; - /* Register the fork generation counter with the libc. */ #ifndef TLS_MULTIPLE_THREADS_IN_TCB __libc_multiple_threads_ptr = diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index 00d2cfe764..0ce63672c4 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -381,8 +381,6 @@ extern int __pthread_multiple_threads attribute_hidden; extern int *__libc_multiple_threads_ptr attribute_hidden; #endif -extern void __pthread_init_static_tls (struct link_map *) attribute_hidden; - extern size_t __pthread_get_minstack (const pthread_attr_t *attr); /* Namespace save aliases. */ diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h index 67c6686015..1b064c5894 100644 --- a/sysdeps/generic/ldsodefs.h +++ b/sysdeps/generic/ldsodefs.h @@ -464,7 +464,9 @@ struct rtld_global /* Generation counter for the dtv. */ EXTERN size_t _dl_tls_generation; +#if !THREAD_GSCOPE_IN_TCB EXTERN void (*_dl_init_static_tls) (struct link_map *); +#endif /* Scopes to free after next THREAD_GSCOPE_WAIT (). */ EXTERN struct dl_scope_free_list @@ -1270,6 +1272,23 @@ extern void _dl_non_dynamic_init (void) extern void _dl_aux_init (ElfW(auxv_t) *av) attribute_hidden; +/* Initialize the static TLS space for the link map in all existing + threads. */ +#if THREAD_GSCOPE_IN_TCB +void _dl_init_static_tls (struct link_map *map) attribute_hidden; +#endif +static inline void +dl_init_static_tls (struct link_map *map) +{ +#if THREAD_GSCOPE_IN_TCB + /* The stack list is available to ld.so, so the initialization can + be handled within ld.so directly. */ + _dl_init_static_tls (map); +#else + GL (dl_init_static_tls) (map); +#endif +} + /* Return true if the ld.so copy in this namespace is actually active and working. If false, the dl_open/dlfcn hooks have to be used to call into the outer dynamic linker (which happens after static