diff mbox series

[28/28] elf: Add glibc-hwcaps subdirectory support to ld.so cache processing

Message ID 704c48fd058d2eb3b4925f4f5b883ec40a2ca421.1601569371.git.fweimer@redhat.com
State New
Headers show
Series glibc-hwcaps support | expand

Commit Message

Florian Weimer Oct. 1, 2020, 4:34 p.m. UTC
This recognizes the DL_CACHE_HWCAP_EXTENSION flag and picks up
the supported cache entry with the highest priority.
---
 elf/dl-cache.c  | 182 +++++++++++++++++++++++++++++++++++++++++++++++-
 elf/dl-hwcaps.c |  78 +++++++++++++++++++++
 elf/dl-hwcaps.h |  19 +++++
 3 files changed, 276 insertions(+), 3 deletions(-)
diff mbox series

Patch

diff --git a/elf/dl-cache.c b/elf/dl-cache.c
index 02c46ffb0c..4714119974 100644
--- a/elf/dl-cache.c
+++ b/elf/dl-cache.c
@@ -35,6 +35,132 @@  static struct cache_file *cache;
 static struct cache_file_new *cache_new;
 static size_t cachesize;
 
+#ifdef SHARED
+/* This is used to cache the priorities of glibc-hwcaps
+   subdirectories.  The elements of _dl_cache_priorities correspond to
+   the strings in the cache_extension_tag_glibc_hwcaps section.  */
+static uint32_t *glibc_hwcaps_priorities;
+static uint32_t glibc_hwcaps_priorities_length;
+static uint32_t glibc_hwcaps_priorities_allocated;
+
+/* True if the full malloc was used to allocated the array.  */
+static bool glibc_hwcaps_priorities_malloced;
+
+/* Deallocate the glibc_hwcaps_priorities array.  */
+static void
+glibc_hwcaps_priorities_free (void)
+{
+  /* When the minimal malloc is in use, free does not do anything,
+     so it does not make sense to call it.  */
+  if (glibc_hwcaps_priorities_malloced)
+    free (glibc_hwcaps_priorities);
+  glibc_hwcaps_priorities = NULL;
+  glibc_hwcaps_priorities_allocated = 0;
+}
+
+/* Return the priority of the cache_extension_tag_glibc_hwcaps section
+   entry at INDEX.  Zero means do not use.  Otherwise, lower values
+   indicate greater preference.  */
+static uint32_t __attribute__ ((noinline, noclone))
+glibc_hwcaps_priority (uint32_t index)
+{
+  /* Using a zero-length array as an indicator that nothing has been
+     loaded is not a problem: It does not lead to repeated
+     initialization attempts because caches without an extension
+     section are processed without calling this function (unless the
+     file is corrupted).  */
+  if (glibc_hwcaps_priorities_length == 0)
+    {
+      struct cache_extension_all_loaded ext;
+      if (!cache_extension_load (cache_new, cache, cachesize, &ext))
+	return 0;
+
+      uint32_t length
+	= (ext.sections[cache_extension_tag_glibc_hwcaps].size
+	   / sizeof (uint32_t));
+      if (length > glibc_hwcaps_priorities_allocated)
+	{
+	  glibc_hwcaps_priorities_free ();
+
+	  glibc_hwcaps_priorities = malloc (length * sizeof (uint32_t));
+	  if (glibc_hwcaps_priorities == NULL)
+	    /* Disable hwcaps on memory allocation error.  */
+	    return 0;
+
+	  glibc_hwcaps_priorities_allocated = length;
+	  glibc_hwcaps_priorities_malloced = __rtld_malloc_is_full ();
+	}
+
+      /* Compute the priorities for the subdirectories by merging the
+	 array in the cache with the dl_hwcaps_priorities array.  */
+      const uint32_t *left
+	= ext.sections[cache_extension_tag_glibc_hwcaps].base;
+      const uint32_t *left_end = left + length;
+      struct dl_hwcaps_priority *right = _dl_hwcaps_priorities;
+      struct dl_hwcaps_priority *right_end
+	= right + _dl_hwcaps_priorities_length;
+      uint32_t *result = glibc_hwcaps_priorities;
+
+      while (left < left_end && right < right_end)
+	{
+	  uint32_t string_table_index = *left;
+	  if (string_table_index < cachesize)
+	    {
+	      const char *left_name
+		= (const char *) cache + string_table_index;
+	      uint32_t left_name_length = strlen (left_name);
+	      uint32_t to_compare;
+	      if (left_name_length < right->name_length)
+		to_compare = left_name_length;
+	      else
+		to_compare = right->name_length;
+	      int cmp = memcmp (left_name, right->name, to_compare);
+	      if (cmp == 0)
+		{
+		  if (left_name_length < right->name_length)
+		    cmp = -1;
+		  else if (left_name_length > right->name_length)
+		    cmp = 1;
+		}
+	      if (cmp == 0)
+		{
+		  *result = right->priority;
+		  ++result;
+		  ++left;
+		  ++right;
+		}
+	      else if (cmp < 0)
+		{
+		  *result = 0;
+		  ++result;
+		  ++left;
+		}
+	      else
+		++right;
+	    }
+	  else
+	    {
+	      *result = 0;
+	      ++result;
+	    }
+	}
+      while (left < left_end)
+	{
+	  *result = 0;
+	  ++result;
+	  ++left;
+	}
+
+      glibc_hwcaps_priorities_length = length;
+    }
+
+  if (index < glibc_hwcaps_priorities_length)
+    return glibc_hwcaps_priorities[index];
+  else
+    return 0;
+}
+#endif /* SHARED */
+
 /* True if PTR is a valid string table index.  */
 static inline bool
 _dl_cache_verify_ptr (uint32_t ptr, size_t string_table_size)
@@ -74,6 +200,9 @@  search_cache (const char *string_table, uint32_t string_table_size,
   int left = 0;
   int right = nlibs - 1;
   const char *best = NULL;
+#ifdef SHARED
+  uint32_t best_priority = 0;
+#endif
 
   while (left <= right)
     {
@@ -129,6 +258,11 @@  search_cache (const char *string_table, uint32_t string_table_size,
 		{
 		  if (best == NULL || flags == GLRO (dl_correct_cache_id))
 		    {
+		      /* Named/extension hwcaps get slightly different
+			 treatment: We keep searching for a better
+			 match.  */
+		      bool named_hwcap = false;
+
 		      if (entry_size >= sizeof (struct file_entry_new))
 			{
 			  /* The entry is large enough to include
@@ -136,7 +270,18 @@  search_cache (const char *string_table, uint32_t string_table_size,
 			  struct file_entry_new *libnew
 			    = (struct file_entry_new *) lib;
 
-			  if (libnew->hwcap & hwcap_exclude)
+#ifdef SHARED
+			  named_hwcap = dl_cache_hwcap_extension (libnew);
+#endif
+
+			  /* The entries with named/extension hwcaps
+			     have been exhausted.  Return the best
+			     match encountered so far if there is
+			     one.  */
+			  if (!named_hwcap && best != NULL)
+			    break;
+
+			  if ((libnew->hwcap & hwcap_exclude) && !named_hwcap)
 			    continue;
 			  if (GLRO (dl_osversion)
 			      && libnew->osversion > GLRO (dl_osversion))
@@ -146,14 +291,41 @@  search_cache (const char *string_table, uint32_t string_table_size,
 			      && ((libnew->hwcap & _DL_HWCAP_PLATFORM)
 				  != platform))
 			    continue;
+
+#ifdef SHARED
+			  /* For named hwcaps, determine the priority
+			     and see if beats what has been found so
+			     far.  */
+			  if (named_hwcap)
+			    {
+			      uint32_t entry_priority
+				= glibc_hwcaps_priority (libnew->hwcap);
+			      if (entry_priority == 0)
+				/* Not usable at all.  Skip.  */
+				continue;
+			      else if (best == NULL
+				       || entry_priority < best_priority)
+				/* This entry is of higher priority
+				   than the previous one, or it is the
+				   first entry.  */
+				best_priority = entry_priority;
+			      else
+				/* An entry has already been found,
+				   but it is a better match.  */
+				continue;
+			    }
+#endif /* SHARED */
 			}
 
 		      best = string_table + lib->value;
 
-		      if (flags == GLRO (dl_correct_cache_id))
+		      if (flags == GLRO (dl_correct_cache_id)
+			  && !named_hwcap)
 			/* We've found an exact match for the shared
 			   object and no general `ELF' release.  Stop
-			   searching.  */
+			   searching, but not if a named (extension)
+			   hwcap is used.  In this case, an entry with
+			   a higher priority may come up later.  */
 			break;
 		    }
 		}
@@ -346,5 +518,9 @@  _dl_unload_cache (void)
       __munmap (cache, cachesize);
       cache = NULL;
     }
+#ifdef SHARED
+  /* This marks the glibc_hwcaps_priorities array as out-of-date.  */
+  glibc_hwcaps_priorities_length = 0;
+#endif
 }
 #endif
diff --git a/elf/dl-hwcaps.c b/elf/dl-hwcaps.c
index 4de94759a2..cb53337025 100644
--- a/elf/dl-hwcaps.c
+++ b/elf/dl-hwcaps.c
@@ -89,6 +89,81 @@  copy_hwcaps (struct copy_hwcaps *target, const char *hwcaps,
     }
 }
 
+struct dl_hwcaps_priority *_dl_hwcaps_priorities;
+uint32_t _dl_hwcaps_priorities_length;
+
+/* Allocate _dl_hwcaps_priorities and fill it with data.  */
+static void
+compute_priorities (size_t total_count, const char *prepend,
+		    int32_t bitmask, const char *mask)
+{
+  _dl_hwcaps_priorities = malloc (total_count
+				  * sizeof (*_dl_hwcaps_priorities));
+  if (_dl_hwcaps_priorities == NULL)
+    _dl_signal_error (ENOMEM, NULL, NULL,
+		      N_("cannot create HWCAP priorities"));
+  _dl_hwcaps_priorities_length = total_count;
+
+  /* First the prepended subdirectories.  */
+  size_t i = 0;
+  {
+    struct dl_hwcaps_split sp;
+    _dl_hwcaps_split_init (&sp, prepend);
+    while (_dl_hwcaps_split (&sp))
+      {
+	_dl_hwcaps_priorities[i].name = sp.segment;
+	_dl_hwcaps_priorities[i].name_length = sp.length;
+	_dl_hwcaps_priorities[i].priority = i + 1;
+	++i;
+      }
+  }
+
+  /* Then the built-in subdirectories that are actually active.  */
+  {
+    struct dl_hwcaps_split_masked sp;
+    _dl_hwcaps_split_masked_init (&sp, _dl_hwcaps_subdirs, bitmask, mask);
+    while (_dl_hwcaps_split_masked (&sp))
+      {
+	_dl_hwcaps_priorities[i].name = sp.split.segment;
+	_dl_hwcaps_priorities[i].name_length = sp.split.length;
+	_dl_hwcaps_priorities[i].priority = i + 1;
+	++i;
+      }
+  }
+  assert (i == total_count);
+}
+
+/* Sort the _dl_hwcaps_priorities array by name.  */
+static void
+sort_priorities_by_name (void)
+{
+  /* Insertion sort.  There is no need to link qsort into the dynamic
+     loader for such a short array.  */
+  for (size_t i = 1; i < _dl_hwcaps_priorities_length; ++i)
+    for (size_t j = i; j > 0; --j)
+      {
+	struct dl_hwcaps_priority *previous = _dl_hwcaps_priorities + j - 1;
+	struct dl_hwcaps_priority *current = _dl_hwcaps_priorities + j;
+
+	/* Bail out if current is greater or equal to the previous
+	   value.  */
+	uint32_t to_compare;
+	if (current->name_length < previous->name_length)
+	  to_compare = current->name_length;
+	else
+	  to_compare = previous->name_length;
+	int cmp = memcmp (current->name, previous->name, to_compare);
+	if (cmp >= 0
+	    || (cmp == 0 && current->name_length >= previous->name_length))
+	  break;
+
+	/* Swap *previous and *current.  */
+	struct dl_hwcaps_priority tmp = *previous;
+	*previous = *current;
+	*current = tmp;
+      }
+}
+
 /* Return an array of useful/necessary hardware capability names.  */
 const struct r_strlenpair *
 _dl_important_hwcaps (const char *glibc_hwcaps_prepend,
@@ -111,6 +186,9 @@  _dl_important_hwcaps (const char *glibc_hwcaps_prepend,
   count_hwcaps (&hwcaps_counts, glibc_hwcaps_prepend, -1, NULL);
   count_hwcaps (&hwcaps_counts, _dl_hwcaps_subdirs, hwcaps_subdirs_active,
 		glibc_hwcaps_mask);
+  compute_priorities (hwcaps_counts.count, glibc_hwcaps_prepend,
+		      hwcaps_subdirs_active, glibc_hwcaps_mask);
+  sort_priorities_by_name ();
 
   /* Each hwcaps subdirectory has a GLIBC_HWCAPS_PREFIX string prefix
      and a "/" suffix once stored in the result.  */
diff --git a/elf/dl-hwcaps.h b/elf/dl-hwcaps.h
index a6453f15f3..4b4c143854 100644
--- a/elf/dl-hwcaps.h
+++ b/elf/dl-hwcaps.h
@@ -110,4 +110,23 @@  extern const char _dl_hwcaps_subdirs[] attribute_hidden;
    bitmask.  */
 int32_t _dl_hwcaps_subdirs_active (void) attribute_hidden;
 
+/* Pre-computed glibc-hwcaps subdirectory priorities.  Used in
+   dl-cache.c to quickly find the proprities for the stored HWCAP
+   names.  */
+struct dl_hwcaps_priority
+{
+  /* The name consists of name_length bytes at name (not necessarily
+     null-terminated).  */
+  const char *name;
+  uint32_t name_length;
+
+  /* Priority of this name.  A positive number.  */
+  uint32_t priority;
+};
+
+/* Pre-computed hwcaps priorities.  Set up by
+   _dl_important_hwcaps.  */
+extern struct dl_hwcaps_priority *_dl_hwcaps_priorities attribute_hidden;
+extern uint32_t _dl_hwcaps_priorities_length attribute_hidden;
+
 #endif /* _DL_HWCAPS_H */