@@ -429,6 +429,7 @@ handle_amd (int name)
}
+#if IS_IN (libc)
/* Get the value of the system variable NAME. */
long int
attribute_hidden
@@ -445,24 +446,25 @@ __cache_sysconf (int name)
/* CPU not known, we have no information. */
return 0;
}
+#endif
/* Data cache size for use in memory and string routines, typically
L1 size, rounded to multiple of 256 bytes. */
-long int __x86_data_cache_size_half attribute_hidden = 32 * 1024 / 2;
-long int __x86_data_cache_size attribute_hidden = 32 * 1024;
+long int __x86_data_cache_size_half attribute_hidden;
+long int __x86_data_cache_size attribute_hidden;
/* Similar to __x86_data_cache_size_half, but not rounded. */
-long int __x86_raw_data_cache_size_half attribute_hidden = 32 * 1024 / 2;
+long int __x86_raw_data_cache_size_half attribute_hidden;
/* Similar to __x86_data_cache_size, but not rounded. */
-long int __x86_raw_data_cache_size attribute_hidden = 32 * 1024;
+long int __x86_raw_data_cache_size attribute_hidden;
/* Shared cache size for use in memory and string routines, typically
L2 or L3 size, rounded to multiple of 256 bytes. */
-long int __x86_shared_cache_size_half attribute_hidden = 1024 * 1024 / 2;
-long int __x86_shared_cache_size attribute_hidden = 1024 * 1024;
+long int __x86_shared_cache_size_half attribute_hidden;
+long int __x86_shared_cache_size attribute_hidden;
/* Similar to __x86_shared_cache_size_half, but not rounded. */
-long int __x86_raw_shared_cache_size_half attribute_hidden = 1024 * 1024 / 2;
+long int __x86_raw_shared_cache_size_half attribute_hidden;
/* Similar to __x86_shared_cache_size, but not rounded. */
-long int __x86_raw_shared_cache_size attribute_hidden = 1024 * 1024;
+long int __x86_raw_shared_cache_size attribute_hidden;
/* Threshold to use non temporal store. */
long int __x86_shared_non_temporal_threshold attribute_hidden;
@@ -473,9 +475,29 @@ int __x86_prefetchw attribute_hidden;
#endif
-static void
-__attribute__((constructor))
-init_cacheinfo (void)
+/* Initialize private copy of cache info in ld.so, libc.so or libc.a
+ from _dl_x86_cpu_features for faster and consistent access. */
+
+static inline void
+init_x86_cacheinfo (const struct cache_info *cache)
+{
+ __x86_data_cache_size_half = cache->data_size_half;
+ __x86_data_cache_size = cache->data_size;
+ __x86_raw_data_cache_size_half = cache->raw_data_size_half;
+ __x86_raw_data_cache_size = cache->raw_data_size;
+ __x86_shared_cache_size_half = cache->shared_size_half;
+ __x86_shared_cache_size = cache->shared_size;
+ __x86_raw_shared_cache_size_half = cache->raw_shared_size_half;
+ __x86_raw_shared_cache_size = cache->raw_shared_size;
+ __x86_shared_non_temporal_threshold
+ = cache->shared_non_temporal_threshold;
+}
+
+#if IS_IN (rtld) || !defined SHARED
+/* Initialize cache info in _dl_x86_cpu_features in ld.so or libc.a. */
+
+void
+__init_cpu_features_cacheinfo (struct cache_info *cache)
{
/* Find out what brand of processor. */
unsigned int eax;
@@ -646,28 +668,58 @@ init_cacheinfo (void)
#endif
}
- if (data > 0)
+ /* Default data cache size to 32KB. */
+ if (data <= 0)
+ data = 32 * 1024;
+
+ if (cache->raw_data_size == 0)
{
- __x86_raw_data_cache_size_half = data / 2;
- __x86_raw_data_cache_size = data;
- /* Round data cache size to multiple of 256 bytes. */
- data = data & ~255L;
- __x86_data_cache_size_half = data / 2;
- __x86_data_cache_size = data;
+ cache->raw_data_size_half = data / 2;
+ cache->raw_data_size = data;
}
+ else
+ data = cache->raw_data_size;
+
+ /* Round data cache size to multiple of 256 bytes. */
+ data = data & ~255L;
+ cache->data_size_half = data / 2;
+ cache->data_size = data;
+
+ /* Default shared cache size to 1MB. */
+ if (shared < 0)
+ shared = 1024 * 1024;
- if (shared > 0)
+ if (cache->raw_shared_size == 0)
{
- __x86_raw_shared_cache_size_half = shared / 2;
- __x86_raw_shared_cache_size = shared;
- /* Round shared cache size to multiple of 256 bytes. */
- shared = shared & ~255L;
- __x86_shared_cache_size_half = shared / 2;
- __x86_shared_cache_size = shared;
+ cache->raw_shared_size_half = shared / 2;
+ cache->raw_shared_size = shared;
}
+ else
+ shared = cache->raw_shared_size;
+
+ /* Round shared cache size to multiple of 256 bytes. */
+ shared = shared & ~255L;
+ cache->shared_size_half = shared / 2;
+ cache->shared_size = shared;
/* The large memcpy micro benchmark in glibc shows that 6 times of
shared cache size is the approximate value above which non-temporal
store becomes faster. */
- __x86_shared_non_temporal_threshold = __x86_shared_cache_size * 6;
+ if (cache->shared_non_temporal_threshold == 0)
+ cache->shared_non_temporal_threshold = cache->shared_size * 6;
+
+ /* Initialize private copy of cache info in ld.so or libc.a. */
+ init_x86_cacheinfo (cache);
}
+#endif
+
+#if IS_IN (libc) && defined SHARED
+/* Initialize private copy of cache info in libc.so. */
+
+static void
+__attribute__((constructor))
+init_cacheinfo (void)
+{
+ init_x86_cacheinfo (&GLRO(dl_x86_cpu_features).cache);
+}
+#endif
@@ -263,4 +263,6 @@ no_cpuid:
cpu_features->family = family;
cpu_features->model = model;
cpu_features->kind = kind;
+
+ __init_cpu_features_cacheinfo (&cpu_features->cache);
}
@@ -164,6 +164,28 @@
#else /* __ASSEMBLER__ */
+struct cache_info
+{
+ /* Data cache size for use in memory and string routines, typically
+ L1 size, rounded to multiple of 256 bytes. */
+ long int data_size_half;
+ long int data_size;
+ /* Similar to data_size_half, but not rounded. */
+ long int raw_data_size_half;
+ /* Similar to data_size, but not rounded. */
+ long int raw_data_size;
+ /* Shared cache size for use in memory and string routines, typically
+ L2 or L3 size, rounded to multiple of 256 bytes. */
+ long int shared_size_half;
+ long int shared_size;
+ /* Similar to shared_size_half, but not rounded. */
+ long int raw_shared_size_half;
+ /* Similar to shared_size, but not rounded. */
+ long int raw_shared_size;
+ /* Threshold to use non temporal store. */
+ long int shared_non_temporal_threshold;
+};
+
enum
{
COMMON_CPUID_INDEX_1 = 0,
@@ -193,8 +215,12 @@ struct cpu_features
unsigned int family;
unsigned int model;
unsigned int feature[FEATURE_INDEX_MAX];
+ struct cache_info cache;
};
+extern void __init_cpu_features_cacheinfo (struct cache_info *)
+ attribute_hidden;
+
/* Used from outside of glibc to get access to the CPU features
structure. */
extern const struct cpu_features *__get_cpu_features (void)