diff mbox

[fortran] Enable FMA for AVX2 and AVX512F for matmul

Message ID abd2b337-c6a6-541a-113c-1783b9815ad9@netcologne.de
State New
Headers show

Commit Message

Thomas Koenig March 2, 2017, 6:15 a.m. UTC 
Hi Jerry,

> I would prefer that it was tested on the actual expected platform. Does
> anyone anywhere on this list have access to one of these machines to test?

If anybody wants to test who does not have --enable-maintainer-mode
activated, here is a patch that works "out of the box".

Regards

	Thomas
diff mbox

Patch

Index: generated/matmul_c10.c
===================================================================
--- generated/matmul_c10.c	(Revision 245760)
+++ generated/matmul_c10.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_c10 (gfc_array_c10 * const rest
 	int blas_limit, blas_call gemm);
 export_proto(matmul_c10);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_c10_avx (gfc_array_c10 * const restrict ret
 static void
 matmul_c10_avx2 (gfc_array_c10 * const restrict retarray, 
 	gfc_array_c10 * const restrict a, gfc_array_c10 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_c10_avx2 (gfc_array_c10 * const restrict retarray, 
 	gfc_array_c10 * const restrict a, gfc_array_c10 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_c10_avx2 (gfc_array_c10 * const restrict re
 static void
 matmul_c10_avx512f (gfc_array_c10 * const restrict retarray, 
 	gfc_array_c10 * const restrict a, gfc_array_c10 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_c10_avx512f (gfc_array_c10 * const restrict retarray, 
 	gfc_array_c10 * const restrict a, gfc_array_c10 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_c10 (gfc_array_c10 * const restrict re
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_c10_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_c10 (gfc_array_c10 * const restrict re
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_c10_avx2;
 	      goto tailcall;
Index: generated/matmul_c16.c
===================================================================
--- generated/matmul_c16.c	(Revision 245760)
+++ generated/matmul_c16.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_c16 (gfc_array_c16 * const rest
 	int blas_limit, blas_call gemm);
 export_proto(matmul_c16);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_c16_avx (gfc_array_c16 * const restrict ret
 static void
 matmul_c16_avx2 (gfc_array_c16 * const restrict retarray, 
 	gfc_array_c16 * const restrict a, gfc_array_c16 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_c16_avx2 (gfc_array_c16 * const restrict retarray, 
 	gfc_array_c16 * const restrict a, gfc_array_c16 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_c16_avx2 (gfc_array_c16 * const restrict re
 static void
 matmul_c16_avx512f (gfc_array_c16 * const restrict retarray, 
 	gfc_array_c16 * const restrict a, gfc_array_c16 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_c16_avx512f (gfc_array_c16 * const restrict retarray, 
 	gfc_array_c16 * const restrict a, gfc_array_c16 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_c16 (gfc_array_c16 * const restrict re
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_c16_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_c16 (gfc_array_c16 * const restrict re
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_c16_avx2;
 	      goto tailcall;
Index: generated/matmul_c4.c
===================================================================
--- generated/matmul_c4.c	(Revision 245760)
+++ generated/matmul_c4.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_c4 (gfc_array_c4 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_c4);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_c4_avx (gfc_array_c4 * const restrict retar
 static void
 matmul_c4_avx2 (gfc_array_c4 * const restrict retarray, 
 	gfc_array_c4 * const restrict a, gfc_array_c4 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_c4_avx2 (gfc_array_c4 * const restrict retarray, 
 	gfc_array_c4 * const restrict a, gfc_array_c4 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_c4_avx2 (gfc_array_c4 * const restrict reta
 static void
 matmul_c4_avx512f (gfc_array_c4 * const restrict retarray, 
 	gfc_array_c4 * const restrict a, gfc_array_c4 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_c4_avx512f (gfc_array_c4 * const restrict retarray, 
 	gfc_array_c4 * const restrict a, gfc_array_c4 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_c4 (gfc_array_c4 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_c4_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_c4 (gfc_array_c4 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_c4_avx2;
 	      goto tailcall;
Index: generated/matmul_c8.c
===================================================================
--- generated/matmul_c8.c	(Revision 245760)
+++ generated/matmul_c8.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_c8 (gfc_array_c8 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_c8);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_c8_avx (gfc_array_c8 * const restrict retar
 static void
 matmul_c8_avx2 (gfc_array_c8 * const restrict retarray, 
 	gfc_array_c8 * const restrict a, gfc_array_c8 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_c8_avx2 (gfc_array_c8 * const restrict retarray, 
 	gfc_array_c8 * const restrict a, gfc_array_c8 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_c8_avx2 (gfc_array_c8 * const restrict reta
 static void
 matmul_c8_avx512f (gfc_array_c8 * const restrict retarray, 
 	gfc_array_c8 * const restrict a, gfc_array_c8 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_c8_avx512f (gfc_array_c8 * const restrict retarray, 
 	gfc_array_c8 * const restrict a, gfc_array_c8 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_c8 (gfc_array_c8 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_c8_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_c8 (gfc_array_c8 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_c8_avx2;
 	      goto tailcall;
Index: generated/matmul_i1.c
===================================================================
--- generated/matmul_i1.c	(Revision 245760)
+++ generated/matmul_i1.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_i1 (gfc_array_i1 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_i1);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_i1_avx (gfc_array_i1 * const restrict retar
 static void
 matmul_i1_avx2 (gfc_array_i1 * const restrict retarray, 
 	gfc_array_i1 * const restrict a, gfc_array_i1 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_i1_avx2 (gfc_array_i1 * const restrict retarray, 
 	gfc_array_i1 * const restrict a, gfc_array_i1 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_i1_avx2 (gfc_array_i1 * const restrict reta
 static void
 matmul_i1_avx512f (gfc_array_i1 * const restrict retarray, 
 	gfc_array_i1 * const restrict a, gfc_array_i1 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_i1_avx512f (gfc_array_i1 * const restrict retarray, 
 	gfc_array_i1 * const restrict a, gfc_array_i1 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_i1 (gfc_array_i1 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_i1_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_i1 (gfc_array_i1 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_i1_avx2;
 	      goto tailcall;
Index: generated/matmul_i16.c
===================================================================
--- generated/matmul_i16.c	(Revision 245760)
+++ generated/matmul_i16.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_i16 (gfc_array_i16 * const rest
 	int blas_limit, blas_call gemm);
 export_proto(matmul_i16);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_i16_avx (gfc_array_i16 * const restrict ret
 static void
 matmul_i16_avx2 (gfc_array_i16 * const restrict retarray, 
 	gfc_array_i16 * const restrict a, gfc_array_i16 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_i16_avx2 (gfc_array_i16 * const restrict retarray, 
 	gfc_array_i16 * const restrict a, gfc_array_i16 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_i16_avx2 (gfc_array_i16 * const restrict re
 static void
 matmul_i16_avx512f (gfc_array_i16 * const restrict retarray, 
 	gfc_array_i16 * const restrict a, gfc_array_i16 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_i16_avx512f (gfc_array_i16 * const restrict retarray, 
 	gfc_array_i16 * const restrict a, gfc_array_i16 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_i16 (gfc_array_i16 * const restrict re
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_i16_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_i16 (gfc_array_i16 * const restrict re
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_i16_avx2;
 	      goto tailcall;
Index: generated/matmul_i2.c
===================================================================
--- generated/matmul_i2.c	(Revision 245760)
+++ generated/matmul_i2.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_i2 (gfc_array_i2 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_i2);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_i2_avx (gfc_array_i2 * const restrict retar
 static void
 matmul_i2_avx2 (gfc_array_i2 * const restrict retarray, 
 	gfc_array_i2 * const restrict a, gfc_array_i2 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_i2_avx2 (gfc_array_i2 * const restrict retarray, 
 	gfc_array_i2 * const restrict a, gfc_array_i2 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_i2_avx2 (gfc_array_i2 * const restrict reta
 static void
 matmul_i2_avx512f (gfc_array_i2 * const restrict retarray, 
 	gfc_array_i2 * const restrict a, gfc_array_i2 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_i2_avx512f (gfc_array_i2 * const restrict retarray, 
 	gfc_array_i2 * const restrict a, gfc_array_i2 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_i2 (gfc_array_i2 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_i2_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_i2 (gfc_array_i2 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_i2_avx2;
 	      goto tailcall;
Index: generated/matmul_i4.c
===================================================================
--- generated/matmul_i4.c	(Revision 245760)
+++ generated/matmul_i4.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_i4 (gfc_array_i4 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_i4);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_i4_avx (gfc_array_i4 * const restrict retar
 static void
 matmul_i4_avx2 (gfc_array_i4 * const restrict retarray, 
 	gfc_array_i4 * const restrict a, gfc_array_i4 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_i4_avx2 (gfc_array_i4 * const restrict retarray, 
 	gfc_array_i4 * const restrict a, gfc_array_i4 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_i4_avx2 (gfc_array_i4 * const restrict reta
 static void
 matmul_i4_avx512f (gfc_array_i4 * const restrict retarray, 
 	gfc_array_i4 * const restrict a, gfc_array_i4 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_i4_avx512f (gfc_array_i4 * const restrict retarray, 
 	gfc_array_i4 * const restrict a, gfc_array_i4 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_i4 (gfc_array_i4 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_i4_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_i4 (gfc_array_i4 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_i4_avx2;
 	      goto tailcall;
Index: generated/matmul_i8.c
===================================================================
--- generated/matmul_i8.c	(Revision 245760)
+++ generated/matmul_i8.c	(Arbeitskopie)
@@ -74,9 +74,6 @@  extern void matmul_i8 (gfc_array_i8 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_i8);
 
-
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -628,7 +625,7 @@  matmul_i8_avx (gfc_array_i8 * const restrict retar
 static void
 matmul_i8_avx2 (gfc_array_i8 * const restrict retarray, 
 	gfc_array_i8 * const restrict a, gfc_array_i8 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_i8_avx2 (gfc_array_i8 * const restrict retarray, 
 	gfc_array_i8 * const restrict a, gfc_array_i8 * const restrict b, int try_blas,
@@ -1171,7 +1168,7 @@  matmul_i8_avx2 (gfc_array_i8 * const restrict reta
 static void
 matmul_i8_avx512f (gfc_array_i8 * const restrict retarray, 
 	gfc_array_i8 * const restrict a, gfc_array_i8 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_i8_avx512f (gfc_array_i8 * const restrict retarray, 
 	gfc_array_i8 * const restrict a, gfc_array_i8 * const restrict b, int try_blas,
@@ -2268,7 +2265,9 @@  void matmul_i8 (gfc_array_i8 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_i8_avx512f;
 	      goto tailcall;
@@ -2277,7 +2276,8 @@  void matmul_i8 (gfc_array_i8 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_i8_avx2;
 	      goto tailcall;
Index: generated/matmul_r10.c
===================================================================
--- generated/matmul_r10.c	(Revision 245760)
+++ generated/matmul_r10.c	(Arbeitskopie)
@@ -74,13 +74,6 @@  extern void matmul_r10 (gfc_array_r10 * const rest
 	int blas_limit, blas_call gemm);
 export_proto(matmul_r10);
 
-#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2.  Only generate
-   an AVX2 function if we are dealing with integer.  */
-#undef HAVE_AVX2
-#endif
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -632,7 +625,7 @@  matmul_r10_avx (gfc_array_r10 * const restrict ret
 static void
 matmul_r10_avx2 (gfc_array_r10 * const restrict retarray, 
 	gfc_array_r10 * const restrict a, gfc_array_r10 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_r10_avx2 (gfc_array_r10 * const restrict retarray, 
 	gfc_array_r10 * const restrict a, gfc_array_r10 * const restrict b, int try_blas,
@@ -1175,7 +1168,7 @@  matmul_r10_avx2 (gfc_array_r10 * const restrict re
 static void
 matmul_r10_avx512f (gfc_array_r10 * const restrict retarray, 
 	gfc_array_r10 * const restrict a, gfc_array_r10 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_r10_avx512f (gfc_array_r10 * const restrict retarray, 
 	gfc_array_r10 * const restrict a, gfc_array_r10 * const restrict b, int try_blas,
@@ -2272,7 +2265,9 @@  void matmul_r10 (gfc_array_r10 * const restrict re
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_r10_avx512f;
 	      goto tailcall;
@@ -2281,7 +2276,8 @@  void matmul_r10 (gfc_array_r10 * const restrict re
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_r10_avx2;
 	      goto tailcall;
Index: generated/matmul_r16.c
===================================================================
--- generated/matmul_r16.c	(Revision 245760)
+++ generated/matmul_r16.c	(Arbeitskopie)
@@ -74,13 +74,6 @@  extern void matmul_r16 (gfc_array_r16 * const rest
 	int blas_limit, blas_call gemm);
 export_proto(matmul_r16);
 
-#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2.  Only generate
-   an AVX2 function if we are dealing with integer.  */
-#undef HAVE_AVX2
-#endif
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -632,7 +625,7 @@  matmul_r16_avx (gfc_array_r16 * const restrict ret
 static void
 matmul_r16_avx2 (gfc_array_r16 * const restrict retarray, 
 	gfc_array_r16 * const restrict a, gfc_array_r16 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_r16_avx2 (gfc_array_r16 * const restrict retarray, 
 	gfc_array_r16 * const restrict a, gfc_array_r16 * const restrict b, int try_blas,
@@ -1175,7 +1168,7 @@  matmul_r16_avx2 (gfc_array_r16 * const restrict re
 static void
 matmul_r16_avx512f (gfc_array_r16 * const restrict retarray, 
 	gfc_array_r16 * const restrict a, gfc_array_r16 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_r16_avx512f (gfc_array_r16 * const restrict retarray, 
 	gfc_array_r16 * const restrict a, gfc_array_r16 * const restrict b, int try_blas,
@@ -2272,7 +2265,9 @@  void matmul_r16 (gfc_array_r16 * const restrict re
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_r16_avx512f;
 	      goto tailcall;
@@ -2281,7 +2276,8 @@  void matmul_r16 (gfc_array_r16 * const restrict re
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_r16_avx2;
 	      goto tailcall;
Index: generated/matmul_r4.c
===================================================================
--- generated/matmul_r4.c	(Revision 245760)
+++ generated/matmul_r4.c	(Arbeitskopie)
@@ -74,13 +74,6 @@  extern void matmul_r4 (gfc_array_r4 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_r4);
 
-#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2.  Only generate
-   an AVX2 function if we are dealing with integer.  */
-#undef HAVE_AVX2
-#endif
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -632,7 +625,7 @@  matmul_r4_avx (gfc_array_r4 * const restrict retar
 static void
 matmul_r4_avx2 (gfc_array_r4 * const restrict retarray, 
 	gfc_array_r4 * const restrict a, gfc_array_r4 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_r4_avx2 (gfc_array_r4 * const restrict retarray, 
 	gfc_array_r4 * const restrict a, gfc_array_r4 * const restrict b, int try_blas,
@@ -1175,7 +1168,7 @@  matmul_r4_avx2 (gfc_array_r4 * const restrict reta
 static void
 matmul_r4_avx512f (gfc_array_r4 * const restrict retarray, 
 	gfc_array_r4 * const restrict a, gfc_array_r4 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_r4_avx512f (gfc_array_r4 * const restrict retarray, 
 	gfc_array_r4 * const restrict a, gfc_array_r4 * const restrict b, int try_blas,
@@ -2272,7 +2265,9 @@  void matmul_r4 (gfc_array_r4 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_r4_avx512f;
 	      goto tailcall;
@@ -2281,7 +2276,8 @@  void matmul_r4 (gfc_array_r4 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_r4_avx2;
 	      goto tailcall;
Index: generated/matmul_r8.c
===================================================================
--- generated/matmul_r8.c	(Revision 245760)
+++ generated/matmul_r8.c	(Arbeitskopie)
@@ -74,13 +74,6 @@  extern void matmul_r8 (gfc_array_r8 * const restri
 	int blas_limit, blas_call gemm);
 export_proto(matmul_r8);
 
-#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2.  Only generate
-   an AVX2 function if we are dealing with integer.  */
-#undef HAVE_AVX2
-#endif
-
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -632,7 +625,7 @@  matmul_r8_avx (gfc_array_r8 * const restrict retar
 static void
 matmul_r8_avx2 (gfc_array_r8 * const restrict retarray, 
 	gfc_array_r8 * const restrict a, gfc_array_r8 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static void
 matmul_r8_avx2 (gfc_array_r8 * const restrict retarray, 
 	gfc_array_r8 * const restrict a, gfc_array_r8 * const restrict b, int try_blas,
@@ -1175,7 +1168,7 @@  matmul_r8_avx2 (gfc_array_r8 * const restrict reta
 static void
 matmul_r8_avx512f (gfc_array_r8 * const restrict retarray, 
 	gfc_array_r8 * const restrict a, gfc_array_r8 * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static void
 matmul_r8_avx512f (gfc_array_r8 * const restrict retarray, 
 	gfc_array_r8 * const restrict a, gfc_array_r8 * const restrict b, int try_blas,
@@ -2272,7 +2265,9 @@  void matmul_r8 (gfc_array_r8 * const restrict reta
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_r8_avx512f;
 	      goto tailcall;
@@ -2281,7 +2276,8 @@  void matmul_r8 (gfc_array_r8 * const restrict reta
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_r8_avx2;
 	      goto tailcall;
Index: m4/matmul.m4
===================================================================
--- m4/matmul.m4	(Revision 245760)
+++ m4/matmul.m4	(Arbeitskopie)
@@ -75,14 +75,6 @@  extern void matmul_'rtype_code` ('rtype` * const r
 	int blas_limit, blas_call gemm);
 export_proto(matmul_'rtype_code`);
 
-'ifelse(rtype_letter,`r',dnl
-`#if defined(HAVE_AVX) && defined(HAVE_AVX2)
-/* REAL types generate identical code for AVX and AVX2.  Only generate
-   an AVX2 function if we are dealing with integer.  */
-#undef HAVE_AVX2
-#endif')
-`
-
 /* Put exhaustive list of possible architectures here here, ORed together.  */
 
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_AVX512F)
@@ -101,7 +93,7 @@  static' include(matmul_internal.m4)dnl
 `static void
 'matmul_name` ('rtype` * const restrict retarray, 
 	'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx2")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx2,fma")));
 static' include(matmul_internal.m4)dnl
 `#endif /* HAVE_AVX2 */
 
@@ -110,7 +102,7 @@  static' include(matmul_internal.m4)dnl
 `static void
 'matmul_name` ('rtype` * const restrict retarray, 
 	'rtype` * const restrict a, 'rtype` * const restrict b, int try_blas,
-	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f")));
+	int blas_limit, blas_call gemm) __attribute__((__target__("avx512f,fma")));
 static' include(matmul_internal.m4)dnl
 `#endif  /* HAVE_AVX512F */
 
@@ -138,7 +130,9 @@  void matmul_'rtype_code` ('rtype` * const restrict
 	{
           /* Run down the available processors in order of preference.  */
 #ifdef HAVE_AVX512F
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX512F))
+	      && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
+
 	    {
 	      matmul_p = matmul_'rtype_code`_avx512f;
 	      goto tailcall;
@@ -147,7 +141,8 @@  void matmul_'rtype_code` ('rtype` * const restrict
 #endif  /* HAVE_AVX512F */
 
 #ifdef HAVE_AVX2
-      	  if (__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+      	  if ((__cpu_model.__cpu_features[0] & (1 << FEATURE_AVX2))
+	     && (__cpu_model.__cpu_features[0] & (1 << FEATURE_FMA)))
 	    {
 	      matmul_p = matmul_'rtype_code`_avx2;
 	      goto tailcall;