===================================================================
@@ -2182,6 +2182,38 @@ dump_generic_node (pretty_printer *buffe
pp_string (buffer, " > ");
break;
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ pp_string (buffer, " REDUC_MIN_FIRST_LOC_EXPR < ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
+ pp_string (buffer, ", ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 1), spc, flags, false);
+ pp_string (buffer, " > ");
+ break;
+
+ case REDUC_MIN_LAST_LOC_EXPR:
+ pp_string (buffer, " REDUC_MIN_LAST_LOC_EXPR < ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
+ pp_string (buffer, ", ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 1), spc, flags, false);
+ pp_string (buffer, " > ");
+ break;
+
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ pp_string (buffer, " REDUC_MAX_FIRST_LOC_EXPR < ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
+ pp_string (buffer, ", ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 1), spc, flags, false);
+ pp_string (buffer, " > ");
+ break;
+
+ case REDUC_MAX_LAST_LOC_EXPR:
+ pp_string (buffer, " REDUC_MAX_LAST_LOC_EXPR < ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
+ pp_string (buffer, ", ");
+ dump_generic_node (buffer, TREE_OPERAND (node, 1), spc, flags, false);
+ pp_string (buffer, " > ");
+ break;
+
case VEC_WIDEN_MULT_HI_EXPR:
pp_string (buffer, " VEC_WIDEN_MULT_HI_EXPR < ");
dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
===================================================================
@@ -383,6 +383,30 @@ optab_for_tree_code (enum tree_code code
case REDUC_PLUS_EXPR:
return TYPE_UNSIGNED (type) ? reduc_uplus_optab : reduc_splus_optab;
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ if (VECTOR_FLOAT_TYPE_P (type))
+ return (optab) reduc_min_first_loc_optab;
+ else
+ return NULL;
+
+ case REDUC_MIN_LAST_LOC_EXPR:
+ if (VECTOR_FLOAT_TYPE_P (type))
+ return (optab) reduc_min_last_loc_optab;
+ else
+ return NULL;
+
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ if (VECTOR_FLOAT_TYPE_P (type))
+ return (optab) reduc_max_first_loc_optab;
+ else
+ return NULL;
+
+ case REDUC_MAX_LAST_LOC_EXPR:
+ if (VECTOR_FLOAT_TYPE_P (type))
+ return (optab) reduc_max_last_loc_optab;
+ else
+ return NULL;
+
case VEC_LSHIFT_EXPR:
return vec_shl_optab;
@@ -6314,6 +6338,13 @@ init_optabs (void)
init_convert_optab (satfract_optab, SAT_FRACT);
init_convert_optab (satfractuns_optab, UNSIGNED_SAT_FRACT);
+ init_convert_optab (vcondc_optab, UNKNOWN);
+ init_convert_optab (vcondcu_optab, UNKNOWN);
+ init_convert_optab (reduc_min_first_loc_optab, UNKNOWN);
+ init_convert_optab (reduc_min_last_loc_optab, UNKNOWN);
+ init_convert_optab (reduc_max_first_loc_optab, UNKNOWN);
+ init_convert_optab (reduc_max_last_loc_optab, UNKNOWN);
+
/* Fill in the optabs with the insns we support. */
init_all_optabs ();
@@ -6762,14 +6793,25 @@ vector_compare_rtx (tree cond, bool unsi
/* Return insn code for TYPE, the type of a VEC_COND_EXPR. */
static inline enum insn_code
-get_vcond_icode (tree type, enum machine_mode mode)
+get_vcond_icode (tree type, enum machine_mode mode, tree vec_cmp_type,
+ enum machine_mode cmp_mode)
{
enum insn_code icode = CODE_FOR_nothing;
- if (TYPE_UNSIGNED (type))
- icode = direct_optab_handler (vcondu_optab, mode);
+ if (type != vec_cmp_type)
+ {
+ if (TYPE_UNSIGNED (type))
+ icode = convert_optab_handler (vcondcu_optab, mode, cmp_mode);
+ else
+ icode = convert_optab_handler (vcondc_optab, mode, cmp_mode);
+ }
else
- icode = direct_optab_handler (vcond_optab, mode);
+ {
+ if (TYPE_UNSIGNED (type))
+ icode = direct_optab_handler (vcondu_optab, mode);
+ else
+ icode = direct_optab_handler (vcond_optab, mode);
+ }
return icode;
}
@@ -6777,9 +6819,11 @@ get_vcond_icode (tree type, enum machine
for vector cond expr with type TYPE in VMODE mode. */
bool
-expand_vec_cond_expr_p (tree type, enum machine_mode vmode)
+expand_vec_cond_expr_p (tree type, enum machine_mode vmode,
+ tree vec_cmp_type, enum machine_mode cmp_mode)
{
- if (get_vcond_icode (type, vmode) == CODE_FOR_nothing)
+ if (get_vcond_icode (type, vmode, vec_cmp_type, cmp_mode)
+ == CODE_FOR_nothing)
return false;
return true;
}
@@ -6794,13 +6838,15 @@ expand_vec_cond_expr (tree vec_cond_type
enum insn_code icode;
rtx comparison, rtx_op1, rtx_op2, cc_op0, cc_op1;
enum machine_mode mode = TYPE_MODE (vec_cond_type);
- bool unsignedp = TYPE_UNSIGNED (vec_cond_type);
+ tree vec_cmp_type = TREE_TYPE (op0);
+ enum machine_mode cmp_mode = TYPE_MODE (vec_cmp_type);
+ bool unsignedp = TYPE_UNSIGNED (vec_cmp_type);
- icode = get_vcond_icode (vec_cond_type, mode);
+ icode = get_vcond_icode (vec_cond_type, mode, vec_cmp_type, cmp_mode);
if (icode == CODE_FOR_nothing)
return 0;
- if (!target || !insn_data[icode].operand[0].predicate (target, mode))
+ if (!target || !insn_data[icode].operand[0].predicate (target, cmp_mode))
target = gen_reg_rtx (mode);
/* Get comparison rtx. First expand both cond expr operands. */
@@ -6826,6 +6872,55 @@ expand_vec_cond_expr (tree vec_cond_type
return target;
}
+/* Return instruction code for vector reduction epilogue for CODE. */
+enum insn_code
+get_vec_reduc_minloc_expr_code (enum tree_code code, tree type0, tree type1)
+{
+ enum machine_mode mode0 = TYPE_MODE (type0);
+ enum machine_mode mode1 = TYPE_MODE (type1);
+ convert_optab this_optab;
+
+ this_optab = (convert_optab) optab_for_tree_code (code, type0,
+ optab_default);
+ return convert_optab_handler (this_optab, mode1, mode0);
+}
+
+/* Expand vector reduction epilogue for min/max location. */
+rtx
+expand_vec_reduc_minloc_expr (enum tree_code code, tree op0, tree op1,
+ rtx target)
+{
+ enum insn_code icode;
+ rtx rtx_op0, rtx_op1;
+ enum machine_mode mode0 = TYPE_MODE (TREE_TYPE (op0));
+ enum machine_mode mode1 = TYPE_MODE (TREE_TYPE (op1));
+
+ icode = get_vec_reduc_minloc_expr_code (code, TREE_TYPE (op0),
+ TREE_TYPE (op1));
+ if (icode == CODE_FOR_nothing)
+ return 0;
+
+ if (!target || !insn_data[icode].operand[0].predicate (target, mode1))
+ target = gen_reg_rtx (mode1);
+
+ /* Expand both operands and force them in reg, if required. */
+ rtx_op0 = expand_normal (op0);
+ if (!insn_data[icode].operand[1].predicate (rtx_op0, mode0)
+ && mode0 != VOIDmode)
+ rtx_op0 = force_reg (mode0, rtx_op0);
+
+ rtx_op1 = expand_normal (op1);
+ if (!insn_data[icode].operand[2].predicate (rtx_op1, mode1)
+ && mode1 != VOIDmode)
+ rtx_op1 = force_reg (mode1, rtx_op1);
+
+ /* Emit instruction! */
+ emit_insn (GEN_FCN (icode) (target, rtx_op0, rtx_op1));
+
+ return target;
+}
+
+
/* This is an internal subroutine of the other compare_and_swap expanders.
MEM, OLD_VAL and NEW_VAL are as you'd expect for a compare-and-swap
===================================================================
@@ -569,6 +569,15 @@ enum convert_optab_index
COI_satfract,
COI_satfractuns,
+ COI_vcondc,
+ COI_vcondcu,
+
+ COI_reduc_min_first_loc,
+ COI_reduc_min_last_loc,
+ COI_reduc_max_first_loc,
+ COI_reduc_max_last_loc,
+
+
COI_MAX
};
@@ -589,6 +598,13 @@ enum convert_optab_index
#define fractuns_optab (&convert_optab_table[COI_fractuns])
#define satfract_optab (&convert_optab_table[COI_satfract])
#define satfractuns_optab (&convert_optab_table[COI_satfractuns])
+#define vcondc_optab (&convert_optab_table[COI_vcondc])
+#define vcondcu_optab (&convert_optab_table[COI_vcondcu])
+#define reduc_min_first_loc_optab (&convert_optab_table[COI_reduc_min_first_loc])
+#define reduc_min_last_loc_optab (&convert_optab_table[COI_reduc_min_last_loc])
+#define reduc_max_first_loc_optab (&convert_optab_table[COI_reduc_max_first_loc])
+#define reduc_max_last_loc_optab (&convert_optab_table[COI_reduc_max_last_loc])
+
/* Contains the optab used for each rtx code. */
extern optab code_to_optab[NUM_RTX_CODE + 1];
@@ -842,14 +858,20 @@ extern bool expand_sfix_optab (rtx, rtx,
/* Generate code for a widening multiply. */
extern rtx expand_widening_mult (enum machine_mode, rtx, rtx, rtx, int, optab);
-/* Return tree if target supports vector operations for COND_EXPR. */
-bool expand_vec_cond_expr_p (tree, enum machine_mode);
+/* Return true if target supports vector operations for COND_EXPR. */
+bool expand_vec_cond_expr_p (tree, enum machine_mode, tree, enum machine_mode);
/* Generate code for VEC_COND_EXPR. */
extern rtx expand_vec_cond_expr (tree, tree, tree, tree, rtx);
/* Generate code for VEC_LSHIFT_EXPR and VEC_RSHIFT_EXPR. */
extern rtx expand_vec_shift_expr (sepops, rtx);
+/* Return the instruction for REDUC_MIN/MAX_FIRST/LAST_EXPR. */
+enum insn_code get_vec_reduc_minloc_expr_code (enum tree_code, tree, tree);
+/* Generate code for REDUC_MIN/MAX_FIRST/LAST_EXPR. */
+extern rtx expand_vec_reduc_minloc_expr (enum tree_code, tree, tree, rtx);
+
+
/* Return the insn used to implement mode MODE of OP, or CODE_FOR_nothing
if the target does not have such an insn. */
===================================================================
@@ -246,6 +246,8 @@ static const char * const optabs[] =
"set_optab_handler (vec_realign_load_optab, $A, CODE_FOR_$(vec_realign_load_$a$))",
"set_direct_optab_handler (vcond_optab, $A, CODE_FOR_$(vcond$a$))",
"set_direct_optab_handler (vcondu_optab, $A, CODE_FOR_$(vcondu$a$))",
+ "set_convert_optab_handler (vcondc_optab, $B, $A, CODE_FOR_$(vcondc$F$a$I$b$))",
+ "set_convert_optab_handler (vcondcu_optab, $B, $A, CODE_FOR_$(vcondcu$F$a$I$b$))",
"set_optab_handler (ssum_widen_optab, $A, CODE_FOR_$(widen_ssum$I$a3$))",
"set_optab_handler (usum_widen_optab, $A, CODE_FOR_$(widen_usum$I$a3$))",
"set_optab_handler (udot_prod_optab, $A, CODE_FOR_$(udot_prod$I$a$))",
@@ -256,6 +258,10 @@ static const char * const optabs[] =
"set_optab_handler (reduc_umin_optab, $A, CODE_FOR_$(reduc_umin_$a$))",
"set_optab_handler (reduc_splus_optab, $A, CODE_FOR_$(reduc_splus_$a$))" ,
"set_optab_handler (reduc_uplus_optab, $A, CODE_FOR_$(reduc_uplus_$a$))",
+ "set_convert_optab_handler (reduc_min_first_loc_optab, $B, $A, CODE_FOR_$(reduc_min_first_loc_$F$a$I$b$))",
+ "set_convert_optab_handler (reduc_min_last_loc_optab, $B, $A, CODE_FOR_$(reduc_min_last_loc_$F$a$I$b$))",
+ "set_convert_optab_handler (reduc_max_first_loc_optab, $B, $A, CODE_FOR_$(reduc_max_first_loc_$F$a$I$b$))",
+ "set_convert_optab_handler (reduc_max_last_loc_optab, $B, $A, CODE_FOR_$(reduc_max_last_loc_$F$a$I$b$))",
"set_optab_handler (vec_widen_umult_hi_optab, $A, CODE_FOR_$(vec_widen_umult_hi_$a$))",
"set_optab_handler (vec_widen_umult_lo_optab, $A, CODE_FOR_$(vec_widen_umult_lo_$a$))",
"set_optab_handler (vec_widen_smult_hi_optab, $A, CODE_FOR_$(vec_widen_smult_hi_$a$))",
===================================================================
@@ -0,0 +1,47 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+__attribute__ ((noinline))
+int foo ()
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = 12;
+
+ for (i = 0; i < N; i++)
+ if (arr[i] > limit)
+ {
+ pos = i + 1;
+ limit = arr[i];
+ }
+
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr [2] = N + 15.8;
+
+ pos = foo ();
+ if (pos != 3)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_cmp } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,47 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+__attribute__ ((noinline))
+int foo ()
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = N+N;
+
+ for (i = 0; i < N; i++)
+ if (arr[i] < limit)
+ {
+ pos = i + 1;
+ limit = arr[i];
+ }
+
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr [2] = -5.8;
+
+ pos = foo ();
+ if (pos != 3)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_cmp } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,49 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+__attribute__ ((noinline))
+int foo (unsigned int n, float *min)
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = N+N;
+
+ for (i = 0; i < N; i++)
+ if (arr[i] < limit)
+ {
+ pos = i + 1;
+ limit = arr[i];
+ }
+
+ *min = limit;
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+ float min;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr[2] = -5.8;
+
+ pos = foo (N, &min);
+ if (pos != 3 || min != arr[2])
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_cmp } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,55 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+short a[N];
+
+/* Loop with multiple types - currently not supported. */
+__attribute__ ((noinline))
+int foo (unsigned int n, float *min, short x)
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = N+N;
+
+ for (i = 0; i < n; i++)
+ {
+ if (arr[i] < limit)
+ {
+ limit = arr[i];
+ pos = i + 1;
+ }
+
+ a[i] = x;
+ }
+
+ *min = limit;
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+ float min;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr[2] = -5.8;
+
+ pos = foo (N, &min, 6);
+ if (pos != 3 || min != arr[2])
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,47 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+__attribute__ ((noinline))
+int foo ()
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = N+N;
+
+ for (i = 0; i < N; i++)
+ if (arr[i] <= limit)
+ {
+ pos = i + 1;
+ limit = arr[i];
+ }
+
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr [2] = -5.8;
+
+ pos = foo ();
+ if (pos != 3)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_cmp } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,52 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+#define MAX_VALUE N+N
+float arr[N];
+
+/* Not minloc pattern - different conditions. */
+__attribute__ ((noinline))
+int foo ()
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = MAX_VALUE;
+
+ for (i = 0; i < N; i++)
+ {
+ if (arr[i] < limit)
+ pos = i + 1;
+
+ if (arr[i] > limit)
+ limit = arr[i];
+ }
+
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr[2] = -5.8;
+
+ pos = foo ();
+
+ if (pos != N)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,50 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+/* Not minloc pattern: position is not induction. */
+__attribute__ ((noinline))
+int foo (unsigned int n, float *min)
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = N+N;
+
+ for (i = 0; i < n; i++)
+ if (arr[i] < limit)
+ {
+ pos = 5;
+ limit = arr[i];
+ }
+
+ *min = limit;
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+ float min;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr[2] = -5.8;
+
+ pos = foo (N, &min);
+ if (pos != 5 || min != arr[2])
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,50 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+/* Position and minimum are of types of different sizes - not supported. */
+__attribute__ ((noinline))
+int foo (unsigned short n, float *min)
+{
+ unsigned short pos = 1;
+ unsigned short i;
+ float limit = N+N;
+
+ for (i = 0; i < n; i++)
+ if (arr[i] < limit)
+ {
+ pos = i + 1;
+ limit = arr[i];
+ }
+
+ *min = limit;
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+ float min;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr[2] = -5.8;
+
+ pos = foo (N, &min);
+ if (pos != 3 || min != arr[2])
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -159,9 +159,27 @@ dg-runtest [lsort [glob -nocomplain $src
# -ffast-math tests
set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
lappend DEFAULT_VECTCFLAGS "-ffast-math"
-dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/fast-math-*.\[cS\]]] \
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/fast-math-pr*.\[cS\]]] \
"" $DEFAULT_VECTCFLAGS
+# -ffast-math tests
+set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
+lappend DEFAULT_VECTCFLAGS "-ffast-math"
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/fast-math-slp*.\[cS\]]] \
+ "" $DEFAULT_VECTCFLAGS
+
+# -ffast-math tests
+set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
+lappend DEFAULT_VECTCFLAGS "-ffast-math"
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/fast-math-vect*.\[cS\]]] \
+ "" $DEFAULT_VECTCFLAGS
+
+# -ffast-math and -fno-tree-pre tests
+set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
+lappend DEFAULT_VECTCFLAGS "-ffast-math" "-fno-tree-pre"
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/fast-math-no-pre*.\[cS\]]] \
+ "" $DEFAULT_VECTCFLAGS
+
# -fno-math-errno tests
set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
lappend DEFAULT_VECTCFLAGS "-fno-math-errno"
===================================================================
@@ -0,0 +1,49 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N][N];
+
+/* Double reduction. */
+__attribute__ ((noinline))
+int foo ()
+{
+ unsigned int pos = 1;
+ unsigned int i, j;
+ float limit = N+N;
+
+ for (j = 0; j < N; j++)
+ for (i = 0; i < N; i++)
+ if (arr[i][j] < limit)
+ {
+ pos = i + 1;
+ limit = arr[i][j];
+ }
+
+ return pos;
+}
+
+int main (void)
+{
+ int i, j, pos;
+
+ check_vect();
+
+ for (j = 0; j < N; j++)
+ for (i = 0; i < N; i++)
+ arr[j][i] = (float)(i+j+1);
+
+ arr[8][2] = 0;
+ pos = foo ();
+ if (pos != 9)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_cmp } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,48 @@
+/* { dg-require-effective-target vect_float } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define N 64
+float arr[N];
+
+__attribute__ ((noinline))
+int foo ()
+{
+ unsigned int pos = 1;
+ unsigned int i;
+ float limit = 7;
+
+ for (i = 0; i < N; i++)
+ if (arr[i] >= limit)
+ {
+ pos = i + 1;
+ limit = arr[i];
+ }
+
+ return pos;
+}
+
+int main (void)
+{
+ int i, pos;
+
+ check_vect();
+
+ for (i = 0; i < N; i++)
+ arr[i] = (float)(i);
+
+ arr [2] = N + 5.8;
+ arr [12] = N + 5.8;
+
+ pos = foo ();
+ if (pos != 13)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_cmp } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -2966,6 +2966,23 @@ proc check_effective_target_vect_strided
return $et_vect_strided_wide_saved
}
+# Return 1 if the target supports vector comparison, 0 otherwise.
+proc check_effective_target_vect_cmp { } {
+ global et_vect_cmp_saved
+
+ if [info exists et_vect_cmp_saved] {
+ verbose "check_effective_target_vect_cmp: using cached result" 2
+ } else {
+ set et_vect_cmp_saved 0
+ if { [istarget powerpc*-*-*] } {
+ set et_vect_cmp_saved 1
+ }
+ }
+
+ verbose "check_effective_target_vect_cmp: returning $et_vect_cmp_saved" 2
+ return $et_vect_cmp_saved
+}
+
# Return 1 if the target supports section-anchors
proc check_effective_target_section_anchors { } {
===================================================================
@@ -519,6 +519,12 @@ DEF_RTL_EXPR(GEU, "geu", "ee", RTX_COMPA
DEF_RTL_EXPR(GTU, "gtu", "ee", RTX_COMPARE)
DEF_RTL_EXPR(LEU, "leu", "ee", RTX_COMPARE)
DEF_RTL_EXPR(LTU, "ltu", "ee", RTX_COMPARE)
+DEF_RTL_EXPR(GEF, "gef", "ee", RTX_COMPARE)
+DEF_RTL_EXPR(GTF, "gtf", "ee", RTX_COMPARE)
+DEF_RTL_EXPR(LEF, "lef", "ee", RTX_COMPARE)
+DEF_RTL_EXPR(LTF, "ltf", "ee", RTX_COMPARE)
+DEF_RTL_EXPR(EQF, "eqf", "ee", RTX_COMM_COMPARE)
+DEF_RTL_EXPR(NEQF, "neqf", "ee", RTX_COMM_COMPARE)
/* Additional floating point unordered comparison flavors. */
DEF_RTL_EXPR(UNORDERED, "unordered", "ee", RTX_COMM_COMPARE)
===================================================================
@@ -460,6 +460,19 @@ reverse_condition (enum rtx_code code)
return GEU;
case LEU:
return GTU;
+ case GTF:
+ return LEF;
+ case GEF:
+ return LTF;
+ case LTF:
+ return GEF;
+ case LEF:
+ return GTF;
+ case EQF:
+ return NEQF;
+ case NEQF:
+ return EQF;
+
case UNORDERED:
return ORDERED;
case ORDERED:
@@ -535,6 +548,8 @@ swap_condition (enum rtx_code code)
case ORDERED:
case UNEQ:
case LTGT:
+ case EQF:
+ case NEQF:
return code;
case GT:
@@ -561,6 +576,14 @@ swap_condition (enum rtx_code code)
return UNLT;
case UNGE:
return UNLE;
+ case GTF:
+ return LTF;
+ case GEF:
+ return LEF;
+ case LTF:
+ return GTF;
+ case LEF:
+ return GEF;
default:
gcc_unreachable ();
===================================================================
@@ -8110,6 +8110,15 @@ expand_expr_real_2 (sepops ops, rtx targ
return temp;
}
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ case REDUC_MIN_LAST_LOC_EXPR:
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ case REDUC_MAX_LAST_LOC_EXPR:
+ {
+ target = expand_vec_reduc_minloc_expr (code, treeop0, treeop1, target);
+ return target;
+ }
+
case VEC_EXTRACT_EVEN_EXPR:
case VEC_EXTRACT_ODD_EXPR:
{
===================================================================
@@ -343,6 +343,10 @@ dump_binary_rhs (pretty_printer *buffer,
case VEC_EXTRACT_ODD_EXPR:
case VEC_INTERLEAVE_HIGH_EXPR:
case VEC_INTERLEAVE_LOW_EXPR:
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ case REDUC_MIN_LAST_LOC_EXPR:
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ case REDUC_MAX_LAST_LOC_EXPR:
for (p = tree_code_name [(int) code]; *p; p++)
pp_character (buffer, TOUPPER (*p));
pp_string (buffer, " <");
===================================================================
@@ -409,6 +409,17 @@ enum slp_vect_type {
hybrid
};
+/* Compound pattern is a pattern consisting more than one statement that need
+ to be vectorized. Currenty min/max location pattern is the only supported
+ compound pattern. It has two statements: the first statement calculates the
+ minimum (marked MINMAX_STMT) and the second one calculates the location
+ (marked MINMAX_LOC_STMT). */
+enum vect_compound_pattern {
+ not_in_pattern = 0,
+ minmax_stmt,
+ minmax_loc_stmt
+};
+
typedef struct data_reference *dr_p;
DEF_VEC_P(dr_p);
@@ -425,6 +436,10 @@ typedef struct _stmt_vec_info {
/* Stmt is part of some pattern (computation idiom) */
bool in_pattern_p;
+ /* Statement is a part of a compound pattern, i.e., a pattern consisting
+ more than one statement. */
+ enum vect_compound_pattern compound_pattern;
+
/* For loads only, if there is a store with the same location, this field is
TRUE. */
bool read_write_dep;
@@ -535,6 +550,7 @@ typedef struct _stmt_vec_info {
#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to
#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
+#define STMT_VINFO_COMPOUND_PATTERN(S) (S)->compound_pattern
#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
#define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
@@ -642,7 +658,8 @@ is_pattern_stmt_p (stmt_vec_info stmt_in
related_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
if (related_stmt
&& (related_stmt_info = vinfo_for_stmt (related_stmt))
- && STMT_VINFO_IN_PATTERN_P (related_stmt_info))
+ && (STMT_VINFO_IN_PATTERN_P (related_stmt_info)
+ || STMT_VINFO_COMPOUND_PATTERN (related_stmt_info)))
return true;
return false;
@@ -709,6 +726,29 @@ known_alignment_for_access_p (struct dat
return (DR_MISALIGNMENT (data_ref_info) != -1);
}
+
+static inline enum tree_code
+get_minloc_reduc_epilogue_code (enum tree_code code)
+{
+ switch (code)
+ {
+ case LT_EXPR:
+ return REDUC_MIN_FIRST_LOC_EXPR;
+
+ case LE_EXPR:
+ return REDUC_MIN_LAST_LOC_EXPR;
+
+ case GT_EXPR:
+ return REDUC_MAX_FIRST_LOC_EXPR;
+
+ case GE_EXPR:
+ return REDUC_MAX_LAST_LOC_EXPR;
+
+ default:
+ return ERROR_MARK;
+ }
+}
+
/* vect_dump will be set to stderr or dump_file if exist. */
extern FILE *vect_dump;
extern LOC vect_loop_location;
@@ -764,7 +804,7 @@ extern bool vect_transform_stmt (gimple,
extern void vect_remove_stores (gimple);
extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *,
- tree, int);
+ tree, int, tree, int);
extern void vect_get_load_cost (struct data_reference *, int, bool,
unsigned int *, unsigned int *);
extern void vect_get_store_cost (struct data_reference *, int, unsigned int *);
@@ -844,8 +884,11 @@ extern void vect_slp_transform_bb (basic
Additional pattern recognition functions can (and will) be added
in the future. */
typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *);
-#define NUM_PATTERNS 4
+typedef bool (* vect_recog_compound_func_ptr) (unsigned int, va_list);
+#define NUM_PATTERNS 4
+#define NUM_COMPOUND_PATTERNS 1
void vect_pattern_recog (loop_vec_info);
+void vect_compound_pattern_recog (unsigned int, ...);
/* In tree-vectorizer.c. */
unsigned vectorize_loops (void);
===================================================================
@@ -296,7 +296,8 @@ vect_determine_vectorization_factor (loo
else
{
gcc_assert (!STMT_VINFO_DATA_REF (stmt_info)
- && !is_pattern_stmt_p (stmt_info));
+ && (!is_pattern_stmt_p (stmt_info)
+ || STMT_VINFO_COMPOUND_PATTERN (stmt_info)));
scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
if (vect_print_dump_info (REPORT_DETAILS))
@@ -445,10 +446,15 @@ static void
vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
{
basic_block bb = loop->header;
- tree dumy;
+ tree dummy;
VEC(gimple,heap) *worklist = VEC_alloc (gimple, heap, 64);
gimple_stmt_iterator gsi;
- bool double_reduc;
+ bool double_reduc, found, minmax_loc = false;
+ gimple first_cond_stmt = NULL, second_cond_stmt = NULL;
+ gimple first_phi = NULL, second_phi = NULL, phi, use_stmt;
+ int i;
+ imm_use_iterator imm_iter;
+ use_operand_p use_p;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
@@ -485,7 +491,8 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
}
if (!access_fn
- || !vect_is_simple_iv_evolution (loop->num, access_fn, &dumy, &dumy))
+ || !vect_is_simple_iv_evolution (loop->num, access_fn, &dummy,
+ &dummy))
{
VEC_safe_push (gimple, heap, worklist, phi);
continue;
@@ -496,8 +503,56 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
}
+ /* Detect compound reduction patterns (before reduction detection):
+ we currently support only min/max location pattern, so we look for two
+ reduction condition statements. */
+ for (i = 0; VEC_iterate (gimple, worklist, i, phi); i++)
+ {
+ tree def = PHI_RESULT (phi);
+
+ found = false;
+ FOR_EACH_IMM_USE_FAST (use_p, imm_iter, def)
+ {
+ use_stmt = USE_STMT (use_p);
+ if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
+ && vinfo_for_stmt (use_stmt)
+ && is_gimple_assign (use_stmt)
+ && gimple_assign_rhs_code (use_stmt) == COND_EXPR)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found)
+ continue;
+
+ if (!first_cond_stmt)
+ {
+ first_cond_stmt = use_stmt;
+ first_phi = phi;
+ }
+ else
+ {
+ if (second_cond_stmt)
+ {
+ /* This one is the third reduction condition statement in the
+ loop. This is too confusing, we bail out. */
+ minmax_loc = false;
+ break;
+ }
+
+ second_cond_stmt = use_stmt;
+ second_phi = phi;
+ minmax_loc = true;
+ }
+ }
+
+ if (minmax_loc)
+ vect_compound_pattern_recog (4, first_phi, first_cond_stmt,
+ second_phi, second_cond_stmt);
- /* Second - identify all reductions and nested cycles. */
+ /* Identify all reductions and nested cycles. */
while (VEC_length (gimple, worklist) > 0)
{
gimple phi = VEC_pop (gimple, worklist);
@@ -596,11 +651,9 @@ vect_analyze_scalar_cycles (loop_vec_inf
/* When vectorizing an outer-loop, the inner-loop is executed sequentially.
Reductions in such inner-loop therefore have different properties than
the reductions in the nest that gets vectorized:
- 1. When vectorized, they are executed in the same order as in the original
- scalar loop, so we can't change the order of computation when
- vectorizing them.
- 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
- current checks are too strict. */
+ when vectorized, they are executed in the same order as in the original
+ scalar loop, so we can't change the order of computation when
+ vectorizing them. */
if (loop->inner)
vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
@@ -821,7 +874,15 @@ destroy_loop_vec_info (loop_vec_info loo
if (orig_stmt_info
&& STMT_VINFO_IN_PATTERN_P (orig_stmt_info))
remove_stmt_p = true;
- }
+
+ /* We are removing statement inserted by the pattern
+ detection pass. Update the original statement to be the
+ def stmt of the statement's LHS. */
+ if (remove_stmt_p && is_gimple_assign (orig_stmt)
+ && TREE_CODE (gimple_assign_lhs (orig_stmt)) == SSA_NAME)
+ SSA_NAME_DEF_STMT (gimple_assign_lhs (orig_stmt))
+ = orig_stmt;
+ }
/* Free stmt_vec_info. */
free_stmt_vec_info (stmt);
@@ -1671,13 +1732,16 @@ vect_is_simple_reduction_1 (loop_vec_inf
gimple use_stmt = USE_STMT (use_p);
if (is_gimple_debug (use_stmt))
continue;
+
if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
&& vinfo_for_stmt (use_stmt)
- && !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
+ && !is_pattern_stmt_p (vinfo_for_stmt (use_stmt))
+ && !STMT_VINFO_COMPOUND_PATTERN (vinfo_for_stmt (use_stmt)))
nloop_uses++;
+
if (nloop_uses > 1)
{
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "reduction used in loop.");
return NULL;
}
@@ -1725,10 +1789,12 @@ vect_is_simple_reduction_1 (loop_vec_inf
gimple use_stmt = USE_STMT (use_p);
if (is_gimple_debug (use_stmt))
continue;
+
if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
&& vinfo_for_stmt (use_stmt)
&& !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
nloop_uses++;
+
if (nloop_uses > 1)
{
if (vect_print_dump_info (REPORT_DETAILS))
@@ -1778,6 +1844,9 @@ vect_is_simple_reduction_1 (loop_vec_inf
code = PLUS_EXPR;
if (check_reduction
+ && (!vinfo_for_stmt (def_stmt)
+ || STMT_VINFO_COMPOUND_PATTERN (vinfo_for_stmt (def_stmt))
+ != minmax_loc_stmt)
&& (!commutative_tree_code (code) || !associative_tree_code (code)))
{
if (vect_print_dump_info (REPORT_DETAILS))
@@ -1828,14 +1897,16 @@ vect_is_simple_reduction_1 (loop_vec_inf
}
type = TREE_TYPE (gimple_assign_lhs (def_stmt));
- if ((TREE_CODE (op1) == SSA_NAME
- && !types_compatible_p (type,TREE_TYPE (op1)))
- || (TREE_CODE (op2) == SSA_NAME
- && !types_compatible_p (type, TREE_TYPE (op2)))
- || (op3 && TREE_CODE (op3) == SSA_NAME
- && !types_compatible_p (type, TREE_TYPE (op3)))
- || (op4 && TREE_CODE (op4) == SSA_NAME
- && !types_compatible_p (type, TREE_TYPE (op4))))
+ if (STMT_VINFO_COMPOUND_PATTERN (vinfo_for_stmt (def_stmt))
+ != minmax_loc_stmt
+ && ((TREE_CODE (op1) == SSA_NAME
+ && !types_compatible_p (type, TREE_TYPE (op1)))
+ || (TREE_CODE (op2) == SSA_NAME
+ && !types_compatible_p (type, TREE_TYPE (op2)))
+ || (op3 && TREE_CODE (op3) == SSA_NAME
+ && !types_compatible_p (type, TREE_TYPE (op3)))
+ || (op4 && TREE_CODE (op4) == SSA_NAME
+ && !types_compatible_p (type, TREE_TYPE (op4)))))
{
if (vect_print_dump_info (REPORT_DETAILS))
{
@@ -1843,17 +1914,17 @@ vect_is_simple_reduction_1 (loop_vec_inf
print_generic_expr (vect_dump, type, TDF_SLIM);
fprintf (vect_dump, ", operands types: ");
print_generic_expr (vect_dump, TREE_TYPE (op1), TDF_SLIM);
- fprintf (vect_dump, ",");
+ fprintf (vect_dump, ", ");
print_generic_expr (vect_dump, TREE_TYPE (op2), TDF_SLIM);
if (op3)
{
- fprintf (vect_dump, ",");
+ fprintf (vect_dump, ", ");
print_generic_expr (vect_dump, TREE_TYPE (op3), TDF_SLIM);
}
if (op4)
{
- fprintf (vect_dump, ",");
+ fprintf (vect_dump, ", ");
print_generic_expr (vect_dump, TREE_TYPE (op4), TDF_SLIM);
}
}
@@ -1961,7 +2032,7 @@ vect_is_simple_reduction_1 (loop_vec_inf
== vect_internal_def
&& !is_loop_header_bb_p (gimple_bb (def2)))))))
{
- if (check_reduction)
+ if (check_reduction && code != COND_EXPR)
{
/* Swap operands (just for simplicity - so that the rest of the code
can assume that the reduction variable is always the last (second)
@@ -2432,7 +2503,6 @@ vect_model_reduction_cost (stmt_vec_info
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-
/* Cost of reduction op inside loop. */
STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info)
+= ncopies * vect_get_cost (vector_stmt);
@@ -2469,11 +2539,15 @@ vect_model_reduction_cost (stmt_vec_info
mode = TYPE_MODE (vectype);
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
- if (!orig_stmt)
+ if (!orig_stmt || STMT_VINFO_COMPOUND_PATTERN (stmt_info))
orig_stmt = STMT_VINFO_STMT (stmt_info);
code = gimple_assign_rhs_code (orig_stmt);
+ if (STMT_VINFO_COMPOUND_PATTERN (stmt_info) == minmax_loc_stmt)
+ STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info)
+ += ncopies * 5 * vect_get_cost (vector_stmt);
+
/* Add in cost for initial definition. */
outer_cost += vect_get_cost (scalar_to_vec);
@@ -2489,28 +2563,34 @@ vect_model_reduction_cost (stmt_vec_info
+ vect_get_cost (vec_to_scalar);
else
{
- int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
- tree bitsize =
- TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt)));
- int element_bitsize = tree_low_cst (bitsize, 1);
- int nelements = vec_size_in_bits / element_bitsize;
-
- optab = optab_for_tree_code (code, vectype, optab_default);
-
- /* We have a whole vector shift available. */
- if (VECTOR_MODE_P (mode)
- && optab_handler (optab, mode) != CODE_FOR_nothing
- && optab_handler (vec_shr_optab, mode) != CODE_FOR_nothing)
- /* Final reduction via vector shifts and the reduction operator. Also
- requires scalar extract. */
- outer_cost += ((exact_log2(nelements) * 2)
- * vect_get_cost (vector_stmt)
- + vect_get_cost (vec_to_scalar));
- else
- /* Use extracts and reduction op for final reduction. For N elements,
- we have N extracts and N-1 reduction ops. */
- outer_cost += ((nelements + nelements - 1)
- * vect_get_cost (vector_stmt));
+ if (STMT_VINFO_COMPOUND_PATTERN (stmt_info))
+ outer_cost += 6 * vect_get_cost (vector_stmt)
+ + vect_get_cost (vec_to_scalar);
+ else
+ {
+ int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
+ tree bitsize =
+ TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt)));
+ int element_bitsize = tree_low_cst (bitsize, 1);
+ int nelements = vec_size_in_bits / element_bitsize;
+
+ optab = optab_for_tree_code (code, vectype, optab_default);
+
+ /* We have a whole vector shift available. */
+ if (VECTOR_MODE_P (mode)
+ && optab_handler (optab, mode) != CODE_FOR_nothing
+ && optab_handler (vec_shr_optab, mode) != CODE_FOR_nothing)
+ /* Final reduction via vector shifts and the reduction
+ operator. Also requires scalar extract. */
+ outer_cost += ((exact_log2(nelements) * 2)
+ * vect_get_cost (vector_stmt)
+ + vect_get_cost (vec_to_scalar));
+ else
+ /* Use extracts and reduction op for final reduction. For N
+ elements, we have N extracts and N-1 reduction ops. */
+ outer_cost += ((nelements + nelements - 1)
+ * vect_get_cost (vector_stmt));
+ }
}
}
@@ -3113,6 +3193,8 @@ vect_create_epilog_for_reduction (VEC (t
unsigned int group_size = 1, k, ratio;
VEC (tree, heap) *vec_initial_defs = NULL;
VEC (gimple, heap) *phis;
+ tree vec_temp;
+ tree min_max_res = NULL_TREE;
if (slp_node)
group_size = VEC_length (gimple, SLP_TREE_SCALAR_STMTS (slp_node));
@@ -3170,9 +3252,9 @@ vect_create_epilog_for_reduction (VEC (t
else
{
vec_initial_defs = VEC_alloc (tree, heap, 1);
- /* For the case of reduction, vect_get_vec_def_for_operand returns
- the scalar def before the loop, that defines the initial value
- of the reduction variable. */
+ /* For the case of reduction, vect_get_vec_def_for_operand returns
+ the scalar def before the loop, that defines the initial value
+ of the reduction variable. */
vec_initial_def = vect_get_vec_def_for_operand (reduction_op, stmt,
&adjustment_def);
VEC_quick_push (tree, vec_initial_defs, vec_initial_def);
@@ -3280,18 +3362,18 @@ vect_create_epilog_for_reduction (VEC (t
defined in the loop. In case STMT is a "pattern-stmt" (i.e. - it
represents a reduction pattern), the tree-code and scalar-def are
taken from the original stmt that the pattern-stmt (STMT) replaces.
- Otherwise (it is a regular reduction) - the tree-code and scalar-def
- are taken from STMT. */
+ Otherwise (it is a regular reduction or a compound pattern) - the
+ tree-code and scalar-def are taken from STMT. */
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
- if (!orig_stmt)
+ if (!orig_stmt || STMT_VINFO_COMPOUND_PATTERN (stmt_info))
{
- /* Regular reduction */
+ /* Regular reduction or compound pattern. */
orig_stmt = stmt;
}
else
{
- /* Reduction pattern */
+ /* Reduction pattern. */
stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt);
gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
@@ -3318,6 +3400,27 @@ vect_create_epilog_for_reduction (VEC (t
if (nested_in_vect_loop && !double_reduc)
goto vect_finalize_reduction;
+ /* Get reduction code for compound pattern. */
+ if (STMT_VINFO_COMPOUND_PATTERN (stmt_info) == minmax_loc_stmt)
+ {
+ gimple related = STMT_VINFO_RELATED_STMT (stmt_info);
+ gimple min_max_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (related));
+
+ tree cond = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
+ enum tree_code cond_code;
+
+ min_max_res = gimple_assign_lhs (min_max_stmt);
+ if (TREE_CODE (cond) == SSA_NAME)
+ {
+ gimple cond_def_stmt = SSA_NAME_DEF_STMT (cond);
+ cond_code = gimple_assign_rhs_code (cond_def_stmt);
+ }
+ else
+ cond_code = TREE_CODE (cond);
+
+ reduc_code = get_minloc_reduc_epilogue_code (cond_code);
+ }
+
/* 2.3 Create the reduction code, using one of the three schemes described
above. In SLP we simply need to extract all the elements from the
vector (without reducing them), so we use scalar shifts. */
@@ -3333,7 +3436,12 @@ vect_create_epilog_for_reduction (VEC (t
vec_dest = vect_create_destination_var (scalar_dest, vectype);
new_phi = VEC_index (gimple, new_phis, 0);
- tmp = build1 (reduc_code, vectype, PHI_RESULT (new_phi));
+ vec_temp = PHI_RESULT (new_phi);
+ if (min_max_res)
+ tmp = build2 (reduc_code, vectype, min_max_res, vec_temp);
+ else
+ tmp = build1 (reduc_code, vectype, vec_temp);
+
epilog_stmt = gimple_build_assign (vec_dest, tmp);
new_temp = make_ssa_name (vec_dest, epilog_stmt);
gimple_assign_set_lhs (epilog_stmt, new_temp);
@@ -3348,7 +3456,6 @@ vect_create_epilog_for_reduction (VEC (t
int bit_offset;
int element_bitsize = tree_low_cst (bitsize, 1);
int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
- tree vec_temp;
if (optab_handler (vec_shr_optab, mode) != CODE_FOR_nothing)
shift_code = VEC_RSHIFT_EXPR;
@@ -3366,7 +3473,7 @@ vect_create_epilog_for_reduction (VEC (t
else
{
optab optab = optab_for_tree_code (code, vectype, optab_default);
- if (optab_handler (optab, mode) == CODE_FOR_nothing)
+ if (!optab || optab_handler (optab, mode) == CODE_FOR_nothing)
have_whole_vector_shift = false;
}
@@ -3644,8 +3751,10 @@ vect_finalize_reduction:
VEC_safe_push (gimple, heap, phis, USE_STMT (use_p));
/* We expect to have found an exit_phi because of loop-closed-ssa
- form. */
- gcc_assert (!VEC_empty (gimple, phis));
+ form,unless it's a min/max statement of min/max location pattern,
+ which is inserted by the pattern recognition phase. */
+ gcc_assert (!VEC_empty (gimple, phis)
+ || STMT_VINFO_COMPOUND_PATTERN (stmt_info) == minmax_stmt);
for (i = 0; VEC_iterate (gimple, phis, i, exit_phi); i++)
{
@@ -3887,11 +3996,11 @@ vectorizable_reduction (gimple stmt, gim
basic_block def_bb;
struct loop * def_stmt_loop, *outer_loop = NULL;
tree def_arg;
- gimple def_arg_stmt;
+ gimple def_arg_stmt, related;
VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL, *vect_defs = NULL;
VEC (gimple, heap) *phis = NULL;
- int vec_num;
- tree def0, def1;
+ int vec_num, cond_reduc_index = 0;
+ tree def0, def1, cond_reduc_def = NULL_TREE;
if (nested_in_vect_loop_p (loop, stmt))
{
@@ -3901,8 +4010,10 @@ vectorizable_reduction (gimple stmt, gim
}
/* 1. Is vectorizable reduction? */
- /* Not supportable if the reduction variable is used in the loop. */
- if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer)
+ /* Not supportable if the reduction variable is used in the loop,
+ unless it's a pattern. */
+ if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer
+ && !STMT_VINFO_COMPOUND_PATTERN (stmt_info))
return false;
/* Reductions that are not used even in an enclosing outer-loop,
@@ -3924,14 +4035,17 @@ vectorizable_reduction (gimple stmt, gim
the original sequence that constitutes the pattern. */
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
- if (orig_stmt)
+ if (orig_stmt
+ && !STMT_VINFO_COMPOUND_PATTERN (stmt_info))
{
orig_stmt_info = vinfo_for_stmt (orig_stmt);
gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt);
gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
}
-
+ else
+ orig_stmt = NULL;
+
/* 3. Check the operands of the operation. The first operands are defined
inside the loop body. The last operand is the reduction variable,
which is defined by the loop-header-phi. */
@@ -4044,12 +4158,13 @@ vectorizable_reduction (gimple stmt, gim
if (code == COND_EXPR)
{
- if (!vectorizable_condition (stmt, gsi, NULL, ops[reduc_index], 0))
+ if (!vectorizable_condition (stmt, gsi, NULL, ops[reduc_index], 0,
+ cond_reduc_def, cond_reduc_index))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "unsupported condition in reduction");
- return false;
+ return false;
}
}
else
@@ -4182,7 +4297,12 @@ vectorizable_reduction (gimple stmt, gim
}
else
{
- if (!nested_cycle || double_reduc)
+ /* There is no need in reduction epilogue in case of a nested cycle,
+ unless it is double reduction. For reduction pattern, we assume that
+ we know how to create an epilogue even if there is no reduction code
+ for it. */
+ if ((!nested_cycle || double_reduc)
+ && !STMT_VINFO_COMPOUND_PATTERN (stmt_info))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "no reduc code for scalar code.");
@@ -4202,8 +4322,9 @@ vectorizable_reduction (gimple stmt, gim
if (!vec_stmt) /* transformation not required. */
{
STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
- if (!vect_model_reduction_cost (stmt_info, epilog_reduc_code, ncopies))
+ if (!vect_model_reduction_cost (stmt_info, epilog_reduc_code, ncopies))
return false;
+
return true;
}
@@ -4254,6 +4375,32 @@ vectorizable_reduction (gimple stmt, gim
else
epilog_copies = ncopies;
+ /* Prepare vector operands for min/max location. */
+ if (STMT_VINFO_COMPOUND_PATTERN (stmt_info) == minmax_loc_stmt)
+ {
+ tree cond_op;
+ gimple cond_def_stmt;
+
+ related = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt));
+ cond_op = TREE_OPERAND (ops[0], 0);
+ cond_def_stmt = SSA_NAME_DEF_STMT (cond_op);
+ if (gimple_code (cond_def_stmt) == GIMPLE_PHI)
+ {
+ cond_reduc_index = 1;
+ cond_reduc_def = gimple_assign_rhs1 (STMT_VINFO_VEC_STMT (
+ vinfo_for_stmt (related)));
+ }
+ else
+ {
+ cond_op = TREE_OPERAND (ops[0], 1);
+ cond_def_stmt = SSA_NAME_DEF_STMT (cond_op);
+ gcc_assert (gimple_code (cond_def_stmt) == GIMPLE_PHI);
+ cond_reduc_index = 2;
+ cond_reduc_def = gimple_assign_rhs2 (STMT_VINFO_VEC_STMT (
+ vinfo_for_stmt (related)));
+ }
+ }
+
prev_stmt_info = NULL;
prev_phi_info = NULL;
if (slp_node)
@@ -4297,7 +4444,8 @@ vectorizable_reduction (gimple stmt, gim
gcc_assert (!slp_node);
vectorizable_condition (stmt, gsi, vec_stmt,
PHI_RESULT (VEC_index (gimple, phis, 0)),
- reduc_index);
+ reduc_index, cond_reduc_def,
+ cond_reduc_index);
/* Multiple types are not supported for condition. */
break;
}
@@ -4533,6 +4681,9 @@ vectorizable_live_operation (gimple stmt
gcc_assert (STMT_VINFO_LIVE_P (stmt_info));
+ if (STMT_VINFO_COMPOUND_PATTERN (stmt_info))
+ return true;
+
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
return false;
===================================================================
@@ -1054,10 +1054,14 @@ DEFTREECODE (OMP_CLAUSE, "omp_clause", t
result (e.g. summing the elements of the vector, finding the minimum over
the vector elements, etc).
Operand 0 is a vector; the first element in the vector has the result.
- Operand 1 is a vector. */
+ Operands 1 and 2 are vectors. */
DEFTREECODE (REDUC_MAX_EXPR, "reduc_max_expr", tcc_unary, 1)
DEFTREECODE (REDUC_MIN_EXPR, "reduc_min_expr", tcc_unary, 1)
DEFTREECODE (REDUC_PLUS_EXPR, "reduc_plus_expr", tcc_unary, 1)
+DEFTREECODE (REDUC_MIN_FIRST_LOC_EXPR, "reduc_min_first_loc_expr", tcc_binary, 2)
+DEFTREECODE (REDUC_MIN_LAST_LOC_EXPR, "reduc_min_last_loc_expr", tcc_binary, 2)
+DEFTREECODE (REDUC_MAX_FIRST_LOC_EXPR, "reduc_max_first_loc_expr", tcc_binary, 2)
+DEFTREECODE (REDUC_MAX_LAST_LOC_EXPR, "reduc_max_last_loc_expr", tcc_binary, 2)
/* Widening dot-product.
The first two arguments are of type t1.
===================================================================
@@ -2991,6 +2991,10 @@ expand_debug_expr (tree exp)
case REDUC_MAX_EXPR:
case REDUC_MIN_EXPR:
case REDUC_PLUS_EXPR:
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ case REDUC_MIN_LAST_LOC_EXPR:
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ case REDUC_MAX_LAST_LOC_EXPR:
case VEC_COND_EXPR:
case VEC_EXTRACT_EVEN_EXPR:
case VEC_EXTRACT_ODD_EXPR:
===================================================================
@@ -54,6 +54,10 @@ static vect_recog_func_ptr vect_vect_rec
vect_recog_widen_sum_pattern,
vect_recog_dot_prod_pattern,
vect_recog_pow_pattern};
+static bool vect_recog_min_max_loc_pattern (unsigned int, va_list);
+static vect_recog_compound_func_ptr
+ vect_recog_compound_func_ptrs[NUM_COMPOUND_PATTERNS] = {
+ vect_recog_min_max_loc_pattern};
/* Function widened_name_p
@@ -847,3 +851,286 @@ vect_pattern_recog (loop_vec_info loop_v
}
}
}
+
+
+/* Detect min/max location pattern.
+ Given two reducton condition statements and their phi nodes, we check
+ if one of the statements calculates minimum or maximum, and the other one
+ records its location. If the pattern is detected, we replace the min/max
+ condition statement with MIN_EXPR or MAX_EXPR, and mark the old statement
+ as pattern statement.
+
+ The pattern we are looking for:
+
+ s1: min = [cond_expr] a < min ? a : min
+ s2: index = [cond_expr] a < min ? new_index : index
+
+ We add MIN_EXPR statement before the index calculation statement:
+
+ s1: min = [cond_expr] a < min ? a : min
+ s1': min = [min_expr] <a, min>
+ s2: index = [cond_expr] a < min ? new_index : index
+
+ s1 is marked as pattern statement
+ s1' points to s1 via related_stmt field
+ s1 points to s1' via related_stmt field
+ s2 points to s1' via related_stmt field.
+ s1' and s2 are marked as compound pattern min/max and min/max location
+ statements. */
+
+static bool
+vect_recog_min_max_loc_pattern (unsigned int nargs, va_list args)
+{
+ gimple first_phi, first_stmt, second_phi, second_stmt, loop_op_def_stmt;
+ stmt_vec_info stmt_vinfo, new_stmt_info, minmax_stmt_info, pos_stmt_info;
+ loop_vec_info loop_info;
+ struct loop *loop;
+ enum tree_code code, first_code, second_code;
+ gimple first_cond_def_stmt = NULL, second_cond_def_stmt = NULL;
+ tree first_cond_op0, first_cond_op1, second_cond_op0, second_cond_op1;
+ tree first_stmt_oprnd0, first_stmt_oprnd1, second_stmt_oprnd0;
+ tree second_stmt_oprnd1, first_cond, second_cond;
+ int phi_def_index;
+ tree first_loop_op, second_loop_op, pos_stmt_loop_op, def, result;
+ gimple pos_stmt, min_max_stmt, new_stmt, def_stmt;
+ gimple_stmt_iterator gsi;
+
+ if (nargs < 4)
+ return false;
+
+ first_phi = va_arg (args, gimple);
+ first_stmt = va_arg (args, gimple);
+ second_phi = va_arg (args, gimple);
+ second_stmt = va_arg (args, gimple);
+
+ stmt_vinfo = vinfo_for_stmt (first_stmt);
+ loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ loop = LOOP_VINFO_LOOP (loop_info);
+
+ /* Check that the condition is the same and is GT or LT. */
+ first_cond = TREE_OPERAND (gimple_assign_rhs1 (first_stmt), 0);
+ if (TREE_CODE (first_cond) == SSA_NAME)
+ {
+ first_cond_def_stmt = SSA_NAME_DEF_STMT (first_cond);
+ first_code = gimple_assign_rhs_code (first_cond_def_stmt);
+ first_cond_op0 = gimple_assign_rhs1 (first_cond_def_stmt);
+ first_cond_op1 = gimple_assign_rhs2 (first_cond_def_stmt);
+ }
+ else
+ {
+ first_code = TREE_CODE (first_cond);
+ first_cond_op0 = TREE_OPERAND (first_cond, 0);
+ first_cond_op1 = TREE_OPERAND (first_cond, 1);
+ }
+
+ if (first_code != GT_EXPR && first_code != LT_EXPR
+ && first_code != GE_EXPR && first_code != LE_EXPR)
+ return false;
+
+ second_cond = TREE_OPERAND (gimple_assign_rhs1 (second_stmt), 0);
+ if (TREE_CODE (second_cond) == SSA_NAME)
+ {
+ second_cond_def_stmt = SSA_NAME_DEF_STMT (second_cond);
+ second_code = gimple_assign_rhs_code (second_cond_def_stmt);
+ second_cond_op0 = gimple_assign_rhs1 (second_cond_def_stmt);
+ second_cond_op1 = gimple_assign_rhs2 (second_cond_def_stmt);
+ }
+ else
+ {
+ second_code = TREE_CODE (second_cond);
+ second_cond_op0 = TREE_OPERAND (second_cond, 0);
+ second_cond_op1 = TREE_OPERAND (second_cond, 1);
+ }
+
+ if (first_code != second_code)
+ return false;
+
+ if (first_cond_def_stmt
+ && (!second_cond_def_stmt
+ || first_cond_def_stmt != second_cond_def_stmt
+ || !operand_equal_p (first_cond_op0, second_cond_op0, 0)
+ || !operand_equal_p (first_cond_op1, second_cond_op1, 0)))
+ return false;
+
+ /* Both statements have the same condition. */
+
+ first_stmt_oprnd0 = TREE_OPERAND (gimple_assign_rhs1 (first_stmt), 1);
+ first_stmt_oprnd1 = TREE_OPERAND (gimple_assign_rhs1 (first_stmt), 2);
+
+ second_stmt_oprnd0 = TREE_OPERAND (gimple_assign_rhs1 (second_stmt), 1);
+ second_stmt_oprnd1 = TREE_OPERAND (gimple_assign_rhs1 (second_stmt), 2);
+
+ if (TREE_CODE (first_stmt_oprnd0) != SSA_NAME
+ || TREE_CODE (first_stmt_oprnd1) != SSA_NAME
+ || TREE_CODE (second_stmt_oprnd0) != SSA_NAME
+ || TREE_CODE (second_stmt_oprnd1) != SSA_NAME)
+ return false;
+
+ if (operand_equal_p (PHI_RESULT (first_phi), first_stmt_oprnd0, 0)
+ && operand_equal_p (PHI_RESULT (second_phi), second_stmt_oprnd0, 0))
+ {
+ phi_def_index = 0;
+ first_loop_op = first_stmt_oprnd1;
+ second_loop_op = second_stmt_oprnd1;
+ }
+ else
+ {
+ if (operand_equal_p (PHI_RESULT (first_phi), first_stmt_oprnd1, 0)
+ && operand_equal_p (PHI_RESULT (second_phi), second_stmt_oprnd1, 0))
+ {
+ phi_def_index = 1;
+ first_loop_op = first_stmt_oprnd0;
+ second_loop_op = second_stmt_oprnd0;
+ }
+ else
+ return false;
+ }
+
+ /* Now we know which operand is defined by phi node. Analyze the second
+ one. */
+
+ /* The min/max stmt must be x < y ? x : y. */
+ if (operand_equal_p (first_cond_op0, first_stmt_oprnd0, 0)
+ && operand_equal_p (first_cond_op1, first_stmt_oprnd1, 0))
+ {
+ pos_stmt = second_stmt;
+ min_max_stmt = first_stmt;
+ pos_stmt_loop_op = second_loop_op;
+ }
+ else
+ {
+ if (operand_equal_p (second_cond_op0, second_stmt_oprnd0, 0)
+ && operand_equal_p (second_cond_op1, second_stmt_oprnd1, 0))
+ {
+ pos_stmt = first_stmt;
+ min_max_stmt = second_stmt;
+ pos_stmt_loop_op = first_loop_op;
+ }
+ else
+ return false;
+ }
+
+ /* Analyze the position stmt. We expect it to be either induction or
+ induction plus constant. */
+ loop_op_def_stmt = SSA_NAME_DEF_STMT (pos_stmt_loop_op);
+
+ if (!flow_bb_inside_loop_p (loop, gimple_bb (loop_op_def_stmt)))
+ return false;
+
+ if (gimple_code (loop_op_def_stmt) == GIMPLE_PHI)
+ {
+ if (STMT_VINFO_DEF_TYPE (vinfo_for_stmt (loop_op_def_stmt))
+ != vect_induction_def)
+ return false;
+ }
+ else
+ {
+ if (!is_gimple_assign (loop_op_def_stmt))
+ return false;
+
+ if (get_gimple_rhs_class (gimple_assign_rhs_code (loop_op_def_stmt))
+ == GIMPLE_UNARY_RHS)
+ def = gimple_assign_rhs1 (loop_op_def_stmt);
+ else
+ {
+ tree op1, op2;
+
+ if (get_gimple_rhs_class (gimple_assign_rhs_code (loop_op_def_stmt))
+ != GIMPLE_BINARY_RHS
+ || gimple_assign_rhs_code (loop_op_def_stmt) != PLUS_EXPR)
+ return false;
+
+ op1 = gimple_assign_rhs1 (loop_op_def_stmt);
+ op2 = gimple_assign_rhs2 (loop_op_def_stmt);
+
+ if (TREE_CONSTANT (op1))
+ def = op2;
+ else
+ {
+ if (TREE_CONSTANT (op2))
+ def = op1;
+ else
+ return false;
+ }
+ }
+
+ if (TREE_CODE (def) != SSA_NAME)
+ return false;
+
+ def_stmt = SSA_NAME_DEF_STMT (def);
+ if (!flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ || gimple_code (def_stmt) != GIMPLE_PHI
+ || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
+ != vect_induction_def)
+ return false;
+ }
+
+ /* Pattern detected. Create a stmt to be used to replace the pattern. */
+ if (first_code == GT_EXPR || first_code == GE_EXPR)
+ code = phi_def_index ? MAX_EXPR : MIN_EXPR;
+ else
+ code = phi_def_index ? MIN_EXPR : MAX_EXPR;
+
+ result = gimple_assign_lhs (min_max_stmt);
+ new_stmt = gimple_build_assign_with_ops (code, result,
+ TREE_OPERAND (gimple_assign_rhs1 (min_max_stmt), 1),
+ TREE_OPERAND (gimple_assign_rhs1 (min_max_stmt), 2));
+ gsi = gsi_for_stmt (pos_stmt);
+ gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
+ SSA_NAME_DEF_STMT (result) = new_stmt;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Detected min/max location pattern:\nmin/max stmt ");
+ print_gimple_stmt (vect_dump, min_max_stmt, 0, TDF_SLIM);
+ fprintf (vect_dump, "\nlocation stmt ");
+ print_gimple_stmt (vect_dump, pos_stmt, 0, TDF_SLIM);
+ fprintf (vect_dump, "\nCreated stmt: ");
+ print_gimple_stmt (vect_dump, new_stmt, 0, TDF_SLIM);
+ }
+
+ /* Mark the stmts that are involved in the pattern. */
+ set_vinfo_for_stmt (new_stmt,
+ new_stmt_vec_info (new_stmt, loop_info, NULL));
+ new_stmt_info = vinfo_for_stmt (new_stmt);
+
+ pos_stmt_info = vinfo_for_stmt (pos_stmt);
+ minmax_stmt_info = vinfo_for_stmt (min_max_stmt);
+
+ STMT_VINFO_DEF_TYPE (new_stmt_info) = STMT_VINFO_DEF_TYPE (minmax_stmt_info);
+ STMT_VINFO_VECTYPE (new_stmt_info) = STMT_VINFO_VECTYPE (minmax_stmt_info);
+
+ STMT_VINFO_IN_PATTERN_P (minmax_stmt_info) = true;
+ STMT_VINFO_COMPOUND_PATTERN (new_stmt_info) = minmax_stmt;
+ STMT_VINFO_COMPOUND_PATTERN (pos_stmt_info) = minmax_loc_stmt;
+ STMT_VINFO_RELATED_STMT (new_stmt_info) = min_max_stmt;
+ STMT_VINFO_RELATED_STMT (minmax_stmt_info) = new_stmt;
+ STMT_VINFO_RELATED_STMT (pos_stmt_info) = new_stmt;
+
+ return true;
+}
+
+/* Detect patterns consisting of two more statements to be vectorized.
+ Currently the only supported pattern is min/max location. */
+
+void
+vect_compound_pattern_recog (unsigned int nargs, ...)
+{
+ unsigned int j;
+ va_list args;
+ bool detected = false;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "=== vect_compound_pattern_recog ===");
+
+ /* Scan over all generic vect_recog__compound_xxx_pattern functions. */
+ for (j = 0; j < NUM_COMPOUND_PATTERNS; j++)
+ {
+ va_start (args, nargs);
+ detected = (* vect_recog_compound_func_ptrs[j]) (nargs, args);
+ va_end (args);
+ if (detected)
+ break;
+ }
+}
+
===================================================================
@@ -272,8 +272,10 @@ process_use (gimple stmt, tree use, loop
/* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
DEF_STMT must have already been processed, because this should be the
only way that STMT, which is a reduction-phi, was put in the worklist,
- as there should be no other uses for DEF_STMT in the loop. So we just
- check that everything is as expected, and we are done. */
+ as there should be no other uses for DEF_STMT in the loop, unless it is
+ min/max location pattern. So we just check that everything is as
+ as expected, and mark the min/max stmt of the location pattern stmt as
+ used by reduction (it is used by the reduction of location). */
dstmt_vinfo = vinfo_for_stmt (def_stmt);
bb = gimple_bb (stmt);
if (gimple_code (stmt) == GIMPLE_PHI
@@ -284,11 +286,22 @@ process_use (gimple stmt, tree use, loop
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "reduc-stmt defining reduc-phi in the same nest.");
+
+ /* Compound reduction pattern: is used by reduction. */
+ if (STMT_VINFO_COMPOUND_PATTERN (dstmt_vinfo))
+ {
+ relevant = vect_used_by_reduction;
+ vect_mark_relevant (worklist, def_stmt, relevant, live_p);
+ return true;
+ }
+
if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
+
gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
- gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
- || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
+ gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
+ || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope
+ || STMT_VINFO_COMPOUND_PATTERN (dstmt_vinfo));
return true;
}
@@ -482,7 +495,8 @@ vect_mark_stmts_to_be_vectorized (loop_v
break;
case vect_used_by_reduction:
- if (gimple_code (stmt) == GIMPLE_PHI)
+ if (gimple_code (stmt) == GIMPLE_PHI
+ || STMT_VINFO_COMPOUND_PATTERN (stmt_vinfo))
break;
/* fall through */
@@ -3993,11 +4007,16 @@ vect_is_simple_cond (tree cond, loop_vec
to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
else caluse if it is 2).
+ In min/max location pattern, reduction defs are used in both condition part
+ and then/else clause. In that case COND_REDUC_DEF contains such vector def,
+ and COND_REDUC_INDEX specifies its place in the condition.
+
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
bool
vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi,
- gimple *vec_stmt, tree reduc_def, int reduc_index)
+ gimple *vec_stmt, tree reduc_def, int reduc_index,
+ tree cond_reduc_def, int cond_reduc_index)
{
tree scalar_dest = NULL_TREE;
tree vec_dest = NULL_TREE;
@@ -4015,6 +4034,7 @@ vectorizable_condition (gimple stmt, gim
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
enum tree_code code;
+ tree comparison_type;
/* FORNOW: unsupported in basic block SLP. */
gcc_assert (loop_vinfo);
@@ -4023,20 +4043,23 @@ vectorizable_condition (gimple stmt, gim
if (ncopies > 1)
return false; /* FORNOW */
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ && !STMT_VINFO_COMPOUND_PATTERN (stmt_info))
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
- && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
+ && !((STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
+ || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
&& reduc_def))
- return false;
+ return false;
/* FORNOW: SLP not supported. */
if (STMT_SLP_TYPE (stmt_info))
return false;
/* FORNOW: not yet supported. */
- if (STMT_VINFO_LIVE_P (stmt_info))
+ if (STMT_VINFO_LIVE_P (stmt_info)
+ && !STMT_VINFO_COMPOUND_PATTERN (stmt_info))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "value used after loop.");
@@ -4062,10 +4085,13 @@ vectorizable_condition (gimple stmt, gim
return false;
/* We do not handle two different vector types for the condition
- and the values. */
+ and the values. */
if (!types_compatible_p (TREE_TYPE (TREE_OPERAND (cond_expr, 0)),
- TREE_TYPE (vectype)))
- return false;
+ TREE_TYPE (vectype))
+ && !(STMT_VINFO_COMPOUND_PATTERN (stmt_info)
+ && TYPE_SIZE_UNIT (TREE_TYPE (TREE_OPERAND (cond_expr, 0)))
+ == TYPE_SIZE_UNIT (TREE_TYPE (then_clause))))
+ return false;
if (TREE_CODE (then_clause) == SSA_NAME)
{
@@ -4094,43 +4120,75 @@ vectorizable_condition (gimple stmt, gim
vec_mode = TYPE_MODE (vectype);
- if (!vec_stmt)
+ comparison_type =
+ get_vectype_for_scalar_type (TREE_TYPE (TREE_OPERAND (cond_expr, 0)));
+
+ /* Check that min/max location pattern is supported. Here we check if the
+ reduction epilogue is supported. The condition itself is checked in
+ expand_vec_cond_expr_p () below. */
+ if (STMT_VINFO_COMPOUND_PATTERN (stmt_info) == minmax_loc_stmt)
+ {
+ enum tree_code reduc_code;
+ enum insn_code icode;
+
+ reduc_code = get_minloc_reduc_epilogue_code (TREE_CODE (cond_expr));
+ icode = get_vec_reduc_minloc_expr_code (reduc_code, comparison_type,
+ vectype);
+ if (icode == CODE_FOR_nothing)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "unsupported reduction epilogue");
+
+ return false;
+ }
+ }
+
+ if (!vec_stmt)
{
STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
- return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode);
+ return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode,
+ TREE_TYPE (TREE_OPERAND (cond_expr, 0)),
+ TYPE_MODE (comparison_type));
}
- /* Transform */
+ /* Transform. */
/* Handle def. */
scalar_dest = gimple_assign_lhs (stmt);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
/* Handle cond expr. */
- vec_cond_lhs =
- vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt, NULL);
- vec_cond_rhs =
- vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt, NULL);
+ if (cond_reduc_index == 1)
+ vec_cond_lhs = cond_reduc_def;
+ else
+ vec_cond_lhs =
+ vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt, NULL);
+
+ if (cond_reduc_index == 2)
+ vec_cond_rhs = cond_reduc_def;
+ else
+ vec_cond_rhs =
+ vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt, NULL);
+
if (reduc_index == 1)
vec_then_clause = reduc_def;
else
vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt, NULL);
+
if (reduc_index == 2)
vec_else_clause = reduc_def;
else
vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt, NULL);
/* Arguments are ready. Create the new vector stmt. */
- vec_compare = build2 (TREE_CODE (cond_expr), vectype,
- vec_cond_lhs, vec_cond_rhs);
- vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
- vec_compare, vec_then_clause, vec_else_clause);
-
+ vec_compare = build2 (TREE_CODE (cond_expr), comparison_type, vec_cond_lhs,
+ vec_cond_rhs);
+ vec_cond_expr = build3 (VEC_COND_EXPR, vectype, vec_compare,
+ vec_then_clause, vec_else_clause);
*vec_stmt = gimple_build_assign (vec_dest, vec_cond_expr);
new_temp = make_ssa_name (vec_dest, *vec_stmt);
gimple_assign_set_lhs (*vec_stmt, new_temp);
vect_finish_stmt_generation (stmt, *vec_stmt, gsi);
-
return true;
}
@@ -4186,7 +4244,8 @@ vect_analyze_stmt (gimple stmt, bool *ne
case vect_nested_cycle:
gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
|| relevance == vect_used_in_outer_by_reduction
- || relevance == vect_unused_in_scope));
+ || relevance == vect_unused_in_scope
+ || relevance == vect_used_by_reduction));
break;
case vect_induction_def:
@@ -4248,7 +4307,7 @@ vect_analyze_stmt (gimple stmt, bool *ne
|| vectorizable_call (stmt, NULL, NULL)
|| vectorizable_store (stmt, NULL, NULL, NULL)
|| vectorizable_reduction (stmt, NULL, NULL, NULL)
- || vectorizable_condition (stmt, NULL, NULL, NULL, 0));
+ || vectorizable_condition (stmt, NULL, NULL, NULL, 0, NULL, 0));
else
{
if (bb_vinfo)
@@ -4389,7 +4448,7 @@ vect_transform_stmt (gimple stmt, gimple
case condition_vec_info_type:
gcc_assert (!slp_node);
- done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0);
+ done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0, NULL, 0);
gcc_assert (done);
break;
@@ -4527,6 +4586,7 @@ new_stmt_vec_info (gimple stmt, loop_vec
STMT_VINFO_VEC_STMT (res) = NULL;
STMT_VINFO_VECTORIZABLE (res) = true;
STMT_VINFO_IN_PATTERN_P (res) = false;
+ STMT_VINFO_COMPOUND_PATTERN (res) = not_in_pattern;
STMT_VINFO_RELATED_STMT (res) = NULL;
STMT_VINFO_DATA_REF (res) = NULL;
===================================================================
@@ -3336,6 +3336,10 @@ estimate_operator_cost (enum tree_code c
case REDUC_MAX_EXPR:
case REDUC_MIN_EXPR:
case REDUC_PLUS_EXPR:
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ case REDUC_MIN_LAST_LOC_EXPR:
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ case REDUC_MAX_LAST_LOC_EXPR:
case WIDEN_SUM_EXPR:
case WIDEN_MULT_EXPR:
case DOT_PROD_EXPR:
===================================================================
@@ -395,6 +395,7 @@ expand_vector_operations_1 (gimple_stmt_
optab op;
enum gimple_rhs_class rhs_class;
tree new_rhs;
+ enum insn_code icode = CODE_FOR_nothing;
if (gimple_code (stmt) != GIMPLE_ASSIGN)
return;
@@ -471,6 +472,16 @@ expand_vector_operations_1 (gimple_stmt_
&& INTEGRAL_TYPE_P (TREE_TYPE (type)))
op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
+ if (code == REDUC_MIN_FIRST_LOC_EXPR
+ || code == REDUC_MIN_LAST_LOC_EXPR
+ || code == REDUC_MAX_FIRST_LOC_EXPR
+ || code == REDUC_MAX_LAST_LOC_EXPR)
+ {
+ type = TREE_TYPE (rhs1);
+ icode = get_vec_reduc_minloc_expr_code (code, TREE_TYPE (rhs1),
+ TREE_TYPE (rhs2));
+ }
+
/* For very wide vectors, try using a smaller vector mode. */
compute_type = type;
if (TYPE_MODE (type) == BLKmode && op)
@@ -496,8 +507,9 @@ expand_vector_operations_1 (gimple_stmt_
|| GET_MODE_CLASS (compute_mode) == MODE_VECTOR_UFRACT
|| GET_MODE_CLASS (compute_mode) == MODE_VECTOR_ACCUM
|| GET_MODE_CLASS (compute_mode) == MODE_VECTOR_UACCUM)
- && op != NULL
- && optab_handler (op, compute_mode) != CODE_FOR_nothing)
+ && ((op != NULL
+ && optab_handler (op, compute_mode) != CODE_FOR_nothing)
+ || icode != CODE_FOR_nothing))
return;
else
/* There is no operation in hardware, so fall back to scalars. */
===================================================================
@@ -3574,6 +3574,22 @@ do_pointer_plus_expr_check:
/* Continue with generic binary expression handling. */
break;
+ case REDUC_MIN_FIRST_LOC_EXPR:
+ case REDUC_MIN_LAST_LOC_EXPR:
+ case REDUC_MAX_FIRST_LOC_EXPR:
+ case REDUC_MAX_LAST_LOC_EXPR:
+ if (!useless_type_conversion_p (lhs_type, rhs2_type)
+ || TYPE_VECTOR_SUBPARTS (lhs_type)
+ != TYPE_VECTOR_SUBPARTS (rhs1_type))
+ {
+ error ("type mismatch in binary expression");
+ debug_generic_stmt (lhs_type);
+ debug_generic_stmt (rhs2_type);
+ return true;
+ }
+
+ return false;
+
default:
gcc_unreachable ();
}
===================================================================
@@ -1187,7 +1187,8 @@ static tree rs6000_gimplify_va_arg (tree
static bool rs6000_must_pass_in_stack (enum machine_mode, const_tree);
static bool rs6000_scalar_mode_supported_p (enum machine_mode);
static bool rs6000_vector_mode_supported_p (enum machine_mode);
-static rtx rs6000_emit_vector_compare_inner (enum rtx_code, rtx, rtx);
+static rtx rs6000_emit_vector_compare_inner (enum rtx_code, rtx, rtx,
+ enum machine_mode);
static rtx rs6000_emit_vector_compare (enum rtx_code, rtx, rtx,
enum machine_mode);
static tree rs6000_stack_protect_fail (void);
@@ -16389,7 +16390,8 @@ output_e500_flip_gt_bit (rtx dst, rtx sr
/* Return insn for VSX or Altivec comparisons. */
static rtx
-rs6000_emit_vector_compare_inner (enum rtx_code code, rtx op0, rtx op1)
+rs6000_emit_vector_compare_inner (enum rtx_code code, rtx op0, rtx op1,
+ enum machine_mode mask_mode)
{
rtx mask;
enum machine_mode mode = GET_MODE (op0);
@@ -16400,9 +16402,15 @@ rs6000_emit_vector_compare_inner (enum r
break;
case GE:
+ case GEF:
if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
return NULL_RTX;
+ case EQF:
+ case GTF:
+ mode = mask_mode;
+ /* Fall through. */
+
case EQ:
case GT:
case GTU:
@@ -16432,7 +16440,7 @@ rs6000_emit_vector_compare (enum rtx_cod
gcc_assert (GET_MODE (op0) == GET_MODE (op1));
/* See if the comparison works as is. */
- mask = rs6000_emit_vector_compare_inner (rcode, op0, op1);
+ mask = rs6000_emit_vector_compare_inner (rcode, op0, op1, dmode);
if (mask)
return mask;
@@ -16448,6 +16456,11 @@ rs6000_emit_vector_compare (enum rtx_cod
swap_operands = true;
try_again = true;
break;
+ case LTF:
+ rcode = GTF;
+ swap_operands = true;
+ try_again = true;
+ break;
case NE:
case UNLE:
case UNLT:
@@ -16481,11 +16494,12 @@ rs6000_emit_vector_compare (enum rtx_cod
case GEU:
case LE:
case LEU:
+ case LEF:
/* Try GT/GTU/LT/LTU OR EQ */
{
rtx c_rtx, eq_rtx;
enum insn_code ior_code;
- enum rtx_code new_code;
+ enum rtx_code new_code, eq_code = EQ;
switch (rcode)
{
@@ -16501,6 +16515,11 @@ rs6000_emit_vector_compare (enum rtx_cod
new_code = LT;
break;
+ case LEF:
+ new_code = LTF;
+ eq_code = EQF;
+ break;
+
case LEU:
new_code = LTU;
break;
@@ -16517,7 +16536,7 @@ rs6000_emit_vector_compare (enum rtx_cod
if (!c_rtx)
return NULL_RTX;
- eq_rtx = rs6000_emit_vector_compare (EQ, op0, op1, dmode);
+ eq_rtx = rs6000_emit_vector_compare (eq_code, op0, op1, dmode);
if (!eq_rtx)
return NULL_RTX;
@@ -16540,7 +16559,7 @@ rs6000_emit_vector_compare (enum rtx_cod
op1 = tmp;
}
- mask = rs6000_emit_vector_compare_inner (rcode, op0, op1);
+ mask = rs6000_emit_vector_compare_inner (rcode, op0, op1, dmode);
if (mask)
return mask;
}
@@ -16558,6 +16577,7 @@ rs6000_emit_vector_cond_expr (rtx dest,
rtx cond, rtx cc_op0, rtx cc_op1)
{
enum machine_mode dest_mode = GET_MODE (dest);
+ enum machine_mode cond_mode = GET_MODE (cc_op0);
enum rtx_code rcode = GET_CODE (cond);
enum machine_mode cc_mode = CCmode;
rtx mask;
@@ -16595,8 +16615,34 @@ rs6000_emit_vector_cond_expr (rtx dest,
default:
break;
- }
+ }
+
+ if (!FLOAT_MODE_P (dest_mode) && FLOAT_MODE_P (cond_mode))
+ {
+ switch (rcode)
+ {
+ case GE:
+ rcode = GEF;
+ break;
+
+ case GT:
+ rcode = GTF;
+ break;
+
+ case LE:
+ rcode = LEF;
+ break;
+
+ case LT:
+ rcode = LTF;
+ break;
+
+ default:
+ break;
+ }
+ }
+
/* Get the vector mask for the given relational operations. */
mask = rs6000_emit_vector_compare (rcode, cc_op0, cc_op1, dest_mode);
===================================================================
@@ -144,6 +144,7 @@
(UNSPEC_VUPKHU_V4SF 326)
(UNSPEC_VUPKLU_V4SF 327)
(UNSPEC_VNMSUBFP 328)
+ (UNSPEC_REDUC_MINLOC 329)
])
(define_constants
@@ -475,6 +476,14 @@
"vcmpeqfp %0,%1,%2"
[(set_attr "type" "veccmp")])
+(define_insn "altivec_eqfv4sf"
+ [(set (match_operand:V4SI 0 "altivec_register_operand" "=v")
+ (eqf:V4SI (match_operand:V4SF 1 "altivec_register_operand" "v")
+ (match_operand:V4SF 2 "altivec_register_operand" "v")))]
+ "VECTOR_UNIT_ALTIVEC_P (V4SImode)"
+ "vcmpeqfp %0,%1,%2"
+ [(set_attr "type" "veccmp")])
+
(define_insn "*altivec_gtv4sf"
[(set (match_operand:V4SF 0 "altivec_register_operand" "=v")
(gt:V4SF (match_operand:V4SF 1 "altivec_register_operand" "v")
@@ -483,6 +492,14 @@
"vcmpgtfp %0,%1,%2"
[(set_attr "type" "veccmp")])
+(define_insn "*altivec_gtfv4sf"
+ [(set (match_operand:V4SI 0 "altivec_register_operand" "=v")
+ (gtf:V4SI (match_operand:V4SF 1 "altivec_register_operand" "v")
+ (match_operand:V4SF 2 "altivec_register_operand" "v")))]
+ "VECTOR_UNIT_ALTIVEC_P (V4SImode)"
+ "vcmpgtfp %0,%1,%2"
+ [(set_attr "type" "veccmp")])
+
(define_insn "*altivec_gev4sf"
[(set (match_operand:V4SF 0 "altivec_register_operand" "=v")
(ge:V4SF (match_operand:V4SF 1 "altivec_register_operand" "v")
@@ -491,6 +508,14 @@
"vcmpgefp %0,%1,%2"
[(set_attr "type" "veccmp")])
+(define_insn "*altivec_gefv4sf"
+ [(set (match_operand:V4SI 0 "altivec_register_operand" "=v")
+ (gef:V4SI (match_operand:V4SF 1 "altivec_register_operand" "v")
+ (match_operand:V4SF 2 "altivec_register_operand" "v")))]
+ "VECTOR_UNIT_ALTIVEC_P (V4SImode)"
+ "vcmpgefp %0,%1,%2"
+ [(set_attr "type" "veccmp")])
+
(define_insn "*altivec_vsel<mode>"
[(set (match_operand:VM 0 "altivec_register_operand" "=v")
(if_then_else:VM
@@ -1942,6 +1967,172 @@
DONE;
}")
+(define_expand "reduc_min_first_loc_v4sfv4si"
+ [(set (match_operand:V4SI 0 "register_operand" "")
+ (unspec:V4SI [(match_operand:V4SF 1 "register_operand" "")
+ (match_operand:V4SI 2 "register_operand" "")]
+ UNSPEC_REDUC_MINLOC))]
+ "TARGET_ALTIVEC"
+ "
+{
+ rtx shift1 = gen_rtx_CONST_INT (QImode, 8);
+ rtx shift2 = gen_rtx_CONST_INT (QImode, 4);
+ rtx vr1 = gen_reg_rtx (V4SFmode);
+ rtx vr2 = gen_reg_rtx (V4SFmode);
+ rtx vr3 = gen_reg_rtx (V4SFmode);
+ rtx mask = gen_reg_rtx (V4SImode);
+ rtx not_mask = gen_reg_rtx (V4SImode);
+ rtx vr4 = gen_reg_rtx (V4SImode);
+ rtx vr5 = gen_reg_rtx (V4SImode);
+ rtx vr6 = gen_reg_rtx (V4SImode);
+ rtx max_val = gen_reg_rtx (V4SImode);
+
+ emit_insn (gen_altivec_vsldoi_v4sf (vr1, operands[1], operands[1], shift1));
+ emit_insn (gen_sminv4sf3 (vr2, vr1, operands[1]));
+ emit_insn (gen_altivec_vsldoi_v4sf (vr3, vr2, vr2, shift2));
+ emit_insn (gen_sminv4sf3 (vr1, vr2, vr3));
+
+ emit_insn (gen_altivec_eqfv4sf (mask, operands[1], vr1));
+
+ emit_insn (gen_andv4si3 (vr4, mask, operands[2]));
+ emit_insn (gen_norv4si3 (not_mask, mask, mask));
+ emit_insn (gen_altivec_vspltisw (max_val, constm1_rtx));
+ emit_insn (gen_andv4si3 (vr5, not_mask, max_val));
+ emit_insn (gen_iorv4si3 (vr6, vr4, vr5));
+
+ emit_insn (gen_altivec_vsldoi_v4si (vr4, vr6, vr6, shift1));
+ emit_insn (gen_uminv4si3 (vr5, vr4, vr6));
+ emit_insn (gen_altivec_vsldoi_v4si (vr6, vr5, vr5, shift2));
+ emit_insn (gen_uminv4si3 (operands[0], vr5, vr6));
+
+ DONE;
+}")
+
+(define_expand "reduc_min_last_loc_v4sfv4si"
+ [(set (match_operand:V4SI 0 "register_operand" "")
+ (unspec:V4SI [(match_operand:V4SF 1 "register_operand" "")
+ (match_operand:V4SI 2 "register_operand" "")]
+ UNSPEC_REDUC_MINLOC))]
+ "TARGET_ALTIVEC"
+ "
+{
+ rtx shift1 = gen_rtx_CONST_INT (QImode, 8);
+ rtx shift2 = gen_rtx_CONST_INT (QImode, 4);
+ rtx vr1 = gen_reg_rtx (V4SFmode);
+ rtx vr2 = gen_reg_rtx (V4SFmode);
+ rtx vr3 = gen_reg_rtx (V4SFmode);
+ rtx mask = gen_reg_rtx (V4SImode);
+ rtx not_mask = gen_reg_rtx (V4SImode);
+ rtx vr4 = gen_reg_rtx (V4SImode);
+ rtx vr5 = gen_reg_rtx (V4SImode);
+ rtx vr6 = gen_reg_rtx (V4SImode);
+ rtx min_val = gen_reg_rtx (V4SImode);
+
+ emit_insn (gen_altivec_vsldoi_v4sf (vr1, operands[1], operands[1], shift1));
+ emit_insn (gen_sminv4sf3 (vr2, vr1, operands[1]));
+ emit_insn (gen_altivec_vsldoi_v4sf (vr3, vr2, vr2, shift2));
+ emit_insn (gen_sminv4sf3 (vr1, vr2, vr3));
+
+ emit_insn (gen_altivec_eqfv4sf (mask, operands[1], vr1));
+
+ emit_insn (gen_andv4si3 (vr4, mask, operands[2]));
+ emit_insn (gen_norv4si3 (not_mask, mask, mask));
+ emit_insn (gen_altivec_vspltisw (min_val, const0_rtx));
+ emit_insn (gen_andv4si3 (vr5, not_mask, min_val));
+ emit_insn (gen_iorv4si3 (vr6, vr4, vr5));
+
+ emit_insn (gen_altivec_vsldoi_v4si (vr4, vr6, vr6, shift1));
+ emit_insn (gen_umaxv4si3 (vr5, vr4, vr6));
+ emit_insn (gen_altivec_vsldoi_v4si (vr6, vr5, vr5, shift2));
+ emit_insn (gen_umaxv4si3 (operands[0], vr5, vr6));
+
+ DONE;
+}")
+
+
+(define_expand "reduc_max_first_loc_v4sfv4si"
+ [(set (match_operand:V4SI 0 "register_operand" "")
+ (unspec:V4SI [(match_operand:V4SF 1 "register_operand" "")
+ (match_operand:V4SI 2 "register_operand" "")]
+ UNSPEC_REDUC_MINLOC))]
+ "TARGET_ALTIVEC"
+ "
+{
+ rtx shift1 = gen_rtx_CONST_INT (QImode, 8);
+ rtx shift2 = gen_rtx_CONST_INT (QImode, 4);
+ rtx vr1 = gen_reg_rtx (V4SFmode);
+ rtx vr2 = gen_reg_rtx (V4SFmode);
+ rtx vr3 = gen_reg_rtx (V4SFmode);
+ rtx mask = gen_reg_rtx (V4SImode);
+ rtx not_mask = gen_reg_rtx (V4SImode);
+ rtx vr4 = gen_reg_rtx (V4SImode);
+ rtx vr5 = gen_reg_rtx (V4SImode);
+ rtx vr6 = gen_reg_rtx (V4SImode);
+ rtx max_val = gen_reg_rtx (V4SImode);
+
+ emit_insn (gen_altivec_vsldoi_v4sf (vr1, operands[1], operands[1], shift1));
+ emit_insn (gen_smaxv4sf3 (vr2, vr1, operands[1]));
+ emit_insn (gen_altivec_vsldoi_v4sf (vr3, vr2, vr2, shift2));
+ emit_insn (gen_smaxv4sf3 (vr1, vr2, vr3));
+
+ emit_insn (gen_altivec_eqfv4sf (mask, operands[1], vr1));
+
+ emit_insn (gen_andv4si3 (vr4, mask, operands[2]));
+ emit_insn (gen_norv4si3 (not_mask, mask, mask));
+ emit_insn (gen_altivec_vspltisw (max_val, constm1_rtx));
+ emit_insn (gen_andv4si3 (vr5, not_mask, max_val));
+ emit_insn (gen_iorv4si3 (vr6, vr4, vr5));
+
+ emit_insn (gen_altivec_vsldoi_v4si (vr4, vr6, vr6, shift1));
+ emit_insn (gen_uminv4si3 (vr5, vr4, vr6));
+ emit_insn (gen_altivec_vsldoi_v4si (vr6, vr5, vr5, shift2));
+ emit_insn (gen_uminv4si3 (operands[0], vr5, vr6));
+
+ DONE;
+}")
+
+(define_expand "reduc_max_last_loc_v4sfv4si"
+ [(set (match_operand:V4SI 0 "register_operand" "")
+ (unspec:V4SI [(match_operand:V4SF 1 "register_operand" "")
+ (match_operand:V4SI 2 "register_operand" "")]
+ UNSPEC_REDUC_MINLOC))]
+ "TARGET_ALTIVEC"
+ "
+{
+ rtx shift1 = gen_rtx_CONST_INT (QImode, 8);
+ rtx shift2 = gen_rtx_CONST_INT (QImode, 4);
+ rtx vr1 = gen_reg_rtx (V4SFmode);
+ rtx vr2 = gen_reg_rtx (V4SFmode);
+ rtx vr3 = gen_reg_rtx (V4SFmode);
+ rtx mask = gen_reg_rtx (V4SImode);
+ rtx not_mask = gen_reg_rtx (V4SImode);
+ rtx vr4 = gen_reg_rtx (V4SImode);
+ rtx vr5 = gen_reg_rtx (V4SImode);
+ rtx vr6 = gen_reg_rtx (V4SImode);
+ rtx min_val = gen_reg_rtx (V4SImode);
+
+ emit_insn (gen_altivec_vsldoi_v4sf (vr1, operands[1], operands[1], shift1));
+ emit_insn (gen_smaxv4sf3 (vr2, vr1, operands[1]));
+ emit_insn (gen_altivec_vsldoi_v4sf (vr3, vr2, vr2, shift2));
+ emit_insn (gen_smaxv4sf3 (vr1, vr2, vr3));
+
+ emit_insn (gen_altivec_eqfv4sf (mask, operands[1], vr1));
+
+ emit_insn (gen_andv4si3 (vr4, mask, operands[2]));
+ emit_insn (gen_norv4si3 (not_mask, mask, mask));
+ emit_insn (gen_altivec_vspltisw (min_val, const0_rtx));
+ emit_insn (gen_andv4si3 (vr5, not_mask, min_val));
+ emit_insn (gen_iorv4si3 (vr6, vr4, vr5));
+
+ emit_insn (gen_altivec_vsldoi_v4si (vr4, vr6, vr6, shift1));
+ emit_insn (gen_umaxv4si3 (vr5, vr4, vr6));
+ emit_insn (gen_altivec_vsldoi_v4si (vr6, vr5, vr5, shift2));
+ emit_insn (gen_umaxv4si3 (operands[0], vr5, vr6));
+
+ DONE;
+}")
+
+
(define_expand "neg<mode>2"
[(use (match_operand:VI 0 "register_operand" ""))
(use (match_operand:VI 1 "register_operand" ""))]
@@ -2802,3 +2993,39 @@
emit_insn (gen_altivec_vcfux (operands[0], tmp, const0_rtx));
DONE;
}")
+
+(define_expand "vcondcv4sfv4si"
+ [(set (match_operand:V4SI 0 "vint_operand" "")
+ (if_then_else:V4SI
+ (match_operator 3 "comparison_operator"
+ [(match_operand:V4SF 4 "vfloat_operand" "")
+ (match_operand:V4SF 5 "vfloat_operand" "")])
+ (match_operand:V4SI 1 "vint_operand" "")
+ (match_operand:V4SI 2 "vint_operand" "")))]
+ "TARGET_ALTIVEC"
+ "
+{
+ if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
+ operands[3], operands[4], operands[5]))
+ DONE;
+ else
+ FAIL;
+}")
+
+(define_expand "vcondcuv4sfv4si"
+ [(set (match_operand:V4SI 0 "vint_operand" "")
+ (if_then_else:V4SI
+ (match_operator 3 "comparison_operator"
+ [(match_operand:V4SF 4 "vfloat_operand" "")
+ (match_operand:V4SF 5 "vfloat_operand" "")])
+ (match_operand:V4SI 1 "vint_operand" "")
+ (match_operand:V4SI 2 "vint_operand" "")))]
+ "TARGET_ALTIVEC"
+ "
+{
+ if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
+ operands[3], operands[4], operands[5]))
+ DONE;
+ else
+ FAIL;
+}")
===================================================================
@@ -146,6 +146,18 @@ vect_get_and_check_slp_defs (loop_vec_in
return false;
}
+ if (def_stmt && vinfo_for_stmt (def_stmt)
+ && STMT_VINFO_COMPOUND_PATTERN (vinfo_for_stmt (def_stmt)))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump, "Build SLP failed: compound pattern ");
+ print_gimple_stmt (vect_dump, def_stmt, 0, TDF_SLIM);
+ }
+
+ return false;
+ }
+
/* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
from the pattern. Check that all the stmts of the node are in the
pattern. */