@@ -2536,7 +2536,7 @@ report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
overflow must wrap. */
-static bool
+bool
needs_fold_left_reduction_p (tree type, tree_code code)
{
/* CHECKME: check for !flag_finite_math_only too? */
@@ -2888,13 +2888,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
op1 = gimple_assign_rhs2 (def_stmt);
op2 = gimple_assign_rhs3 (def_stmt);
}
- else if (!commutative_tree_code (code) || !associative_tree_code (code))
- {
- if (dump_enabled_p ())
- report_vect_op (MSG_MISSED_OPTIMIZATION, def_stmt,
- "reduction: not commutative/associative: ");
- return NULL;
- }
else if (get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS)
{
op1 = gimple_assign_rhs1 (def_stmt);
@@ -2917,18 +2910,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
return NULL;
}
- /* Check whether it's ok to change the order of the computation.
- Generally, when vectorizing a reduction we change the order of the
- computation. This may change the behavior of the program in some
- cases, so we need to check that this is ok. One exception is when
- vectorizing an outer-loop: the inner-loop is executed sequentially,
- and therefore vectorizing reductions in the inner-loop during
- outer-loop vectorization is safe. */
- tree type = TREE_TYPE (gimple_assign_lhs (def_stmt));
- if (STMT_VINFO_REDUC_TYPE (phi_info) == TREE_CODE_REDUCTION
- && needs_fold_left_reduction_p (type, code))
- STMT_VINFO_REDUC_TYPE (phi_info) = FOLD_LEFT_REDUCTION;
-
/* Reduction is safe. We're dealing with one of the following:
1) integer arithmetic and no trapv
2) floating point arithmetic, and special flags permit this optimization
@@ -5633,7 +5614,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum tree_code code;
- internal_fn reduc_fn;
int op_type;
enum vect_def_type dt, cond_reduc_dt = vect_unknown_def_type;
stmt_vec_info cond_stmt_vinfo = NULL;
@@ -5872,19 +5852,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
operation in the reduction meta. */
STMT_VINFO_REDUC_IDX (reduc_info) = reduc_index;
- /* When vectorizing a reduction chain w/o SLP the reduction PHI is not
- directy used in stmt. */
- if (reduc_index == -1)
- {
- if (STMT_VINFO_REDUC_TYPE (phi_info) == FOLD_LEFT_REDUCTION)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "in-order reduction chain without SLP.\n");
- return false;
- }
- }
-
if (!(reduc_index == -1
|| dts[reduc_index] == vect_reduction_def
|| dts[reduc_index] == vect_nested_cycle
@@ -6047,17 +6014,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
double_reduc = true;
}
- vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
- if ((double_reduc || reduction_type != TREE_CODE_REDUCTION)
- && ncopies > 1)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "multiple types in double reduction or condition "
- "reduction.\n");
- return false;
- }
-
/* 4.2. Check support for the epilog operation.
If STMT represents a reduction pattern, then the type of the
@@ -6093,38 +6049,75 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
(and also the same tree-code) when generating the epilog code and
when generating the code inside the loop. */
- enum tree_code orig_code;
- if (orig_stmt_info
- && (reduction_type == TREE_CODE_REDUCTION
- || reduction_type == FOLD_LEFT_REDUCTION))
+ vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
+ enum tree_code orig_code = ERROR_MARK;
+ if (reduction_type == CONST_COND_REDUCTION
+ || reduction_type == INTEGER_INDUC_COND_REDUCTION)
{
- /* This is a reduction pattern: get the vectype from the type of the
- reduction variable, and get the tree-code from orig_stmt. */
- orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt);
- gcc_assert (vectype_out);
+ /* For simple condition reductions, replace with the actual expression
+ we want to base our reduction around. */
+ orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info);
+ gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
}
- else
+ else if (reduction_type == COND_REDUCTION)
+ orig_code = COND_EXPR;
+ else if (reduction_type == TREE_CODE_REDUCTION
+ || reduction_type == FOLD_LEFT_REDUCTION)
{
- /* Regular reduction: use the same vectype and tree-code as used for
- the vector code inside the loop can be used for the epilog code. */
- orig_code = code;
-
- if (code == MINUS_EXPR)
+ if (orig_stmt_info)
+ orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt);
+ else
+ orig_code = code;
+ gcc_assert (vectype_out);
+ if (orig_code == MINUS_EXPR)
orig_code = PLUS_EXPR;
+ }
+ STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
- /* For simple condition reductions, replace with the actual expression
- we want to base our reduction around. */
- if (reduction_type == CONST_COND_REDUCTION
- || reduction_type == INTEGER_INDUC_COND_REDUCTION)
+ if (reduction_type == TREE_CODE_REDUCTION)
+ {
+ /* Check whether it's ok to change the order of the computation.
+ Generally, when vectorizing a reduction we change the order of the
+ computation. This may change the behavior of the program in some
+ cases, so we need to check that this is ok. One exception is when
+ vectorizing an outer-loop: the inner-loop is executed sequentially,
+ and therefore vectorizing reductions in the inner-loop during
+ outer-loop vectorization is safe. */
+ if (needs_fold_left_reduction_p (scalar_type, orig_code))
+ {
+ STMT_VINFO_REDUC_TYPE (reduc_info)
+ = reduction_type = FOLD_LEFT_REDUCTION;
+ /* When vectorizing a reduction chain w/o SLP the reduction PHI is not
+ directy used in stmt. */
+ if (reduc_index == -1)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "in-order reduction chain without SLP.\n");
+ return false;
+ }
+ }
+ else if (!commutative_tree_code (orig_code)
+ || !associative_tree_code (orig_code))
{
- orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info);
- gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "reduction: not commutative/associative");
+ return false;
}
}
- STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
- reduc_fn = IFN_LAST;
+ if ((double_reduc || reduction_type != TREE_CODE_REDUCTION)
+ && ncopies > 1)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "multiple types in double reduction or condition "
+ "reduction or fold-left reduction.\n");
+ return false;
+ }
+ internal_fn reduc_fn = IFN_LAST;
if (reduction_type == TREE_CODE_REDUCTION
|| reduction_type == FOLD_LEFT_REDUCTION
|| reduction_type == INTEGER_INDUC_COND_REDUCTION
@@ -833,19 +833,8 @@ vect_convert_output (stmt_vec_info stmt_info, tree type, gimple *pattern_stmt,
/* Return true if STMT_VINFO describes a reduction for which reassociation
is allowed. If STMT_INFO is part of a group, assume that it's part of
a reduction chain and optimistically assume that all statements
- except the last allow reassociation. */
-
-static bool
-vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo)
-{
- if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
- return (STMT_VINFO_REDUC_TYPE (STMT_VINFO_REDUC_DEF (stmt_vinfo))
- != FOLD_LEFT_REDUCTION);
- else
- return REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL;
-}
-
-/* As above, but also require it to have code CODE and to be a reduction
+ except the last allow reassociation.
+ Also require it to have code CODE and to be a reduction
in the outermost loop. When returning true, store the operands in
*OP0_OUT and *OP1_OUT. */
@@ -867,7 +856,13 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_info, tree_code code,
if (loop && nested_in_vect_loop_p (loop, stmt_info))
return false;
- if (!vect_reassociating_reduction_p (stmt_info))
+ if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+ {
+ if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
+ code))
+ return false;
+ }
+ else if (REDUC_GROUP_FIRST_ELEMENT (stmt_info) == NULL)
return false;
*op0_out = gimple_assign_rhs1 (assign);
@@ -1632,6 +1632,7 @@ extern widest_int vect_iv_limit_for_full_masking (loop_vec_info loop_vinfo);
/* Used in gimple-loop-interchange.c and tree-parloops.c. */
extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
enum tree_code);
+extern bool needs_fold_left_reduction_p (tree, tree_code);
/* Drive for loop analysis stage. */
extern opt_loop_vec_info vect_analyze_loop (class loop *,
loop_vec_info,