@@ -5783,7 +5783,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
bool is_simple_use;
int i;
int ncopies;
- stmt_vec_info prev_phi_info;
bool single_defuse_cycle = false;
int j;
tree ops[3];
@@ -5811,207 +5810,15 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcc_assert (slp_node
&& REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info);
- if (gphi *phi = dyn_cast <gphi *> (stmt_info->stmt))
+ if (is_a <gphi *> (stmt_info->stmt))
{
- tree phi_result = gimple_phi_result (phi);
/* Analysis is fully done on the reduction stmt invocation. */
- if (! vec_stmt)
- {
- if (slp_node)
- slp_node_instance->reduc_phis = slp_node;
-
- STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
- return true;
- }
-
- if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION)
- /* Leave the scalar phi in place. Note that checking
- STMT_VINFO_VEC_REDUCTION_TYPE (as below) only works
- for reductions involving a single statement. */
- return true;
-
- stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
- reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info);
-
- if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
- == EXTRACT_LAST_REDUCTION)
- /* Leave the scalar phi in place. */
- return true;
-
- if (gassign *reduc_stmt = dyn_cast <gassign *> (reduc_stmt_info->stmt))
- for (unsigned k = 1; k < gimple_num_ops (reduc_stmt); ++k)
- {
- tree op = gimple_op (reduc_stmt, k);
- if (op == phi_result)
- continue;
- if (k == 1 && gimple_assign_rhs_code (reduc_stmt) == COND_EXPR)
- continue;
- bool is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt);
- gcc_assert (is_simple_use);
- if (dt == vect_constant_def || dt == vect_external_def)
- continue;
- if (!vectype_in
- || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
- < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (op)))))
- vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op));
- break;
- }
- /* For a nested cycle we might end up with an operation like
- phi_result * phi_result. */
- if (!vectype_in)
- vectype_in = STMT_VINFO_VECTYPE (stmt_info);
- gcc_assert (vectype_in);
+ gcc_assert (! vec_stmt);
if (slp_node)
- {
- /* The size vect_schedule_slp_instance computes is off for us. */
- vec_num = vect_get_num_vectors
- (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
- * SLP_TREE_SCALAR_STMTS (slp_node).length (), vectype_in);
- ncopies = 1;
- }
- else
- {
- vec_num = 1;
- ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
- }
-
- /* Check whether we can use a single PHI node and accumulate
- vectors to one before the backedge. */
- stmt_vec_info use_stmt_info;
- if (ncopies > 1
- && STMT_VINFO_RELEVANT (reduc_stmt_info) <= vect_used_only_live
- && (use_stmt_info = loop_vinfo->lookup_single_use (phi_result))
- && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info)
- || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info))
- && vect_stmt_to_vectorize (use_stmt_info) == reduc_stmt_info)
- {
- single_defuse_cycle = true;
- ncopies = 1;
- }
-
- /* Create the destination vector */
- tree vec_dest = vect_create_destination_var (phi_result, vectype_out);
-
- /* Get the loop-entry arguments. */
- tree vec_initial_def;
- auto_vec<tree> vec_initial_defs;
- if (slp_node)
- {
- vec_initial_defs.reserve (vec_num);
- gcc_assert (slp_node == slp_node_instance->reduc_phis);
- stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info);
- tree neutral_op
- = neutral_op_for_slp_reduction (slp_node,
- STMT_VINFO_REDUC_CODE
- (first ? first : reduc_stmt_info),
- first != NULL);
- get_initial_defs_for_reduction (slp_node_instance->reduc_phis,
- &vec_initial_defs, vec_num,
- first != NULL, neutral_op);
- }
- else
- {
- /* Get at the scalar def before the loop, that defines the initial
- value of the reduction variable. */
- tree initial_def = PHI_ARG_DEF_FROM_EDGE (phi,
- loop_preheader_edge (loop));
- /* Optimize: if initial_def is for REDUC_MAX smaller than the base
- and we can't use zero for induc_val, use initial_def. Similarly
- for REDUC_MIN and initial_def larger than the base. */
- if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
- == INTEGER_INDUC_COND_REDUCTION)
- {
- tree induc_val
- = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info);
- if (TREE_CODE (initial_def) == INTEGER_CST
- && (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
- == INTEGER_INDUC_COND_REDUCTION)
- && !integer_zerop (induc_val)
- && (((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info)
- == MAX_EXPR)
- && tree_int_cst_lt (initial_def, induc_val))
- || ((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info)
- == MIN_EXPR)
- && tree_int_cst_lt (induc_val, initial_def))))
- {
- induc_val = initial_def;
- /* Communicate we used the initial_def to epilouge
- generation. */
- STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info)
- = NULL_TREE;
- }
- vec_initial_def = build_vector_from_val (vectype_out, induc_val);
- }
- else if (nested_cycle)
- {
- /* Do not use an adjustment def as that case is not supported
- correctly if ncopies is not one. */
- vec_initial_def = vect_get_vec_def_for_operand (initial_def,
- reduc_stmt_info);
- }
- else
- {
- tree adjustment_def = NULL_TREE;
- tree *adjustment_defp = &adjustment_def;
- enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_stmt_info);
- /* ??? For the outer loop PHI we have to do a bit of searching
- to find the stmt with the code. reduc_stmt_info here is the
- loop-closed PHI of the inner reduction which means we can look
- at its single-arg def. */
- if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
- {
- tree def = gimple_phi_arg_def
- (as_a <gphi *> (reduc_stmt_info->stmt), 0);
- code = STMT_VINFO_REDUC_CODE
- (vect_stmt_to_vectorize (loop_vinfo->lookup_def (def)));
- adjustment_defp = NULL;
- }
- vec_initial_def
- = get_initial_def_for_reduction (reduc_stmt_info, code,
- initial_def, adjustment_defp);
- STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_stmt_info)
- = adjustment_def;
- }
- vec_initial_defs.create (1);
- vec_initial_defs.quick_push (vec_initial_def);
- }
-
- /* Generate the reduction PHIs upfront. */
- prev_phi_info = NULL;
- for (i = 0; i < vec_num; i++)
- {
- tree vec_init_def = vec_initial_defs[i];
- for (j = 0; j < ncopies; j++)
- {
- /* Create the reduction-phi that defines the reduction
- operand. */
- gphi *new_phi = create_phi_node (vec_dest, loop->header);
- stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
-
- /* Set the loop-entry arg of the reduction-phi. */
- if (j != 0 && nested_cycle)
- vec_init_def = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_init_def);
- add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
- UNKNOWN_LOCATION);
-
- /* The loop-latch arg is set in epilogue processing. */
-
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
- else
- {
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info)
- = *vec_stmt = new_phi_info;
- else
- STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info;
- prev_phi_info = new_phi_info;
- }
- }
- }
+ slp_node_instance->reduc_phis = slp_node;
+ STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
return true;
}
@@ -6841,7 +6648,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
prev_stmt_info = NULL;
- prev_phi_info = NULL;
if (!slp_node)
{
vec_oprnds0.create (1);
@@ -6998,7 +6804,221 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
return true;
}
-/* Vectorizes LC PHIs of nested cycles (sofar). */
+/* Transform phase of a cycle PHI. */
+
+bool
+vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
+ slp_tree slp_node, slp_instance slp_node_instance)
+{
+ tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+ tree vectype_in = NULL_TREE;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ enum vect_def_type dt;
+ int i;
+ int ncopies;
+ stmt_vec_info prev_phi_info;
+ int j;
+ bool nested_cycle = false;
+ int vec_num;
+
+ if (nested_in_vect_loop_p (loop, stmt_info))
+ {
+ loop = loop->inner;
+ nested_cycle = true;
+ }
+
+ gphi *phi = as_a <gphi *> (stmt_info->stmt);
+ tree phi_result = gimple_phi_result (phi);
+
+ if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION)
+ /* Leave the scalar phi in place. Note that checking
+ STMT_VINFO_VEC_REDUCTION_TYPE (as below) only works
+ for reductions involving a single statement. */
+ return true;
+
+ stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
+ reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info);
+
+ if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
+ == EXTRACT_LAST_REDUCTION)
+ /* Leave the scalar phi in place. */
+ return true;
+
+ if (gassign *reduc_stmt = dyn_cast <gassign *> (reduc_stmt_info->stmt))
+ for (unsigned k = 1; k < gimple_num_ops (reduc_stmt); ++k)
+ {
+ tree op = gimple_op (reduc_stmt, k);
+ if (op == phi_result)
+ continue;
+ if (k == 1 && gimple_assign_rhs_code (reduc_stmt) == COND_EXPR)
+ continue;
+ bool is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt);
+ gcc_assert (is_simple_use);
+ if (dt == vect_constant_def || dt == vect_external_def)
+ continue;
+ if (!vectype_in
+ || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
+ < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (op)))))
+ vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op));
+ break;
+ }
+ /* For a nested cycle we might end up with an operation like
+ phi_result * phi_result. */
+ if (!vectype_in)
+ vectype_in = STMT_VINFO_VECTYPE (stmt_info);
+ gcc_assert (vectype_in);
+
+ if (slp_node)
+ {
+ /* The size vect_schedule_slp_instance computes is off for us. */
+ vec_num = vect_get_num_vectors
+ (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ * SLP_TREE_SCALAR_STMTS (slp_node).length (), vectype_in);
+ ncopies = 1;
+ }
+ else
+ {
+ vec_num = 1;
+ ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
+ }
+
+ /* Check whether we can use a single PHI node and accumulate
+ vectors to one before the backedge. */
+ stmt_vec_info use_stmt_info;
+ if (ncopies > 1
+ && STMT_VINFO_RELEVANT (reduc_stmt_info) <= vect_used_only_live
+ && (use_stmt_info = loop_vinfo->lookup_single_use (phi_result))
+ && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info)
+ || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info))
+ && vect_stmt_to_vectorize (use_stmt_info) == reduc_stmt_info)
+ ncopies = 1;
+
+ /* Create the destination vector */
+ tree vec_dest = vect_create_destination_var (phi_result, vectype_out);
+
+ /* Get the loop-entry arguments. */
+ tree vec_initial_def;
+ auto_vec<tree> vec_initial_defs;
+ if (slp_node)
+ {
+ vec_initial_defs.reserve (vec_num);
+ gcc_assert (slp_node == slp_node_instance->reduc_phis);
+ stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info);
+ tree neutral_op
+ = neutral_op_for_slp_reduction (slp_node,
+ STMT_VINFO_REDUC_CODE
+ (first ? first : reduc_stmt_info),
+ first != NULL);
+ get_initial_defs_for_reduction (slp_node_instance->reduc_phis,
+ &vec_initial_defs, vec_num,
+ first != NULL, neutral_op);
+ }
+ else
+ {
+ /* Get at the scalar def before the loop, that defines the initial
+ value of the reduction variable. */
+ tree initial_def = PHI_ARG_DEF_FROM_EDGE (phi,
+ loop_preheader_edge (loop));
+ /* Optimize: if initial_def is for REDUC_MAX smaller than the base
+ and we can't use zero for induc_val, use initial_def. Similarly
+ for REDUC_MIN and initial_def larger than the base. */
+ if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
+ == INTEGER_INDUC_COND_REDUCTION)
+ {
+ tree induc_val
+ = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info);
+ if (TREE_CODE (initial_def) == INTEGER_CST
+ && (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
+ == INTEGER_INDUC_COND_REDUCTION)
+ && !integer_zerop (induc_val)
+ && (((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info)
+ == MAX_EXPR)
+ && tree_int_cst_lt (initial_def, induc_val))
+ || ((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info)
+ == MIN_EXPR)
+ && tree_int_cst_lt (induc_val, initial_def))))
+ {
+ induc_val = initial_def;
+ /* Communicate we used the initial_def to epilouge
+ generation. */
+ STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info)
+ = NULL_TREE;
+ }
+ vec_initial_def = build_vector_from_val (vectype_out, induc_val);
+ }
+ else if (nested_cycle)
+ {
+ /* Do not use an adjustment def as that case is not supported
+ correctly if ncopies is not one. */
+ vec_initial_def = vect_get_vec_def_for_operand (initial_def,
+ reduc_stmt_info);
+ }
+ else
+ {
+ tree adjustment_def = NULL_TREE;
+ tree *adjustment_defp = &adjustment_def;
+ enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_stmt_info);
+ /* ??? For the outer loop PHI we have to do a bit of searching
+ to find the stmt with the code. reduc_stmt_info here is the
+ loop-closed PHI of the inner reduction which means we can look
+ at its single-arg def. */
+ if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
+ {
+ tree def = gimple_phi_arg_def
+ (as_a <gphi *> (reduc_stmt_info->stmt), 0);
+ code = STMT_VINFO_REDUC_CODE
+ (vect_stmt_to_vectorize (loop_vinfo->lookup_def (def)));
+ adjustment_defp = NULL;
+ }
+ vec_initial_def
+ = get_initial_def_for_reduction (reduc_stmt_info, code,
+ initial_def, adjustment_defp);
+ STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_stmt_info)
+ = adjustment_def;
+ }
+ vec_initial_defs.create (1);
+ vec_initial_defs.quick_push (vec_initial_def);
+ }
+
+ /* Generate the reduction PHIs upfront. */
+ prev_phi_info = NULL;
+ for (i = 0; i < vec_num; i++)
+ {
+ tree vec_init_def = vec_initial_defs[i];
+ for (j = 0; j < ncopies; j++)
+ {
+ /* Create the reduction-phi that defines the reduction
+ operand. */
+ gphi *new_phi = create_phi_node (vec_dest, loop->header);
+ stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
+
+ /* Set the loop-entry arg of the reduction-phi. */
+ if (j != 0 && nested_cycle)
+ vec_init_def = vect_get_vec_def_for_stmt_copy (loop_vinfo,
+ vec_init_def);
+ add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
+ UNKNOWN_LOCATION);
+
+ /* The loop-latch arg is set in epilogue processing. */
+
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
+ else
+ {
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_phi_info;
+ else
+ STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info;
+ prev_phi_info = new_phi_info;
+ }
+ }
+ }
+
+ return true;
+}
+
+/* Vectorizes LC PHIs. */
bool
vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
@@ -10824,6 +10824,12 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcc_assert (done);
break;
+ case cycle_phi_info_type:
+ done = vect_transform_cycle_phi (stmt_info, &vec_stmt, slp_node,
+ slp_node_instance);
+ gcc_assert (done);
+ break;
+
case lc_phi_info_type:
done = vectorizable_lc_phi (stmt_info, &vec_stmt, slp_node);
gcc_assert (done);
@@ -694,6 +694,7 @@ enum stmt_vec_info_type {
type_promotion_vec_info_type,
type_demotion_vec_info_type,
type_conversion_vec_info_type,
+ cycle_phi_info_type,
lc_phi_info_type,
loop_exit_ctrl_vec_info_type
};
@@ -1658,6 +1659,8 @@ extern bool vectorizable_reduction (stmt_vec_info, gimple_stmt_iterator *,
extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *,
stmt_vec_info *, slp_tree,
stmt_vector_for_cost *);
+extern bool vect_transform_cycle_phi (stmt_vec_info, stmt_vec_info *,
+ slp_tree, slp_instance);
extern bool vectorizable_lc_phi (stmt_vec_info, stmt_vec_info *, slp_tree);
extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code);
extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,