| Message ID | 87va95osxd.fsf@arm.com |
|---|---|
| State | New |
| Headers | show |
| Series | Remove vinfo_for_stmt etc. | expand |
On Tue, Jul 24, 2018 at 11:59 AM Richard Sandiford <richard.sandiford@arm.com> wrote: > > This patch changes LOOP_VINFO_REDUCTIONS from an auto_vec<gimple *> > to an auto_vec<stmt_vec_info>. It also changes the associated > vect_force_simple_reduction so that it takes and returns stmt_vec_infos > instead of gimple stmts. OK. Highlights that reduction detection needs refactoring to be usable outside of the vectorizer (see tree-parloops.c). Exposing vinfos doesn't make the situation better here... Richard. > > 2018-07-24 Richard Sandiford <richard.sandiford@arm.com> > > gcc/ > * tree-vectorizer.h (_loop_vec_info::reductions): Change from an > auto_vec<gimple *> to an auto_vec<stmt_vec_info>. > (vect_force_simple_reduction): Take and return stmt_vec_infos rather > than gimple stmts. > * tree-parloops.c (valid_reduction_p): Take a stmt_vec_info instead > of a gimple stmt. > (gather_scalar_reductions): Update after above interface changes. > * tree-vect-loop.c (vect_analyze_scalar_cycles_1): Likewise. > (vect_is_simple_reduction): Take and return stmt_vec_infos rather > than gimple stmts. > (vect_force_simple_reduction): Likewise. > * tree-vect-patterns.c (vect_pattern_recog_1): Update use of > LOOP_VINFO_REDUCTIONS. > * tree-vect-slp.c (vect_analyze_slp_instance): Likewise. > > Index: gcc/tree-vectorizer.h > =================================================================== > --- gcc/tree-vectorizer.h 2018-07-24 10:22:53.909100298 +0100 > +++ gcc/tree-vectorizer.h 2018-07-24 10:22:57.277070390 +0100 > @@ -475,7 +475,7 @@ typedef struct _loop_vec_info : public v > auto_vec<gimple *> may_misalign_stmts; > > /* Reduction cycles detected in the loop. Used in loop-aware SLP. */ > - auto_vec<gimple *> reductions; > + auto_vec<stmt_vec_info> reductions; > > /* All reduction chains in the loop, represented by the first > stmt in the chain. */ > @@ -1627,8 +1627,8 @@ extern tree vect_create_addr_base_for_ve > > /* In tree-vect-loop.c. */ > /* FORNOW: Used in tree-parloops.c. */ > -extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *, > - bool *, bool); > +extern stmt_vec_info vect_force_simple_reduction (loop_vec_info, stmt_vec_info, > + bool *, bool); > /* Used in gimple-loop-interchange.c. */ > extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree, > enum tree_code); > Index: gcc/tree-parloops.c > =================================================================== > --- gcc/tree-parloops.c 2018-06-27 10:27:09.778650686 +0100 > +++ gcc/tree-parloops.c 2018-07-24 10:22:57.273070426 +0100 > @@ -2570,15 +2570,14 @@ set_reduc_phi_uids (reduction_info **slo > return 1; > } > > -/* Return true if the type of reduction performed by STMT is suitable > +/* Return true if the type of reduction performed by STMT_INFO is suitable > for this pass. */ > > static bool > -valid_reduction_p (gimple *stmt) > +valid_reduction_p (stmt_vec_info stmt_info) > { > /* Parallelization would reassociate the operation, which isn't > allowed for in-order reductions. */ > - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); > vect_reduction_type reduc_type = STMT_VINFO_REDUC_TYPE (stmt_info); > return reduc_type != FOLD_LEFT_REDUCTION; > } > @@ -2615,10 +2614,11 @@ gather_scalar_reductions (loop_p loop, r > if (simple_iv (loop, loop, res, &iv, true)) > continue; > > - gimple *reduc_stmt > - = vect_force_simple_reduction (simple_loop_info, phi, > + stmt_vec_info reduc_stmt_info > + = vect_force_simple_reduction (simple_loop_info, > + simple_loop_info->lookup_stmt (phi), > &double_reduc, true); > - if (!reduc_stmt || !valid_reduction_p (reduc_stmt)) > + if (!reduc_stmt_info || !valid_reduction_p (reduc_stmt_info)) > continue; > > if (double_reduc) > @@ -2627,11 +2627,11 @@ gather_scalar_reductions (loop_p loop, r > continue; > > double_reduc_phis.safe_push (phi); > - double_reduc_stmts.safe_push (reduc_stmt); > + double_reduc_stmts.safe_push (reduc_stmt_info->stmt); > continue; > } > > - build_new_reduction (reduction_list, reduc_stmt, phi); > + build_new_reduction (reduction_list, reduc_stmt_info->stmt, phi); > } > delete simple_loop_info; > > @@ -2661,12 +2661,15 @@ gather_scalar_reductions (loop_p loop, r > &iv, true)) > continue; > > - gimple *inner_reduc_stmt > - = vect_force_simple_reduction (simple_loop_info, inner_phi, > + stmt_vec_info inner_phi_info > + = simple_loop_info->lookup_stmt (inner_phi); > + stmt_vec_info inner_reduc_stmt_info > + = vect_force_simple_reduction (simple_loop_info, > + inner_phi_info, > &double_reduc, true); > gcc_assert (!double_reduc); > - if (inner_reduc_stmt == NULL > - || !valid_reduction_p (inner_reduc_stmt)) > + if (!inner_reduc_stmt_info > + || !valid_reduction_p (inner_reduc_stmt_info)) > continue; > > build_new_reduction (reduction_list, double_reduc_stmts[i], phi); > Index: gcc/tree-vect-loop.c > =================================================================== > --- gcc/tree-vect-loop.c 2018-07-24 10:22:53.909100298 +0100 > +++ gcc/tree-vect-loop.c 2018-07-24 10:22:57.273070426 +0100 > @@ -546,7 +546,6 @@ vect_analyze_scalar_cycles_1 (loop_vec_i > gimple *phi = worklist.pop (); > tree def = PHI_RESULT (phi); > stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi); > - gimple *reduc_stmt; > > if (dump_enabled_p ()) > { > @@ -557,9 +556,10 @@ vect_analyze_scalar_cycles_1 (loop_vec_i > gcc_assert (!virtual_operand_p (def) > && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type); > > - reduc_stmt = vect_force_simple_reduction (loop_vinfo, phi, > - &double_reduc, false); > - if (reduc_stmt) > + stmt_vec_info reduc_stmt_info > + = vect_force_simple_reduction (loop_vinfo, stmt_vinfo, > + &double_reduc, false); > + if (reduc_stmt_info) > { > if (double_reduc) > { > @@ -568,8 +568,8 @@ vect_analyze_scalar_cycles_1 (loop_vec_i > "Detected double reduction.\n"); > > STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def; > - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) = > - vect_double_reduction_def; > + STMT_VINFO_DEF_TYPE (reduc_stmt_info) > + = vect_double_reduction_def; > } > else > { > @@ -580,8 +580,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_i > "Detected vectorizable nested cycle.\n"); > > STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle; > - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) = > - vect_nested_cycle; > + STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_nested_cycle; > } > else > { > @@ -590,13 +589,13 @@ vect_analyze_scalar_cycles_1 (loop_vec_i > "Detected reduction.\n"); > > STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def; > - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) = > - vect_reduction_def; > + STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def; > /* Store the reduction cycles for possible vectorization in > loop-aware SLP if it was not detected as reduction > chain. */ > - if (! REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (reduc_stmt))) > - LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt); > + if (! REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info)) > + LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push > + (reduc_stmt_info); > } > } > } > @@ -2530,8 +2529,8 @@ vect_is_slp_reduction (loop_vec_info loo > struct loop *loop = (gimple_bb (phi))->loop_father; > struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info); > enum tree_code code; > - gimple *current_stmt = NULL, *loop_use_stmt = NULL, *first, *next_stmt; > - stmt_vec_info use_stmt_info, current_stmt_info; > + gimple *loop_use_stmt = NULL, *first, *next_stmt; > + stmt_vec_info use_stmt_info, current_stmt_info = NULL; > tree lhs; > imm_use_iterator imm_iter; > use_operand_p use_p; > @@ -2593,9 +2592,8 @@ vect_is_slp_reduction (loop_vec_info loo > > /* Insert USE_STMT into reduction chain. */ > use_stmt_info = loop_info->lookup_stmt (loop_use_stmt); > - if (current_stmt) > + if (current_stmt_info) > { > - current_stmt_info = vinfo_for_stmt (current_stmt); > REDUC_GROUP_NEXT_ELEMENT (current_stmt_info) = loop_use_stmt; > REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) > = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); > @@ -2604,7 +2602,7 @@ vect_is_slp_reduction (loop_vec_info loo > REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) = loop_use_stmt; > > lhs = gimple_assign_lhs (loop_use_stmt); > - current_stmt = loop_use_stmt; > + current_stmt_info = use_stmt_info; > size++; > } > > @@ -2614,7 +2612,7 @@ vect_is_slp_reduction (loop_vec_info loo > /* Swap the operands, if needed, to make the reduction operand be the second > operand. */ > lhs = PHI_RESULT (phi); > - next_stmt = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt)); > + next_stmt = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); > while (next_stmt) > { > if (gimple_assign_rhs2 (next_stmt) == lhs) > @@ -2671,7 +2669,7 @@ vect_is_slp_reduction (loop_vec_info loo > } > > /* Save the chain for further analysis in SLP detection. */ > - first = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt)); > + first = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); > LOOP_VINFO_REDUCTION_CHAINS (loop_info).safe_push (first); > REDUC_GROUP_SIZE (vinfo_for_stmt (first)) = size; > > @@ -2867,15 +2865,16 @@ check_reduction_path (dump_user_location > > */ > > -static gimple * > -vect_is_simple_reduction (loop_vec_info loop_info, gimple *phi, > +static stmt_vec_info > +vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, > bool *double_reduc, > bool need_wrapping_integral_overflow, > enum vect_reduction_type *v_reduc_type) > { > + gphi *phi = as_a <gphi *> (phi_info->stmt); > struct loop *loop = (gimple_bb (phi))->loop_father; > struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info); > - gimple *def_stmt, *phi_use_stmt = NULL; > + gimple *phi_use_stmt = NULL; > enum tree_code orig_code, code; > tree op1, op2, op3 = NULL_TREE, op4 = NULL_TREE; > tree type; > @@ -2937,13 +2936,16 @@ vect_is_simple_reduction (loop_vec_info > return NULL; > } > > - def_stmt = SSA_NAME_DEF_STMT (loop_arg); > - if (is_gimple_assign (def_stmt)) > + stmt_vec_info def_stmt_info = loop_info->lookup_def (loop_arg); > + if (!def_stmt_info) > + return NULL; > + > + if (gassign *def_stmt = dyn_cast <gassign *> (def_stmt_info->stmt)) > { > name = gimple_assign_lhs (def_stmt); > phi_def = false; > } > - else if (gimple_code (def_stmt) == GIMPLE_PHI) > + else if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt)) > { > name = PHI_RESULT (def_stmt); > phi_def = true; > @@ -2954,14 +2956,12 @@ vect_is_simple_reduction (loop_vec_info > { > dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > "reduction: unhandled reduction operation: "); > - dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, def_stmt, 0); > + dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, > + def_stmt_info->stmt, 0); > } > return NULL; > } > > - if (! flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))) > - return NULL; > - > nloop_uses = 0; > auto_vec<gphi *, 3> lcphis; > FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name) > @@ -2987,6 +2987,7 @@ vect_is_simple_reduction (loop_vec_info > defined in the inner loop. */ > if (phi_def) > { > + gphi *def_stmt = as_a <gphi *> (def_stmt_info->stmt); > op1 = PHI_ARG_DEF (def_stmt, 0); > > if (gimple_phi_num_args (def_stmt) != 1 > @@ -3012,7 +3013,7 @@ vect_is_simple_reduction (loop_vec_info > "detected double reduction: "); > > *double_reduc = true; > - return def_stmt; > + return def_stmt_info; > } > > return NULL; > @@ -3038,6 +3039,7 @@ vect_is_simple_reduction (loop_vec_info > } > } > > + gassign *def_stmt = as_a <gassign *> (def_stmt_info->stmt); > bool nested_in_vect_loop = flow_loop_nested_p (vect_loop, loop); > code = orig_code = gimple_assign_rhs_code (def_stmt); > > @@ -3178,7 +3180,7 @@ vect_is_simple_reduction (loop_vec_info > { > if (dump_enabled_p ()) > report_vect_op (MSG_NOTE, def_stmt, "detected reduction: "); > - return def_stmt; > + return def_stmt_info; > } > > if (def1_info > @@ -3237,7 +3239,7 @@ vect_is_simple_reduction (loop_vec_info > report_vect_op (MSG_NOTE, def_stmt, "detected reduction: "); > } > > - return def_stmt; > + return def_stmt_info; > } > > /* Try to find SLP reduction chain. */ > @@ -3250,7 +3252,7 @@ vect_is_simple_reduction (loop_vec_info > report_vect_op (MSG_NOTE, def_stmt, > "reduction: detected reduction chain: "); > > - return def_stmt; > + return def_stmt_info; > } > > /* Dissolve group eventually half-built by vect_is_slp_reduction. */ > @@ -3264,9 +3266,8 @@ vect_is_simple_reduction (loop_vec_info > } > > /* Look for the expression computing loop_arg from loop PHI result. */ > - if (check_reduction_path (vect_location, loop, as_a <gphi *> (phi), loop_arg, > - code)) > - return def_stmt; > + if (check_reduction_path (vect_location, loop, phi, loop_arg, code)) > + return def_stmt_info; > > if (dump_enabled_p ()) > { > @@ -3281,25 +3282,24 @@ vect_is_simple_reduction (loop_vec_info > in-place if it enables detection of more reductions. Arguments > as there. */ > > -gimple * > -vect_force_simple_reduction (loop_vec_info loop_info, gimple *phi, > +stmt_vec_info > +vect_force_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, > bool *double_reduc, > bool need_wrapping_integral_overflow) > { > enum vect_reduction_type v_reduc_type; > - gimple *def = vect_is_simple_reduction (loop_info, phi, double_reduc, > - need_wrapping_integral_overflow, > - &v_reduc_type); > - if (def) > + stmt_vec_info def_info > + = vect_is_simple_reduction (loop_info, phi_info, double_reduc, > + need_wrapping_integral_overflow, > + &v_reduc_type); > + if (def_info) > { > - stmt_vec_info phi_info = vinfo_for_stmt (phi); > - stmt_vec_info def_info = vinfo_for_stmt (def); > STMT_VINFO_REDUC_TYPE (phi_info) = v_reduc_type; > STMT_VINFO_REDUC_DEF (phi_info) = def_info; > STMT_VINFO_REDUC_TYPE (def_info) = v_reduc_type; > STMT_VINFO_REDUC_DEF (def_info) = phi_info; > } > - return def; > + return def_info; > } > > /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times. */ > Index: gcc/tree-vect-patterns.c > =================================================================== > --- gcc/tree-vect-patterns.c 2018-07-24 10:22:44.289185723 +0100 > +++ gcc/tree-vect-patterns.c 2018-07-24 10:22:57.277070390 +0100 > @@ -4851,9 +4851,9 @@ vect_pattern_recog_1 (vect_recog_func *r > if (loop_vinfo) > { > unsigned ix, ix2; > - gimple **elem_ptr; > + stmt_vec_info *elem_ptr; > VEC_ORDERED_REMOVE_IF (LOOP_VINFO_REDUCTIONS (loop_vinfo), ix, ix2, > - elem_ptr, *elem_ptr == stmt); > + elem_ptr, *elem_ptr == stmt_info); > } > } > > Index: gcc/tree-vect-slp.c > =================================================================== > --- gcc/tree-vect-slp.c 2018-07-24 10:22:50.777128110 +0100 > +++ gcc/tree-vect-slp.c 2018-07-24 10:22:57.277070390 +0100 > @@ -1931,6 +1931,7 @@ vect_analyze_slp_instance (vec_info *vin > unsigned int group_size; > tree vectype, scalar_type = NULL_TREE; > gimple *next; > + stmt_vec_info next_info; > unsigned int i; > vec<slp_tree> loads; > struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)); > @@ -2008,9 +2009,9 @@ vect_analyze_slp_instance (vec_info *vin > else > { > /* Collect reduction statements. */ > - vec<gimple *> reductions = as_a <loop_vec_info> (vinfo)->reductions; > - for (i = 0; reductions.iterate (i, &next); i++) > - scalar_stmts.safe_push (next); > + vec<stmt_vec_info> reductions = as_a <loop_vec_info> (vinfo)->reductions; > + for (i = 0; reductions.iterate (i, &next_info); i++) > + scalar_stmts.safe_push (next_info); > } > > loads.create (group_size);
Index: gcc/tree-vectorizer.h =================================================================== --- gcc/tree-vectorizer.h 2018-07-24 10:22:53.909100298 +0100 +++ gcc/tree-vectorizer.h 2018-07-24 10:22:57.277070390 +0100 @@ -475,7 +475,7 @@ typedef struct _loop_vec_info : public v auto_vec<gimple *> may_misalign_stmts; /* Reduction cycles detected in the loop. Used in loop-aware SLP. */ - auto_vec<gimple *> reductions; + auto_vec<stmt_vec_info> reductions; /* All reduction chains in the loop, represented by the first stmt in the chain. */ @@ -1627,8 +1627,8 @@ extern tree vect_create_addr_base_for_ve /* In tree-vect-loop.c. */ /* FORNOW: Used in tree-parloops.c. */ -extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *, - bool *, bool); +extern stmt_vec_info vect_force_simple_reduction (loop_vec_info, stmt_vec_info, + bool *, bool); /* Used in gimple-loop-interchange.c. */ extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree, enum tree_code); Index: gcc/tree-parloops.c =================================================================== --- gcc/tree-parloops.c 2018-06-27 10:27:09.778650686 +0100 +++ gcc/tree-parloops.c 2018-07-24 10:22:57.273070426 +0100 @@ -2570,15 +2570,14 @@ set_reduc_phi_uids (reduction_info **slo return 1; } -/* Return true if the type of reduction performed by STMT is suitable +/* Return true if the type of reduction performed by STMT_INFO is suitable for this pass. */ static bool -valid_reduction_p (gimple *stmt) +valid_reduction_p (stmt_vec_info stmt_info) { /* Parallelization would reassociate the operation, which isn't allowed for in-order reductions. */ - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); vect_reduction_type reduc_type = STMT_VINFO_REDUC_TYPE (stmt_info); return reduc_type != FOLD_LEFT_REDUCTION; } @@ -2615,10 +2614,11 @@ gather_scalar_reductions (loop_p loop, r if (simple_iv (loop, loop, res, &iv, true)) continue; - gimple *reduc_stmt - = vect_force_simple_reduction (simple_loop_info, phi, + stmt_vec_info reduc_stmt_info + = vect_force_simple_reduction (simple_loop_info, + simple_loop_info->lookup_stmt (phi), &double_reduc, true); - if (!reduc_stmt || !valid_reduction_p (reduc_stmt)) + if (!reduc_stmt_info || !valid_reduction_p (reduc_stmt_info)) continue; if (double_reduc) @@ -2627,11 +2627,11 @@ gather_scalar_reductions (loop_p loop, r continue; double_reduc_phis.safe_push (phi); - double_reduc_stmts.safe_push (reduc_stmt); + double_reduc_stmts.safe_push (reduc_stmt_info->stmt); continue; } - build_new_reduction (reduction_list, reduc_stmt, phi); + build_new_reduction (reduction_list, reduc_stmt_info->stmt, phi); } delete simple_loop_info; @@ -2661,12 +2661,15 @@ gather_scalar_reductions (loop_p loop, r &iv, true)) continue; - gimple *inner_reduc_stmt - = vect_force_simple_reduction (simple_loop_info, inner_phi, + stmt_vec_info inner_phi_info + = simple_loop_info->lookup_stmt (inner_phi); + stmt_vec_info inner_reduc_stmt_info + = vect_force_simple_reduction (simple_loop_info, + inner_phi_info, &double_reduc, true); gcc_assert (!double_reduc); - if (inner_reduc_stmt == NULL - || !valid_reduction_p (inner_reduc_stmt)) + if (!inner_reduc_stmt_info + || !valid_reduction_p (inner_reduc_stmt_info)) continue; build_new_reduction (reduction_list, double_reduc_stmts[i], phi); Index: gcc/tree-vect-loop.c =================================================================== --- gcc/tree-vect-loop.c 2018-07-24 10:22:53.909100298 +0100 +++ gcc/tree-vect-loop.c 2018-07-24 10:22:57.273070426 +0100 @@ -546,7 +546,6 @@ vect_analyze_scalar_cycles_1 (loop_vec_i gimple *phi = worklist.pop (); tree def = PHI_RESULT (phi); stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi); - gimple *reduc_stmt; if (dump_enabled_p ()) { @@ -557,9 +556,10 @@ vect_analyze_scalar_cycles_1 (loop_vec_i gcc_assert (!virtual_operand_p (def) && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type); - reduc_stmt = vect_force_simple_reduction (loop_vinfo, phi, - &double_reduc, false); - if (reduc_stmt) + stmt_vec_info reduc_stmt_info + = vect_force_simple_reduction (loop_vinfo, stmt_vinfo, + &double_reduc, false); + if (reduc_stmt_info) { if (double_reduc) { @@ -568,8 +568,8 @@ vect_analyze_scalar_cycles_1 (loop_vec_i "Detected double reduction.\n"); STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def; - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) = - vect_double_reduction_def; + STMT_VINFO_DEF_TYPE (reduc_stmt_info) + = vect_double_reduction_def; } else { @@ -580,8 +580,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_i "Detected vectorizable nested cycle.\n"); STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle; - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) = - vect_nested_cycle; + STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_nested_cycle; } else { @@ -590,13 +589,13 @@ vect_analyze_scalar_cycles_1 (loop_vec_i "Detected reduction.\n"); STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def; - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) = - vect_reduction_def; + STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def; /* Store the reduction cycles for possible vectorization in loop-aware SLP if it was not detected as reduction chain. */ - if (! REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (reduc_stmt))) - LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt); + if (! REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info)) + LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push + (reduc_stmt_info); } } } @@ -2530,8 +2529,8 @@ vect_is_slp_reduction (loop_vec_info loo struct loop *loop = (gimple_bb (phi))->loop_father; struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info); enum tree_code code; - gimple *current_stmt = NULL, *loop_use_stmt = NULL, *first, *next_stmt; - stmt_vec_info use_stmt_info, current_stmt_info; + gimple *loop_use_stmt = NULL, *first, *next_stmt; + stmt_vec_info use_stmt_info, current_stmt_info = NULL; tree lhs; imm_use_iterator imm_iter; use_operand_p use_p; @@ -2593,9 +2592,8 @@ vect_is_slp_reduction (loop_vec_info loo /* Insert USE_STMT into reduction chain. */ use_stmt_info = loop_info->lookup_stmt (loop_use_stmt); - if (current_stmt) + if (current_stmt_info) { - current_stmt_info = vinfo_for_stmt (current_stmt); REDUC_GROUP_NEXT_ELEMENT (current_stmt_info) = loop_use_stmt; REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); @@ -2604,7 +2602,7 @@ vect_is_slp_reduction (loop_vec_info loo REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) = loop_use_stmt; lhs = gimple_assign_lhs (loop_use_stmt); - current_stmt = loop_use_stmt; + current_stmt_info = use_stmt_info; size++; } @@ -2614,7 +2612,7 @@ vect_is_slp_reduction (loop_vec_info loo /* Swap the operands, if needed, to make the reduction operand be the second operand. */ lhs = PHI_RESULT (phi); - next_stmt = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt)); + next_stmt = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); while (next_stmt) { if (gimple_assign_rhs2 (next_stmt) == lhs) @@ -2671,7 +2669,7 @@ vect_is_slp_reduction (loop_vec_info loo } /* Save the chain for further analysis in SLP detection. */ - first = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt)); + first = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); LOOP_VINFO_REDUCTION_CHAINS (loop_info).safe_push (first); REDUC_GROUP_SIZE (vinfo_for_stmt (first)) = size; @@ -2867,15 +2865,16 @@ check_reduction_path (dump_user_location */ -static gimple * -vect_is_simple_reduction (loop_vec_info loop_info, gimple *phi, +static stmt_vec_info +vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, bool *double_reduc, bool need_wrapping_integral_overflow, enum vect_reduction_type *v_reduc_type) { + gphi *phi = as_a <gphi *> (phi_info->stmt); struct loop *loop = (gimple_bb (phi))->loop_father; struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info); - gimple *def_stmt, *phi_use_stmt = NULL; + gimple *phi_use_stmt = NULL; enum tree_code orig_code, code; tree op1, op2, op3 = NULL_TREE, op4 = NULL_TREE; tree type; @@ -2937,13 +2936,16 @@ vect_is_simple_reduction (loop_vec_info return NULL; } - def_stmt = SSA_NAME_DEF_STMT (loop_arg); - if (is_gimple_assign (def_stmt)) + stmt_vec_info def_stmt_info = loop_info->lookup_def (loop_arg); + if (!def_stmt_info) + return NULL; + + if (gassign *def_stmt = dyn_cast <gassign *> (def_stmt_info->stmt)) { name = gimple_assign_lhs (def_stmt); phi_def = false; } - else if (gimple_code (def_stmt) == GIMPLE_PHI) + else if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt)) { name = PHI_RESULT (def_stmt); phi_def = true; @@ -2954,14 +2956,12 @@ vect_is_simple_reduction (loop_vec_info { dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "reduction: unhandled reduction operation: "); - dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, def_stmt, 0); + dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, + def_stmt_info->stmt, 0); } return NULL; } - if (! flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))) - return NULL; - nloop_uses = 0; auto_vec<gphi *, 3> lcphis; FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name) @@ -2987,6 +2987,7 @@ vect_is_simple_reduction (loop_vec_info defined in the inner loop. */ if (phi_def) { + gphi *def_stmt = as_a <gphi *> (def_stmt_info->stmt); op1 = PHI_ARG_DEF (def_stmt, 0); if (gimple_phi_num_args (def_stmt) != 1 @@ -3012,7 +3013,7 @@ vect_is_simple_reduction (loop_vec_info "detected double reduction: "); *double_reduc = true; - return def_stmt; + return def_stmt_info; } return NULL; @@ -3038,6 +3039,7 @@ vect_is_simple_reduction (loop_vec_info } } + gassign *def_stmt = as_a <gassign *> (def_stmt_info->stmt); bool nested_in_vect_loop = flow_loop_nested_p (vect_loop, loop); code = orig_code = gimple_assign_rhs_code (def_stmt); @@ -3178,7 +3180,7 @@ vect_is_simple_reduction (loop_vec_info { if (dump_enabled_p ()) report_vect_op (MSG_NOTE, def_stmt, "detected reduction: "); - return def_stmt; + return def_stmt_info; } if (def1_info @@ -3237,7 +3239,7 @@ vect_is_simple_reduction (loop_vec_info report_vect_op (MSG_NOTE, def_stmt, "detected reduction: "); } - return def_stmt; + return def_stmt_info; } /* Try to find SLP reduction chain. */ @@ -3250,7 +3252,7 @@ vect_is_simple_reduction (loop_vec_info report_vect_op (MSG_NOTE, def_stmt, "reduction: detected reduction chain: "); - return def_stmt; + return def_stmt_info; } /* Dissolve group eventually half-built by vect_is_slp_reduction. */ @@ -3264,9 +3266,8 @@ vect_is_simple_reduction (loop_vec_info } /* Look for the expression computing loop_arg from loop PHI result. */ - if (check_reduction_path (vect_location, loop, as_a <gphi *> (phi), loop_arg, - code)) - return def_stmt; + if (check_reduction_path (vect_location, loop, phi, loop_arg, code)) + return def_stmt_info; if (dump_enabled_p ()) { @@ -3281,25 +3282,24 @@ vect_is_simple_reduction (loop_vec_info in-place if it enables detection of more reductions. Arguments as there. */ -gimple * -vect_force_simple_reduction (loop_vec_info loop_info, gimple *phi, +stmt_vec_info +vect_force_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, bool *double_reduc, bool need_wrapping_integral_overflow) { enum vect_reduction_type v_reduc_type; - gimple *def = vect_is_simple_reduction (loop_info, phi, double_reduc, - need_wrapping_integral_overflow, - &v_reduc_type); - if (def) + stmt_vec_info def_info + = vect_is_simple_reduction (loop_info, phi_info, double_reduc, + need_wrapping_integral_overflow, + &v_reduc_type); + if (def_info) { - stmt_vec_info phi_info = vinfo_for_stmt (phi); - stmt_vec_info def_info = vinfo_for_stmt (def); STMT_VINFO_REDUC_TYPE (phi_info) = v_reduc_type; STMT_VINFO_REDUC_DEF (phi_info) = def_info; STMT_VINFO_REDUC_TYPE (def_info) = v_reduc_type; STMT_VINFO_REDUC_DEF (def_info) = phi_info; } - return def; + return def_info; } /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times. */ Index: gcc/tree-vect-patterns.c =================================================================== --- gcc/tree-vect-patterns.c 2018-07-24 10:22:44.289185723 +0100 +++ gcc/tree-vect-patterns.c 2018-07-24 10:22:57.277070390 +0100 @@ -4851,9 +4851,9 @@ vect_pattern_recog_1 (vect_recog_func *r if (loop_vinfo) { unsigned ix, ix2; - gimple **elem_ptr; + stmt_vec_info *elem_ptr; VEC_ORDERED_REMOVE_IF (LOOP_VINFO_REDUCTIONS (loop_vinfo), ix, ix2, - elem_ptr, *elem_ptr == stmt); + elem_ptr, *elem_ptr == stmt_info); } } Index: gcc/tree-vect-slp.c =================================================================== --- gcc/tree-vect-slp.c 2018-07-24 10:22:50.777128110 +0100 +++ gcc/tree-vect-slp.c 2018-07-24 10:22:57.277070390 +0100 @@ -1931,6 +1931,7 @@ vect_analyze_slp_instance (vec_info *vin unsigned int group_size; tree vectype, scalar_type = NULL_TREE; gimple *next; + stmt_vec_info next_info; unsigned int i; vec<slp_tree> loads; struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)); @@ -2008,9 +2009,9 @@ vect_analyze_slp_instance (vec_info *vin else { /* Collect reduction statements. */ - vec<gimple *> reductions = as_a <loop_vec_info> (vinfo)->reductions; - for (i = 0; reductions.iterate (i, &next); i++) - scalar_stmts.safe_push (next); + vec<stmt_vec_info> reductions = as_a <loop_vec_info> (vinfo)->reductions; + for (i = 0; reductions.iterate (i, &next_info); i++) + scalar_stmts.safe_push (next_info); } loads.create (group_size);