@@ -3512,11 +3512,14 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
epilog = vect_epilogues ? get_loop_copy (loop) : scalar_loop;
edge epilog_e = vect_epilogues ? e : scalar_e;
edge new_epilog_e = NULL;
- epilog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog,
- epilog_e, e,
- &new_epilog_e);
+ auto_vec<basic_block> doms;
+ epilog
+ = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog, epilog_e, e,
+ &new_epilog_e, true, &doms);
+
LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo) = new_epilog_e;
gcc_assert (epilog);
+ gcc_assert (new_epilog_e);
epilog->force_vectorize = false;
bb_before_epilog = loop_preheader_edge (epilog)->src;
@@ -3610,10 +3613,16 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
}
- if (skip_epilog)
+ if (skip_epilog || LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
{
- guard_cond = fold_build2 (EQ_EXPR, boolean_type_node,
+ /* For the case where a different exit was chosen we must execute
+ the scalar loop with the remaining iterations. */
+ if (inversed_iv)
+ guard_cond = boolean_false_node;
+ else
+ guard_cond = fold_build2 (EQ_EXPR, boolean_type_node,
niters, niters_vector_mult_vf);
+
guard_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest;
edge epilog_e = LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo);
guard_to = split_edge (epilog_e);
@@ -3621,11 +3630,13 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
skip_vector ? anchor : guard_bb,
prob_epilog.invert (),
irred_flag);
+ doms.safe_push (guard_to);
if (vect_epilogues)
epilogue_vinfo->skip_this_loop_edge = guard_e;
edge main_iv = LOOP_VINFO_IV_EXIT (loop_vinfo);
slpeel_update_phi_nodes_for_guard2 (loop, epilog, main_iv, guard_e,
epilog_e);
+
/* Only need to handle basic block before epilog loop if it's not
the guard_bb, which is the case when skip_vector is true. */
if (guard_bb != bb_before_epilog)
@@ -3637,6 +3648,10 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
scale_loop_profile (epilog, prob_epilog, -1);
}
+ /* Recalculate the dominators after adding the guard edge. */
+ if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
+ iterate_fix_dominators (CDI_DOMINATORS, doms, false);
+
unsigned HOST_WIDE_INT bound;
if (bound_scalar.is_constant (&bound))
{
Hi All, This wires through the final bits to support adding the guard block between the loop and epilog. For an "inverted loop", i.e. one where an early exit was chosen as the main exit then we can never skip the scalar loop since we know we have side effects to still perform. For those cases we always restart the scalar loop regardless of which exit is taken. When we add the guard we also need to update the dominators calculated by the peeling code as the bypass edge changes the dominators. Bootstrapped Regtested on aarch64-none-linux-gnu and no issues. Ok for master? Thanks, Tamar gcc/ChangeLog: * tree-vect-loop-manip.cc (vect_do_peeling): --- inline copy of patch -- diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index abd905b78f3661f80168c3866d7c3e68a9c15521..eef2bb50c1505f5cf802d5d80300affc2cbe69f6 100644 -- diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index abd905b78f3661f80168c3866d7c3e68a9c15521..eef2bb50c1505f5cf802d5d80300affc2cbe69f6 100644 --- a/gcc/tree-vect-loop-manip.cc +++ b/gcc/tree-vect-loop-manip.cc @@ -3512,11 +3512,14 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, epilog = vect_epilogues ? get_loop_copy (loop) : scalar_loop; edge epilog_e = vect_epilogues ? e : scalar_e; edge new_epilog_e = NULL; - epilog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog, - epilog_e, e, - &new_epilog_e); + auto_vec<basic_block> doms; + epilog + = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog, epilog_e, e, + &new_epilog_e, true, &doms); + LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo) = new_epilog_e; gcc_assert (epilog); + gcc_assert (new_epilog_e); epilog->force_vectorize = false; bb_before_epilog = loop_preheader_edge (epilog)->src; @@ -3610,10 +3613,16 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, } - if (skip_epilog) + if (skip_epilog || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) { - guard_cond = fold_build2 (EQ_EXPR, boolean_type_node, + /* For the case where a different exit was chosen we must execute + the scalar loop with the remaining iterations. */ + if (inversed_iv) + guard_cond = boolean_false_node; + else + guard_cond = fold_build2 (EQ_EXPR, boolean_type_node, niters, niters_vector_mult_vf); + guard_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest; edge epilog_e = LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo); guard_to = split_edge (epilog_e); @@ -3621,11 +3630,13 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, skip_vector ? anchor : guard_bb, prob_epilog.invert (), irred_flag); + doms.safe_push (guard_to); if (vect_epilogues) epilogue_vinfo->skip_this_loop_edge = guard_e; edge main_iv = LOOP_VINFO_IV_EXIT (loop_vinfo); slpeel_update_phi_nodes_for_guard2 (loop, epilog, main_iv, guard_e, epilog_e); + /* Only need to handle basic block before epilog loop if it's not the guard_bb, which is the case when skip_vector is true. */ if (guard_bb != bb_before_epilog) @@ -3637,6 +3648,10 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, scale_loop_profile (epilog, prob_epilog, -1); } + /* Recalculate the dominators after adding the guard edge. */ + if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) + iterate_fix_dominators (CDI_DOMINATORS, doms, false); + unsigned HOST_WIDE_INT bound; if (bound_scalar.is_constant (&bound)) {