===================================================================
@@ -3101,8 +3101,8 @@ vect_create_epilog_for_reduction (VEC (t
tree vec_initial_def = NULL;
tree reduction_op, expr, def;
tree orig_name, scalar_result;
- imm_use_iterator imm_iter;
- use_operand_p use_p;
+ imm_use_iterator imm_iter, phi_imm_iter;
+ use_operand_p use_p, phi_use_p;
bool extract_scalar_result = false;
gimple use_stmt, orig_stmt, reduction_phi = NULL;
bool nested_in_vect_loop = false;
@@ -3264,6 +3264,14 @@ vect_create_epilog_for_reduction (VEC (t
}
}
+ /* The epilogue is created for the outer-loop, i.e., for the loop being
+ vectorized. */
+ if (double_reduc)
+ {
+ loop = outer_loop;
+ exit_bb = single_exit (loop)->dest;
+ }
+
exit_gsi = gsi_after_labels (exit_bb);
/* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
@@ -3519,6 +3527,9 @@ vect_create_epilog_for_reduction (VEC (t
vect_finalize_reduction:
+ if (double_reduc)
+ loop = loop->inner;
+
/* 2.5 Adjust the final result by the initial value of the reduction
variable. (When such adjustment is not needed, then
'adjustment_def' is zero). For example, if code is PLUS we