@@ -429,6 +429,8 @@ struct iv_group
struct iv_cand *selected;
/* To indicate this is a doloop use group. */
bool doloop_p;
+ /* To indicate this group is D-form preferred. */
+ bool dform_p;
/* Uses in the group. */
vec<struct iv_use *> vuses;
};
@@ -470,6 +472,7 @@ struct iv_cand
struct iv *orig_iv; /* The original iv if this cand is added from biv with
smaller type. */
bool doloop_p; /* Whether this is a doloop candidate. */
+ bool dform_p; /* Derived from one D-form preferred group. */
};
/* Hashtable entry for common candidate derived from iv uses. */
@@ -650,6 +653,10 @@ struct ivopts_data
/* Whether the loop has doloop comparison use. */
bool doloop_use_p;
+
+ /* Whether the loop is likely to unroll and need to check and mark
+ D-form group for better step cost modeling. */
+ bool mark_dform_p;
};
/* An assignment of iv candidates to uses. */
@@ -1575,6 +1582,7 @@ record_group (struct ivopts_data *data, enum use_type type)
group->related_cands = BITMAP_ALLOC (NULL);
group->vuses.create (1);
group->doloop_p = false;
+ group->dform_p = false;
data->vgroups.safe_push (group);
return group;
@@ -2724,6 +2732,59 @@ split_address_groups (struct ivopts_data *data)
}
}
+/* Go through all address type groups, check and mark D-form preferred. */
+static void
+mark_dform_groups (struct ivopts_data *data)
+{
+ if (!data->mark_dform_p)
+ return;
+
+ class loop *loop = data->current_loop;
+ bool dump_details = (dump_file && (dump_flags & TDF_DETAILS));
+ for (unsigned i = 0; i < data->vgroups.length (); i++)
+ {
+ struct iv_group *group = data->vgroups[i];
+ if (address_p (group->type))
+ {
+ bool found = true;
+ for (unsigned j = 0; j < group->vuses.length (); j++)
+ {
+ struct iv_use *use = group->vuses[j];
+ gcc_assert (use->mem_type);
+ /* Ensure the step fit into D-form field. */
+ if (TREE_CODE (use->iv->step) != INTEGER_CST
+ || !tree_fits_shwi_p (use->iv->step))
+ {
+ found = false;
+ if (dump_details)
+ fprintf (dump_file,
+ " Group use %u.%u doesn't"
+ "have constant step for D-form.\n",
+ i, j);
+ break;
+ }
+ bool is_store
+ = TREE_CODE (gimple_assign_lhs (use->stmt)) == SSA_NAME;
+ if (!targetm.stride_dform_valid_p (TYPE_MODE (use->mem_type),
+ tree_to_shwi (use->iv->step),
+ TYPE_UNSIGNED (use->mem_type),
+ is_store, loop->estimated_uf))
+ {
+ found = false;
+ if (dump_details)
+ fprintf (dump_file,
+ " Group use %u.%u isn't"
+ "suitable for D-form.\n",
+ i, j);
+ break;
+ }
+ }
+ if (found)
+ group->dform_p = true;
+ }
+ }
+}
+
/* Finds uses of the induction variables that are interesting. */
static void
@@ -2755,6 +2816,8 @@ find_interesting_uses (struct ivopts_data *data)
split_address_groups (data);
+ mark_dform_groups (data);
+
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "\n<IV Groups>:\n");
@@ -3137,6 +3200,7 @@ add_candidate_1 (struct ivopts_data *data, tree base, tree step, bool important,
cand->important = important;
cand->incremented_at = incremented_at;
cand->doloop_p = doloop;
+ cand->dform_p = false;
data->vcands.safe_push (cand);
if (!poly_int_tree_p (step))
@@ -3173,7 +3237,11 @@ add_candidate_1 (struct ivopts_data *data, tree base, tree step, bool important,
/* Relate candidate to the group for which it is added. */
if (use)
- bitmap_set_bit (data->vgroups[use->group_id]->related_cands, i);
+ {
+ bitmap_set_bit (data->vgroups[use->group_id]->related_cands, i);
+ if (data->vgroups[use->group_id]->dform_p)
+ cand->dform_p = true;
+ }
return cand;
}
@@ -5867,6 +5935,10 @@ determine_iv_cost (struct ivopts_data *data, struct iv_cand *cand)
cost_step = add_cost (data->speed, TYPE_MODE (TREE_TYPE (base)));
cost = cost_step + adjust_setup_cost (data, cost_base.cost);
+ /* Consider the stride update cost during unrolling. */
+ if (!cand->dform_p)
+ cost += (data->current_loop->estimated_uf - 1) * cost_step;
+
/* Prefer the original ivs unless we may gain something by replacing it.
The reason is to make debugging simpler; so this is not relevant for
artificial ivs created by other optimization passes. */
@@ -7953,6 +8025,7 @@ tree_ssa_iv_optimize_loop (struct ivopts_data *data, class loop *loop,
data->current_loop = loop;
data->loop_loc = find_loop_location (loop).get_location_t ();
data->speed = optimize_loop_for_speed_p (loop);
+ data->mark_dform_p = false;
if (dump_file && (dump_flags & TDF_DETAILS))
{
@@ -7984,6 +8057,15 @@ tree_ssa_iv_optimize_loop (struct ivopts_data *data, class loop *loop,
if (!find_induction_variables (data))
goto finish;
+ if (targetm.stride_dform_valid_p && exit)
+ {
+ tree_niter_desc *desc = niter_for_exit (data, exit);
+ estimate_unroll_factor (loop, desc);
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file, "estimated_uf:%u\n", loop->estimated_uf);
+ data->mark_dform_p = loop->estimated_uf > 1;
+ }
+
/* Finds interesting uses (item 1). */
find_interesting_uses (data);
if (data->vgroups.length () > MAX_CONSIDERED_GROUPS)