===================================================================
@@ -240,6 +240,7 @@ struct vec_info {
stmt_vec_info lookup_stmt (gimple *);
stmt_vec_info lookup_def (tree);
stmt_vec_info lookup_single_use (tree);
+ stmt_vec_info lookup_dr (data_reference *);
/* The type of vectorization. */
vec_kind kind;
@@ -1327,22 +1328,6 @@ vect_dr_behavior (stmt_vec_info stmt_inf
return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
}
-/* Return the stmt DR is in. For DR_STMT that have been replaced by
- a pattern this returns the corresponding pattern stmt. Otherwise
- DR_STMT is returned. */
-
-inline stmt_vec_info
-vect_dr_stmt (data_reference *dr)
-{
- gimple *stmt = DR_STMT (dr);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- if (STMT_VINFO_IN_PATTERN_P (stmt_info))
- return STMT_VINFO_RELATED_STMT (stmt_info);
- /* DR_STMT should never refer to a stmt in a pattern replacement. */
- gcc_checking_assert (!STMT_VINFO_RELATED_STMT (stmt_info));
- return stmt_info;
-}
-
/* Return true if the vect cost model is unlimited. */
static inline bool
unlimited_cost_model (loop_p loop)
===================================================================
@@ -562,6 +562,21 @@ vec_info::lookup_single_use (tree lhs)
return NULL;
}
+/* Return the stmt DR is in. For DR_STMT that have been replaced by
+ a pattern this returns the corresponding pattern stmt. Otherwise
+ it returns the information for DR_STMT itself. */
+
+stmt_vec_info
+vec_info::lookup_dr (data_reference *dr)
+{
+ stmt_vec_info stmt_info = lookup_stmt (DR_STMT (dr));
+ if (STMT_VINFO_IN_PATTERN_P (stmt_info))
+ return STMT_VINFO_RELATED_STMT (stmt_info);
+ /* DR_STMT should never refer to a stmt in a pattern replacement. */
+ gcc_checking_assert (!STMT_VINFO_RELATED_STMT (stmt_info));
+ return stmt_info;
+}
+
/* A helper function to free scev and LOOP niter information, as well as
clear loop constraint LOOP_C_FINITE. */
===================================================================
@@ -1752,8 +1752,8 @@ vect_update_inits_of_drs (loop_vec_info
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- gimple *stmt = DR_STMT (dr);
- if (!STMT_VINFO_GATHER_SCATTER_P (vinfo_for_stmt (stmt)))
+ stmt_vec_info stmt_info = loop_vinfo->lookup_dr (dr);
+ if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
vect_update_init_of_dr (dr, niters, code);
}
}
===================================================================
@@ -267,10 +267,10 @@ vect_analyze_possibly_independent_ddr (d
Note that the alias checks will be removed if the VF ends up
being small enough. */
- return (!STMT_VINFO_GATHER_SCATTER_P
- (vinfo_for_stmt (DR_STMT (DDR_A (ddr))))
- && !STMT_VINFO_GATHER_SCATTER_P
- (vinfo_for_stmt (DR_STMT (DDR_B (ddr))))
+ stmt_vec_info stmt_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
+ stmt_vec_info stmt_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
+ return (!STMT_VINFO_GATHER_SCATTER_P (stmt_info_a)
+ && !STMT_VINFO_GATHER_SCATTER_P (stmt_info_b)
&& vect_mark_for_runtime_alias_test (ddr, loop_vinfo));
}
}
@@ -294,8 +294,8 @@ vect_analyze_data_ref_dependence (struct
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
- stmt_vec_info stmtinfo_a = vect_dr_stmt (dra);
- stmt_vec_info stmtinfo_b = vect_dr_stmt (drb);
+ stmt_vec_info stmtinfo_a = loop_vinfo->lookup_dr (dra);
+ stmt_vec_info stmtinfo_b = loop_vinfo->lookup_dr (drb);
lambda_vector dist_v;
unsigned int loop_depth;
@@ -600,12 +600,13 @@ vect_analyze_data_ref_dependences (loop_
/* Function vect_slp_analyze_data_ref_dependence.
Return TRUE if there (might) exist a dependence between a memory-reference
- DRA and a memory-reference DRB. When versioning for alias may check a
- dependence at run-time, return FALSE. Adjust *MAX_VF according to
- the data dependence. */
+ DRA and a memory-reference DRB for VINFO. When versioning for alias
+ may check a dependence at run-time, return FALSE. Adjust *MAX_VF
+ according to the data dependence. */
static bool
-vect_slp_analyze_data_ref_dependence (struct data_dependence_relation *ddr)
+vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
+ struct data_dependence_relation *ddr)
{
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
@@ -626,9 +627,10 @@ vect_slp_analyze_data_ref_dependence (st
/* If dra and drb are part of the same interleaving chain consider
them independent. */
- if (STMT_VINFO_GROUPED_ACCESS (vect_dr_stmt (dra))
- && (DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (dra))
- == DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (drb))))
+ stmt_vec_info stmt_info_a = vinfo->lookup_dr (dra);
+ if (STMT_VINFO_GROUPED_ACCESS (stmt_info_a)
+ && (DR_GROUP_FIRST_ELEMENT (stmt_info_a)
+ == DR_GROUP_FIRST_ELEMENT (vinfo->lookup_dr (drb))))
return false;
/* Unknown data dependence. */
@@ -720,7 +722,8 @@ vect_slp_analyze_node_dependences (slp_i
data_reference *store_dr = STMT_VINFO_DATA_REF (store_info);
ddr_p ddr = initialize_data_dependence_relation
(dr_a, store_dr, vNULL);
- dependent = vect_slp_analyze_data_ref_dependence (ddr);
+ dependent
+ = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
free_dependence_relation (ddr);
if (dependent)
break;
@@ -730,7 +733,7 @@ vect_slp_analyze_node_dependences (slp_i
{
ddr_p ddr = initialize_data_dependence_relation (dr_a,
dr_b, vNULL);
- dependent = vect_slp_analyze_data_ref_dependence (ddr);
+ dependent = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
free_dependence_relation (ddr);
}
if (dependent)
@@ -842,7 +845,7 @@ vect_record_base_alignments (vec_info *v
unsigned int i;
FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = vinfo->lookup_dr (dr);
if (!DR_IS_CONDITIONAL_IN_STMT (dr)
&& STMT_VINFO_VECTORIZABLE (stmt_info)
&& !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
@@ -1167,7 +1170,7 @@ vect_verify_datarefs_alignment (loop_vec
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = vinfo->lookup_dr (dr);
if (!STMT_VINFO_RELEVANT_P (stmt_info))
continue;
@@ -1392,12 +1395,12 @@ vect_peeling_hash_get_most_frequent (_ve
return 1;
}
-/* Get the costs of peeling NPEEL iterations checking data access costs
- for all data refs. If UNKNOWN_MISALIGNMENT is true, we assume
- PEEL_STMT_INFO's misalignment will be zero after peeling. */
+/* Get the costs of peeling NPEEL iterations for LOOP_VINFO, checking
+ data access costs for all data refs. If UNKNOWN_MISALIGNMENT is true,
+ we assume PEEL_STMT_INFO's misalignment will be zero after peeling. */
static void
-vect_get_peeling_costs_all_drs (vec<data_reference_p> datarefs,
+vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
stmt_vec_info peel_stmt_info,
unsigned int *inside_cost,
unsigned int *outside_cost,
@@ -1406,12 +1409,13 @@ vect_get_peeling_costs_all_drs (vec<data
unsigned int npeel,
bool unknown_misalignment)
{
+ vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
unsigned i;
data_reference *dr;
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_dr (dr);
if (!STMT_VINFO_RELEVANT_P (stmt_info))
continue;
@@ -1460,10 +1464,9 @@ vect_peeling_hash_get_lowest_cost (_vect
body_cost_vec.create (2);
epilogue_cost_vec.create (2);
- vect_get_peeling_costs_all_drs (LOOP_VINFO_DATAREFS (loop_vinfo),
- elem->stmt_info, &inside_cost, &outside_cost,
- &body_cost_vec, &prologue_cost_vec,
- elem->npeel, false);
+ vect_get_peeling_costs_all_drs (loop_vinfo, elem->stmt_info, &inside_cost,
+ &outside_cost, &body_cost_vec,
+ &prologue_cost_vec, elem->npeel, false);
body_cost_vec.release ();
@@ -1541,7 +1544,7 @@ vect_peeling_supportable (loop_vec_info
{
int save_misalignment;
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_dr (dr);
if (stmt_info == peel_stmt_info)
continue;
@@ -1725,7 +1728,7 @@ vect_enhance_data_refs_alignment (loop_v
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_dr (dr);
if (!STMT_VINFO_RELEVANT_P (stmt_info))
continue;
@@ -1891,7 +1894,7 @@ vect_enhance_data_refs_alignment (loop_v
stmt_vector_for_cost dummy;
dummy.create (2);
- vect_get_peeling_costs_all_drs (datarefs, peel_stmt_info,
+ vect_get_peeling_costs_all_drs (loop_vinfo, peel_stmt_info,
&load_inside_cost,
&load_outside_cost,
&dummy, &dummy, estimated_npeels, true);
@@ -1900,7 +1903,7 @@ vect_enhance_data_refs_alignment (loop_v
if (first_store_info)
{
dummy.create (2);
- vect_get_peeling_costs_all_drs (datarefs, first_store_info,
+ vect_get_peeling_costs_all_drs (loop_vinfo, first_store_info,
&store_inside_cost,
&store_outside_cost,
&dummy, &dummy,
@@ -1991,7 +1994,7 @@ vect_enhance_data_refs_alignment (loop_v
stmt_vector_for_cost dummy;
dummy.create (2);
- vect_get_peeling_costs_all_drs (datarefs, NULL, &nopeel_inside_cost,
+ vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost,
&nopeel_outside_cost, &dummy, &dummy,
0, false);
dummy.release ();
@@ -2125,7 +2128,7 @@ vect_enhance_data_refs_alignment (loop_v
{
/* Strided accesses perform only component accesses, alignment
is irrelevant for them. */
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_dr (dr);
if (STMT_VINFO_STRIDED_P (stmt_info)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
@@ -2175,7 +2178,7 @@ vect_enhance_data_refs_alignment (loop_v
{
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_dr (dr);
/* For interleaving, only the alignment of the first access
matters. */
@@ -2288,16 +2291,16 @@ vect_enhance_data_refs_alignment (loop_v
/* Function vect_find_same_alignment_drs.
- Update group and alignment relations according to the chosen
+ Update group and alignment relations in VINFO according to the chosen
vectorization factor. */
static void
-vect_find_same_alignment_drs (struct data_dependence_relation *ddr)
+vect_find_same_alignment_drs (vec_info *vinfo, data_dependence_relation *ddr)
{
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
- stmt_vec_info stmtinfo_a = vect_dr_stmt (dra);
- stmt_vec_info stmtinfo_b = vect_dr_stmt (drb);
+ stmt_vec_info stmtinfo_a = vinfo->lookup_dr (dra);
+ stmt_vec_info stmtinfo_b = vinfo->lookup_dr (drb);
if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
return;
@@ -2362,7 +2365,7 @@ vect_analyze_data_refs_alignment (loop_v
unsigned int i;
FOR_EACH_VEC_ELT (ddrs, i, ddr)
- vect_find_same_alignment_drs (ddr);
+ vect_find_same_alignment_drs (vinfo, ddr);
vec<data_reference_p> datarefs = vinfo->shared->datarefs;
struct data_reference *dr;
@@ -2370,7 +2373,7 @@ vect_analyze_data_refs_alignment (loop_v
vect_record_base_alignments (vinfo);
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = vinfo->lookup_dr (dr);
if (STMT_VINFO_VECTORIZABLE (stmt_info))
vect_compute_data_ref_alignment (stmt_info);
}
@@ -2933,7 +2936,7 @@ vect_analyze_data_ref_accesses (vec_info
for (i = 0; i < datarefs_copy.length () - 1;)
{
data_reference_p dra = datarefs_copy[i];
- stmt_vec_info stmtinfo_a = vect_dr_stmt (dra);
+ stmt_vec_info stmtinfo_a = vinfo->lookup_dr (dra);
stmt_vec_info lastinfo = NULL;
if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
|| STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a))
@@ -2944,7 +2947,7 @@ vect_analyze_data_ref_accesses (vec_info
for (i = i + 1; i < datarefs_copy.length (); ++i)
{
data_reference_p drb = datarefs_copy[i];
- stmt_vec_info stmtinfo_b = vect_dr_stmt (drb);
+ stmt_vec_info stmtinfo_b = vinfo->lookup_dr (drb);
if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b)
|| STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
break;
@@ -3068,7 +3071,7 @@ vect_analyze_data_ref_accesses (vec_info
FOR_EACH_VEC_ELT (datarefs_copy, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = vinfo->lookup_dr (dr);
if (STMT_VINFO_VECTORIZABLE (stmt_info)
&& !vect_analyze_data_ref_access (stmt_info))
{
@@ -3430,10 +3433,10 @@ vect_prune_runtime_alias_test_list (loop
}
dr_a = DDR_A (ddr);
- stmt_vec_info stmt_info_a = vect_dr_stmt (DDR_A (ddr));
+ stmt_vec_info stmt_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
dr_b = DDR_B (ddr);
- stmt_vec_info stmt_info_b = vect_dr_stmt (DDR_B (ddr));
+ stmt_vec_info stmt_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
/* Skip the pair if inter-iteration dependencies are irrelevant
and intra-iteration dependencies are guaranteed to be honored. */
@@ -4149,7 +4152,7 @@ vect_analyze_data_refs (vec_info *vinfo,
poly_uint64 vf;
gcc_assert (DR_REF (dr));
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = vinfo->lookup_dr (dr);
/* Check that analysis of the data-ref succeeded. */
if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)