[37/46] Associate alignment information with stmt_vec_infos

Alignment information is really a property of a stmt_vec_info
(and the way we want to vectorise it) rather than the original scalar dr.
I think that was true even before the recent dr sharing.

This patch therefore makes the alignment-related interfaces take
stmt_vec_infos rather than data_references.


2018-07-24  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (STMT_VINFO_TARGET_ALIGNMENT): New macro.
	(DR_VECT_AUX, DR_MISALIGNMENT, SET_DR_MISALIGNMENT)
	(DR_TARGET_ALIGNMENT): Delete.
	(set_dr_misalignment, dr_misalignment, aligned_access_p)
	(known_alignment_for_access_p, vect_known_alignment_in_bytes)
	(vect_dr_behavior): Take a stmt_vec_info rather than a data_reference.
	* tree-vect-data-refs.c (vect_calculate_target_alignment)
	(vect_compute_data_ref_alignment, vect_update_misalignment_for_peel)
	(vector_alignment_reachable_p, vect_get_peeling_costs_all_drs)
	(vect_peeling_supportable, vect_enhance_data_refs_alignment)
	(vect_duplicate_ssa_name_ptr_info): Update after above changes.
	(vect_create_addr_base_for_vector_ref, vect_create_data_ref_ptr)
	(vect_setup_realignment, vect_supportable_dr_alignment): Likewise.
	* tree-vect-loop-manip.c (get_misalign_in_elems): Likewise.
	(vect_gen_prolog_loop_niters): Likewise.
	* tree-vect-stmts.c (vect_get_store_cost, vect_get_load_cost)
	(compare_step_with_zero, get_group_load_store_type): Likewise.
	(vect_get_data_ptr_increment, ensure_base_align, vectorizable_store)
	(vectorizable_load): Likewise.

On Tue, Jul 24, 2018 at 12:08 PM Richard Sandiford
<richard.sandiford@arm.com> wrote:
>
> Alignment information is really a property of a stmt_vec_info
> (and the way we want to vectorise it) rather than the original scalar dr.
> I think that was true even before the recent dr sharing.

But that is only so as long as we handle only stmts with a single DR.
In reality alignment info _is_ a property of the DR and not of the stmt.

So you're doing a shortcut here, shouldn't we rename
dr_misalignment to stmt_dr_misalignment then?

Otherwise I don't see how this makes sense semantically.

> This patch therefore makes the alignment-related interfaces take
> stmt_vec_infos rather than data_references.
>
>
> 2018-07-24  Richard Sandiford  <richard.sandiford@arm.com>
>
> gcc/
>         * tree-vectorizer.h (STMT_VINFO_TARGET_ALIGNMENT): New macro.
>         (DR_VECT_AUX, DR_MISALIGNMENT, SET_DR_MISALIGNMENT)
>         (DR_TARGET_ALIGNMENT): Delete.
>         (set_dr_misalignment, dr_misalignment, aligned_access_p)
>         (known_alignment_for_access_p, vect_known_alignment_in_bytes)
>         (vect_dr_behavior): Take a stmt_vec_info rather than a data_reference.
>         * tree-vect-data-refs.c (vect_calculate_target_alignment)
>         (vect_compute_data_ref_alignment, vect_update_misalignment_for_peel)
>         (vector_alignment_reachable_p, vect_get_peeling_costs_all_drs)
>         (vect_peeling_supportable, vect_enhance_data_refs_alignment)
>         (vect_duplicate_ssa_name_ptr_info): Update after above changes.
>         (vect_create_addr_base_for_vector_ref, vect_create_data_ref_ptr)
>         (vect_setup_realignment, vect_supportable_dr_alignment): Likewise.
>         * tree-vect-loop-manip.c (get_misalign_in_elems): Likewise.
>         (vect_gen_prolog_loop_niters): Likewise.
>         * tree-vect-stmts.c (vect_get_store_cost, vect_get_load_cost)
>         (compare_step_with_zero, get_group_load_store_type): Likewise.
>         (vect_get_data_ptr_increment, ensure_base_align, vectorizable_store)
>         (vectorizable_load): Likewise.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h       2018-07-24 10:24:02.364492386 +0100
> +++ gcc/tree-vectorizer.h       2018-07-24 10:24:05.744462369 +0100
> @@ -1031,6 +1031,9 @@ #define STMT_VINFO_NUM_SLP_USES(S)        (S)->
>  #define STMT_VINFO_REDUC_TYPE(S)       (S)->reduc_type
>  #define STMT_VINFO_REDUC_DEF(S)                (S)->reduc_def
>
> +/* Only defined once dr_misalignment is defined.  */
> +#define STMT_VINFO_TARGET_ALIGNMENT(S) (S)->dr_aux.target_alignment
> +
>  #define DR_GROUP_FIRST_ELEMENT(S)  (gcc_checking_assert ((S)->data_ref_info), (S)->first_element)
>  #define DR_GROUP_NEXT_ELEMENT(S)   (gcc_checking_assert ((S)->data_ref_info), (S)->next_element)
>  #define DR_GROUP_SIZE(S)           (gcc_checking_assert ((S)->data_ref_info), (S)->size)
> @@ -1048,8 +1051,6 @@ #define HYBRID_SLP_STMT(S)
>  #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
>  #define STMT_SLP_TYPE(S)                   (S)->slp_type
>
> -#define DR_VECT_AUX(dr) (&vinfo_for_stmt (DR_STMT (dr))->dr_aux)
> -
>  #define VECT_MAX_COST 1000
>
>  /* The maximum number of intermediate steps required in multi-step type
> @@ -1256,73 +1257,72 @@ add_stmt_costs (void *data, stmt_vector_
>  #define DR_MISALIGNMENT_UNKNOWN (-1)
>  #define DR_MISALIGNMENT_UNINITIALIZED (-2)
>
> +/* Record that the vectorized form of the data access in STMT_INFO
> +   will be misaligned by VAL bytes wrt its target alignment.
> +   Negative values have the meanings above.  */
> +
>  inline void
> -set_dr_misalignment (struct data_reference *dr, int val)
> +set_dr_misalignment (stmt_vec_info stmt_info, int val)
>  {
> -  dataref_aux *data_aux = DR_VECT_AUX (dr);
> -  data_aux->misalignment = val;
> +  stmt_info->dr_aux.misalignment = val;
>  }
>
> +/* Return the misalignment in bytes of the vectorized form of the data
> +   access in STMT_INFO, relative to its target alignment.  Negative
> +   values have the meanings above.  */
> +
>  inline int
> -dr_misalignment (struct data_reference *dr)
> +dr_misalignment (stmt_vec_info stmt_info)
>  {
> -  int misalign = DR_VECT_AUX (dr)->misalignment;
> +  int misalign = stmt_info->dr_aux.misalignment;
>    gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED);
>    return misalign;
>  }
>
> -/* Reflects actual alignment of first access in the vectorized loop,
> -   taking into account peeling/versioning if applied.  */
> -#define DR_MISALIGNMENT(DR) dr_misalignment (DR)
> -#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
> -
> -/* Only defined once DR_MISALIGNMENT is defined.  */
> -#define DR_TARGET_ALIGNMENT(DR) DR_VECT_AUX (DR)->target_alignment
> -
> -/* Return true if data access DR is aligned to its target alignment
> -   (which may be less than a full vector).  */
> +/* Return true if the vectorized form of the data access in STMT_INFO is
> +   aligned to its target alignment (which may be less than a full vector).  */
>
>  static inline bool
> -aligned_access_p (struct data_reference *data_ref_info)
> +aligned_access_p (stmt_vec_info stmt_info)
>  {
> -  return (DR_MISALIGNMENT (data_ref_info) == 0);
> +  return (dr_misalignment (stmt_info) == 0);
>  }
>
> -/* Return TRUE if the alignment of the data access is known, and FALSE
> -   otherwise.  */
> +/* Return true if the alignment of the vectorized form of the data
> +   access in STMT_INFO is known at compile time.  */
>
>  static inline bool
> -known_alignment_for_access_p (struct data_reference *data_ref_info)
> +known_alignment_for_access_p (stmt_vec_info stmt_info)
>  {
> -  return (DR_MISALIGNMENT (data_ref_info) != DR_MISALIGNMENT_UNKNOWN);
> +  return (dr_misalignment (stmt_info) != DR_MISALIGNMENT_UNKNOWN);
>  }
>
>  /* Return the minimum alignment in bytes that the vectorized version
> -   of DR is guaranteed to have.  */
> +   of the data reference in STMT_INFO is guaranteed to have.  */
>
>  static inline unsigned int
> -vect_known_alignment_in_bytes (struct data_reference *dr)
> +vect_known_alignment_in_bytes (stmt_vec_info stmt_info)
>  {
> -  if (DR_MISALIGNMENT (dr) == DR_MISALIGNMENT_UNKNOWN)
> +  data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
> +  int misalignment = dr_misalignment (stmt_info);
> +  if (misalignment == DR_MISALIGNMENT_UNKNOWN)
>      return TYPE_ALIGN_UNIT (TREE_TYPE (DR_REF (dr)));
> -  if (DR_MISALIGNMENT (dr) == 0)
> -    return DR_TARGET_ALIGNMENT (dr);
> -  return DR_MISALIGNMENT (dr) & -DR_MISALIGNMENT (dr);
> +  if (misalignment == 0)
> +    return STMT_VINFO_TARGET_ALIGNMENT (stmt_info);
> +  return misalignment & -misalignment;
>  }
>
> -/* Return the behavior of DR with respect to the vectorization context
> -   (which for outer loop vectorization might not be the behavior recorded
> -   in DR itself).  */
> +/* Return the data reference behavior of STMT_INFO with respect to the
> +   vectorization context (which for outer loop vectorization might not
> +   be the behavior recorded in STMT_VINFO_DATA_DEF).  */
>
>  static inline innermost_loop_behavior *
> -vect_dr_behavior (data_reference *dr)
> +vect_dr_behavior (stmt_vec_info stmt_info)
>  {
> -  gimple *stmt = DR_STMT (dr);
> -  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
>    loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
>    if (loop_vinfo == NULL
>        || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
> -    return &DR_INNERMOST (dr);
> +    return &DR_INNERMOST (STMT_VINFO_DATA_REF (stmt_info));
>    else
>      return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
>  }
> Index: gcc/tree-vect-data-refs.c
> ===================================================================
> --- gcc/tree-vect-data-refs.c   2018-07-24 10:24:02.356492457 +0100
> +++ gcc/tree-vect-data-refs.c   2018-07-24 10:24:05.740462405 +0100
> @@ -873,7 +873,7 @@ vect_calculate_target_alignment (struct
>     Compute the misalignment of the data reference DR.
>
>     Output:
> -   1. DR_MISALIGNMENT (DR) is defined.
> +   1. dr_misalignment (STMT_INFO) is defined.
>
>     FOR NOW: No analysis is actually performed. Misalignment is calculated
>     only for trivial cases. TODO.  */
> @@ -896,17 +896,17 @@ vect_compute_data_ref_alignment (struct
>      loop = LOOP_VINFO_LOOP (loop_vinfo);
>
>    /* Initialize misalignment to unknown.  */
> -  SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN);
> +  set_dr_misalignment (stmt_info, DR_MISALIGNMENT_UNKNOWN);
>
>    if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
>      return;
>
> -  innermost_loop_behavior *drb = vect_dr_behavior (dr);
> +  innermost_loop_behavior *drb = vect_dr_behavior (stmt_info);
>    bool step_preserves_misalignment_p;
>
>    unsigned HOST_WIDE_INT vector_alignment
>      = vect_calculate_target_alignment (dr) / BITS_PER_UNIT;
> -  DR_TARGET_ALIGNMENT (dr) = vector_alignment;
> +  STMT_VINFO_TARGET_ALIGNMENT (stmt_info) = vector_alignment;
>
>    /* No step for BB vectorization.  */
>    if (!loop)
> @@ -1009,8 +1009,8 @@ vect_compute_data_ref_alignment (struct
>            dump_printf (MSG_NOTE, "\n");
>          }
>
> -      DR_VECT_AUX (dr)->base_decl = base;
> -      DR_VECT_AUX (dr)->base_misaligned = true;
> +      stmt_info->dr_aux.base_decl = base;
> +      stmt_info->dr_aux.base_misaligned = true;
>        base_misalignment = 0;
>      }
>    poly_int64 misalignment
> @@ -1038,12 +1038,13 @@ vect_compute_data_ref_alignment (struct
>        return;
>      }
>
> -  SET_DR_MISALIGNMENT (dr, const_misalignment);
> +  set_dr_misalignment (stmt_info, const_misalignment);
>
>    if (dump_enabled_p ())
>      {
>        dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                       "misalign = %d bytes of ref ", DR_MISALIGNMENT (dr));
> +                      "misalign = %d bytes of ref ",
> +                      dr_misalignment (stmt_info));
>        dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, ref);
>        dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
>      }
> @@ -1089,29 +1090,29 @@ vect_update_misalignment_for_peel (struc
>      {
>        if (current_dr != dr)
>          continue;
> -      gcc_assert (!known_alignment_for_access_p (dr)
> -                 || !known_alignment_for_access_p (dr_peel)
> -                 || (DR_MISALIGNMENT (dr) / dr_size
> -                     == DR_MISALIGNMENT (dr_peel) / dr_peel_size));
> -      SET_DR_MISALIGNMENT (dr, 0);
> +      gcc_assert (!known_alignment_for_access_p (stmt_info)
> +                 || !known_alignment_for_access_p (peel_stmt_info)
> +                 || (dr_misalignment (stmt_info) / dr_size
> +                     == dr_misalignment (peel_stmt_info) / dr_peel_size));
> +      set_dr_misalignment (stmt_info, 0);
>        return;
>      }
>
> -  if (known_alignment_for_access_p (dr)
> -      && known_alignment_for_access_p (dr_peel))
> +  if (known_alignment_for_access_p (stmt_info)
> +      && known_alignment_for_access_p (peel_stmt_info))
>      {
>        bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
> -      int misal = DR_MISALIGNMENT (dr);
> +      int misal = dr_misalignment (stmt_info);
>        misal += negative ? -npeel * dr_size : npeel * dr_size;
> -      misal &= DR_TARGET_ALIGNMENT (dr) - 1;
> -      SET_DR_MISALIGNMENT (dr, misal);
> +      misal &= STMT_VINFO_TARGET_ALIGNMENT (stmt_info) - 1;
> +      set_dr_misalignment (stmt_info, misal);
>        return;
>      }
>
>    if (dump_enabled_p ())
>      dump_printf_loc (MSG_NOTE, vect_location, "Setting misalignment " \
>                      "to unknown (-1).\n");
> -  SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN);
> +  set_dr_misalignment (stmt_info, DR_MISALIGNMENT_UNKNOWN);
>  }
>
>
> @@ -1219,13 +1220,13 @@ vector_alignment_reachable_p (struct dat
>        int elem_size, mis_in_elements;
>
>        /* FORNOW: handle only known alignment.  */
> -      if (!known_alignment_for_access_p (dr))
> +      if (!known_alignment_for_access_p (stmt_info))
>         return false;
>
>        poly_uint64 nelements = TYPE_VECTOR_SUBPARTS (vectype);
>        poly_uint64 vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
>        elem_size = vector_element_size (vector_size, nelements);
> -      mis_in_elements = DR_MISALIGNMENT (dr) / elem_size;
> +      mis_in_elements = dr_misalignment (stmt_info) / elem_size;
>
>        if (!multiple_p (nelements - mis_in_elements, DR_GROUP_SIZE (stmt_info)))
>         return false;
> @@ -1233,7 +1234,8 @@ vector_alignment_reachable_p (struct dat
>
>    /* If misalignment is known at the compile time then allow peeling
>       only if natural alignment is reachable through peeling.  */
> -  if (known_alignment_for_access_p (dr) && !aligned_access_p (dr))
> +  if (known_alignment_for_access_p (stmt_info)
> +      && !aligned_access_p (stmt_info))
>      {
>        HOST_WIDE_INT elmsize =
>                 int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
> @@ -1241,10 +1243,10 @@ vector_alignment_reachable_p (struct dat
>         {
>           dump_printf_loc (MSG_NOTE, vect_location,
>                            "data size =" HOST_WIDE_INT_PRINT_DEC, elmsize);
> -         dump_printf (MSG_NOTE,
> -                      ". misalignment = %d.\n", DR_MISALIGNMENT (dr));
> +         dump_printf (MSG_NOTE, ". misalignment = %d.\n",
> +                      dr_misalignment (stmt_info));
>         }
> -      if (DR_MISALIGNMENT (dr) % elmsize)
> +      if (dr_misalignment (stmt_info) % elmsize)
>         {
>           if (dump_enabled_p ())
>             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -1253,7 +1255,7 @@ vector_alignment_reachable_p (struct dat
>         }
>      }
>
> -  if (!known_alignment_for_access_p (dr))
> +  if (!known_alignment_for_access_p (stmt_info))
>      {
>        tree type = TREE_TYPE (DR_REF (dr));
>        bool is_packed = not_size_aligned (DR_REF (dr));
> @@ -1401,6 +1403,8 @@ vect_get_peeling_costs_all_drs (vec<data
>                                 unsigned int npeel,
>                                 bool unknown_misalignment)
>  {
> +  stmt_vec_info peel_stmt_info = (dr0 ? vect_dr_stmt (dr0)
> +                                 : NULL_STMT_VEC_INFO);
>    unsigned i;
>    data_reference *dr;
>
> @@ -1423,16 +1427,16 @@ vect_get_peeling_costs_all_drs (vec<data
>         continue;
>
>        int save_misalignment;
> -      save_misalignment = DR_MISALIGNMENT (dr);
> +      save_misalignment = dr_misalignment (stmt_info);
>        if (npeel == 0)
>         ;
> -      else if (unknown_misalignment && dr == dr0)
> -       SET_DR_MISALIGNMENT (dr, 0);
> +      else if (unknown_misalignment && stmt_info == peel_stmt_info)
> +       set_dr_misalignment (stmt_info, 0);
>        else
>         vect_update_misalignment_for_peel (dr, dr0, npeel);
>        vect_get_data_access_cost (dr, inside_cost, outside_cost,
>                                  body_cost_vec, prologue_cost_vec);
> -      SET_DR_MISALIGNMENT (dr, save_misalignment);
> +      set_dr_misalignment (stmt_info, save_misalignment);
>      }
>  }
>
> @@ -1552,10 +1556,10 @@ vect_peeling_supportable (loop_vec_info
>           && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
>         continue;
>
> -      save_misalignment = DR_MISALIGNMENT (dr);
> +      save_misalignment = dr_misalignment (stmt_info);
>        vect_update_misalignment_for_peel (dr, dr0, npeel);
>        supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
> -      SET_DR_MISALIGNMENT (dr, save_misalignment);
> +      set_dr_misalignment (stmt_info, save_misalignment);
>
>        if (!supportable_dr_alignment)
>         return false;
> @@ -1598,27 +1602,27 @@ vect_peeling_supportable (loop_vec_info
>
>       -- original loop, before alignment analysis:
>         for (i=0; i<N; i++){
> -         x = q[i];                     # DR_MISALIGNMENT(q) = unknown
> -         p[i] = y;                     # DR_MISALIGNMENT(p) = unknown
> +         x = q[i];                     # dr_misalignment(q) = unknown
> +         p[i] = y;                     # dr_misalignment(p) = unknown
>         }
>
>       -- After vect_compute_data_refs_alignment:
>         for (i=0; i<N; i++){
> -         x = q[i];                     # DR_MISALIGNMENT(q) = 3
> -         p[i] = y;                     # DR_MISALIGNMENT(p) = unknown
> +         x = q[i];                     # dr_misalignment(q) = 3
> +         p[i] = y;                     # dr_misalignment(p) = unknown
>         }
>
>       -- Possibility 1: we do loop versioning:
>       if (p is aligned) {
>         for (i=0; i<N; i++){    # loop 1A
> -         x = q[i];                     # DR_MISALIGNMENT(q) = 3
> -         p[i] = y;                     # DR_MISALIGNMENT(p) = 0
> +         x = q[i];                     # dr_misalignment(q) = 3
> +         p[i] = y;                     # dr_misalignment(p) = 0
>         }
>       }
>       else {
>         for (i=0; i<N; i++){    # loop 1B
> -         x = q[i];                     # DR_MISALIGNMENT(q) = 3
> -         p[i] = y;                     # DR_MISALIGNMENT(p) = unaligned
> +         x = q[i];                     # dr_misalignment(q) = 3
> +         p[i] = y;                     # dr_misalignment(p) = unaligned
>         }
>       }
>
> @@ -1628,8 +1632,8 @@ vect_peeling_supportable (loop_vec_info
>         p[i] = y;
>       }
>       for (i = 3; i < N; i++){  # loop 2A
> -       x = q[i];                       # DR_MISALIGNMENT(q) = 0
> -       p[i] = y;                       # DR_MISALIGNMENT(p) = unknown
> +       x = q[i];                       # dr_misalignment(q) = 0
> +       p[i] = y;                       # dr_misalignment(p) = unknown
>       }
>
>       -- Possibility 3: combination of loop peeling and versioning:
> @@ -1639,14 +1643,14 @@ vect_peeling_supportable (loop_vec_info
>       }
>       if (p is aligned) {
>         for (i = 3; i<N; i++){  # loop 3A
> -         x = q[i];                     # DR_MISALIGNMENT(q) = 0
> -         p[i] = y;                     # DR_MISALIGNMENT(p) = 0
> +         x = q[i];                     # dr_misalignment(q) = 0
> +         p[i] = y;                     # dr_misalignment(p) = 0
>         }
>       }
>       else {
>         for (i = 3; i<N; i++){  # loop 3B
> -         x = q[i];                     # DR_MISALIGNMENT(q) = 0
> -         p[i] = y;                     # DR_MISALIGNMENT(p) = unaligned
> +         x = q[i];                     # dr_misalignment(q) = 0
> +         p[i] = y;                     # dr_misalignment(p) = unaligned
>         }
>       }
>
> @@ -1745,17 +1749,20 @@ vect_enhance_data_refs_alignment (loop_v
>        do_peeling = vector_alignment_reachable_p (dr);
>        if (do_peeling)
>          {
> -          if (known_alignment_for_access_p (dr))
> +         if (known_alignment_for_access_p (stmt_info))
>              {
>               unsigned int npeel_tmp = 0;
>               bool negative = tree_int_cst_compare (DR_STEP (dr),
>                                                     size_zero_node) < 0;
>
>               vectype = STMT_VINFO_VECTYPE (stmt_info);
> -             unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
> +             unsigned int target_align
> +               = STMT_VINFO_TARGET_ALIGNMENT (stmt_info);
>               unsigned int dr_size = vect_get_scalar_dr_size (dr);
> -             mis = (negative ? DR_MISALIGNMENT (dr) : -DR_MISALIGNMENT (dr));
> -             if (DR_MISALIGNMENT (dr) != 0)
> +             mis = (negative
> +                    ? dr_misalignment (stmt_info)
> +                    : -dr_misalignment (stmt_info));
> +             if (mis != 0)
>                 npeel_tmp = (mis & (target_align - 1)) / dr_size;
>
>                /* For multiple types, it is possible that the bigger type access
> @@ -1780,7 +1787,7 @@ vect_enhance_data_refs_alignment (loop_v
>
>                   /* NPEEL_TMP is 0 when there is no misalignment, but also
>                      allow peeling NELEMENTS.  */
> -                 if (DR_MISALIGNMENT (dr) == 0)
> +                 if (dr_misalignment (stmt_info) == 0)
>                     possible_npeel_number++;
>                 }
>
> @@ -1841,7 +1848,7 @@ vect_enhance_data_refs_alignment (loop_v
>          }
>        else
>          {
> -          if (!aligned_access_p (dr))
> +         if (!aligned_access_p (stmt_info))
>              {
>                if (dump_enabled_p ())
>                  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -2010,10 +2017,10 @@ vect_enhance_data_refs_alignment (loop_v
>
>    if (do_peeling)
>      {
> -      stmt_vec_info stmt_info = vect_dr_stmt (dr0);
> -      vectype = STMT_VINFO_VECTYPE (stmt_info);
> +      stmt_vec_info peel_stmt_info = vect_dr_stmt (dr0);
> +      vectype = STMT_VINFO_VECTYPE (peel_stmt_info);
>
> -      if (known_alignment_for_access_p (dr0))
> +      if (known_alignment_for_access_p (peel_stmt_info))
>          {
>           bool negative = tree_int_cst_compare (DR_STEP (dr0),
>                                                 size_zero_node) < 0;
> @@ -2021,11 +2028,14 @@ vect_enhance_data_refs_alignment (loop_v
>              {
>                /* Since it's known at compile time, compute the number of
>                   iterations in the peeled loop (the peeling factor) for use in
> -                 updating DR_MISALIGNMENT values.  The peeling factor is the
> +                 updating dr_misalignment values.  The peeling factor is the
>                   vectorization factor minus the misalignment as an element
>                   count.  */
> -             mis = negative ? DR_MISALIGNMENT (dr0) : -DR_MISALIGNMENT (dr0);
> -             unsigned int target_align = DR_TARGET_ALIGNMENT (dr0);
> +             mis = (negative
> +                    ? dr_misalignment (peel_stmt_info)
> +                    : -dr_misalignment (peel_stmt_info));
> +             unsigned int target_align
> +               = STMT_VINFO_TARGET_ALIGNMENT (peel_stmt_info);
>               npeel = ((mis & (target_align - 1))
>                        / vect_get_scalar_dr_size (dr0));
>              }
> @@ -2033,9 +2043,8 @@ vect_enhance_data_refs_alignment (loop_v
>           /* For interleaved data access every iteration accesses all the
>              members of the group, therefore we divide the number of iterations
>              by the group size.  */
> -         stmt_info = vect_dr_stmt (dr0);
> -         if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
> -           npeel /= DR_GROUP_SIZE (stmt_info);
> +         if (STMT_VINFO_GROUPED_ACCESS (peel_stmt_info))
> +           npeel /= DR_GROUP_SIZE (peel_stmt_info);
>
>            if (dump_enabled_p ())
>              dump_printf_loc (MSG_NOTE, vect_location,
> @@ -2047,7 +2056,9 @@ vect_enhance_data_refs_alignment (loop_v
>         do_peeling = false;
>
>        /* Check if all datarefs are supportable and log.  */
> -      if (do_peeling && known_alignment_for_access_p (dr0) && npeel == 0)
> +      if (do_peeling
> +         && known_alignment_for_access_p (peel_stmt_info)
> +         && npeel == 0)
>          {
>            stat = vect_verify_datarefs_alignment (loop_vinfo);
>            if (!stat)
> @@ -2066,7 +2077,8 @@ vect_enhance_data_refs_alignment (loop_v
>                unsigned max_peel = npeel;
>                if (max_peel == 0)
>                  {
> -                 unsigned int target_align = DR_TARGET_ALIGNMENT (dr0);
> +                 unsigned int target_align
> +                   = STMT_VINFO_TARGET_ALIGNMENT (peel_stmt_info);
>                   max_peel = target_align / vect_get_scalar_dr_size (dr0) - 1;
>                  }
>                if (max_peel > max_allowed_peel)
> @@ -2095,19 +2107,20 @@ vect_enhance_data_refs_alignment (loop_v
>
>        if (do_peeling)
>          {
> -          /* (1.2) Update the DR_MISALIGNMENT of each data reference DR_i.
> -             If the misalignment of DR_i is identical to that of dr0 then set
> -             DR_MISALIGNMENT (DR_i) to zero.  If the misalignment of DR_i and
> -             dr0 are known at compile time then increment DR_MISALIGNMENT (DR_i)
> -             by the peeling factor times the element size of DR_i (MOD the
> -             vectorization factor times the size).  Otherwise, the
> -             misalignment of DR_i must be set to unknown.  */
> +         /* (1.2) Update the dr_misalignment of each data reference
> +            statement STMT_i.  If the misalignment of STMT_i is identical
> +            to that of PEEL_STMT_INFO then set dr_misalignment (STMT_i)
> +            to zero.  If the misalignment of STMT_i and PEEL_STMT_INFO are
> +            known at compile time then increment dr_misalignment (STMT_i)
> +            by the peeling factor times the element size of STMT_i (MOD
> +            the vectorization factor times the size).  Otherwise, the
> +            misalignment of STMT_i must be set to unknown.  */
>           FOR_EACH_VEC_ELT (datarefs, i, dr)
>             if (dr != dr0)
>               {
>                 /* Strided accesses perform only component accesses, alignment
>                    is irrelevant for them.  */
> -               stmt_info = vect_dr_stmt (dr);
> +               stmt_vec_info stmt_info = vect_dr_stmt (dr);
>                 if (STMT_VINFO_STRIDED_P (stmt_info)
>                     && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
>                   continue;
> @@ -2120,8 +2133,8 @@ vect_enhance_data_refs_alignment (loop_v
>              LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = npeel;
>            else
>              LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
> -             = DR_MISALIGNMENT (dr0);
> -         SET_DR_MISALIGNMENT (dr0, 0);
> +             = dr_misalignment (peel_stmt_info);
> +         set_dr_misalignment (peel_stmt_info, 0);
>           if (dump_enabled_p ())
>              {
>                dump_printf_loc (MSG_NOTE, vect_location,
> @@ -2160,7 +2173,7 @@ vect_enhance_data_refs_alignment (loop_v
>
>           /* For interleaving, only the alignment of the first access
>              matters.  */
> -         if (aligned_access_p (dr)
> +         if (aligned_access_p (stmt_info)
>               || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
>                   && DR_GROUP_FIRST_ELEMENT (stmt_info) != stmt_info))
>             continue;
> @@ -2182,7 +2195,7 @@ vect_enhance_data_refs_alignment (loop_v
>                int mask;
>                tree vectype;
>
> -              if (known_alignment_for_access_p (dr)
> +              if (known_alignment_for_access_p (stmt_info)
>                    || LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
>                       >= (unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIGNMENT_CHECKS))
>                  {
> @@ -2241,8 +2254,7 @@ vect_enhance_data_refs_alignment (loop_v
>           of the loop being vectorized.  */
>        FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
>          {
> -          dr = STMT_VINFO_DATA_REF (stmt_info);
> -         SET_DR_MISALIGNMENT (dr, 0);
> +         set_dr_misalignment (stmt_info, 0);
>           if (dump_enabled_p ())
>              dump_printf_loc (MSG_NOTE, vect_location,
>                               "Alignment of access forced using versioning.\n");
> @@ -4456,13 +4468,14 @@ vect_get_new_ssa_name (tree type, enum v
>  static void
>  vect_duplicate_ssa_name_ptr_info (tree name, data_reference *dr)
>  {
> +  stmt_vec_info stmt_info = vect_dr_stmt (dr);
>    duplicate_ssa_name_ptr_info (name, DR_PTR_INFO (dr));
> -  int misalign = DR_MISALIGNMENT (dr);
> +  int misalign = dr_misalignment (stmt_info);
>    if (misalign == DR_MISALIGNMENT_UNKNOWN)
>      mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (name));
>    else
>      set_ptr_info_alignment (SSA_NAME_PTR_INFO (name),
> -                           DR_TARGET_ALIGNMENT (dr), misalign);
> +                           STMT_VINFO_TARGET_ALIGNMENT (stmt_info), misalign);
>  }
>
>  /* Function vect_create_addr_base_for_vector_ref.
> @@ -4513,7 +4526,7 @@ vect_create_addr_base_for_vector_ref (st
>    tree vect_ptr_type;
>    tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
>    loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
> -  innermost_loop_behavior *drb = vect_dr_behavior (dr);
> +  innermost_loop_behavior *drb = vect_dr_behavior (stmt_info);
>
>    tree data_ref_base = unshare_expr (drb->base_address);
>    tree base_offset = unshare_expr (drb->offset);
> @@ -4687,7 +4700,7 @@ vect_create_data_ref_ptr (stmt_vec_info
>
>    /* Check the step (evolution) of the load in LOOP, and record
>       whether it's invariant.  */
> -  step = vect_dr_behavior (dr)->step;
> +  step = vect_dr_behavior (stmt_info)->step;
>    if (integer_zerop (step))
>      *inv_p = true;
>    else
> @@ -5519,7 +5532,7 @@ vect_setup_realignment (stmt_vec_info st
>         new_temp = copy_ssa_name (ptr);
>        else
>         new_temp = make_ssa_name (TREE_TYPE (ptr));
> -      unsigned int align = DR_TARGET_ALIGNMENT (dr);
> +      unsigned int align = STMT_VINFO_TARGET_ALIGNMENT (stmt_info);
>        new_stmt = gimple_build_assign
>                    (new_temp, BIT_AND_EXPR, ptr,
>                     build_int_cst (TREE_TYPE (ptr), -(HOST_WIDE_INT) align));
> @@ -6438,7 +6451,7 @@ vect_supportable_dr_alignment (struct da
>    struct loop *vect_loop = NULL;
>    bool nested_in_vect_loop = false;
>
> -  if (aligned_access_p (dr) && !check_aligned_accesses)
> +  if (aligned_access_p (stmt_info) && !check_aligned_accesses)
>      return dr_aligned;
>
>    /* For now assume all conditional loads/stores support unaligned
> @@ -6546,11 +6559,11 @@ vect_supportable_dr_alignment (struct da
>           else
>             return dr_explicit_realign_optimized;
>         }
> -      if (!known_alignment_for_access_p (dr))
> +      if (!known_alignment_for_access_p (stmt_info))
>         is_packed = not_size_aligned (DR_REF (dr));
>
>        if (targetm.vectorize.support_vector_misalignment
> -           (mode, type, DR_MISALIGNMENT (dr), is_packed))
> +           (mode, type, dr_misalignment (stmt_info), is_packed))
>         /* Can't software pipeline the loads, but can at least do them.  */
>         return dr_unaligned_supported;
>      }
> @@ -6559,11 +6572,11 @@ vect_supportable_dr_alignment (struct da
>        bool is_packed = false;
>        tree type = (TREE_TYPE (DR_REF (dr)));
>
> -      if (!known_alignment_for_access_p (dr))
> +      if (!known_alignment_for_access_p (stmt_info))
>         is_packed = not_size_aligned (DR_REF (dr));
>
>       if (targetm.vectorize.support_vector_misalignment
> -          (mode, type, DR_MISALIGNMENT (dr), is_packed))
> +          (mode, type, dr_misalignment (stmt_info), is_packed))
>         return dr_unaligned_supported;
>      }
>
> Index: gcc/tree-vect-loop-manip.c
> ===================================================================
> --- gcc/tree-vect-loop-manip.c  2018-07-24 10:23:46.112636713 +0100
> +++ gcc/tree-vect-loop-manip.c  2018-07-24 10:24:05.740462405 +0100
> @@ -1564,7 +1564,7 @@ get_misalign_in_elems (gimple **seq, loo
>    stmt_vec_info stmt_info = vect_dr_stmt (dr);
>    tree vectype = STMT_VINFO_VECTYPE (stmt_info);
>
> -  unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
> +  unsigned int target_align = STMT_VINFO_TARGET_ALIGNMENT (stmt_info);
>    gcc_assert (target_align != 0);
>
>    bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
> @@ -1600,7 +1600,7 @@ get_misalign_in_elems (gimple **seq, loo
>     refer to an aligned location.  The following computation is generated:
>
>     If the misalignment of DR is known at compile time:
> -     addr_mis = int mis = DR_MISALIGNMENT (dr);
> +     addr_mis = int mis = dr_misalignment (stmt-containing-DR);
>     Else, compute address misalignment in bytes:
>       addr_mis = addr & (target_align - 1)
>
> @@ -1633,7 +1633,7 @@ vect_gen_prolog_loop_niters (loop_vec_in
>    tree iters, iters_name;
>    stmt_vec_info stmt_info = vect_dr_stmt (dr);
>    tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> -  unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
> +  unsigned int target_align = STMT_VINFO_TARGET_ALIGNMENT (stmt_info);
>
>    if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
>      {
> Index: gcc/tree-vect-stmts.c
> ===================================================================
> --- gcc/tree-vect-stmts.c       2018-07-24 10:24:02.364492386 +0100
> +++ gcc/tree-vect-stmts.c       2018-07-24 10:24:05.744462369 +0100
> @@ -1079,7 +1079,8 @@ vect_get_store_cost (stmt_vec_info stmt_
>          /* Here, we assign an additional cost for the unaligned store.  */
>         *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
>                                           unaligned_store, stmt_info,
> -                                         DR_MISALIGNMENT (dr), vect_body);
> +                                         dr_misalignment (stmt_info),
> +                                         vect_body);
>          if (dump_enabled_p ())
>            dump_printf_loc (MSG_NOTE, vect_location,
>                             "vect_model_store_cost: unaligned supported by "
> @@ -1257,7 +1258,8 @@ vect_get_load_cost (stmt_vec_info stmt_i
>          /* Here, we assign an additional cost for the unaligned load.  */
>         *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
>                                           unaligned_load, stmt_info,
> -                                         DR_MISALIGNMENT (dr), vect_body);
> +                                         dr_misalignment (stmt_info),
> +                                         vect_body);
>
>          if (dump_enabled_p ())
>            dump_printf_loc (MSG_NOTE, vect_location,
> @@ -2102,8 +2104,7 @@ vect_use_strided_gather_scatters_p (stmt
>  static int
>  compare_step_with_zero (stmt_vec_info stmt_info)
>  {
> -  data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
> -  return tree_int_cst_compare (vect_dr_behavior (dr)->step,
> +  return tree_int_cst_compare (vect_dr_behavior (stmt_info)->step,
>                                size_zero_node);
>  }
>
> @@ -2218,7 +2219,7 @@ get_group_load_store_type (stmt_vec_info
>              be a multiple of B and so we are guaranteed to access a
>              non-gap element in the same B-sized block.  */
>           if (overrun_p
> -             && gap < (vect_known_alignment_in_bytes (first_dr)
> +             && gap < (vect_known_alignment_in_bytes (first_stmt_info)
>                         / vect_get_scalar_dr_size (first_dr)))
>             overrun_p = false;
>           if (overrun_p && !can_overrun_p)
> @@ -2246,7 +2247,7 @@ get_group_load_store_type (stmt_vec_info
>          same B-sized block.  */
>        if (would_overrun_p
>           && !masked_p
> -         && gap < (vect_known_alignment_in_bytes (first_dr)
> +         && gap < (vect_known_alignment_in_bytes (first_stmt_info)
>                     / vect_get_scalar_dr_size (first_dr)))
>         would_overrun_p = false;
>
> @@ -2931,11 +2932,12 @@ vect_get_strided_load_store_ops (stmt_ve
>  vect_get_data_ptr_increment (data_reference *dr, tree aggr_type,
>                              vect_memory_access_type memory_access_type)
>  {
> +  stmt_vec_info stmt_info = vect_dr_stmt (dr);
>    if (memory_access_type == VMAT_INVARIANT)
>      return size_zero_node;
>
>    tree iv_step = TYPE_SIZE_UNIT (aggr_type);
> -  tree step = vect_dr_behavior (dr)->step;
> +  tree step = vect_dr_behavior (stmt_info)->step;
>    if (tree_int_cst_sgn (step) == -1)
>      iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
>    return iv_step;
> @@ -6174,14 +6176,16 @@ vectorizable_operation (stmt_vec_info st
>  static void
>  ensure_base_align (struct data_reference *dr)
>  {
> -  if (DR_VECT_AUX (dr)->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
> +  stmt_vec_info stmt_info = vect_dr_stmt (dr);
> +  if (stmt_info->dr_aux.misalignment == DR_MISALIGNMENT_UNINITIALIZED)
>      return;
>
> -  if (DR_VECT_AUX (dr)->base_misaligned)
> +  if (stmt_info->dr_aux.base_misaligned)
>      {
> -      tree base_decl = DR_VECT_AUX (dr)->base_decl;
> +      tree base_decl = stmt_info->dr_aux.base_decl;
>
> -      unsigned int align_base_to = DR_TARGET_ALIGNMENT (dr) * BITS_PER_UNIT;
> +      unsigned int align_base_to = (stmt_info->dr_aux.target_alignment
> +                                   * BITS_PER_UNIT);
>
>        if (decl_in_symtab_p (base_decl))
>         symtab_node::get (base_decl)->increase_alignment (align_base_to);
> @@ -6190,7 +6194,7 @@ ensure_base_align (struct data_reference
>           SET_DECL_ALIGN (base_decl, align_base_to);
>            DECL_USER_ALIGN (base_decl) = 1;
>         }
> -      DR_VECT_AUX (dr)->base_misaligned = false;
> +      stmt_info->dr_aux.base_misaligned = false;
>      }
>  }
>
> @@ -7175,16 +7179,16 @@ vectorizable_store (stmt_vec_info stmt_i
>                    vect_permute_store_chain().  */
>                 vec_oprnd = result_chain[i];
>
> -             align = DR_TARGET_ALIGNMENT (first_dr);
> -             if (aligned_access_p (first_dr))
> +             align = STMT_VINFO_TARGET_ALIGNMENT (first_stmt_info);
> +             if (aligned_access_p (first_stmt_info))
>                 misalign = 0;
> -             else if (DR_MISALIGNMENT (first_dr) == -1)
> +             else if (dr_misalignment (first_stmt_info) == -1)
>                 {
> -                 align = dr_alignment (vect_dr_behavior (first_dr));
> +                 align = dr_alignment (vect_dr_behavior (first_stmt_info));
>                   misalign = 0;
>                 }
>               else
> -               misalign = DR_MISALIGNMENT (first_dr);
> +               misalign = dr_misalignment (first_stmt_info);
>               if (dataref_offset == NULL_TREE
>                   && TREE_CODE (dataref_ptr) == SSA_NAME)
>                 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> @@ -7227,9 +7231,9 @@ vectorizable_store (stmt_vec_info stmt_i
>                                           dataref_offset
>                                           ? dataref_offset
>                                           : build_int_cst (ref_type, 0));
> -                 if (aligned_access_p (first_dr))
> +                 if (aligned_access_p (first_stmt_info))
>                     ;
> -                 else if (DR_MISALIGNMENT (first_dr) == -1)
> +                 else if (dr_misalignment (first_stmt_info) == -1)
>                     TREE_TYPE (data_ref)
>                       = build_aligned_type (TREE_TYPE (data_ref),
>                                             align * BITS_PER_UNIT);
> @@ -8326,19 +8330,20 @@ vectorizable_load (stmt_vec_info stmt_in
>                         break;
>                       }
>
> -                   align = DR_TARGET_ALIGNMENT (dr);
> +                   align = STMT_VINFO_TARGET_ALIGNMENT (stmt_info);
>                     if (alignment_support_scheme == dr_aligned)
>                       {
> -                       gcc_assert (aligned_access_p (first_dr));
> +                       gcc_assert (aligned_access_p (first_stmt_info));
>                         misalign = 0;
>                       }
> -                   else if (DR_MISALIGNMENT (first_dr) == -1)
> +                   else if (dr_misalignment (first_stmt_info) == -1)
>                       {
> -                       align = dr_alignment (vect_dr_behavior (first_dr));
> +                       align = dr_alignment
> +                         (vect_dr_behavior (first_stmt_info));
>                         misalign = 0;
>                       }
>                     else
> -                     misalign = DR_MISALIGNMENT (first_dr);
> +                     misalign = dr_misalignment (first_stmt_info);
>                     if (dataref_offset == NULL_TREE
>                         && TREE_CODE (dataref_ptr) == SSA_NAME)
>                       set_ptr_info_alignment (get_ptr_info (dataref_ptr),
> @@ -8365,7 +8370,7 @@ vectorizable_load (stmt_vec_info stmt_in
>                                          : build_int_cst (ref_type, 0));
>                         if (alignment_support_scheme == dr_aligned)
>                           ;
> -                       else if (DR_MISALIGNMENT (first_dr) == -1)
> +                       else if (dr_misalignment (first_stmt_info) == -1)
>                           TREE_TYPE (data_ref)
>                             = build_aligned_type (TREE_TYPE (data_ref),
>                                                   align * BITS_PER_UNIT);
> @@ -8392,7 +8397,8 @@ vectorizable_load (stmt_vec_info stmt_in
>                       ptr = copy_ssa_name (dataref_ptr);
>                     else
>                       ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
> -                   unsigned int align = DR_TARGET_ALIGNMENT (first_dr);
> +                   unsigned int align
> +                     = STMT_VINFO_TARGET_ALIGNMENT (first_stmt_info);
>                     new_stmt = gimple_build_assign
>                                  (ptr, BIT_AND_EXPR, dataref_ptr,
>                                   build_int_cst
> @@ -8436,7 +8442,8 @@ vectorizable_load (stmt_vec_info stmt_in
>                       new_temp = copy_ssa_name (dataref_ptr);
>                     else
>                       new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
> -                   unsigned int align = DR_TARGET_ALIGNMENT (first_dr);
> +                   unsigned int align
> +                     = STMT_VINFO_TARGET_ALIGNMENT (first_stmt_info);
>                     new_stmt = gimple_build_assign
>                       (new_temp, BIT_AND_EXPR, dataref_ptr,
>                        build_int_cst (TREE_TYPE (dataref_ptr),

Richard Biener <richard.guenther@gmail.com> writes:
> On Tue, Jul 24, 2018 at 12:08 PM Richard Sandiford
> <richard.sandiford@arm.com> wrote:
>>
>> Alignment information is really a property of a stmt_vec_info
>> (and the way we want to vectorise it) rather than the original scalar dr.
>> I think that was true even before the recent dr sharing.
>
> But that is only so as long as we handle only stmts with a single DR.
> In reality alignment info _is_ a property of the DR and not of the stmt.
>
> So you're doing a shortcut here, shouldn't we rename
> dr_misalignment to stmt_dr_misalignment then?
>
> Otherwise I don't see how this makes sense semantically.

OK, the patch below takes a different approach, suggested in the
38/46 thread.  The idea is to make dr_aux link back to both the scalar
data_reference and the containing stmt_vec_info, so that it becomes a
lookup-free key for a vectorisable reference.

The data_reference link is just STMT_VINFO_DATA_REF, moved from
_stmt_vec_info.  The stmt pointer is a new field and always tracks
the current stmt_vec_info for the reference (which might be a pattern
stmt or the original stmt).

Then 38/40 can use dr_aux instead of data_reference (compared to current
sources) and instead of stmt_vec_info (compared to the original series).
This still avoids the repeated lookups that the series is trying to avoid.

The patch also makes the dr_aux in the current (possibly pattern) stmt
be the one that counts, rather than have the information stay with the
original DR_STMT.  A new macro (STMT_VINFO_DR_INFO) gives this
information for a given stmt_vec_info.

The changes together should make it easier to have multiple dr_auxs
in a single statement.

Thanks,
Richard


2018-07-26  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (vec_info::move_dr): New member function.
	(dataref_aux): Rename to...
	(dr_vec_info): ...this and add "dr" and "stmt" fields.
	(_stmt_vec_info::dr_aux): Update accordingly.
	(_stmt_vec_info::data_ref_info): Delete.
	(STMT_VINFO_GROUPED_ACCESS, DR_GROUP_FIRST_ELEMENT)
	(DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, DR_GROUP_STORE_COUNT)
	(DR_GROUP_GAP, DR_GROUP_SAME_DR_STMT, REDUC_GROUP_FIRST_ELEMENT):
	(REDUC_GROUP_NEXT_ELEMENT, REDUC_GROUP_SIZE): Use dr_aux.dr instead
	of data_ref.
	(STMT_VINFO_DATA_REF): Likewise.  Turn into an lvalue.
	(STMT_VINFO_DR_INFO): New macro.
	(DR_VECT_AUX): Use STMT_VINFO_DR_INKFO and vect_dr_stmt.
	(set_dr_misalignment): Update after rename of dataref_aux.
	(vect_dr_stmt): Move earlier in file.  Return dr_aux.stmt.
	* tree-vect-stmts.c (new_stmt_vec_info): Remove redundant
	initialization of STMT_VINFO_DATA_REF.
	* tree-vectorizer.c (vec_info::move_dr): New function.
	* tree-vect-patterns.c (vect_recog_bool_pattern)
	(vect_recog_mask_conversion_pattern)
	(vect_recog_gather_scatter_pattern): Use it.
	* tree-vect-data-refs.c (vect_analyze_data_refs): Initialize
	the "dr" and "stmt" fields of dr_vec_info instead of
	STMT_VINFO_DATA_REF.

Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h	2018-07-26 11:30:55.000000000 +0100
+++ gcc/tree-vectorizer.h	2018-07-26 11:30:56.197256524 +0100
@@ -240,6 +240,7 @@ struct vec_info {
   stmt_vec_info lookup_stmt (gimple *);
   stmt_vec_info lookup_def (tree);
   stmt_vec_info lookup_single_use (tree);
+  void move_dr (stmt_vec_info, stmt_vec_info);
 
   /* The type of vectorization.  */
   vec_kind kind;
@@ -767,7 +768,11 @@ enum vect_memory_access_type {
   VMAT_GATHER_SCATTER
 };
 
-struct dataref_aux {
+struct dr_vec_info {
+  /* The data reference itself.  */
+  data_reference *dr;
+  /* The statement that contains the data reference.  */
+  stmt_vec_info stmt;
   /* The misalignment in bytes of the reference, or -1 if not known.  */
   int misalignment;
   /* The byte alignment that we'd ideally like the reference to have,
@@ -818,11 +823,7 @@ struct _stmt_vec_info {
      data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
      at most one such data-ref.  */
 
-  /* Information about the data-ref (access function, etc),
-     relative to the inner-most containing loop.  */
-  struct data_reference *data_ref_info;
-
-  dataref_aux dr_aux;
+  dr_vec_info dr_aux;
 
   /* Information about the data-ref relative to this loop
      nest (the loop that is being considered for vectorization).  */
@@ -996,7 +997,7 @@ #define STMT_VINFO_LIVE_P(S)
 #define STMT_VINFO_VECTYPE(S)              (S)->vectype
 #define STMT_VINFO_VEC_STMT(S)             (S)->vectorized_stmt
 #define STMT_VINFO_VECTORIZABLE(S)         (S)->vectorizable
-#define STMT_VINFO_DATA_REF(S)             (S)->data_ref_info
+#define STMT_VINFO_DATA_REF(S)             ((S)->dr_aux.dr + 0)
 #define STMT_VINFO_GATHER_SCATTER_P(S)	   (S)->gather_scatter_p
 #define STMT_VINFO_STRIDED_P(S)	   	   (S)->strided_p
 #define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
@@ -1017,13 +1018,17 @@ #define STMT_VINFO_DR_OFFSET_ALIGNMENT(S
 #define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
   (S)->dr_wrt_vec_loop.step_alignment
 
+#define STMT_VINFO_DR_INFO(S) \
+  (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
+
 #define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p
 #define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt
 #define STMT_VINFO_PATTERN_DEF_SEQ(S)      (S)->pattern_def_seq
 #define STMT_VINFO_SAME_ALIGN_REFS(S)      (S)->same_align_refs
 #define STMT_VINFO_SIMD_CLONE_INFO(S)	   (S)->simd_clone_info
 #define STMT_VINFO_DEF_TYPE(S)             (S)->def_type
-#define STMT_VINFO_GROUPED_ACCESS(S)      ((S)->data_ref_info && DR_GROUP_FIRST_ELEMENT(S))
+#define STMT_VINFO_GROUPED_ACCESS(S) \
+  ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
 #define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
 #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
 #define STMT_VINFO_MIN_NEG_DIST(S)	(S)->min_neg_dist
@@ -1031,16 +1036,25 @@ #define STMT_VINFO_NUM_SLP_USES(S)	(S)->
 #define STMT_VINFO_REDUC_TYPE(S)	(S)->reduc_type
 #define STMT_VINFO_REDUC_DEF(S)		(S)->reduc_def
 
-#define DR_GROUP_FIRST_ELEMENT(S)  (gcc_checking_assert ((S)->data_ref_info), (S)->first_element)
-#define DR_GROUP_NEXT_ELEMENT(S)   (gcc_checking_assert ((S)->data_ref_info), (S)->next_element)
-#define DR_GROUP_SIZE(S)           (gcc_checking_assert ((S)->data_ref_info), (S)->size)
-#define DR_GROUP_STORE_COUNT(S)    (gcc_checking_assert ((S)->data_ref_info), (S)->store_count)
-#define DR_GROUP_GAP(S)            (gcc_checking_assert ((S)->data_ref_info), (S)->gap)
-#define DR_GROUP_SAME_DR_STMT(S)   (gcc_checking_assert ((S)->data_ref_info), (S)->same_dr_stmt)
-
-#define REDUC_GROUP_FIRST_ELEMENT(S)	(gcc_checking_assert (!(S)->data_ref_info), (S)->first_element)
-#define REDUC_GROUP_NEXT_ELEMENT(S)	(gcc_checking_assert (!(S)->data_ref_info), (S)->next_element)
-#define REDUC_GROUP_SIZE(S)		(gcc_checking_assert (!(S)->data_ref_info), (S)->size)
+#define DR_GROUP_FIRST_ELEMENT(S) \
+  (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
+#define DR_GROUP_NEXT_ELEMENT(S) \
+  (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
+#define DR_GROUP_SIZE(S) \
+  (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
+#define DR_GROUP_STORE_COUNT(S) \
+  (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
+#define DR_GROUP_GAP(S) \
+  (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
+#define DR_GROUP_SAME_DR_STMT(S) \
+  (gcc_checking_assert ((S)->dr_aux.dr), (S)->same_dr_stmt)
+
+#define REDUC_GROUP_FIRST_ELEMENT(S) \
+  (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
+#define REDUC_GROUP_NEXT_ELEMENT(S) \
+  (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
+#define REDUC_GROUP_SIZE(S) \
+  (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
 
 #define STMT_VINFO_RELEVANT_P(S)          ((S)->relevant != vect_unused_in_scope)
 
@@ -1048,7 +1062,7 @@ #define HYBRID_SLP_STMT(S)
 #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
 #define STMT_SLP_TYPE(S)                   (S)->slp_type
 
-#define DR_VECT_AUX(dr) (&vinfo_for_stmt (DR_STMT (dr))->dr_aux)
+#define DR_VECT_AUX(dr) (STMT_VINFO_DR_INFO (vect_dr_stmt (dr)))
 
 #define VECT_MAX_COST 1000
 
@@ -1259,6 +1273,20 @@ add_stmt_costs (void *data, stmt_vector_
 		   cost->misalign, cost->where);
 }
 
+/* Return the stmt DR is in.  For DR_STMT that have been replaced by
+   a pattern this returns the corresponding pattern stmt.  Otherwise
+   DR_STMT is returned.  */
+
+inline stmt_vec_info
+vect_dr_stmt (data_reference *dr)
+{
+  gimple *stmt = DR_STMT (dr);
+  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+  /* DR_STMT should never refer to a stmt in a pattern replacement.  */
+  gcc_checking_assert (!is_pattern_stmt_p (stmt_info));
+  return stmt_info->dr_aux.stmt;
+}
+
 /*-----------------------------------------------------------------*/
 /* Info on data references alignment.                              */
 /*-----------------------------------------------------------------*/
@@ -1268,8 +1296,7 @@ #define DR_MISALIGNMENT_UNINITIALIZED (-
 inline void
 set_dr_misalignment (struct data_reference *dr, int val)
 {
-  dataref_aux *data_aux = DR_VECT_AUX (dr);
-  data_aux->misalignment = val;
+  DR_VECT_AUX (dr)->misalignment = val;
 }
 
 inline int
@@ -1336,22 +1363,6 @@ vect_dr_behavior (data_reference *dr)
     return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
 }
 
-/* Return the stmt DR is in.  For DR_STMT that have been replaced by
-   a pattern this returns the corresponding pattern stmt.  Otherwise
-   DR_STMT is returned.  */
-
-inline stmt_vec_info
-vect_dr_stmt (data_reference *dr)
-{
-  gimple *stmt = DR_STMT (dr);
-  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
-  if (STMT_VINFO_IN_PATTERN_P (stmt_info))
-    return STMT_VINFO_RELATED_STMT (stmt_info);
-  /* DR_STMT should never refer to a stmt in a pattern replacement.  */
-  gcc_checking_assert (!STMT_VINFO_RELATED_STMT (stmt_info));
-  return stmt_info;
-}
-
 /* Return true if the vect cost model is unlimited.  */
 static inline bool
 unlimited_cost_model (loop_p loop)
Index: gcc/tree-vect-stmts.c
===================================================================
--- gcc/tree-vect-stmts.c	2018-07-26 11:30:55.000000000 +0100
+++ gcc/tree-vect-stmts.c	2018-07-26 11:30:56.197256524 +0100
@@ -9872,7 +9872,6 @@ new_stmt_vec_info (gimple *stmt, vec_inf
   STMT_VINFO_VECTORIZABLE (res) = true;
   STMT_VINFO_IN_PATTERN_P (res) = false;
   STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
-  STMT_VINFO_DATA_REF (res) = NULL;
   STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
   STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK;
 
Index: gcc/tree-vectorizer.c
===================================================================
--- gcc/tree-vectorizer.c	2018-07-26 11:30:55.000000000 +0100
+++ gcc/tree-vectorizer.c	2018-07-26 11:30:56.197256524 +0100
@@ -562,6 +562,21 @@ vec_info::lookup_single_use (tree lhs)
   return NULL;
 }
 
+/* Record that NEW_STMT_INFO now implements the same data reference
+   as OLD_STMT_INFO.  */
+
+void
+vec_info::move_dr (stmt_vec_info new_stmt_info, stmt_vec_info old_stmt_info)
+{
+  gcc_assert (!is_pattern_stmt_p (old_stmt_info));
+  STMT_VINFO_DR_INFO (old_stmt_info)->stmt = new_stmt_info;
+  new_stmt_info->dr_aux = old_stmt_info->dr_aux;
+  STMT_VINFO_DR_WRT_VEC_LOOP (new_stmt_info)
+    = STMT_VINFO_DR_WRT_VEC_LOOP (old_stmt_info);
+  STMT_VINFO_GATHER_SCATTER_P (new_stmt_info)
+    = STMT_VINFO_GATHER_SCATTER_P (old_stmt_info);
+}
+
 /* A helper function to free scev and LOOP niter information, as well as
    clear loop constraint LOOP_C_FINITE.  */
 
Index: gcc/tree-vect-patterns.c
===================================================================
--- gcc/tree-vect-patterns.c	2018-07-26 11:30:55.000000000 +0100
+++ gcc/tree-vect-patterns.c	2018-07-26 11:30:56.193256600 +0100
@@ -3828,10 +3828,7 @@ vect_recog_bool_pattern (stmt_vec_info s
 	}
       pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
       pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
-      STMT_VINFO_DATA_REF (pattern_stmt_info)
-	= STMT_VINFO_DATA_REF (stmt_vinfo);
-      STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
-	= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
+      vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
       *type_out = vectype;
       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
 
@@ -3954,14 +3951,7 @@ vect_recog_mask_conversion_pattern (stmt
 
       pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
       if (STMT_VINFO_DATA_REF (stmt_vinfo))
-	{
-	  STMT_VINFO_DATA_REF (pattern_stmt_info)
-	    = STMT_VINFO_DATA_REF (stmt_vinfo);
-	  STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
-	    = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
-	  STMT_VINFO_GATHER_SCATTER_P (pattern_stmt_info)
-	    = STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo);
-	}
+	vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
 
       *type_out = vectype1;
       vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4283,11 +4273,7 @@ vect_recog_gather_scatter_pattern (stmt_
   /* Copy across relevant vectorization info and associate DR with the
      new pattern statement instead of the original statement.  */
   stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
-  STMT_VINFO_DATA_REF (pattern_stmt_info) = dr;
-  STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
-    = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
-  STMT_VINFO_GATHER_SCATTER_P (pattern_stmt_info)
-    = STMT_VINFO_GATHER_SCATTER_P (stmt_info);
+  loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
 
   tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   *type_out = vectype;
Index: gcc/tree-vect-data-refs.c
===================================================================
--- gcc/tree-vect-data-refs.c	2018-07-26 11:30:55.000000000 +0100
+++ gcc/tree-vect-data-refs.c	2018-07-26 11:30:56.193256600 +0100
@@ -4120,7 +4120,10 @@ vect_analyze_data_refs (vec_info *vinfo,
       poly_uint64 vf;
 
       gcc_assert (DR_REF (dr));
-      stmt_vec_info stmt_info = vect_dr_stmt (dr);
+      stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
+      gcc_assert (!stmt_info->dr_aux.dr);
+      stmt_info->dr_aux.dr = dr;
+      stmt_info->dr_aux.stmt = stmt_info;
 
       /* Check that analysis of the data-ref succeeded.  */
       if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
@@ -4292,9 +4295,6 @@ vect_analyze_data_refs (vec_info *vinfo,
 	    }
 	}
 
-      gcc_assert (!STMT_VINFO_DATA_REF (stmt_info));
-      STMT_VINFO_DATA_REF (stmt_info) = dr;
-
       /* Set vectype for STMT.  */
       scalar_type = TREE_TYPE (DR_REF (dr));
       STMT_VINFO_VECTYPE (stmt_info)

On Thu, Jul 26, 2018 at 12:55 PM Richard Sandiford
<richard.sandiford@arm.com> wrote:
>
> Richard Biener <richard.guenther@gmail.com> writes:
> > On Tue, Jul 24, 2018 at 12:08 PM Richard Sandiford
> > <richard.sandiford@arm.com> wrote:
> >>
> >> Alignment information is really a property of a stmt_vec_info
> >> (and the way we want to vectorise it) rather than the original scalar dr.
> >> I think that was true even before the recent dr sharing.
> >
> > But that is only so as long as we handle only stmts with a single DR.
> > In reality alignment info _is_ a property of the DR and not of the stmt.
> >
> > So you're doing a shortcut here, shouldn't we rename
> > dr_misalignment to stmt_dr_misalignment then?
> >
> > Otherwise I don't see how this makes sense semantically.
>
> OK, the patch below takes a different approach, suggested in the
> 38/46 thread.  The idea is to make dr_aux link back to both the scalar
> data_reference and the containing stmt_vec_info, so that it becomes a
> lookup-free key for a vectorisable reference.
>
> The data_reference link is just STMT_VINFO_DATA_REF, moved from
> _stmt_vec_info.  The stmt pointer is a new field and always tracks
> the current stmt_vec_info for the reference (which might be a pattern
> stmt or the original stmt).
>
> Then 38/40 can use dr_aux instead of data_reference (compared to current
> sources) and instead of stmt_vec_info (compared to the original series).
> This still avoids the repeated lookups that the series is trying to avoid.
>
> The patch also makes the dr_aux in the current (possibly pattern) stmt
> be the one that counts, rather than have the information stay with the
> original DR_STMT.  A new macro (STMT_VINFO_DR_INFO) gives this
> information for a given stmt_vec_info.
>
> The changes together should make it easier to have multiple dr_auxs
> in a single statement.

I like this.

OK.
Richard.

> Thanks,
> Richard
>
>
> 2018-07-26  Richard Sandiford  <richard.sandiford@arm.com>
>
> gcc/
>         * tree-vectorizer.h (vec_info::move_dr): New member function.
>         (dataref_aux): Rename to...
>         (dr_vec_info): ...this and add "dr" and "stmt" fields.
>         (_stmt_vec_info::dr_aux): Update accordingly.
>         (_stmt_vec_info::data_ref_info): Delete.
>         (STMT_VINFO_GROUPED_ACCESS, DR_GROUP_FIRST_ELEMENT)
>         (DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, DR_GROUP_STORE_COUNT)
>         (DR_GROUP_GAP, DR_GROUP_SAME_DR_STMT, REDUC_GROUP_FIRST_ELEMENT):
>         (REDUC_GROUP_NEXT_ELEMENT, REDUC_GROUP_SIZE): Use dr_aux.dr instead
>         of data_ref.
>         (STMT_VINFO_DATA_REF): Likewise.  Turn into an lvalue.
>         (STMT_VINFO_DR_INFO): New macro.
>         (DR_VECT_AUX): Use STMT_VINFO_DR_INKFO and vect_dr_stmt.
>         (set_dr_misalignment): Update after rename of dataref_aux.
>         (vect_dr_stmt): Move earlier in file.  Return dr_aux.stmt.
>         * tree-vect-stmts.c (new_stmt_vec_info): Remove redundant
>         initialization of STMT_VINFO_DATA_REF.
>         * tree-vectorizer.c (vec_info::move_dr): New function.
>         * tree-vect-patterns.c (vect_recog_bool_pattern)
>         (vect_recog_mask_conversion_pattern)
>         (vect_recog_gather_scatter_pattern): Use it.
>         * tree-vect-data-refs.c (vect_analyze_data_refs): Initialize
>         the "dr" and "stmt" fields of dr_vec_info instead of
>         STMT_VINFO_DATA_REF.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h       2018-07-26 11:30:55.000000000 +0100
> +++ gcc/tree-vectorizer.h       2018-07-26 11:30:56.197256524 +0100
> @@ -240,6 +240,7 @@ struct vec_info {
>    stmt_vec_info lookup_stmt (gimple *);
>    stmt_vec_info lookup_def (tree);
>    stmt_vec_info lookup_single_use (tree);
> +  void move_dr (stmt_vec_info, stmt_vec_info);
>
>    /* The type of vectorization.  */
>    vec_kind kind;
> @@ -767,7 +768,11 @@ enum vect_memory_access_type {
>    VMAT_GATHER_SCATTER
>  };
>
> -struct dataref_aux {
> +struct dr_vec_info {
> +  /* The data reference itself.  */
> +  data_reference *dr;
> +  /* The statement that contains the data reference.  */
> +  stmt_vec_info stmt;
>    /* The misalignment in bytes of the reference, or -1 if not known.  */
>    int misalignment;
>    /* The byte alignment that we'd ideally like the reference to have,
> @@ -818,11 +823,7 @@ struct _stmt_vec_info {
>       data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
>       at most one such data-ref.  */
>
> -  /* Information about the data-ref (access function, etc),
> -     relative to the inner-most containing loop.  */
> -  struct data_reference *data_ref_info;
> -
> -  dataref_aux dr_aux;
> +  dr_vec_info dr_aux;
>
>    /* Information about the data-ref relative to this loop
>       nest (the loop that is being considered for vectorization).  */
> @@ -996,7 +997,7 @@ #define STMT_VINFO_LIVE_P(S)
>  #define STMT_VINFO_VECTYPE(S)              (S)->vectype
>  #define STMT_VINFO_VEC_STMT(S)             (S)->vectorized_stmt
>  #define STMT_VINFO_VECTORIZABLE(S)         (S)->vectorizable
> -#define STMT_VINFO_DATA_REF(S)             (S)->data_ref_info
> +#define STMT_VINFO_DATA_REF(S)             ((S)->dr_aux.dr + 0)
>  #define STMT_VINFO_GATHER_SCATTER_P(S)    (S)->gather_scatter_p
>  #define STMT_VINFO_STRIDED_P(S)                   (S)->strided_p
>  #define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
> @@ -1017,13 +1018,17 @@ #define STMT_VINFO_DR_OFFSET_ALIGNMENT(S
>  #define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
>    (S)->dr_wrt_vec_loop.step_alignment
>
> +#define STMT_VINFO_DR_INFO(S) \
> +  (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
> +
>  #define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p
>  #define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt
>  #define STMT_VINFO_PATTERN_DEF_SEQ(S)      (S)->pattern_def_seq
>  #define STMT_VINFO_SAME_ALIGN_REFS(S)      (S)->same_align_refs
>  #define STMT_VINFO_SIMD_CLONE_INFO(S)     (S)->simd_clone_info
>  #define STMT_VINFO_DEF_TYPE(S)             (S)->def_type
> -#define STMT_VINFO_GROUPED_ACCESS(S)      ((S)->data_ref_info && DR_GROUP_FIRST_ELEMENT(S))
> +#define STMT_VINFO_GROUPED_ACCESS(S) \
> +  ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
>  #define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
>  #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
>  #define STMT_VINFO_MIN_NEG_DIST(S)     (S)->min_neg_dist
> @@ -1031,16 +1036,25 @@ #define STMT_VINFO_NUM_SLP_USES(S)      (S)->
>  #define STMT_VINFO_REDUC_TYPE(S)       (S)->reduc_type
>  #define STMT_VINFO_REDUC_DEF(S)                (S)->reduc_def
>
> -#define DR_GROUP_FIRST_ELEMENT(S)  (gcc_checking_assert ((S)->data_ref_info), (S)->first_element)
> -#define DR_GROUP_NEXT_ELEMENT(S)   (gcc_checking_assert ((S)->data_ref_info), (S)->next_element)
> -#define DR_GROUP_SIZE(S)           (gcc_checking_assert ((S)->data_ref_info), (S)->size)
> -#define DR_GROUP_STORE_COUNT(S)    (gcc_checking_assert ((S)->data_ref_info), (S)->store_count)
> -#define DR_GROUP_GAP(S)            (gcc_checking_assert ((S)->data_ref_info), (S)->gap)
> -#define DR_GROUP_SAME_DR_STMT(S)   (gcc_checking_assert ((S)->data_ref_info), (S)->same_dr_stmt)
> -
> -#define REDUC_GROUP_FIRST_ELEMENT(S)   (gcc_checking_assert (!(S)->data_ref_info), (S)->first_element)
> -#define REDUC_GROUP_NEXT_ELEMENT(S)    (gcc_checking_assert (!(S)->data_ref_info), (S)->next_element)
> -#define REDUC_GROUP_SIZE(S)            (gcc_checking_assert (!(S)->data_ref_info), (S)->size)
> +#define DR_GROUP_FIRST_ELEMENT(S) \
> +  (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
> +#define DR_GROUP_NEXT_ELEMENT(S) \
> +  (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
> +#define DR_GROUP_SIZE(S) \
> +  (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
> +#define DR_GROUP_STORE_COUNT(S) \
> +  (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
> +#define DR_GROUP_GAP(S) \
> +  (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
> +#define DR_GROUP_SAME_DR_STMT(S) \
> +  (gcc_checking_assert ((S)->dr_aux.dr), (S)->same_dr_stmt)
> +
> +#define REDUC_GROUP_FIRST_ELEMENT(S) \
> +  (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
> +#define REDUC_GROUP_NEXT_ELEMENT(S) \
> +  (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
> +#define REDUC_GROUP_SIZE(S) \
> +  (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
>
>  #define STMT_VINFO_RELEVANT_P(S)          ((S)->relevant != vect_unused_in_scope)
>
> @@ -1048,7 +1062,7 @@ #define HYBRID_SLP_STMT(S)
>  #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
>  #define STMT_SLP_TYPE(S)                   (S)->slp_type
>
> -#define DR_VECT_AUX(dr) (&vinfo_for_stmt (DR_STMT (dr))->dr_aux)
> +#define DR_VECT_AUX(dr) (STMT_VINFO_DR_INFO (vect_dr_stmt (dr)))
>
>  #define VECT_MAX_COST 1000
>
> @@ -1259,6 +1273,20 @@ add_stmt_costs (void *data, stmt_vector_
>                    cost->misalign, cost->where);
>  }
>
> +/* Return the stmt DR is in.  For DR_STMT that have been replaced by
> +   a pattern this returns the corresponding pattern stmt.  Otherwise
> +   DR_STMT is returned.  */
> +
> +inline stmt_vec_info
> +vect_dr_stmt (data_reference *dr)
> +{
> +  gimple *stmt = DR_STMT (dr);
> +  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> +  /* DR_STMT should never refer to a stmt in a pattern replacement.  */
> +  gcc_checking_assert (!is_pattern_stmt_p (stmt_info));
> +  return stmt_info->dr_aux.stmt;
> +}
> +
>  /*-----------------------------------------------------------------*/
>  /* Info on data references alignment.                              */
>  /*-----------------------------------------------------------------*/
> @@ -1268,8 +1296,7 @@ #define DR_MISALIGNMENT_UNINITIALIZED (-
>  inline void
>  set_dr_misalignment (struct data_reference *dr, int val)
>  {
> -  dataref_aux *data_aux = DR_VECT_AUX (dr);
> -  data_aux->misalignment = val;
> +  DR_VECT_AUX (dr)->misalignment = val;
>  }
>
>  inline int
> @@ -1336,22 +1363,6 @@ vect_dr_behavior (data_reference *dr)
>      return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
>  }
>
> -/* Return the stmt DR is in.  For DR_STMT that have been replaced by
> -   a pattern this returns the corresponding pattern stmt.  Otherwise
> -   DR_STMT is returned.  */
> -
> -inline stmt_vec_info
> -vect_dr_stmt (data_reference *dr)
> -{
> -  gimple *stmt = DR_STMT (dr);
> -  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> -  if (STMT_VINFO_IN_PATTERN_P (stmt_info))
> -    return STMT_VINFO_RELATED_STMT (stmt_info);
> -  /* DR_STMT should never refer to a stmt in a pattern replacement.  */
> -  gcc_checking_assert (!STMT_VINFO_RELATED_STMT (stmt_info));
> -  return stmt_info;
> -}
> -
>  /* Return true if the vect cost model is unlimited.  */
>  static inline bool
>  unlimited_cost_model (loop_p loop)
> Index: gcc/tree-vect-stmts.c
> ===================================================================
> --- gcc/tree-vect-stmts.c       2018-07-26 11:30:55.000000000 +0100
> +++ gcc/tree-vect-stmts.c       2018-07-26 11:30:56.197256524 +0100
> @@ -9872,7 +9872,6 @@ new_stmt_vec_info (gimple *stmt, vec_inf
>    STMT_VINFO_VECTORIZABLE (res) = true;
>    STMT_VINFO_IN_PATTERN_P (res) = false;
>    STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
> -  STMT_VINFO_DATA_REF (res) = NULL;
>    STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
>    STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK;
>
> Index: gcc/tree-vectorizer.c
> ===================================================================
> --- gcc/tree-vectorizer.c       2018-07-26 11:30:55.000000000 +0100
> +++ gcc/tree-vectorizer.c       2018-07-26 11:30:56.197256524 +0100
> @@ -562,6 +562,21 @@ vec_info::lookup_single_use (tree lhs)
>    return NULL;
>  }
>
> +/* Record that NEW_STMT_INFO now implements the same data reference
> +   as OLD_STMT_INFO.  */
> +
> +void
> +vec_info::move_dr (stmt_vec_info new_stmt_info, stmt_vec_info old_stmt_info)
> +{
> +  gcc_assert (!is_pattern_stmt_p (old_stmt_info));
> +  STMT_VINFO_DR_INFO (old_stmt_info)->stmt = new_stmt_info;
> +  new_stmt_info->dr_aux = old_stmt_info->dr_aux;
> +  STMT_VINFO_DR_WRT_VEC_LOOP (new_stmt_info)
> +    = STMT_VINFO_DR_WRT_VEC_LOOP (old_stmt_info);
> +  STMT_VINFO_GATHER_SCATTER_P (new_stmt_info)
> +    = STMT_VINFO_GATHER_SCATTER_P (old_stmt_info);
> +}
> +
>  /* A helper function to free scev and LOOP niter information, as well as
>     clear loop constraint LOOP_C_FINITE.  */
>
> Index: gcc/tree-vect-patterns.c
> ===================================================================
> --- gcc/tree-vect-patterns.c    2018-07-26 11:30:55.000000000 +0100
> +++ gcc/tree-vect-patterns.c    2018-07-26 11:30:56.193256600 +0100
> @@ -3828,10 +3828,7 @@ vect_recog_bool_pattern (stmt_vec_info s
>         }
>        pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
>        pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
> -      STMT_VINFO_DATA_REF (pattern_stmt_info)
> -       = STMT_VINFO_DATA_REF (stmt_vinfo);
> -      STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
> -       = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
> +      vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
>        *type_out = vectype;
>        vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
>
> @@ -3954,14 +3951,7 @@ vect_recog_mask_conversion_pattern (stmt
>
>        pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
>        if (STMT_VINFO_DATA_REF (stmt_vinfo))
> -       {
> -         STMT_VINFO_DATA_REF (pattern_stmt_info)
> -           = STMT_VINFO_DATA_REF (stmt_vinfo);
> -         STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
> -           = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
> -         STMT_VINFO_GATHER_SCATTER_P (pattern_stmt_info)
> -           = STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo);
> -       }
> +       vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
>
>        *type_out = vectype1;
>        vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
> @@ -4283,11 +4273,7 @@ vect_recog_gather_scatter_pattern (stmt_
>    /* Copy across relevant vectorization info and associate DR with the
>       new pattern statement instead of the original statement.  */
>    stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
> -  STMT_VINFO_DATA_REF (pattern_stmt_info) = dr;
> -  STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
> -    = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
> -  STMT_VINFO_GATHER_SCATTER_P (pattern_stmt_info)
> -    = STMT_VINFO_GATHER_SCATTER_P (stmt_info);
> +  loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
>
>    tree vectype = STMT_VINFO_VECTYPE (stmt_info);
>    *type_out = vectype;
> Index: gcc/tree-vect-data-refs.c
> ===================================================================
> --- gcc/tree-vect-data-refs.c   2018-07-26 11:30:55.000000000 +0100
> +++ gcc/tree-vect-data-refs.c   2018-07-26 11:30:56.193256600 +0100
> @@ -4120,7 +4120,10 @@ vect_analyze_data_refs (vec_info *vinfo,
>        poly_uint64 vf;
>
>        gcc_assert (DR_REF (dr));
> -      stmt_vec_info stmt_info = vect_dr_stmt (dr);
> +      stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
> +      gcc_assert (!stmt_info->dr_aux.dr);
> +      stmt_info->dr_aux.dr = dr;
> +      stmt_info->dr_aux.stmt = stmt_info;
>
>        /* Check that analysis of the data-ref succeeded.  */
>        if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
> @@ -4292,9 +4295,6 @@ vect_analyze_data_refs (vec_info *vinfo,
>             }
>         }
>
> -      gcc_assert (!STMT_VINFO_DATA_REF (stmt_info));
> -      STMT_VINFO_DATA_REF (stmt_info) = dr;
> -
>        /* Set vectype for STMT.  */
>        scalar_type = TREE_TYPE (DR_REF (dr));
>        STMT_VINFO_VECTYPE (stmt_info)