Message ID | 87twgults1.fsf@e105548-lin.cambridge.arm.com |
---|---|
State | New |
Headers | show |
On Wed, Jun 15, 2016 at 10:51 AM, Richard Sandiford <richard.sandiford@arm.com> wrote: > This patch just refactors the gather/scatter support so that all > information is in a single structure, rather than separate variables. > This reduces the number of arguments to a function added in patch 6. > > Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install? Please pack it by moving offset_dt after offset_vectype. Ok with that change. Thanks, Richard. > Thanks, > Richard > > > gcc/ > * tree-vectorizer.h (gather_scatter_info): New structure. > (vect_check_gather_scatter): Return a bool rather than a decl. > Replace return-by-pointer arguments with a single > gather_scatter_info *. > * tree-vect-data-refs.c (vect_check_gather_scatter): Likewise. > (vect_analyze_data_refs): Update call accordingly. > * tree-vect-stmts.c (vect_mark_stmts_to_be_vectorized): Likewise. > (vectorizable_mask_load_store): Likewise. Also record the > offset dt and vectype in the gather_scatter_info. > (vectorizable_store): Likewise. > (vectorizable_load): Likewise. > > Index: gcc/tree-vectorizer.h > =================================================================== > --- gcc/tree-vectorizer.h > +++ gcc/tree-vectorizer.h > @@ -612,6 +612,28 @@ typedef struct _stmt_vec_info { > unsigned int num_slp_uses; > } *stmt_vec_info; > > +/* Information about a gather/scatter call. */ > +struct gather_scatter_info { > + /* The FUNCTION_DECL for the built-in gather/scatter function. */ > + tree decl; > + > + /* The loop-invariant base value. */ > + tree base; > + > + /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */ > + tree offset; > + > + /* The definition type for the vectorized offset. */ > + enum vect_def_type offset_dt; > + > + /* The type of the vectorized offset. */ > + tree offset_vectype; > + > + /* Each offset element should be multiplied by this amount before > + being added to the base. */ > + int scale; > +}; > + > /* Access Functions. */ > #define STMT_VINFO_TYPE(S) (S)->type > #define STMT_VINFO_STMT(S) (S)->stmt > @@ -1035,8 +1057,8 @@ extern bool vect_verify_datarefs_alignment (loop_vec_info); > extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance); > extern bool vect_analyze_data_ref_accesses (vec_info *); > extern bool vect_prune_runtime_alias_test_list (loop_vec_info); > -extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree *, > - int *); > +extern bool vect_check_gather_scatter (gimple *, loop_vec_info, > + gather_scatter_info *); > extern bool vect_analyze_data_refs (vec_info *, int *); > extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree, > tree *, gimple_stmt_iterator *, > Index: gcc/tree-vect-data-refs.c > =================================================================== > --- gcc/tree-vect-data-refs.c > +++ gcc/tree-vect-data-refs.c > @@ -3174,12 +3174,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) > return true; > } > > -/* Check whether a non-affine read or write in stmt is suitable for gather load > - or scatter store and if so, return a builtin decl for that operation. */ > +/* Return true if a non-affine read or write in STMT is suitable for a > + gather load or scatter store. Describe the operation in *INFO if so. */ > > -tree > -vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, > - tree *offp, int *scalep) > +bool > +vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, > + gather_scatter_info *info) > { > HOST_WIDE_INT scale = 1, pbitpos, pbitsize; > struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); > @@ -3253,7 +3253,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, > if (!expr_invariant_in_loop_p (loop, base)) > { > if (!integer_zerop (off)) > - return NULL_TREE; > + return false; > off = base; > base = size_int (pbitpos / BITS_PER_UNIT); > } > @@ -3279,7 +3279,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, > gimple *def_stmt = SSA_NAME_DEF_STMT (off); > > if (expr_invariant_in_loop_p (loop, off)) > - return NULL_TREE; > + return false; > > if (gimple_code (def_stmt) != GIMPLE_ASSIGN) > break; > @@ -3291,7 +3291,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, > else > { > if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS) > - return NULL_TREE; > + return false; > code = TREE_CODE (off); > extract_ops_from_tree (off, &code, &op0, &op1); > } > @@ -3366,7 +3366,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, > defined in the loop, punt. */ > if (TREE_CODE (off) != SSA_NAME > || expr_invariant_in_loop_p (loop, off)) > - return NULL_TREE; > + return false; > > if (offtype == NULL_TREE) > offtype = TREE_TYPE (off); > @@ -3379,15 +3379,15 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, > offtype, scale); > > if (decl == NULL_TREE) > - return NULL_TREE; > - > - if (basep) > - *basep = base; > - if (offp) > - *offp = off; > - if (scalep) > - *scalep = scale; > - return decl; > + return false; > + > + info->decl = decl; > + info->base = base; > + info->offset = off; > + info->offset_dt = vect_unknown_def_type; > + info->offset_vectype = NULL_TREE; > + info->scale = scale; > + return true; > } > > /* Function vect_analyze_data_refs. > @@ -3865,10 +3865,10 @@ again: > > if (gatherscatter != SG_NONE) > { > - tree off; > + gather_scatter_info gs_info; > if (!vect_check_gather_scatter (stmt, as_a <loop_vec_info> (vinfo), > - NULL, &off, NULL) > - || get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE) > + &gs_info) > + || !get_vectype_for_scalar_type (TREE_TYPE (gs_info.offset))) > { > STMT_VINFO_DATA_REF (stmt_info) = NULL; > free_data_ref (dr); > Index: gcc/tree-vect-stmts.c > =================================================================== > --- gcc/tree-vect-stmts.c > +++ gcc/tree-vect-stmts.c > @@ -772,10 +772,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) > > if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo)) > { > - tree off; > - tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL); > - gcc_assert (decl); > - if (!process_use (stmt, off, loop_vinfo, relevant, &worklist, true)) > + gather_scatter_info gs_info; > + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) > + gcc_unreachable (); > + if (!process_use (stmt, gs_info.offset, loop_vinfo, relevant, > + &worklist, true)) > return false; > } > } /* while worklist */ > @@ -1703,10 +1704,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > int ncopies; > int i, j; > bool inv_p; > - tree gather_base = NULL_TREE, gather_off = NULL_TREE; > - tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE; > - int gather_scale = 1; > - enum vect_def_type gather_dt = vect_unknown_def_type; > + gather_scatter_info gs_info; > bool is_store; > tree mask; > gimple *def_stmt; > @@ -1774,11 +1772,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) > { > gimple *def_stmt; > - gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, > - &gather_off, &gather_scale); > - gcc_assert (gather_decl); > - if (!vect_is_simple_use (gather_off, loop_vinfo, &def_stmt, &gather_dt, > - &gather_off_vectype)) > + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) > + gcc_unreachable (); > + if (!vect_is_simple_use (gs_info.offset, loop_vinfo, &def_stmt, > + &gs_info.offset_dt, &gs_info.offset_vectype)) > { > if (dump_enabled_p ()) > dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > @@ -1786,7 +1783,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > return false; > } > > - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); > + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); > tree masktype > = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist)))); > if (TREE_CODE (masktype) == INTEGER_TYPE) > @@ -1825,7 +1822,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) > { > tree vec_oprnd0 = NULL_TREE, op; > - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); > + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); > tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; > tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale; > tree perm_mask = NULL_TREE, prev_res = NULL_TREE; > @@ -1834,9 +1831,9 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > gimple_seq seq; > basic_block new_bb; > enum { NARROW, NONE, WIDEN } modifier; > - int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype); > + int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); > > - rettype = TREE_TYPE (TREE_TYPE (gather_decl)); > + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); > srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > @@ -1855,7 +1852,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > for (i = 0; i < gather_off_nunits; ++i) > sel[i] = i | nunits; > > - perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel); > + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); > } > else if (nunits == gather_off_nunits * 2) > { > @@ -1877,7 +1874,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > > vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype); > > - ptr = fold_convert (ptrtype, gather_base); > + ptr = fold_convert (ptrtype, gs_info.base); > if (!is_gimple_min_invariant (ptr)) > { > ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); > @@ -1885,7 +1882,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > gcc_assert (!new_bb); > } > > - scale = build_int_cst (scaletype, gather_scale); > + scale = build_int_cst (scaletype, gs_info.scale); > > prev_stmt_info = NULL; > for (j = 0; j < ncopies; ++j) > @@ -1895,10 +1892,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > perm_mask, stmt, gsi); > else if (j == 0) > op = vec_oprnd0 > - = vect_get_vec_def_for_operand (gather_off, stmt); > + = vect_get_vec_def_for_operand (gs_info.offset, stmt); > else > op = vec_oprnd0 > - = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0); > + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0); > > if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) > { > @@ -1940,7 +1937,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, > } > > new_stmt > - = gimple_build_call (gather_decl, 5, mask_op, ptr, op, mask_op, > + = gimple_build_call (gs_info.decl, 5, mask_op, ptr, op, mask_op, > scale); > > if (!useless_type_conversion_p (vectype, rettype)) > @@ -5207,10 +5204,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); > vec_info *vinfo = stmt_info->vinfo; > tree aggr_type; > - tree scatter_base = NULL_TREE, scatter_off = NULL_TREE; > - tree scatter_off_vectype = NULL_TREE, scatter_decl = NULL_TREE; > - int scatter_scale = 1; > - enum vect_def_type scatter_idx_dt = vect_unknown_def_type; > + gather_scatter_info gs_info; > enum vect_def_type scatter_src_dt = vect_unknown_def_type; > gimple *new_stmt; > > @@ -5378,11 +5372,10 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) > { > gimple *def_stmt; > - scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo, &scatter_base, > - &scatter_off, &scatter_scale); > - gcc_assert (scatter_decl); > - if (!vect_is_simple_use (scatter_off, vinfo, &def_stmt, &scatter_idx_dt, > - &scatter_off_vectype)) > + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) > + gcc_unreachable (); > + if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt, > + &gs_info.offset_dt, &gs_info.offset_vectype)) > { > if (dump_enabled_p ()) > dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > @@ -5408,14 +5401,14 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) > { > tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src; > - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (scatter_decl)); > + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); > tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; > tree ptr, mask, var, scale, perm_mask = NULL_TREE; > edge pe = loop_preheader_edge (loop); > gimple_seq seq; > basic_block new_bb; > enum { NARROW, NONE, WIDEN } modifier; > - int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (scatter_off_vectype); > + int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); > > if (nunits == (unsigned int) scatter_off_nunits) > modifier = NONE; > @@ -5427,7 +5420,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > for (i = 0; i < (unsigned int) scatter_off_nunits; ++i) > sel[i] = i | nunits; > > - perm_mask = vect_gen_perm_mask_checked (scatter_off_vectype, sel); > + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); > gcc_assert (perm_mask != NULL_TREE); > } > else if (nunits == (unsigned int) scatter_off_nunits * 2) > @@ -5445,7 +5438,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > else > gcc_unreachable (); > > - rettype = TREE_TYPE (TREE_TYPE (scatter_decl)); > + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); > ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > @@ -5455,7 +5448,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE > && TREE_CODE (rettype) == VOID_TYPE); > > - ptr = fold_convert (ptrtype, scatter_base); > + ptr = fold_convert (ptrtype, gs_info.base); > if (!is_gimple_min_invariant (ptr)) > { > ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); > @@ -5468,7 +5461,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > mask = build_int_cst (masktype, -1); > mask = vect_init_vector (stmt, mask, masktype, NULL); > > - scale = build_int_cst (scaletype, scatter_scale); > + scale = build_int_cst (scaletype, gs_info.scale); > > prev_stmt_info = NULL; > for (j = 0; j < ncopies; ++j) > @@ -5478,7 +5471,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > src = vec_oprnd1 > = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt); > op = vec_oprnd0 > - = vect_get_vec_def_for_operand (scatter_off, stmt); > + = vect_get_vec_def_for_operand (gs_info.offset, stmt); > } > else if (modifier != NONE && (j & 1)) > { > @@ -5494,7 +5487,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask, > stmt, gsi); > op = vec_oprnd0 > - = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); > + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, > + vec_oprnd0); > } > else > gcc_unreachable (); > @@ -5504,7 +5498,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > src = vec_oprnd1 > = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1); > op = vec_oprnd0 > - = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); > + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, > + vec_oprnd0); > } > > if (!useless_type_conversion_p (srctype, TREE_TYPE (src))) > @@ -5530,7 +5525,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > } > > new_stmt > - = gimple_build_call (scatter_decl, 5, ptr, mask, op, src, scale); > + = gimple_build_call (gs_info.decl, 5, ptr, mask, op, src, scale); > > vect_finish_stmt_generation (stmt, new_stmt, gsi); > > @@ -6204,10 +6199,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); > int vf; > tree aggr_type; > - tree gather_base = NULL_TREE, gather_off = NULL_TREE; > - tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE; > - int gather_scale = 1; > - enum vect_def_type gather_dt = vect_unknown_def_type; > + gather_scatter_info gs_info; > vec_info *vinfo = stmt_info->vinfo; > > if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) > @@ -6404,11 +6396,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) > { > gimple *def_stmt; > - gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, > - &gather_off, &gather_scale); > - gcc_assert (gather_decl); > - if (!vect_is_simple_use (gather_off, vinfo, &def_stmt, &gather_dt, > - &gather_off_vectype)) > + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) > + gcc_unreachable (); > + if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt, > + &gs_info.offset_dt, &gs_info.offset_vectype)) > { > if (dump_enabled_p ()) > dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > @@ -6483,14 +6474,14 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) > { > tree vec_oprnd0 = NULL_TREE, op; > - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); > + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); > tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; > tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res = NULL_TREE; > edge pe = loop_preheader_edge (loop); > gimple_seq seq; > basic_block new_bb; > enum { NARROW, NONE, WIDEN } modifier; > - int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype); > + int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); > > if (nunits == gather_off_nunits) > modifier = NONE; > @@ -6502,7 +6493,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > for (i = 0; i < gather_off_nunits; ++i) > sel[i] = i | nunits; > > - perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel); > + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); > } > else if (nunits == gather_off_nunits * 2) > { > @@ -6519,7 +6510,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > else > gcc_unreachable (); > > - rettype = TREE_TYPE (TREE_TYPE (gather_decl)); > + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); > srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); > @@ -6529,7 +6520,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > > vec_dest = vect_create_destination_var (scalar_dest, vectype); > > - ptr = fold_convert (ptrtype, gather_base); > + ptr = fold_convert (ptrtype, gs_info.base); > if (!is_gimple_min_invariant (ptr)) > { > ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); > @@ -6561,7 +6552,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > else > gcc_unreachable (); > > - scale = build_int_cst (scaletype, gather_scale); > + scale = build_int_cst (scaletype, gs_info.scale); > > if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE) > merge = build_int_cst (TREE_TYPE (rettype), 0); > @@ -6587,10 +6578,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > perm_mask, stmt, gsi); > else if (j == 0) > op = vec_oprnd0 > - = vect_get_vec_def_for_operand (gather_off, stmt); > + = vect_get_vec_def_for_operand (gs_info.offset, stmt); > else > op = vec_oprnd0 > - = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0); > + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0); > > if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) > { > @@ -6605,7 +6596,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, > } > > new_stmt > - = gimple_build_call (gather_decl, 5, merge, ptr, op, mask, scale); > + = gimple_build_call (gs_info.decl, 5, merge, ptr, op, mask, scale); > > if (!useless_type_conversion_p (vectype, rettype)) > {
Index: gcc/tree-vectorizer.h =================================================================== --- gcc/tree-vectorizer.h +++ gcc/tree-vectorizer.h @@ -612,6 +612,28 @@ typedef struct _stmt_vec_info { unsigned int num_slp_uses; } *stmt_vec_info; +/* Information about a gather/scatter call. */ +struct gather_scatter_info { + /* The FUNCTION_DECL for the built-in gather/scatter function. */ + tree decl; + + /* The loop-invariant base value. */ + tree base; + + /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */ + tree offset; + + /* The definition type for the vectorized offset. */ + enum vect_def_type offset_dt; + + /* The type of the vectorized offset. */ + tree offset_vectype; + + /* Each offset element should be multiplied by this amount before + being added to the base. */ + int scale; +}; + /* Access Functions. */ #define STMT_VINFO_TYPE(S) (S)->type #define STMT_VINFO_STMT(S) (S)->stmt @@ -1035,8 +1057,8 @@ extern bool vect_verify_datarefs_alignment (loop_vec_info); extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance); extern bool vect_analyze_data_ref_accesses (vec_info *); extern bool vect_prune_runtime_alias_test_list (loop_vec_info); -extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree *, - int *); +extern bool vect_check_gather_scatter (gimple *, loop_vec_info, + gather_scatter_info *); extern bool vect_analyze_data_refs (vec_info *, int *); extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree, tree *, gimple_stmt_iterator *, Index: gcc/tree-vect-data-refs.c =================================================================== --- gcc/tree-vect-data-refs.c +++ gcc/tree-vect-data-refs.c @@ -3174,12 +3174,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) return true; } -/* Check whether a non-affine read or write in stmt is suitable for gather load - or scatter store and if so, return a builtin decl for that operation. */ +/* Return true if a non-affine read or write in STMT is suitable for a + gather load or scatter store. Describe the operation in *INFO if so. */ -tree -vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, - tree *offp, int *scalep) +bool +vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, + gather_scatter_info *info) { HOST_WIDE_INT scale = 1, pbitpos, pbitsize; struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -3253,7 +3253,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, if (!expr_invariant_in_loop_p (loop, base)) { if (!integer_zerop (off)) - return NULL_TREE; + return false; off = base; base = size_int (pbitpos / BITS_PER_UNIT); } @@ -3279,7 +3279,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, gimple *def_stmt = SSA_NAME_DEF_STMT (off); if (expr_invariant_in_loop_p (loop, off)) - return NULL_TREE; + return false; if (gimple_code (def_stmt) != GIMPLE_ASSIGN) break; @@ -3291,7 +3291,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, else { if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS) - return NULL_TREE; + return false; code = TREE_CODE (off); extract_ops_from_tree (off, &code, &op0, &op1); } @@ -3366,7 +3366,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, defined in the loop, punt. */ if (TREE_CODE (off) != SSA_NAME || expr_invariant_in_loop_p (loop, off)) - return NULL_TREE; + return false; if (offtype == NULL_TREE) offtype = TREE_TYPE (off); @@ -3379,15 +3379,15 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, offtype, scale); if (decl == NULL_TREE) - return NULL_TREE; - - if (basep) - *basep = base; - if (offp) - *offp = off; - if (scalep) - *scalep = scale; - return decl; + return false; + + info->decl = decl; + info->base = base; + info->offset = off; + info->offset_dt = vect_unknown_def_type; + info->offset_vectype = NULL_TREE; + info->scale = scale; + return true; } /* Function vect_analyze_data_refs. @@ -3865,10 +3865,10 @@ again: if (gatherscatter != SG_NONE) { - tree off; + gather_scatter_info gs_info; if (!vect_check_gather_scatter (stmt, as_a <loop_vec_info> (vinfo), - NULL, &off, NULL) - || get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE) + &gs_info) + || !get_vectype_for_scalar_type (TREE_TYPE (gs_info.offset))) { STMT_VINFO_DATA_REF (stmt_info) = NULL; free_data_ref (dr); Index: gcc/tree-vect-stmts.c =================================================================== --- gcc/tree-vect-stmts.c +++ gcc/tree-vect-stmts.c @@ -772,10 +772,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo)) { - tree off; - tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL); - gcc_assert (decl); - if (!process_use (stmt, off, loop_vinfo, relevant, &worklist, true)) + gather_scatter_info gs_info; + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) + gcc_unreachable (); + if (!process_use (stmt, gs_info.offset, loop_vinfo, relevant, + &worklist, true)) return false; } } /* while worklist */ @@ -1703,10 +1704,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, int ncopies; int i, j; bool inv_p; - tree gather_base = NULL_TREE, gather_off = NULL_TREE; - tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE; - int gather_scale = 1; - enum vect_def_type gather_dt = vect_unknown_def_type; + gather_scatter_info gs_info; bool is_store; tree mask; gimple *def_stmt; @@ -1774,11 +1772,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { gimple *def_stmt; - gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, - &gather_off, &gather_scale); - gcc_assert (gather_decl); - if (!vect_is_simple_use (gather_off, loop_vinfo, &def_stmt, &gather_dt, - &gather_off_vectype)) + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) + gcc_unreachable (); + if (!vect_is_simple_use (gs_info.offset, loop_vinfo, &def_stmt, + &gs_info.offset_dt, &gs_info.offset_vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -1786,7 +1783,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, return false; } - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); tree masktype = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist)))); if (TREE_CODE (masktype) == INTEGER_TYPE) @@ -1825,7 +1822,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { tree vec_oprnd0 = NULL_TREE, op; - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale; tree perm_mask = NULL_TREE, prev_res = NULL_TREE; @@ -1834,9 +1831,9 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple_seq seq; basic_block new_bb; enum { NARROW, NONE, WIDEN } modifier; - int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype); + int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); - rettype = TREE_TYPE (TREE_TYPE (gather_decl)); + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); @@ -1855,7 +1852,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, for (i = 0; i < gather_off_nunits; ++i) sel[i] = i | nunits; - perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel); + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); } else if (nunits == gather_off_nunits * 2) { @@ -1877,7 +1874,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype); - ptr = fold_convert (ptrtype, gather_base); + ptr = fold_convert (ptrtype, gs_info.base); if (!is_gimple_min_invariant (ptr)) { ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); @@ -1885,7 +1882,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, gcc_assert (!new_bb); } - scale = build_int_cst (scaletype, gather_scale); + scale = build_int_cst (scaletype, gs_info.scale); prev_stmt_info = NULL; for (j = 0; j < ncopies; ++j) @@ -1895,10 +1892,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, perm_mask, stmt, gsi); else if (j == 0) op = vec_oprnd0 - = vect_get_vec_def_for_operand (gather_off, stmt); + = vect_get_vec_def_for_operand (gs_info.offset, stmt); else op = vec_oprnd0 - = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0); + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0); if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) { @@ -1940,7 +1937,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, } new_stmt - = gimple_build_call (gather_decl, 5, mask_op, ptr, op, mask_op, + = gimple_build_call (gs_info.decl, 5, mask_op, ptr, op, mask_op, scale); if (!useless_type_conversion_p (vectype, rettype)) @@ -5207,10 +5204,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); vec_info *vinfo = stmt_info->vinfo; tree aggr_type; - tree scatter_base = NULL_TREE, scatter_off = NULL_TREE; - tree scatter_off_vectype = NULL_TREE, scatter_decl = NULL_TREE; - int scatter_scale = 1; - enum vect_def_type scatter_idx_dt = vect_unknown_def_type; + gather_scatter_info gs_info; enum vect_def_type scatter_src_dt = vect_unknown_def_type; gimple *new_stmt; @@ -5378,11 +5372,10 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { gimple *def_stmt; - scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo, &scatter_base, - &scatter_off, &scatter_scale); - gcc_assert (scatter_decl); - if (!vect_is_simple_use (scatter_off, vinfo, &def_stmt, &scatter_idx_dt, - &scatter_off_vectype)) + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) + gcc_unreachable (); + if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt, + &gs_info.offset_dt, &gs_info.offset_vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -5408,14 +5401,14 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src; - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (scatter_decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; tree ptr, mask, var, scale, perm_mask = NULL_TREE; edge pe = loop_preheader_edge (loop); gimple_seq seq; basic_block new_bb; enum { NARROW, NONE, WIDEN } modifier; - int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (scatter_off_vectype); + int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); if (nunits == (unsigned int) scatter_off_nunits) modifier = NONE; @@ -5427,7 +5420,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, for (i = 0; i < (unsigned int) scatter_off_nunits; ++i) sel[i] = i | nunits; - perm_mask = vect_gen_perm_mask_checked (scatter_off_vectype, sel); + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); gcc_assert (perm_mask != NULL_TREE); } else if (nunits == (unsigned int) scatter_off_nunits * 2) @@ -5445,7 +5438,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, else gcc_unreachable (); - rettype = TREE_TYPE (TREE_TYPE (scatter_decl)); + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); @@ -5455,7 +5448,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE && TREE_CODE (rettype) == VOID_TYPE); - ptr = fold_convert (ptrtype, scatter_base); + ptr = fold_convert (ptrtype, gs_info.base); if (!is_gimple_min_invariant (ptr)) { ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); @@ -5468,7 +5461,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, mask = build_int_cst (masktype, -1); mask = vect_init_vector (stmt, mask, masktype, NULL); - scale = build_int_cst (scaletype, scatter_scale); + scale = build_int_cst (scaletype, gs_info.scale); prev_stmt_info = NULL; for (j = 0; j < ncopies; ++j) @@ -5478,7 +5471,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, src = vec_oprnd1 = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt); op = vec_oprnd0 - = vect_get_vec_def_for_operand (scatter_off, stmt); + = vect_get_vec_def_for_operand (gs_info.offset, stmt); } else if (modifier != NONE && (j & 1)) { @@ -5494,7 +5487,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask, stmt, gsi); op = vec_oprnd0 - = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, + vec_oprnd0); } else gcc_unreachable (); @@ -5504,7 +5498,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, src = vec_oprnd1 = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1); op = vec_oprnd0 - = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, + vec_oprnd0); } if (!useless_type_conversion_p (srctype, TREE_TYPE (src))) @@ -5530,7 +5525,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, } new_stmt - = gimple_build_call (scatter_decl, 5, ptr, mask, op, src, scale); + = gimple_build_call (gs_info.decl, 5, ptr, mask, op, src, scale); vect_finish_stmt_generation (stmt, new_stmt, gsi); @@ -6204,10 +6199,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); int vf; tree aggr_type; - tree gather_base = NULL_TREE, gather_off = NULL_TREE; - tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE; - int gather_scale = 1; - enum vect_def_type gather_dt = vect_unknown_def_type; + gather_scatter_info gs_info; vec_info *vinfo = stmt_info->vinfo; if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) @@ -6404,11 +6396,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { gimple *def_stmt; - gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, - &gather_off, &gather_scale); - gcc_assert (gather_decl); - if (!vect_is_simple_use (gather_off, vinfo, &def_stmt, &gather_dt, - &gather_off_vectype)) + if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) + gcc_unreachable (); + if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt, + &gs_info.offset_dt, &gs_info.offset_vectype)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -6483,14 +6474,14 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { tree vec_oprnd0 = NULL_TREE, op; - tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res = NULL_TREE; edge pe = loop_preheader_edge (loop); gimple_seq seq; basic_block new_bb; enum { NARROW, NONE, WIDEN } modifier; - int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype); + int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); if (nunits == gather_off_nunits) modifier = NONE; @@ -6502,7 +6493,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, for (i = 0; i < gather_off_nunits; ++i) sel[i] = i | nunits; - perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel); + perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); } else if (nunits == gather_off_nunits * 2) { @@ -6519,7 +6510,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, else gcc_unreachable (); - rettype = TREE_TYPE (TREE_TYPE (gather_decl)); + rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); @@ -6529,7 +6520,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, vec_dest = vect_create_destination_var (scalar_dest, vectype); - ptr = fold_convert (ptrtype, gather_base); + ptr = fold_convert (ptrtype, gs_info.base); if (!is_gimple_min_invariant (ptr)) { ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); @@ -6561,7 +6552,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, else gcc_unreachable (); - scale = build_int_cst (scaletype, gather_scale); + scale = build_int_cst (scaletype, gs_info.scale); if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE) merge = build_int_cst (TREE_TYPE (rettype), 0); @@ -6587,10 +6578,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, perm_mask, stmt, gsi); else if (j == 0) op = vec_oprnd0 - = vect_get_vec_def_for_operand (gather_off, stmt); + = vect_get_vec_def_for_operand (gs_info.offset, stmt); else op = vec_oprnd0 - = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0); + = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0); if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) { @@ -6605,7 +6596,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, } new_stmt - = gimple_build_call (gather_decl, 5, merge, ptr, op, mask, scale); + = gimple_build_call (gs_info.decl, 5, merge, ptr, op, mask, scale); if (!useless_type_conversion_p (vectype, rettype)) {