Message ID | 87o9g8f8b3.fsf@arm.com |
---|---|
State | New |
Headers | show |
Series | [8/n] PR85694: Make patterns check for target support | expand |
On Mon, Jun 18, 2018 at 5:01 PM Richard Sandiford <richard.sandiford@arm.com> wrote: > > This patch makes pattern recognisers do their own checking for vector > types and target support. Previously some recognisers did this > themselves and some left it to vect_pattern_recog_1. > > Doing this means we can get rid of the type_in argument, which was > ignored if the recogniser did its own checking. It also means > we create fewer junk statements. > > Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install? OK. Richard. > Richard > > > 2018-06-18 Richard Sandiford <richard.sandiford@arm.com> > > gcc/ > * tree-vectorizer.h (NUM_PATTERNS, vect_recog_func_ptr): Move to > tree-vect-patterns.c. > * tree-vect-patterns.c (vect_supportable_direct_optab_p): New function. > (vect_recog_dot_prod_pattern): Use it. Remove the type_in argument. > (vect_recog_sad_pattern): Likewise. > (vect_recog_widen_sum_pattern): Likewise. > (vect_recog_pow_pattern): Likewise. Check for a null vectype. > (vect_recog_widen_shift_pattern): Remove the type_in argument. > (vect_recog_rotate_pattern): Likewise. > (vect_recog_mult_pattern): Likewise. > (vect_recog_vector_vector_shift_pattern): Likewise. > (vect_recog_divmod_pattern): Likewise. > (vect_recog_mixed_size_cond_pattern): Likewise. > (vect_recog_bool_pattern): Likewise. > (vect_recog_mask_conversion_pattern): Likewise. > (vect_try_gather_scatter_pattern): Likewise. > (vect_recog_widen_mult_pattern): Likewise. Check for a null vectype. > (vect_recog_over_widening_pattern): Likewise. > (vect_recog_gather_scatter_pattern): Likewise. > (vect_recog_func_ptr): Move from tree-vectorizer.h > (vect_vect_recog_func_ptrs): Move further down the file. > (vect_recog_func): Likewise. Remove the third argument. > (NUM_PATTERNS): Define based on vect_vect_recog_func_ptrs. > (vect_pattern_recog_1): Expect the pattern function to do any > necessary target tests. Also expect it to provide a vector type. > Remove the type_in handling. > > Index: gcc/tree-vectorizer.h > =================================================================== > --- gcc/tree-vectorizer.h 2018-06-14 12:27:34.104084473 +0100 > +++ gcc/tree-vectorizer.h 2018-06-18 15:43:52.951038712 +0100 > @@ -1616,8 +1616,6 @@ extern int vect_get_place_in_interleavin > /* Pattern recognition functions. > Additional pattern recognition functions can (and will) be added > in the future. */ > -typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *); > -#define NUM_PATTERNS 15 > void vect_pattern_recog (vec_info *); > > /* In tree-vectorizer.c. */ > Index: gcc/tree-vect-patterns.c > =================================================================== > --- gcc/tree-vect-patterns.c 2018-06-18 15:43:52.000000000 +0100 > +++ gcc/tree-vect-patterns.c 2018-06-18 15:43:52.951038712 +0100 > @@ -47,66 +47,6 @@ Software Foundation; either version 3, o > #include "omp-simd-clone.h" > #include "predict.h" > > -/* Pattern recognition functions */ > -static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *, > - tree *); > -static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *, > - tree *); > -static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *, > - tree *); > -static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *, > - tree *); > -static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *); > -static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *, > - tree *); > -static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *, > - tree *, tree *); > -static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *); > -static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *, > - tree *, tree *); > -static gimple *vect_recog_divmod_pattern (vec<gimple *> *, > - tree *, tree *); > - > -static gimple *vect_recog_mult_pattern (vec<gimple *> *, > - tree *, tree *); > - > -static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *, > - tree *, tree *); > -static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *); > -static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *); > -static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *, > - tree *); > - > -struct vect_recog_func > -{ > - vect_recog_func_ptr fn; > - const char *name; > -}; > - > -/* Note that ordering matters - the first pattern matching on a stmt > - is taken which means usually the more complex one needs to preceed > - the less comples onex (widen_sum only after dot_prod or sad for example). */ > -static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = { > - { vect_recog_widen_mult_pattern, "widen_mult" }, > - { vect_recog_dot_prod_pattern, "dot_prod" }, > - { vect_recog_sad_pattern, "sad" }, > - { vect_recog_widen_sum_pattern, "widen_sum" }, > - { vect_recog_pow_pattern, "pow" }, > - { vect_recog_widen_shift_pattern, "widen_shift" }, > - { vect_recog_over_widening_pattern, "over_widening" }, > - { vect_recog_rotate_pattern, "rotate" }, > - { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, > - { vect_recog_divmod_pattern, "divmod" }, > - { vect_recog_mult_pattern, "mult" }, > - { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" }, > - { vect_recog_bool_pattern, "bool" }, > - /* This must come before mask conversion, and includes the parts > - of mask conversion that are needed for gather and scatter > - internal functions. */ > - { vect_recog_gather_scatter_pattern, "gather_scatter" }, > - { vect_recog_mask_conversion_pattern, "mask_conversion" } > -}; > - > /* Report that we've found an instance of pattern PATTERN in > statement STMT. */ > > @@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_ > append_pattern_def_seq (stmt_info, stmt); > } > > +/* Return true if the target supports a vector version of CODE, > + where CODE is known to map to a direct optab. ITYPE specifies > + the type of (some of) the scalar inputs and OTYPE specifies the > + type of the scalar result. > + > + If CODE allows the inputs and outputs to have different type > + (such as for WIDEN_SUM_EXPR), it is the input mode rather > + than the output mode that determines the appropriate target pattern. > + Operand 0 of the target pattern then specifies the mode that the output > + must have. > + > + When returning true, set *VECOTYPE_OUT to the vector version of OTYPE. > + Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT > + is nonnull. */ > + > +static bool > +vect_supportable_direct_optab_p (tree otype, tree_code code, > + tree itype, tree *vecotype_out, > + tree *vecitype_out = NULL) > +{ > + tree vecitype = get_vectype_for_scalar_type (itype); > + if (!vecitype) > + return false; > + > + tree vecotype = get_vectype_for_scalar_type (otype); > + if (!vecotype) > + return false; > + > + optab optab = optab_for_tree_code (code, vecitype, optab_default); > + if (!optab) > + return false; > + > + insn_code icode = optab_handler (optab, TYPE_MODE (vecitype)); > + if (icode == CODE_FOR_nothing > + || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype)) > + return false; > + > + *vecotype_out = vecotype; > + if (vecitype_out) > + *vecitype_out = vecitype; > + return true; > +} > + > /* Check whether STMT2 is in the same loop or basic block as STMT1. > Which of the two applies depends on whether we're currently doing > loop-based or basic-block-based vectorization, as determined by > @@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the sequence of > @@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec > inner-loop nested in an outer-loop that us being vectorized). */ > > static gimple * > -vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *stmt, *last_stmt = (*stmts)[0]; > tree oprnd0, oprnd1; > @@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple > vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt); > > half_type = TREE_TYPE (oprnd00); > - *type_in = half_type; > - *type_out = type; > + if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type, > + type_out)) > + return NULL; > > var = vect_recog_temp_ssa_var (type, NULL); > pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR, > @@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the sequence of > @@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple > */ > > static gimple * > -vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = (*stmts)[0]; > tree sad_oprnd0, sad_oprnd1; > @@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *s > > vect_pattern_detected ("vect_recog_sad_pattern", last_stmt); > > - *type_in = TREE_TYPE (sad_oprnd0); > - *type_out = sum_type; > + if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR, > + TREE_TYPE (sad_oprnd0), type_out)) > + return NULL; > > tree var = vect_recog_temp_ssa_var (sum_type, NULL); > gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0, > @@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *s > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the sequence of > @@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *s > */ > > static gimple * > -vect_recog_widen_mult_pattern (vec<gimple *> *stmts, > - tree *type_in, tree *type_out) > +vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > gimple *def_stmt0, *def_stmt1; > @@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimpl > &dummy_int, &dummy_vec)) > return NULL; > > - *type_in = vectype; > *type_out = get_vectype_for_scalar_type (type); > + if (!*type_out) > + return NULL; > > /* Pattern supported. Create a stmt to be used to replace the pattern: */ > var = vect_recog_temp_ssa_var (itype, NULL); > @@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimpl > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the sequence of > @@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimpl > */ > > static gimple * > -vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = (*stmts)[0]; > tree base, exp; > @@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *s > if (node->simd_clones == NULL) > return NULL; > } > + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base)); > + if (!*type_out) > + return NULL; > stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt); > tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); > gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc); > new_pattern_def_seq (stmt_vinfo, g); > - *type_in = TREE_TYPE (base); > - *type_out = NULL_TREE; > tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); > g = gimple_build_call (exp_decl, 1, def); > gimple_call_set_lhs (g, res); > @@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *s > /* We now have a pow or powi builtin function call with a constant > exponent. */ > > - *type_out = NULL_TREE; > - > /* Catch squaring. */ > if ((tree_fits_shwi_p (exp) > && tree_to_shwi (exp) == 2) > || (TREE_CODE (exp) == REAL_CST > && real_equal (&TREE_REAL_CST (exp), &dconst2))) > { > - *type_in = TREE_TYPE (base); > + if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR, > + TREE_TYPE (base), type_out)) > + return NULL; > > var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); > stmt = gimple_build_assign (var, MULT_EXPR, base, base); > @@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *s > if (TREE_CODE (exp) == REAL_CST > && real_equal (&TREE_REAL_CST (exp), &dconsthalf)) > { > - *type_in = get_vectype_for_scalar_type (TREE_TYPE (base)); > - if (*type_in > - && direct_internal_fn_supported_p (IFN_SQRT, *type_in, > + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base)); > + if (*type_out > + && direct_internal_fn_supported_p (IFN_SQRT, *type_out, > OPTIMIZE_FOR_SPEED)) > { > gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base); > @@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *s > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the sequence of > @@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *s > inner-loop nested in an outer-loop that us being vectorized). */ > > static gimple * > -vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *stmt, *last_stmt = (*stmts)[0]; > tree oprnd0, oprnd1; > @@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple > if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt, > &promotion) > || !promotion) > - return NULL; > + return NULL; > > oprnd0 = gimple_assign_rhs1 (stmt); > > vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt); > > - *type_in = half_type; > - *type_out = type; > + if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type, > + type_out)) > + return NULL; > > var = vect_recog_temp_ssa_var (type, NULL); > pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1); > @@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple > demotion operation. We also check that S3 and S4 have only one use. */ > > static gimple * > -vect_recog_over_widening_pattern (vec<gimple *> *stmts, > - tree *type_in, tree *type_out) > +vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *stmt = stmts->pop (); > gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL, > @@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gi > if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type) > || TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type)) > { > + *type_out = get_vectype_for_scalar_type (use_type); > + if (!*type_out) > + return NULL; > + > /* Create NEW_TYPE->USE_TYPE conversion. */ > new_oprnd = make_ssa_name (use_type); > pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var); > STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt; > > - *type_in = get_vectype_for_scalar_type (new_type); > - *type_out = get_vectype_for_scalar_type (use_type); > - > /* We created a pattern statement for the last statement in the > sequence, so we don't need to associate it with the pattern > statement created for PREV_STMT. Therefore, we add PREV_STMT > @@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gi > STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt)) > = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt)); > > - *type_in = vectype; > - *type_out = NULL_TREE; > + *type_out = vectype; > } > > stmts->safe_push (use_stmt); > @@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gi > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the sequence of > @@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gi > WIDEN_LSHIFT_EXPR <a_t, CONST>. */ > > static gimple * > -vect_recog_widen_shift_pattern (vec<gimple *> *stmts, > - tree *type_in, tree *type_out) > +vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > gimple *def_stmt0; > @@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimp > &dummy_int, &dummy_vec)) > return NULL; > > - *type_in = vectype; > *type_out = vectype_out; > > /* Pattern supported. Create a stmt to be used to replace the pattern. */ > @@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimp > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the rotate > S0 stmt. */ > > static gimple * > -vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out) > +vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2; > @@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *> > return NULL; > } > > - *type_in = vectype; > *type_out = vectype; > - if (*type_in == NULL_TREE) > - return NULL; > > if (dt == vect_external_def > && TREE_CODE (oprnd1) == SSA_NAME > @@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *> > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the shift/rotate > S3 stmt. */ > > static gimple * > -vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, > - tree *type_in, tree *type_out) > +vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > tree oprnd0, oprnd1, lhs, var; > @@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern ( > if (!def_vinfo) > return NULL; > > - *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0)); > - *type_out = *type_in; > - if (*type_in == NULL_TREE) > + *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0)); > + if (*type_out == NULL_TREE) > return NULL; > > tree def = NULL_TREE; > @@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tr > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace > the multiplication. */ > > static gimple * > -vect_recog_mult_pattern (vec<gimple *> *stmts, > - tree *type_in, tree *type_out) > +vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > tree oprnd0, oprnd1, vectype, itype; > @@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> * > vect_pattern_detected ("vect_recog_mult_pattern", last_stmt); > > stmts->safe_push (last_stmt); > - *type_in = vectype; > *type_out = vectype; > > return pattern_stmt; > @@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> * > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the division > S1 or modulo S4 stmt. */ > > static gimple * > -vect_recog_divmod_pattern (vec<gimple *> *stmts, > - tree *type_in, tree *type_out) > +vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > tree oprnd0, oprnd1, vectype, itype, cond; > @@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *> > > stmts->safe_push (last_stmt); > > - *type_in = vectype; > *type_out = vectype; > return pattern_stmt; > } > @@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *> > > stmts->safe_push (last_stmt); > > - *type_in = vectype; > *type_out = vectype; > return pattern_stmt; > } > @@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *> > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the pattern. > @@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *> > a_T = (TYPE) a_it; */ > > static gimple * > -vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = (*stmts)[0]; > tree cond_expr, then_clause, else_clause; > @@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec< > def_stmt_info = new_stmt_vec_info (def_stmt, vinfo); > set_vinfo_for_stmt (def_stmt, def_stmt_info); > STMT_VINFO_VECTYPE (def_stmt_info) = vecitype; > - *type_in = vecitype; > *type_out = vectype; > > vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt); > @@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info > > Output: > > - * TYPE_IN: The type of the input arguments to the pattern. > - > * TYPE_OUT: The type of the output of this pattern. > > * Return value: A new stmt that will be used to replace the pattern. > @@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info > but the above is more efficient. */ > > static gimple * > -vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > enum tree_code rhs_code; > @@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> * > } > > *type_out = vectype; > - *type_in = vectype; > stmts->safe_push (last_stmt); > vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); > > @@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> * > gimple_assign_rhs2 (last_stmt), > gimple_assign_rhs3 (last_stmt)); > *type_out = vectype; > - *type_in = vectype; > stmts->safe_push (last_stmt); > vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); > > @@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> * > STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) > = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); > *type_out = vectype; > - *type_in = vectype; > stmts->safe_push (last_stmt); > vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); > > @@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree v > S4' c_1' = m_3'' ? c_2 : c_3; */ > > static gimple * > -vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > enum tree_code rhs_code; > @@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec< > = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); > > *type_out = vectype1; > - *type_in = vectype1; > stmts->safe_push (last_stmt); > vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); > > @@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec< > gimple_assign_rhs3 (last_stmt)); > > *type_out = vectype1; > - *type_in = vectype1; > stmts->safe_push (last_stmt); > vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); > > @@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec< > pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2); > > *type_out = vectype1; > - *type_in = vectype1; > stmts->safe_push (last_stmt); > vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); > > @@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree typ > > /* Try to convert STMT into a call to a gather load or scatter store > internal function. Return the final statement on success and set > - *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored. > + *TYPE_OUT to the vector type being loaded or stored. > > This function only handles gathers and scatters that were recognized > as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */ > > static gimple * > vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info, > - tree *type_in, tree *type_out) > + tree *type_out) > { > /* Currently we only support this for loop vectorization. */ > stmt_vec_info stmt_info = vinfo_for_stmt (stmt); > @@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple > > tree vectype = STMT_VINFO_VECTYPE (stmt_info); > *type_out = vectype; > - *type_in = vectype; > vect_pattern_detected ("gather/scatter pattern", stmt); > > return pattern_stmt; > @@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple > /* Pattern wrapper around vect_try_gather_scatter_pattern. */ > > static gimple * > -vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in, > - tree *type_out) > +vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out) > { > gimple *last_stmt = stmts->pop (); > stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt); > gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt, > last_stmt_info, > - type_in, type_out); > + type_out); > if (pattern_stmt) > stmts->safe_push (last_stmt); > return pattern_stmt; > } > > +typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *); > + > +struct vect_recog_func > +{ > + vect_recog_func_ptr fn; > + const char *name; > +}; > + > +/* Note that ordering matters - the first pattern matching on a stmt is > + taken which means usually the more complex one needs to preceed the > + less comples onex (widen_sum only after dot_prod or sad for example). */ > +static vect_recog_func vect_vect_recog_func_ptrs[] = { > + { vect_recog_widen_mult_pattern, "widen_mult" }, > + { vect_recog_dot_prod_pattern, "dot_prod" }, > + { vect_recog_sad_pattern, "sad" }, > + { vect_recog_widen_sum_pattern, "widen_sum" }, > + { vect_recog_pow_pattern, "pow" }, > + { vect_recog_widen_shift_pattern, "widen_shift" }, > + { vect_recog_over_widening_pattern, "over_widening" }, > + { vect_recog_rotate_pattern, "rotate" }, > + { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, > + { vect_recog_divmod_pattern, "divmod" }, > + { vect_recog_mult_pattern, "mult" }, > + { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" }, > + { vect_recog_bool_pattern, "bool" }, > + /* This must come before mask conversion, and includes the parts > + of mask conversion that are needed for gather and scatter > + internal functions. */ > + { vect_recog_gather_scatter_pattern, "gather_scatter" }, > + { vect_recog_mask_conversion_pattern, "mask_conversion" } > +}; > + > +const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs); > + > /* Mark statements that are involved in a pattern. */ > > static inline void > @@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_st > computation pattern. > STMT: A stmt from which the pattern search should start. > > - If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an > - expression that computes the same functionality and can be used to > - replace the sequence of stmts that are involved in the pattern. > - > - Output: > - This function checks if the expression returned by PATTERN_RECOG_FUNC is > - supported in vector form by the target. We use 'TYPE_IN' to obtain the > - relevant vector type. If 'TYPE_IN' is already a vector type, then this > - indicates that target support had already been checked by PATTERN_RECOG_FUNC. > - If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits > - to the available target pattern. > + If PATTERN_RECOG_FUNC successfully detected the pattern, it creates > + a sequence of statements that has the same functionality and can be > + used to replace STMT. It returns the last statement in the sequence > + and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ. > + PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final > + statement, having first checked that the target supports the new operation > + in that type. > > This function also does some bookkeeping, as explained in the documentation > for vect_recog_pattern. */ > @@ -4361,56 +4325,19 @@ vect_pattern_recog_1 (vect_recog_func *r > stmt_vec_info stmt_info; > loop_vec_info loop_vinfo; > tree pattern_vectype; > - tree type_in, type_out; > - enum tree_code code; > int i; > > stmts_to_replace->truncate (0); > stmts_to_replace->quick_push (stmt); > - pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out); > + pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype); > if (!pattern_stmt) > return false; > > stmt = stmts_to_replace->last (); > stmt_info = vinfo_for_stmt (stmt); > loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); > + gcc_assert (pattern_vectype); > > - if (VECTOR_BOOLEAN_TYPE_P (type_in) > - || VECTOR_TYPE_P (type_in)) > - { > - /* No need to check target support (already checked by the pattern > - recognition function). */ > - pattern_vectype = type_out ? type_out : type_in; > - } > - else > - { > - /* Check target support */ > - type_in = get_vectype_for_scalar_type (type_in); > - if (!type_in) > - return false; > - if (type_out) > - type_out = get_vectype_for_scalar_type (type_out); > - else > - type_out = type_in; > - if (!type_out) > - return false; > - pattern_vectype = type_out; > - > - if (is_gimple_assign (pattern_stmt)) > - { > - enum insn_code icode; > - code = gimple_assign_rhs_code (pattern_stmt); > - optab optab = optab_for_tree_code (code, type_in, optab_default); > - machine_mode vec_mode = TYPE_MODE (type_in); > - if (!optab > - || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing > - || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out))) > - return false; > - } > - else > - gcc_assert (is_gimple_call (pattern_stmt)); > - } > - > /* Found a vectorizable pattern. */ > if (dump_enabled_p ()) > {
Index: gcc/tree-vectorizer.h =================================================================== --- gcc/tree-vectorizer.h 2018-06-14 12:27:34.104084473 +0100 +++ gcc/tree-vectorizer.h 2018-06-18 15:43:52.951038712 +0100 @@ -1616,8 +1616,6 @@ extern int vect_get_place_in_interleavin /* Pattern recognition functions. Additional pattern recognition functions can (and will) be added in the future. */ -typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *); -#define NUM_PATTERNS 15 void vect_pattern_recog (vec_info *); /* In tree-vectorizer.c. */ Index: gcc/tree-vect-patterns.c =================================================================== --- gcc/tree-vect-patterns.c 2018-06-18 15:43:52.000000000 +0100 +++ gcc/tree-vect-patterns.c 2018-06-18 15:43:52.951038712 +0100 @@ -47,66 +47,6 @@ Software Foundation; either version 3, o #include "omp-simd-clone.h" #include "predict.h" -/* Pattern recognition functions */ -static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *, - tree *); -static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *, - tree *, tree *); -static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *, - tree *, tree *); -static gimple *vect_recog_divmod_pattern (vec<gimple *> *, - tree *, tree *); - -static gimple *vect_recog_mult_pattern (vec<gimple *> *, - tree *, tree *); - -static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *, - tree *, tree *); -static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *); -static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *, - tree *); - -struct vect_recog_func -{ - vect_recog_func_ptr fn; - const char *name; -}; - -/* Note that ordering matters - the first pattern matching on a stmt - is taken which means usually the more complex one needs to preceed - the less comples onex (widen_sum only after dot_prod or sad for example). */ -static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = { - { vect_recog_widen_mult_pattern, "widen_mult" }, - { vect_recog_dot_prod_pattern, "dot_prod" }, - { vect_recog_sad_pattern, "sad" }, - { vect_recog_widen_sum_pattern, "widen_sum" }, - { vect_recog_pow_pattern, "pow" }, - { vect_recog_widen_shift_pattern, "widen_shift" }, - { vect_recog_over_widening_pattern, "over_widening" }, - { vect_recog_rotate_pattern, "rotate" }, - { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, - { vect_recog_divmod_pattern, "divmod" }, - { vect_recog_mult_pattern, "mult" }, - { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" }, - { vect_recog_bool_pattern, "bool" }, - /* This must come before mask conversion, and includes the parts - of mask conversion that are needed for gather and scatter - internal functions. */ - { vect_recog_gather_scatter_pattern, "gather_scatter" }, - { vect_recog_mask_conversion_pattern, "mask_conversion" } -}; - /* Report that we've found an instance of pattern PATTERN in statement STMT. */ @@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_ append_pattern_def_seq (stmt_info, stmt); } +/* Return true if the target supports a vector version of CODE, + where CODE is known to map to a direct optab. ITYPE specifies + the type of (some of) the scalar inputs and OTYPE specifies the + type of the scalar result. + + If CODE allows the inputs and outputs to have different type + (such as for WIDEN_SUM_EXPR), it is the input mode rather + than the output mode that determines the appropriate target pattern. + Operand 0 of the target pattern then specifies the mode that the output + must have. + + When returning true, set *VECOTYPE_OUT to the vector version of OTYPE. + Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT + is nonnull. */ + +static bool +vect_supportable_direct_optab_p (tree otype, tree_code code, + tree itype, tree *vecotype_out, + tree *vecitype_out = NULL) +{ + tree vecitype = get_vectype_for_scalar_type (itype); + if (!vecitype) + return false; + + tree vecotype = get_vectype_for_scalar_type (otype); + if (!vecotype) + return false; + + optab optab = optab_for_tree_code (code, vecitype, optab_default); + if (!optab) + return false; + + insn_code icode = optab_handler (optab, TYPE_MODE (vecitype)); + if (icode == CODE_FOR_nothing + || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype)) + return false; + + *vecotype_out = vecotype; + if (vecitype_out) + *vecitype_out = vecitype; + return true; +} + /* Check whether STMT2 is in the same loop or basic block as STMT1. Which of the two applies depends on whether we're currently doing loop-based or basic-block-based vectorization, as determined by @@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec inner-loop nested in an outer-loop that us being vectorized). */ static gimple * -vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *stmt, *last_stmt = (*stmts)[0]; tree oprnd0, oprnd1; @@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt); half_type = TREE_TYPE (oprnd00); - *type_in = half_type; - *type_out = type; + if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type, + type_out)) + return NULL; var = vect_recog_temp_ssa_var (type, NULL); pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR, @@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple */ static gimple * -vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = (*stmts)[0]; tree sad_oprnd0, sad_oprnd1; @@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *s vect_pattern_detected ("vect_recog_sad_pattern", last_stmt); - *type_in = TREE_TYPE (sad_oprnd0); - *type_out = sum_type; + if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR, + TREE_TYPE (sad_oprnd0), type_out)) + return NULL; tree var = vect_recog_temp_ssa_var (sum_type, NULL); gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0, @@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *s Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *s */ static gimple * -vect_recog_widen_mult_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); gimple *def_stmt0, *def_stmt1; @@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimpl &dummy_int, &dummy_vec)) return NULL; - *type_in = vectype; *type_out = get_vectype_for_scalar_type (type); + if (!*type_out) + return NULL; /* Pattern supported. Create a stmt to be used to replace the pattern: */ var = vect_recog_temp_ssa_var (itype, NULL); @@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimpl Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimpl */ static gimple * -vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = (*stmts)[0]; tree base, exp; @@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *s if (node->simd_clones == NULL) return NULL; } + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base)); + if (!*type_out) + return NULL; stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt); tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc); new_pattern_def_seq (stmt_vinfo, g); - *type_in = TREE_TYPE (base); - *type_out = NULL_TREE; tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); g = gimple_build_call (exp_decl, 1, def); gimple_call_set_lhs (g, res); @@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *s /* We now have a pow or powi builtin function call with a constant exponent. */ - *type_out = NULL_TREE; - /* Catch squaring. */ if ((tree_fits_shwi_p (exp) && tree_to_shwi (exp) == 2) || (TREE_CODE (exp) == REAL_CST && real_equal (&TREE_REAL_CST (exp), &dconst2))) { - *type_in = TREE_TYPE (base); + if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR, + TREE_TYPE (base), type_out)) + return NULL; var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL); stmt = gimple_build_assign (var, MULT_EXPR, base, base); @@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *s if (TREE_CODE (exp) == REAL_CST && real_equal (&TREE_REAL_CST (exp), &dconsthalf)) { - *type_in = get_vectype_for_scalar_type (TREE_TYPE (base)); - if (*type_in - && direct_internal_fn_supported_p (IFN_SQRT, *type_in, + *type_out = get_vectype_for_scalar_type (TREE_TYPE (base)); + if (*type_out + && direct_internal_fn_supported_p (IFN_SQRT, *type_out, OPTIMIZE_FOR_SPEED)) { gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base); @@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *s Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *s inner-loop nested in an outer-loop that us being vectorized). */ static gimple * -vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *stmt, *last_stmt = (*stmts)[0]; tree oprnd0, oprnd1; @@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt, &promotion) || !promotion) - return NULL; + return NULL; oprnd0 = gimple_assign_rhs1 (stmt); vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt); - *type_in = half_type; - *type_out = type; + if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type, + type_out)) + return NULL; var = vect_recog_temp_ssa_var (type, NULL); pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1); @@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple demotion operation. We also check that S3 and S4 have only one use. */ static gimple * -vect_recog_over_widening_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *stmt = stmts->pop (); gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL, @@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gi if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type) || TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type)) { + *type_out = get_vectype_for_scalar_type (use_type); + if (!*type_out) + return NULL; + /* Create NEW_TYPE->USE_TYPE conversion. */ new_oprnd = make_ssa_name (use_type); pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var); STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt; - *type_in = get_vectype_for_scalar_type (new_type); - *type_out = get_vectype_for_scalar_type (use_type); - /* We created a pattern statement for the last statement in the sequence, so we don't need to associate it with the pattern statement created for PREV_STMT. Therefore, we add PREV_STMT @@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gi STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt)) = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt)); - *type_in = vectype; - *type_out = NULL_TREE; + *type_out = vectype; } stmts->safe_push (use_stmt); @@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gi Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the sequence of @@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gi WIDEN_LSHIFT_EXPR <a_t, CONST>. */ static gimple * -vect_recog_widen_shift_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); gimple *def_stmt0; @@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimp &dummy_int, &dummy_vec)) return NULL; - *type_in = vectype; *type_out = vectype_out; /* Pattern supported. Create a stmt to be used to replace the pattern. */ @@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimp Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the rotate S0 stmt. */ static gimple * -vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out) +vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2; @@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *> return NULL; } - *type_in = vectype; *type_out = vectype; - if (*type_in == NULL_TREE) - return NULL; if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME @@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *> Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the shift/rotate S3 stmt. */ static gimple * -vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, lhs, var; @@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern ( if (!def_vinfo) return NULL; - *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0)); - *type_out = *type_in; - if (*type_in == NULL_TREE) + *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0)); + if (*type_out == NULL_TREE) return NULL; tree def = NULL_TREE; @@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tr Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the multiplication. */ static gimple * -vect_recog_mult_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, vectype, itype; @@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> * vect_pattern_detected ("vect_recog_mult_pattern", last_stmt); stmts->safe_push (last_stmt); - *type_in = vectype; *type_out = vectype; return pattern_stmt; @@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> * Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the division S1 or modulo S4 stmt. */ static gimple * -vect_recog_divmod_pattern (vec<gimple *> *stmts, - tree *type_in, tree *type_out) +vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); tree oprnd0, oprnd1, vectype, itype, cond; @@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *> stmts->safe_push (last_stmt); - *type_in = vectype; *type_out = vectype; return pattern_stmt; } @@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *> stmts->safe_push (last_stmt); - *type_in = vectype; *type_out = vectype; return pattern_stmt; } @@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *> Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the pattern. @@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *> a_T = (TYPE) a_it; */ static gimple * -vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = (*stmts)[0]; tree cond_expr, then_clause, else_clause; @@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec< def_stmt_info = new_stmt_vec_info (def_stmt, vinfo); set_vinfo_for_stmt (def_stmt, def_stmt_info); STMT_VINFO_VECTYPE (def_stmt_info) = vecitype; - *type_in = vecitype; *type_out = vectype; vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt); @@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info Output: - * TYPE_IN: The type of the input arguments to the pattern. - * TYPE_OUT: The type of the output of this pattern. * Return value: A new stmt that will be used to replace the pattern. @@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info but the above is more efficient. */ static gimple * -vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); enum tree_code rhs_code; @@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> * } *type_out = vectype; - *type_in = vectype; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); @@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> * gimple_assign_rhs2 (last_stmt), gimple_assign_rhs3 (last_stmt)); *type_out = vectype; - *type_in = vectype; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); @@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> * STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); *type_out = vectype; - *type_in = vectype; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_bool_pattern", last_stmt); @@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree v S4' c_1' = m_3'' ? c_2 : c_3; */ static gimple * -vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); enum tree_code rhs_code; @@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec< = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); *type_out = vectype1; - *type_in = vectype1; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); @@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec< gimple_assign_rhs3 (last_stmt)); *type_out = vectype1; - *type_in = vectype1; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); @@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec< pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2); *type_out = vectype1; - *type_in = vectype1; stmts->safe_push (last_stmt); vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt); @@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree typ /* Try to convert STMT into a call to a gather load or scatter store internal function. Return the final statement on success and set - *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored. + *TYPE_OUT to the vector type being loaded or stored. This function only handles gathers and scatters that were recognized as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */ static gimple * vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info, - tree *type_in, tree *type_out) + tree *type_out) { /* Currently we only support this for loop vectorization. */ stmt_vec_info stmt_info = vinfo_for_stmt (stmt); @@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple tree vectype = STMT_VINFO_VECTYPE (stmt_info); *type_out = vectype; - *type_in = vectype; vect_pattern_detected ("gather/scatter pattern", stmt); return pattern_stmt; @@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple /* Pattern wrapper around vect_try_gather_scatter_pattern. */ static gimple * -vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in, - tree *type_out) +vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out) { gimple *last_stmt = stmts->pop (); stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt); gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt, last_stmt_info, - type_in, type_out); + type_out); if (pattern_stmt) stmts->safe_push (last_stmt); return pattern_stmt; } +typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *); + +struct vect_recog_func +{ + vect_recog_func_ptr fn; + const char *name; +}; + +/* Note that ordering matters - the first pattern matching on a stmt is + taken which means usually the more complex one needs to preceed the + less comples onex (widen_sum only after dot_prod or sad for example). */ +static vect_recog_func vect_vect_recog_func_ptrs[] = { + { vect_recog_widen_mult_pattern, "widen_mult" }, + { vect_recog_dot_prod_pattern, "dot_prod" }, + { vect_recog_sad_pattern, "sad" }, + { vect_recog_widen_sum_pattern, "widen_sum" }, + { vect_recog_pow_pattern, "pow" }, + { vect_recog_widen_shift_pattern, "widen_shift" }, + { vect_recog_over_widening_pattern, "over_widening" }, + { vect_recog_rotate_pattern, "rotate" }, + { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" }, + { vect_recog_divmod_pattern, "divmod" }, + { vect_recog_mult_pattern, "mult" }, + { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" }, + { vect_recog_bool_pattern, "bool" }, + /* This must come before mask conversion, and includes the parts + of mask conversion that are needed for gather and scatter + internal functions. */ + { vect_recog_gather_scatter_pattern, "gather_scatter" }, + { vect_recog_mask_conversion_pattern, "mask_conversion" } +}; + +const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs); + /* Mark statements that are involved in a pattern. */ static inline void @@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_st computation pattern. STMT: A stmt from which the pattern search should start. - If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an - expression that computes the same functionality and can be used to - replace the sequence of stmts that are involved in the pattern. - - Output: - This function checks if the expression returned by PATTERN_RECOG_FUNC is - supported in vector form by the target. We use 'TYPE_IN' to obtain the - relevant vector type. If 'TYPE_IN' is already a vector type, then this - indicates that target support had already been checked by PATTERN_RECOG_FUNC. - If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits - to the available target pattern. + If PATTERN_RECOG_FUNC successfully detected the pattern, it creates + a sequence of statements that has the same functionality and can be + used to replace STMT. It returns the last statement in the sequence + and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ. + PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final + statement, having first checked that the target supports the new operation + in that type. This function also does some bookkeeping, as explained in the documentation for vect_recog_pattern. */ @@ -4361,56 +4325,19 @@ vect_pattern_recog_1 (vect_recog_func *r stmt_vec_info stmt_info; loop_vec_info loop_vinfo; tree pattern_vectype; - tree type_in, type_out; - enum tree_code code; int i; stmts_to_replace->truncate (0); stmts_to_replace->quick_push (stmt); - pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out); + pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype); if (!pattern_stmt) return false; stmt = stmts_to_replace->last (); stmt_info = vinfo_for_stmt (stmt); loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + gcc_assert (pattern_vectype); - if (VECTOR_BOOLEAN_TYPE_P (type_in) - || VECTOR_TYPE_P (type_in)) - { - /* No need to check target support (already checked by the pattern - recognition function). */ - pattern_vectype = type_out ? type_out : type_in; - } - else - { - /* Check target support */ - type_in = get_vectype_for_scalar_type (type_in); - if (!type_in) - return false; - if (type_out) - type_out = get_vectype_for_scalar_type (type_out); - else - type_out = type_in; - if (!type_out) - return false; - pattern_vectype = type_out; - - if (is_gimple_assign (pattern_stmt)) - { - enum insn_code icode; - code = gimple_assign_rhs_code (pattern_stmt); - optab optab = optab_for_tree_code (code, type_in, optab_default); - machine_mode vec_mode = TYPE_MODE (type_in); - if (!optab - || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing - || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out))) - return false; - } - else - gcc_assert (is_gimple_call (pattern_stmt)); - } - /* Found a vectorizable pattern. */ if (dump_enabled_p ()) {