diff mbox series

[2/4] TLC to vect_check_store_rhs and vect_slp_child_index_for_operand

Message ID 20231108150309.85118133F5@imap2.suse-dmz.suse.de
State New
Headers show
Series [1/4] Fix SLP of masked loads | expand

Commit Message

Richard Biener Nov. 8, 2023, 3:03 p.m. UTC 
This prepares us for the SLP of scatters.  We have to tell
vect_slp_child_index_for_operand whether we are dealing with
a scatter/gather stmt so this adds an argument similar to
the one we have for vect_get_operand_map.  This also refactors
vect_check_store_rhs to get the actual rhs and the associated
SLP node instead of leaving that to the caller.

Bootstrap and regtest running on x86_64-unknown-linux-gnu.

	* tree-vectorizer.h (vect_slp_child_index_for_operand):
	Add gatherscatter_p argument.
	* tree-vect-slp.cc (vect_slp_child_index_for_operand): Likewise.
	Pass it on.
	* tree-vect-stmts.cc (vect_check_store_rhs): Turn the rhs
	argument into an output, also output the SLP node associated
	with it.
	(vectorizable_simd_clone_call): Adjust.
	(vectorizable_store): Likewise.
	(vectorizable_load): Likewise.
---
 gcc/tree-vect-slp.cc   |  5 ++--
 gcc/tree-vect-stmts.cc | 52 ++++++++++++++++++++++--------------------
 gcc/tree-vectorizer.h  |  2 +-
 3 files changed, 31 insertions(+), 28 deletions(-)
diff mbox series

Patch

diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 13137ede8d4..176aaf270f4 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -589,9 +589,10 @@  vect_get_operand_map (const gimple *stmt, bool gather_scatter_p = false,
 /* Return the SLP node child index for operand OP of STMT.  */
 
 int
-vect_slp_child_index_for_operand (const gimple *stmt, int op)
+vect_slp_child_index_for_operand (const gimple *stmt, int op,
+				  bool gather_scatter_p)
 {
-  const int *opmap = vect_get_operand_map (stmt);
+  const int *opmap = vect_get_operand_map (stmt, gather_scatter_p);
   if (!opmap)
     return op;
   for (int i = 1; i < 1 + opmap[0]; ++i)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 096a857f2dd..61e23b29516 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2486,42 +2486,33 @@  vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
   return true;
 }
 
-/* Return true if stored value RHS is suitable for vectorizing store
-   statement STMT_INFO.  When returning true, store the type of the
-   definition in *RHS_DT_OUT, the type of the vectorized store value in
+/* Return true if stored value is suitable for vectorizing store
+   statement STMT_INFO.  When returning true, store the scalar stored
+   in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
+   the type of the vectorized store value in
    *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT.  */
 
 static bool
 vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
-		      slp_tree slp_node, tree rhs,
+		      slp_tree slp_node, tree *rhs, slp_tree *rhs_node,
 		      vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
 		      vec_load_store_type *vls_type_out)
 {
-  /* In the case this is a store from a constant make sure
-     native_encode_expr can handle it.  */
-  if (CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "cannot encode constant as a byte sequence.\n");
-      return false;
-    }
-
   int op_no = 0;
   if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
     {
       if (gimple_call_internal_p (call)
 	  && internal_store_fn_p (gimple_call_internal_fn (call)))
 	op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
-      if (slp_node)
-	op_no = vect_slp_child_index_for_operand (call, op_no);
     }
+  if (slp_node)
+    op_no = vect_slp_child_index_for_operand
+	      (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
 
   enum vect_def_type rhs_dt;
   tree rhs_vectype;
-  slp_tree slp_op;
   if (!vect_is_simple_use (vinfo, stmt_info, slp_node, op_no,
-			   &rhs, &slp_op, &rhs_dt, &rhs_vectype))
+			   rhs, rhs_node, &rhs_dt, &rhs_vectype))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2529,6 +2520,16 @@  vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
+  /* In the case this is a store from a constant make sure
+     native_encode_expr can handle it.  */
+  if (CONSTANT_CLASS_P (*rhs) && native_encode_expr (*rhs, NULL, 64) == 0)
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "cannot encode constant as a byte sequence.\n");
+      return false;
+    }
+
   tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
     {
@@ -4052,7 +4053,7 @@  vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
       int op_no = i + masked_call_offset;
       if (slp_node)
-	op_no = vect_slp_child_index_for_operand (stmt, op_no);
+	op_no = vect_slp_child_index_for_operand (stmt, op_no, false);
       if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
 			       op_no, &op, &slp_op[i],
 			       &thisarginfo.dt, &thisarginfo.vectype)
@@ -8173,7 +8174,6 @@  vectorizable_store (vec_info *vinfo,
 		    stmt_vector_for_cost *cost_vec)
 {
   tree data_ref;
-  tree op;
   tree vec_oprnd = NULL_TREE;
   tree elem_type;
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -8236,15 +8236,14 @@  vectorizable_store (vec_info *vinfo,
 
       int mask_index = internal_fn_mask_index (ifn);
       if (mask_index >= 0 && slp_node)
-	mask_index = vect_slp_child_index_for_operand (call, mask_index);
+	mask_index = vect_slp_child_index_for_operand
+		    (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (mask_index >= 0
 	  && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
 				      &mask, NULL, &mask_dt, &mask_vectype))
 	return false;
     }
 
-  op = vect_get_store_rhs (stmt_info);
-
   /* Cannot have hybrid store SLP -- that would mean storing to the
      same location twice.  */
   gcc_assert (slp == PURE_SLP_STMT (stmt_info));
@@ -8279,8 +8278,10 @@  vectorizable_store (vec_info *vinfo,
       return false;
     }
 
+  tree op;
+  slp_tree op_node;
   if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
-			     op, &rhs_dt, &rhs_vectype, &vls_type))
+			     &op, &op_node, &rhs_dt, &rhs_vectype, &vls_type))
     return false;
 
   elem_type = TREE_TYPE (vectype);
@@ -9855,7 +9856,8 @@  vectorizable_load (vec_info *vinfo,
 
       mask_index = internal_fn_mask_index (ifn);
       if (mask_index >= 0 && slp_node)
-	mask_index = vect_slp_child_index_for_operand (call, mask_index);
+	mask_index = vect_slp_child_index_for_operand
+		    (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (mask_index >= 0
 	  && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
 				      &mask, &slp_op, &mask_dt, &mask_vectype))
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index d2ddc2e4ad5..e4d7ab4567c 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2462,7 +2462,7 @@  extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info);
 extern slp_tree vect_create_new_slp_node (unsigned, tree_code);
 extern void vect_free_slp_tree (slp_tree);
 extern bool compatible_calls_p (gcall *, gcall *);
-extern int vect_slp_child_index_for_operand (const gimple *, int op);
+extern int vect_slp_child_index_for_operand (const gimple *, int op, bool);
 
 /* In tree-vect-patterns.cc.  */
 extern void