diff mbox series

[2/6] Set LOOP_VINFO_VECT_FACTOR only when it is final

Message ID 20231213123245.4B08A3845BDA@sourceware.org
State New
Headers show
Series Relax single-vector-size restriction | expand

Commit Message

Richard Biener Dec. 13, 2023, 12:30 p.m. UTC 
The following makes sure to keep LOOP_VINFO_VECT_FACTOR at the
indetermined value zero until it is final, making LOOP_VINFO_VECT_FACTOR
an rvalue and changing some direct references to use the macro.

	* tree-vectorizer.h (LOOP_VINFO_VECT_FACTOR): Make an rvalue.
	* tree-vect-loop.cc (vect_determine_vectorization_factor):
	Do not set LOOP_VINFO_VECT_FACTOR, return value via reference.
	(vect_update_vf_for_slp): Likewise.
	(vect_analyze_loop_2): Set LOOP_VINFO_VECT_FACTOR only
	ever to its final value.  Perform SLP optimization after
	setting the vectorization factor.
	* tree-vect-slp.cc (vect_slp_analyze_node_operations_1):
	Use LOOP_VINFO_VECT_FACTOR.
	(vect_slp_analyze_node_operations): Likewise.
	* tree-vectorizer.cc (vect_transform_loops): Likewise.
---
 gcc/tree-vect-loop.cc  | 43 +++++++++++++++++++++++-------------------
 gcc/tree-vect-slp.cc   |  4 ++--
 gcc/tree-vectorizer.cc |  2 +-
 gcc/tree-vectorizer.h  |  2 +-
 4 files changed, 28 insertions(+), 23 deletions(-)
diff mbox series

Patch

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 3a0731f3bea..3af4160426b 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -282,12 +282,12 @@  vect_determine_vf_for_stmt (vec_info *vinfo,
 */
 
 static opt_result
-vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
+vect_determine_vectorization_factor (loop_vec_info loop_vinfo,
+				     poly_uint64 &vectorization_factor)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
   unsigned nbbs = loop->num_nodes;
-  poly_uint64 vectorization_factor = 1;
   tree scalar_type = NULL_TREE;
   gphi *phi;
   tree vectype;
@@ -296,6 +296,8 @@  vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
 
   DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
 
+  vectorization_factor = 1;
+
   for (i = 0; i < nbbs; i++)
     {
       basic_block bb = bbs[i];
@@ -370,7 +372,6 @@  vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
   if (known_le (vectorization_factor, 1U))
     return opt_result::failure_at (vect_location,
 				   "not vectorized: unsupported data-type\n");
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
   return opt_result::success ();
 }
 
@@ -1937,17 +1938,16 @@  vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
    statements update the vectorization factor.  */
 
 static void
-vect_update_vf_for_slp (loop_vec_info loop_vinfo)
+vect_update_vf_for_slp (loop_vec_info loop_vinfo,
+			poly_uint64 &vectorization_factor)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
   int nbbs = loop->num_nodes;
-  poly_uint64 vectorization_factor;
   int i;
 
   DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
 
-  vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   gcc_assert (known_ne (vectorization_factor, 0U));
 
   /* If all the stmts in the loop can be SLPed, we perform only SLP, and
@@ -2006,7 +2006,6 @@  vect_update_vf_for_slp (loop_vec_info loop_vinfo)
 				 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
     }
 
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
   if (dump_enabled_p ())
     {
       dump_printf_loc (MSG_NOTE, vect_location,
@@ -2809,7 +2808,8 @@  vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
     return opt_result::failure_at (vect_location, "bad data dependence.\n");
   LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
 
-  ok = vect_determine_vectorization_factor (loop_vinfo);
+  poly_uint64 vectorization_factor;
+  ok = vect_determine_vectorization_factor (loop_vinfo, vectorization_factor);
   if (!ok)
     {
       if (dump_enabled_p ())
@@ -2821,7 +2821,7 @@  vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
   /* Compute the scalar iteration cost.  */
   vect_compute_single_scalar_iteration_cost (loop_vinfo);
 
-  poly_uint64 saved_vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+  poly_uint64 saved_vectorization_factor = vectorization_factor;
 
   if (slp)
     {
@@ -2839,13 +2839,7 @@  vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
 	  vect_detect_hybrid_slp (loop_vinfo);
 
 	  /* Update the vectorization factor based on the SLP decision.  */
-	  vect_update_vf_for_slp (loop_vinfo);
-
-	  /* Optimize the SLP graph with the vectorization factor fixed.  */
-	  vect_optimize_slp (loop_vinfo);
-
-	  /* Gather the loads reachable from the SLP graph entries.  */
-	  vect_gather_slp_loads (loop_vinfo);
+	  vect_update_vf_for_slp (loop_vinfo, vectorization_factor);
 	}
     }
 
@@ -2863,11 +2857,12 @@  start_over:
      during finish_cost the first time we ran the analyzis for this
      vector mode.  */
   if (applying_suggested_uf)
-    LOOP_VINFO_VECT_FACTOR (loop_vinfo) *= loop_vinfo->suggested_unroll_factor;
+    vectorization_factor *= loop_vinfo->suggested_unroll_factor;
 
   /* Now the vectorization factor is final.  */
-  poly_uint64 vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   gcc_assert (known_ne (vectorization_factor, 0U));
+  gcc_assert (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 0U));
+  loop_vinfo->vectorization_factor = vectorization_factor;
 
   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && dump_enabled_p ())
     {
@@ -2882,6 +2877,15 @@  start_over:
       && maybe_lt (max_vf, LOOP_VINFO_VECT_FACTOR (loop_vinfo)))
     return opt_result::failure_at (vect_location, "bad data dependence.\n");
 
+  if (slp)
+    {
+      /* Optimize the SLP graph with the vectorization factor fixed.  */
+      vect_optimize_slp (loop_vinfo);
+
+      /* Gather the loads reachable from the SLP graph entries.  */
+      vect_gather_slp_loads (loop_vinfo);
+    }
+
   loop_vinfo->vector_costs = init_cost (loop_vinfo, false);
 
   /* Analyze the alignment of the data-refs in the loop.
@@ -3303,7 +3307,8 @@  again:
   /* Roll back state appropriately.  No SLP this time.  */
   slp = false;
   /* Restore vectorization factor as it were without SLP.  */
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = saved_vectorization_factor;
+  vectorization_factor = saved_vectorization_factor;
+  loop_vinfo->vectorization_factor = 0;
   /* Free the SLP instances.  */
   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
     vect_free_slp_instance (instance);
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 6799b9375ae..efda358a7f6 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -6126,7 +6126,7 @@  vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
     {
       poly_uint64 vf;
       if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
-	vf = loop_vinfo->vectorization_factor;
+	vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
       else
 	vf = 1;
       unsigned int group_size = SLP_TREE_LANES (node);
@@ -6399,7 +6399,7 @@  vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
 	  unsigned group_size = SLP_TREE_LANES (child);
 	  poly_uint64 vf = 1;
 	  if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
-	    vf = loop_vinfo->vectorization_factor;
+	    vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
 	  SLP_TREE_NUMBER_OF_VEC_STMTS (child)
 	    = vect_get_num_vectors (vf * group_size, vector_type);
 	  /* And cost them.  */
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index d97e2b54c25..08ff932fb53 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -1015,7 +1015,7 @@  vect_transform_loops (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
       if (!simduid_to_vf_htab)
 	simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
       simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
-      simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
+      simduid_to_vf_data->vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
       *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
 	  = simduid_to_vf_data;
     }
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 1810833a324..a2bab8676af 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -967,7 +967,7 @@  public:
 #define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L)                             \
   (L)->epil_using_partial_vectors_p
 #define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
-#define LOOP_VINFO_VECT_FACTOR(L)          (L)->vectorization_factor
+#define LOOP_VINFO_VECT_FACTOR(L)          ((L)->vectorization_factor + 0)
 #define LOOP_VINFO_MAX_VECT_FACTOR(L)      (L)->max_vectorization_factor
 #define LOOP_VINFO_MASKS(L)                (L)->masks
 #define LOOP_VINFO_LENS(L)                 (L)->lens