diff mbox series

Transfer STMT_VINFO_REDUC_IDX to patterns

Message ID nycvar.YFH.7.76.1910251003390.5566@zhemvz.fhfr.qr
State New
Headers show
Series Transfer STMT_VINFO_REDUC_IDX to patterns | expand

Commit Message

Richard Biener Oct. 25, 2019, 8:05 a.m. UTC 
Reduction discovery nicely computes STMT_VINFO_REDUC_IDX for all
stmts involved in the reduction but that has been useless somewhat
since pattern recog later will wreck the info.  The following is
an attempt to fix that during pattern recog.  I may very well miss
some cases but hope to fix them...

Seeing Richard using STMT_VINFO_REDUC_IDX more I'm committing this now.

Bootstrapped and tested on x86_64-unknown-linux-gnu, applied.

Richard.

2019-10-25  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (vectorizable_reduction): Verify
	STMT_VINFO_REDUC_IDX on the to be vectorized stmts is set up
	correctly.
	* tree-vect-patterns.c (vect_mark_pattern_stmts): Transfer
	STMT_VINFO_REDUC_IDX from the original stmts to the pattern
	stmts.
diff mbox series

Patch

Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c	(revision 277414)
+++ gcc/tree-vect-loop.c	(working copy)
@@ -5719,7 +5719,19 @@  vectorizable_reduction (stmt_vec_info st
   /* PHIs should not participate in patterns.  */
   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
   gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
-  tree reduc_def = PHI_RESULT (reduc_def_phi);
+
+  /* Verify following REDUC_IDX from the latch def leads us back to the PHI.  */
+  tree reduc_def = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi,
+					  loop_latch_edge (loop));
+  while (reduc_def != PHI_RESULT (reduc_def_phi))
+    {
+      stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
+      def = vect_stmt_to_vectorize (def);
+      gcc_assert (STMT_VINFO_REDUC_IDX (def) != -1);
+      reduc_def = gimple_op (def->stmt, 1 + STMT_VINFO_REDUC_IDX (def));
+    }
+
+  reduc_def = PHI_RESULT (reduc_def_phi);
   int reduc_index = -1;
   for (i = 0; i < op_type; i++)
     {
Index: gcc/tree-vect-patterns.c
===================================================================
--- gcc/tree-vect-patterns.c	(revision 277414)
+++ gcc/tree-vect-patterns.c	(working copy)
@@ -5075,6 +5075,7 @@  static inline void
 vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
                          tree pattern_vectype)
 {
+  stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
   gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
 
   gimple *orig_pattern_stmt = NULL;
@@ -5134,6 +5135,57 @@  vect_mark_pattern_stmts (stmt_vec_info o
     }
   else
     vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
+
+  /* Transfer reduction path info to the pattern.  */
+  if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
+    {
+      vec_info *vinfo = orig_stmt_info_saved->vinfo;
+      tree lookfor = gimple_op (orig_stmt_info_saved->stmt,
+				1 + STMT_VINFO_REDUC_IDX (orig_stmt_info));
+      /* Search the pattern def sequence and the main pattern stmt.  Note
+         we may have inserted all into a containing pattern def sequence
+	 so the following is a bit awkward.  */
+      gimple_stmt_iterator si;
+      gimple *s;
+      if (def_seq)
+	{
+	  si = gsi_start (def_seq);
+	  s = gsi_stmt (si);
+	  gsi_next (&si);
+	}
+      else
+	{
+	  si = gsi_none ();
+	  s = pattern_stmt;
+	}
+      do
+	{
+	  bool found = false;
+	  for (unsigned i = 1; i < gimple_num_ops (s); ++i)
+	    if (gimple_op (s, i) == lookfor)
+	      {
+		STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i - 1;
+		lookfor = gimple_get_lhs (s);
+		found = true;
+		break;
+	      }
+	  if (found && s == pattern_stmt)
+	    break;
+	  if (s == pattern_stmt)
+	    gcc_unreachable ();
+	  if (gsi_end_p (si))
+	    s = pattern_stmt;
+	  else
+	    {
+	      s = gsi_stmt (si);
+	      if (s == pattern_stmt)
+		/* Found the end inside a bigger pattern def seq.  */
+		si = gsi_none ();
+	      else
+		gsi_next (&si);
+	    }
+	} while (1);
+    }
 }
 
 /* Function vect_pattern_recog_1