diff mbox series

[v3] rs6000: Stackoverflow in optimized code on PPC [PR100799]

Message ID 617b4a76-914c-4fff-b0db-d23e633ac444@linux.ibm.com
State New
Headers show
Series [v3] rs6000: Stackoverflow in optimized code on PPC [PR100799] | expand

Commit Message

Ajit Agarwal March 23, 2024, 2:27 p.m. UTC 
Hello All:

When using FlexiBLAS with OpenBLAS, we noticed corruption of the caller
stack frame when calling OpenBLAS functions.  This was caused by the
FlexiBLAS C/C++ caller and OpenBLAS Fortran callee disagreeing on the
number of function parameters in the callee due to hidden Fortran
parameters. This can cause problems when the callee believes the caller
has allocated a parameter save area when the caller has not done so.
That means any writes by the callee into the non-existent parameter save
area will corrupt the caller stack frame.

The workaround implemented here, is for the callee to determine whether
the caller has allocated a parameter save area or not, by ignoring any
unused hidden parameters when counting the number of parameters.

Bootstrapped and regtested on powerpc64-linux-gnu.

Thanks & Regards
Ajit


rs6000: Stackoverflow in optimized code on PPC [PR100799]

When using FlexiBLAS with OpenBLAS, we noticed corruption of the caller
stack frame when calling OpenBLAS functions.  This was caused by the
FlexiBLAS C/C++ caller and OpenBLAS Fortran callee disagreeing on the
number of function parameters in the callee due to hidden Fortran
parameters. This can cause problems when the callee believes the caller
has allocated a parameter save area when the caller has not done so.
That means any writes by the callee into the non-existent parameter save
area will corrupt the caller stack frame.

The workaround implemented here, is for the callee to determine whether
the caller has allocated a parameter save area or not, by ignoring any
unused hidden parameters when counting the number of parameters.

2024-03-23  Ajit Kumar Agarwal  <aagarwa1@linux.ibm.com>

gcc/ChangeLog:

	PR rtl-optimization/100799
	* config/rs6000/rs6000-calls.cc (rs6000_function_arg): Don't
	assume a parameter save area has been allocated if the number of
	formal parameters, excluding unused hidden parameters, is less
	than or equal to GP_ARG_NUM_REG (8).
	(init_cumulative_args): Check for unused hidden Fortran
	parameters and set hidden_string_length and actual_parm_length.
	* config/rs6000/rs6000.h (rs6000_args): Add new field
	hidden_string_length and actual_parm_length.
---
 gcc/config/rs6000/rs6000-call.cc | 38 ++++++++++++++++++++++++++++++--
 gcc/config/rs6000/rs6000.h       |  4 ++++
 2 files changed, 40 insertions(+), 2 deletions(-)
diff mbox series

Patch

diff --git a/gcc/config/rs6000/rs6000-call.cc b/gcc/config/rs6000/rs6000-call.cc
index 1f8f93a2ee7..656735aebaf 100644
--- a/gcc/config/rs6000/rs6000-call.cc
+++ b/gcc/config/rs6000/rs6000-call.cc
@@ -64,7 +64,7 @@ 
 #include "ppc-auxv.h"
 #include "targhooks.h"
 #include "opts.h"
-
+#include "tree-dfa.h"
 #include "rs6000-internal.h"
 
 #ifndef TARGET_PROFILE_KERNEL
@@ -584,6 +584,32 @@  init_cumulative_args (CUMULATIVE_ARGS *cum, tree fntype,
   if (incoming || cum->prototype)
     cum->nargs_prototype = n_named_args;
 
+  /* When the buggy C/C++ wrappers call the function with fewer arguments
+     than it actually has. Check whether this function contains any unused
+     hidden parameters and record how many there are for use in
+     rs6000_function_arg() to determine whether its callers
+     have allocated a parameter save area or not. See PR100799 for
+     details.  */
+  unsigned int num_args = 0;
+  unsigned int hidden_length = 0;
+
+  for (tree arg = DECL_ARGUMENTS (current_function_decl);
+       arg; arg = DECL_CHAIN (arg))
+    {
+      num_args++;
+      if (DECL_HIDDEN_STRING_LENGTH (arg))
+	{
+	  tree parmdef = ssa_default_def (cfun, arg);
+	  if (parmdef == NULL || has_zero_uses (parmdef))
+	    {
+	      cum->hidden_string_length = 1;
+	      hidden_length++;
+	    }
+	}
+   }
+
+  cum->actual_parm_length = num_args - hidden_length;
+
   /* Check for a longcall attribute.  */
   if ((!fntype && rs6000_default_long_calls)
       || (fntype
@@ -1857,7 +1883,15 @@  rs6000_function_arg (cumulative_args_t cum_v, const function_arg_info &arg)
 
 	  return rs6000_finish_function_arg (mode, rvec, k);
 	}
-      else if (align_words < GP_ARG_NUM_REG)
+     /* When the buggy C/C++ wrappers call the function with fewer arguments
+	than it actually has. Check whether this function contains any unused
+	hidden parameters and record how many there are for use in
+	rs6000_function_arg() to determine whether its callers
+	have allocated a parameter save area or not. See PR100799 for
+	details.  */
+      else if (align_words < GP_ARG_NUM_REG
+	       || (cum->hidden_string_length
+	       && cum->actual_parm_length <= GP_ARG_NUM_REG))
 	{
 	  if (TARGET_32BIT && TARGET_POWERPC64)
 	    return rs6000_mixed_function_arg (mode, type, align_words);
diff --git a/gcc/config/rs6000/rs6000.h b/gcc/config/rs6000/rs6000.h
index 68bc45d65ba..a8f91301852 100644
--- a/gcc/config/rs6000/rs6000.h
+++ b/gcc/config/rs6000/rs6000.h
@@ -1490,6 +1490,10 @@  typedef struct rs6000_args
   int named;			/* false for varargs params */
   int escapes;			/* if function visible outside tu */
   int libcall;			/* If this is a compiler generated call.  */
+  /* Actual parameter count ignoring unused hidden parameters.  */
+  unsigned int actual_parm_length;
+  /* Set if there is hidden unused parameters.  */
+  unsigned int hidden_string_length : 1;
 } CUMULATIVE_ARGS;
 
 /* Initialize a variable CUM of type CUMULATIVE_ARGS