c++: Add C++20 #__VA_OPT__ support

Hi!

The following patch implements C++20 # __VA_OPT__ (...) support.
Testcases cover what I came up with myself and what LLVM has for #__VA_OPT__
in its testsuite and the string literals are identical between the two
compilers on the va-opt-5.c testcase.

Haven't looked at the non-#__VA_OPT__ differences between LLVM and GCC
though, I think at least the
http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p1042r1.html
#define H4(X, ...) __VA_OPT__(a X ## X) ## b
H4(, 1)  // replaced by a b
case isn't handled right (we emit ab).

Bootstrapped/regtested on x86_64-linux and i686-linux, ok for trunk?

2021-07-15  Jakub Jelinek  <jakub@redhat.com>

libcpp/
	* macro.c (vaopt_state): Add m_stringify member.
	(vaopt_state::vaopt_state): Initialize it.
	(vaopt_state::update): Overwrite it.
	(vaopt_state::stringify): New method.
	(stringify_arg): Replace arg argument with first, count arguments
	and add va_opt argument.  Use first instead of arg->first and
	count instead of arg->count, for va_opt add paste_tokens handling.
	(paste_tokens): Fix up len calculation.  Don't spell rhs twice,
	instead use %.*s to supply lhs and rhs spelling lengths.  Don't call
	_cpp_backup_tokens here.
	(paste_all_tokens): Call it here instead.
	(replace_args): Adjust stringify_arg caller.  For vaopt_state::END
	if stringify is true handle __VA_OPT__ stringification.
	(create_iso_definition): Handle # __VA_OPT__ similarly to # macro_arg.
gcc/testsuite/
	* c-c++-common/cpp/va-opt-5.c: New test.
	* c-c++-common/cpp/va-opt-6.c: New test.


	Jakub

On 7/15/21 1:52 PM, Jakub Jelinek wrote:
> Hi!
> 
> The following patch implements C++20 # __VA_OPT__ (...) support.
> Testcases cover what I came up with myself and what LLVM has for #__VA_OPT__
> in its testsuite and the string literals are identical between the two
> compilers on the va-opt-5.c testcase.
> 
> Haven't looked at the non-#__VA_OPT__ differences between LLVM and GCC
> though, I think at least the
> http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p1042r1.html
> #define H4(X, ...) __VA_OPT__(a X ## X) ## b
> H4(, 1)  // replaced by a b
> case isn't handled right (we emit ab).
> 
> Bootstrapped/regtested on x86_64-linux and i686-linux, ok for trunk?

OK.

> 2021-07-15  Jakub Jelinek  <jakub@redhat.com>
> 
> libcpp/
> 	* macro.c (vaopt_state): Add m_stringify member.
> 	(vaopt_state::vaopt_state): Initialize it.
> 	(vaopt_state::update): Overwrite it.
> 	(vaopt_state::stringify): New method.
> 	(stringify_arg): Replace arg argument with first, count arguments
> 	and add va_opt argument.  Use first instead of arg->first and
> 	count instead of arg->count, for va_opt add paste_tokens handling.
> 	(paste_tokens): Fix up len calculation.  Don't spell rhs twice,
> 	instead use %.*s to supply lhs and rhs spelling lengths.  Don't call
> 	_cpp_backup_tokens here.
> 	(paste_all_tokens): Call it here instead.
> 	(replace_args): Adjust stringify_arg caller.  For vaopt_state::END
> 	if stringify is true handle __VA_OPT__ stringification.
> 	(create_iso_definition): Handle # __VA_OPT__ similarly to # macro_arg.
> gcc/testsuite/
> 	* c-c++-common/cpp/va-opt-5.c: New test.
> 	* c-c++-common/cpp/va-opt-6.c: New test.
> 
> --- libcpp/macro.c.jj	2021-05-21 10:34:09.328560825 +0200
> +++ libcpp/macro.c	2021-07-15 17:27:30.109631306 +0200
> @@ -118,6 +118,7 @@ class vaopt_state {
>       m_arg (arg),
>       m_variadic (is_variadic),
>       m_last_was_paste (false),
> +    m_stringify (false),
>       m_state (0),
>       m_paste_location (0),
>       m_location (0),
> @@ -145,6 +146,7 @@ class vaopt_state {
>   	  }
>   	++m_state;
>   	m_location = token->src_loc;
> +	m_stringify = (token->flags & STRINGIFY_ARG) != 0;
>   	return BEGIN;
>         }
>       else if (m_state == 1)
> @@ -234,6 +236,11 @@ class vaopt_state {
>       return m_state == 0;
>     }
>   
> +  bool stringify () const
> +  {
> +    return m_stringify;
> +  }
> +
>    private:
>   
>     /* The cpp_reader.  */
> @@ -247,6 +254,8 @@ class vaopt_state {
>     /* If true, the previous token was ##.  This is used to detect when
>        a paste occurs at the end of the sequence.  */
>     bool m_last_was_paste;
> +  /* True for #__VA_OPT__.  */
> +  bool m_stringify;
>   
>     /* The state variable:
>        0 means not parsing
> @@ -284,7 +293,8 @@ static _cpp_buff *collect_args (cpp_read
>   static cpp_context *next_context (cpp_reader *);
>   static const cpp_token *padding_token (cpp_reader *, const cpp_token *);
>   static const cpp_token *new_string_token (cpp_reader *, uchar *, unsigned int);
> -static const cpp_token *stringify_arg (cpp_reader *, macro_arg *);
> +static const cpp_token *stringify_arg (cpp_reader *, const cpp_token **,
> +				       unsigned int, bool);
>   static void paste_all_tokens (cpp_reader *, const cpp_token *);
>   static bool paste_tokens (cpp_reader *, location_t,
>   			  const cpp_token **, const cpp_token *);
> @@ -818,10 +828,11 @@ cpp_quote_string (uchar *dest, const uch
>     return dest;
>   }
>   
> -/* Convert a token sequence ARG to a single string token according to
> -   the rules of the ISO C #-operator.  */
> +/* Convert a token sequence FIRST to FIRST+COUNT-1 to a single string token
> +   according to the rules of the ISO C #-operator.  */
>   static const cpp_token *
> -stringify_arg (cpp_reader *pfile, macro_arg *arg)
> +stringify_arg (cpp_reader *pfile, const cpp_token **first, unsigned int count,
> +	       bool va_opt)
>   {
>     unsigned char *dest;
>     unsigned int i, escape_it, backslash_count = 0;
> @@ -834,9 +845,27 @@ stringify_arg (cpp_reader *pfile, macro_
>     *dest++ = '"';
>   
>     /* Loop, reading in the argument's tokens.  */
> -  for (i = 0; i < arg->count; i++)
> +  for (i = 0; i < count; i++)
>       {
> -      const cpp_token *token = arg->first[i];
> +      const cpp_token *token = first[i];
> +
> +      if (va_opt && (token->flags & PASTE_LEFT))
> +	{
> +	  location_t virt_loc = pfile->invocation_location;
> +	  const cpp_token *rhs;
> +	  do
> +	    {
> +	      if (i == count)
> +		abort ();
> +	      rhs = first[++i];
> +	      if (!paste_tokens (pfile, virt_loc, &token, rhs))
> +		{
> +		  --i;
> +		  break;
> +		}
> +	    }
> +	  while (rhs->flags & PASTE_LEFT);
> +	}
>   
>         if (token->type == CPP_PADDING)
>   	{
> @@ -923,7 +952,7 @@ paste_tokens (cpp_reader *pfile, locatio
>     cpp_token *lhs;
>     unsigned int len;
>   
> -  len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 1;
> +  len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 2;
>     buf = (unsigned char *) alloca (len);
>     end = lhsend = cpp_spell_token (pfile, *plhs, buf, true);
>   
> @@ -949,8 +978,10 @@ paste_tokens (cpp_reader *pfile, locatio
>         location_t saved_loc = lhs->src_loc;
>   
>         _cpp_pop_buffer (pfile);
> -      _cpp_backup_tokens (pfile, 1);
> -      *lhsend = '\0';
> +
> +      unsigned char *rhsstart = lhsend;
> +      if ((*plhs)->type == CPP_DIV && rhs->type != CPP_EQ)
> +	rhsstart++;
>   
>         /* We have to remove the PASTE_LEFT flag from the old lhs, but
>   	 we want to keep the new location.  */
> @@ -962,8 +993,10 @@ paste_tokens (cpp_reader *pfile, locatio
>         /* Mandatory error for all apart from assembler.  */
>         if (CPP_OPTION (pfile, lang) != CLK_ASM)
>   	cpp_error_with_line (pfile, CPP_DL_ERROR, location, 0,
> -	 "pasting \"%s\" and \"%s\" does not give a valid preprocessing token",
> -		   buf, cpp_token_as_text (pfile, rhs));
> +			     "pasting \"%.*s\" and \"%.*s\" does not give "
> +			     "a valid preprocessing token",
> +			     (int) (lhsend - buf), buf,
> +			     (int) (end - rhsstart), rhsstart);
>         return false;
>       }
>   
> @@ -1039,7 +1072,10 @@ paste_all_tokens (cpp_reader *pfile, con
>   	    abort ();
>   	}
>         if (!paste_tokens (pfile, virt_loc, &lhs, rhs))
> -	break;
> +	{
> +	  _cpp_backup_tokens (pfile, 1);
> +	  break;
> +	}
>       }
>     while (rhs->flags & PASTE_LEFT);
>   
> @@ -1906,7 +1942,8 @@ replace_args (cpp_reader *pfile, cpp_has
>   	if (src->flags & STRINGIFY_ARG)
>   	  {
>   	    if (!arg->stringified)
> -	      arg->stringified = stringify_arg (pfile, arg);
> +	      arg->stringified = stringify_arg (pfile, arg->first, arg->count,
> +						false);
>   	  }
>   	else if ((src->flags & PASTE_LEFT)
>   		 || (src != macro->exp.tokens && (src[-1].flags & PASTE_LEFT)))
> @@ -2029,7 +2066,24 @@ replace_args (cpp_reader *pfile, cpp_has
>   		  paste_flag = tokens_buff_last_token_ptr (buff);
>   		}
>   
> -	      if (src->flags & PASTE_LEFT)
> +	      if (vaopt_tracker.stringify ())
> +		{
> +		  unsigned int count
> +		    = start ? paste_flag - start : tokens_buff_count (buff);
> +		  const cpp_token *t
> +		    = stringify_arg (pfile,
> +				     start ? start + 1
> +				     : (const cpp_token **) (buff->base),
> +				     count, true);
> +		  while (count--)
> +		    tokens_buff_remove_last_token (buff);
> +		  if (src->flags & PASTE_LEFT)
> +		    copy_paste_flag (pfile, &t, src);
> +		  tokens_buff_add_token (buff, virt_locs,
> +					 t, t->src_loc, t->src_loc,
> +					 NULL, 0);
> +		}
> +	      else if (src->flags & PASTE_LEFT)
>   		{
>   		  /* With a non-empty __VA_OPT__ on the LHS of ##, the last
>   		     token should be flagged PASTE_LEFT.  */
> @@ -3580,7 +3634,10 @@ create_iso_definition (cpp_reader *pfile
>   	 function-like macros when lexing the subsequent token.  */
>         if (macro->count > 1 && token[-1].type == CPP_HASH && macro->fun_like)
>   	{
> -	  if (token->type == CPP_MACRO_ARG)
> +	  if (token->type == CPP_MACRO_ARG
> +	      || (macro->variadic
> +		  && token->type == CPP_NAME
> +		  && token->val.node.node == pfile->spec_nodes.n__VA_OPT__))
>   	    {
>   	      if (token->flags & PREV_WHITE)
>   		token->flags |= SP_PREV_WHITE;
> --- gcc/testsuite/c-c++-common/cpp/va-opt-5.c.jj	2021-07-15 15:51:33.200834534 +0200
> +++ gcc/testsuite/c-c++-common/cpp/va-opt-5.c	2021-07-15 17:35:17.205283482 +0200
> @@ -0,0 +1,67 @@
> +/* { dg-do run } */
> +/* { dg-options "-std=gnu99" { target c } } */
> +/* { dg-options "-std=c++20" { target c++ } } */
> +
> +#define lparen (
> +#define a0 fooa0
> +#define a1  fooa1 a0
> +#define a2  fooa2 a1
> +#define a3  fooa3 a2
> +#define a() b lparen )
> +#define b() c lparen )
> +#define c() d lparen )
> +#define g h
> +#define i(j) j
> +#define f(...) #__VA_OPT__(g i(0))
> +#define k(x,...) # __VA_OPT__(x) #x #__VA_OPT__(__VA_ARGS__)
> +#define l(x,...) #__VA_OPT__(a1 x)
> +#define m(x,...) "a()" #__VA_OPT__(a3 __VA_ARGS__ x ## __VA_ARGS__ ## x ## c a3) "a()"
> +#define n(x,...) = #__VA_OPT__(a3 __VA_ARGS__ x ## __VA_ARGS__ ## x ## c a3) #x #__VA_OPT__(a0 __VA_ARGS__ x ## __VA_ARGS__ ## x ## c a0) ;
> +#define o(x, ...) #__VA_OPT__(x##x x##x)
> +#define p(x, ...) #__VA_OPT__(_Pragma ("foobar"))
> +#define q(...) #__VA_OPT__(/* foo */x/* bar */)
> +const char *v1 = f();
> +const char *v2 = f(123);
> +const char *v3 = k(1);
> +const char *v4 = k(1, 2, 3 );
> +const char *v5 = l(a());
> +const char *v6 = l(a1 a(), 1);
> +const char *v7 = m();
> +const char *v8 = m(,);
> +const char *v9 = m(,a3);
> +const char *v10 = m(a3,a(),a0);
> +const char *v11 n()
> +const char *v12 n(,)
> +const char *v13 n(,a0)
> +const char *v14 n(a0, a(),a0)
> +const char *v15 = o(, 0);
> +const char *v16 = p(0);
> +const char *v17 = p(0, 1);
> +const char *v18 = q();
> +const char *v19 = q(1);
> +
> +int
> +main ()
> +{
> +  if (__builtin_strcmp (v1, "")
> +      || __builtin_strcmp (v2, "g i(0)")
> +      || __builtin_strcmp (v3, "1")
> +      || __builtin_strcmp (v4, "112, 3")
> +      || __builtin_strcmp (v5, "")
> +      || __builtin_strcmp (v6, "a1 fooa1 fooa0 b ( )")
> +      || __builtin_strcmp (v7, "a()a()")
> +      || __builtin_strcmp (v8, "a()a()")
> +      || __builtin_strcmp (v9, "a()a3 fooa3 fooa2 fooa1 fooa0 a3c a3a()")
> +      || __builtin_strcmp (v10, "a()a3 b ( ),fooa0 a3a(),a0a3c a3a()")
> +      || __builtin_strcmp (v11, "")
> +      || __builtin_strcmp (v12, "")
> +      || __builtin_strcmp (v13, "a3 fooa0 a0c a3a0 fooa0 a0c a0")
> +      || __builtin_strcmp (v14, "a3 b ( ),fooa0 a0a(),a0a0c a3a0a0 b ( ),fooa0 a0a(),a0a0c a0")
> +      || __builtin_strcmp (v15, "")
> +      || __builtin_strcmp (v16, "")
> +      || __builtin_strcmp (v17, "_Pragma (\"foobar\")")
> +      || __builtin_strcmp (v18, "")
> +      || __builtin_strcmp (v19, "x"))
> +    __builtin_abort ();
> +  return 0;
> +}
> --- gcc/testsuite/c-c++-common/cpp/va-opt-6.c.jj	2021-07-15 17:35:05.574441546 +0200
> +++ gcc/testsuite/c-c++-common/cpp/va-opt-6.c	2021-07-15 17:36:34.017239610 +0200
> @@ -0,0 +1,17 @@
> +/* { dg-do preprocess } */
> +/* { dg-options "-std=gnu99" { target c } } */
> +/* { dg-options "-std=c++20" { target c++ } } */
> +
> +#define a ""
> +#define b(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting \"a\" and \"\"\"\" does not give a valid preprocessing token" } */
> +#define c(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting \"a\" and \"\"1\"\" does not give a valid preprocessing token" } */
> +#define d(...) #__VA_OPT__(1) ## !
> +#define e(...) #__VA_OPT__(1) ## !
> +#define f(...) #__VA_OPT__(. ## !)
> +#define g(...) #__VA_OPT__(. ## !)
> +b()
> +c(1)
> +d(   )		/* { dg-error "pasting \"\"\"\" and \"!\" does not give a valid preprocessing token" } */
> +e(  1 )		/* { dg-error "pasting \"\"1\"\" and \"!\" does not give a valid preprocessing token" } */
> +f()
> +g(0)		/* { dg-error "pasting \".\" and \"!\" does not give a valid preprocessing token" } */
> 
> 	Jakub
>