Message ID | 20211126093300.GF2646553@tucnak |
---|---|
State | New |
Headers | show |
Series | libcpp: Fix up #__VA_OPT__ handling [PR103415] | expand |
On 11/26/21 04:33, Jakub Jelinek wrote: > Hi! > > stringify_arg uses pfile->u_buff to create the string literal. > Unfortunately, paste_tokens -> _cpp_lex_direct -> lex_number -> _cpp_unaligned_alloc > can in some cases use pfile->u_buff too, which results in losing everything > prepared for the string literal until the token pasting. > > The following patch fixes that by not calling paste_token during the > construction of the string literal, but doing that before. All the tokens > we are processing have been pushed into a token buffer using > tokens_buff_add_token so it is fine if we paste some of them in that buffer > (successful pasting creates a new token in that buffer), move following > tokens if any to make it contiguous, pop (throw away) the extra tokens at > the end and then do stringify_arg. Please add some of this explanation to the "paste any tokens" comment in the code. > Bootstrapped/regtested on x86_64-linux and i686-linux, ok for trunk? > > 2021-11-26 Jakub Jelinek <jakub@redhat.com> > > PR preprocessor/103415 > libcpp/ > * macro.c (stringify_arg): Remove va_opt argument and va_opt handling. > (replace_args): Adjust callers. For #__VA_OPT__, perform token > pasting in a separate loop before stringify_arg call. > gcc/testsuite/ > * c-c++-common/cpp/va-opt-8.c: New test. > > --- libcpp/macro.c.jj 2021-11-23 10:26:30.595792246 +0100 > +++ libcpp/macro.c 2021-11-25 17:55:39.149217299 +0100 > @@ -295,7 +295,7 @@ static cpp_context *next_context (cpp_re > static const cpp_token *padding_token (cpp_reader *, const cpp_token *); > static const cpp_token *new_string_token (cpp_reader *, uchar *, unsigned int); > static const cpp_token *stringify_arg (cpp_reader *, const cpp_token **, > - unsigned int, bool); > + unsigned int); > static void paste_all_tokens (cpp_reader *, const cpp_token *); > static bool paste_tokens (cpp_reader *, location_t, > const cpp_token **, const cpp_token *); > @@ -834,8 +834,7 @@ cpp_quote_string (uchar *dest, const uch > /* Convert a token sequence FIRST to FIRST+COUNT-1 to a single string token > according to the rules of the ISO C #-operator. */ > static const cpp_token * > -stringify_arg (cpp_reader *pfile, const cpp_token **first, unsigned int count, > - bool va_opt) > +stringify_arg (cpp_reader *pfile, const cpp_token **first, unsigned int count) > { > unsigned char *dest; > unsigned int i, escape_it, backslash_count = 0; > @@ -852,24 +851,6 @@ stringify_arg (cpp_reader *pfile, const > { > const cpp_token *token = first[i]; > > - if (va_opt && (token->flags & PASTE_LEFT)) > - { > - location_t virt_loc = pfile->invocation_location; > - const cpp_token *rhs; > - do > - { > - if (i == count) > - abort (); > - rhs = first[++i]; > - if (!paste_tokens (pfile, virt_loc, &token, rhs)) > - { > - --i; > - break; > - } > - } > - while (rhs->flags & PASTE_LEFT); > - } > - > if (token->type == CPP_PADDING) > { > if (source == NULL > @@ -1945,8 +1926,7 @@ replace_args (cpp_reader *pfile, cpp_has > if (src->flags & STRINGIFY_ARG) > { > if (!arg->stringified) > - arg->stringified = stringify_arg (pfile, arg->first, arg->count, > - false); > + arg->stringified = stringify_arg (pfile, arg->first, arg->count); > } > else if ((src->flags & PASTE_LEFT) > || (src != macro->exp.tokens && (src[-1].flags & PASTE_LEFT))) > @@ -2066,11 +2046,49 @@ replace_args (cpp_reader *pfile, cpp_has > { > unsigned int count > = start ? paste_flag - start : tokens_buff_count (buff); > - const cpp_token *t > - = stringify_arg (pfile, > - start ? start + 1 > - : (const cpp_token **) (buff->base), > - count, true); > + const cpp_token **first > + = start ? start + 1 > + : (const cpp_token **) (buff->base); > + unsigned int i, j; > + > + /* Paste any tokens that need to be pasted. */ > + for (i = 0, j = 0; i < count; i++, j++) > + { > + const cpp_token *token = first[i]; > + > + if (token->flags & PASTE_LEFT) > + { > + location_t virt_loc = pfile->invocation_location; > + const cpp_token *rhs; > + unsigned short flags = token->flags; > + do > + { > + if (i == count) > + abort (); > + rhs = first[++i]; > + if (!paste_tokens (pfile, virt_loc, &token, rhs)) > + { > + --i; > + break; > + } > + } > + while (rhs->flags & PASTE_LEFT); > + if ((flags & PREV_WHITE) > + && (token->flags & PREV_WHITE) == 0) > + const_cast<cpp_token *>(token)->flags > + |= PREV_WHITE; Hmm, shouldn't paste_tokens handle copying PREV_WHITE? > + } > + > + first[j] = token; > + } > + if (j != i) > + { > + while (i-- != j) > + tokens_buff_remove_last_token (buff); > + count = j; > + } > + > + const cpp_token *t = stringify_arg (pfile, first, count); > while (count--) > tokens_buff_remove_last_token (buff); > if (src->flags & PASTE_LEFT) > --- gcc/testsuite/c-c++-common/cpp/va-opt-8.c.jj 2021-11-25 18:03:19.243595714 +0100 > +++ gcc/testsuite/c-c++-common/cpp/va-opt-8.c 2021-11-25 18:02:12.712553219 +0100 > @@ -0,0 +1,18 @@ > +/* PR preprocessor/103415 */ > +/* { dg-do run } */ > +/* { dg-options "-std=gnu99" { target c } } */ > +/* { dg-options "-std=c++20" { target c++ } } */ > + > +#define n(x, ...) = #__VA_OPT__(x##3) > +#define o(x, ...) #__VA_OPT__(x##__VA_ARGS__##9) > +const char *c n(1 2, 4); > +const char *d = o(5 6, 7 8); > + > +int > +main () > +{ > + if (__builtin_strcmp (c, "1 23") > + || __builtin_strcmp (d, "5 67 89")) > + __builtin_abort (); > + return 0; > +} > > Jakub >
--- libcpp/macro.c.jj 2021-11-23 10:26:30.595792246 +0100 +++ libcpp/macro.c 2021-11-25 17:55:39.149217299 +0100 @@ -295,7 +295,7 @@ static cpp_context *next_context (cpp_re static const cpp_token *padding_token (cpp_reader *, const cpp_token *); static const cpp_token *new_string_token (cpp_reader *, uchar *, unsigned int); static const cpp_token *stringify_arg (cpp_reader *, const cpp_token **, - unsigned int, bool); + unsigned int); static void paste_all_tokens (cpp_reader *, const cpp_token *); static bool paste_tokens (cpp_reader *, location_t, const cpp_token **, const cpp_token *); @@ -834,8 +834,7 @@ cpp_quote_string (uchar *dest, const uch /* Convert a token sequence FIRST to FIRST+COUNT-1 to a single string token according to the rules of the ISO C #-operator. */ static const cpp_token * -stringify_arg (cpp_reader *pfile, const cpp_token **first, unsigned int count, - bool va_opt) +stringify_arg (cpp_reader *pfile, const cpp_token **first, unsigned int count) { unsigned char *dest; unsigned int i, escape_it, backslash_count = 0; @@ -852,24 +851,6 @@ stringify_arg (cpp_reader *pfile, const { const cpp_token *token = first[i]; - if (va_opt && (token->flags & PASTE_LEFT)) - { - location_t virt_loc = pfile->invocation_location; - const cpp_token *rhs; - do - { - if (i == count) - abort (); - rhs = first[++i]; - if (!paste_tokens (pfile, virt_loc, &token, rhs)) - { - --i; - break; - } - } - while (rhs->flags & PASTE_LEFT); - } - if (token->type == CPP_PADDING) { if (source == NULL @@ -1945,8 +1926,7 @@ replace_args (cpp_reader *pfile, cpp_has if (src->flags & STRINGIFY_ARG) { if (!arg->stringified) - arg->stringified = stringify_arg (pfile, arg->first, arg->count, - false); + arg->stringified = stringify_arg (pfile, arg->first, arg->count); } else if ((src->flags & PASTE_LEFT) || (src != macro->exp.tokens && (src[-1].flags & PASTE_LEFT))) @@ -2066,11 +2046,49 @@ replace_args (cpp_reader *pfile, cpp_has { unsigned int count = start ? paste_flag - start : tokens_buff_count (buff); - const cpp_token *t - = stringify_arg (pfile, - start ? start + 1 - : (const cpp_token **) (buff->base), - count, true); + const cpp_token **first + = start ? start + 1 + : (const cpp_token **) (buff->base); + unsigned int i, j; + + /* Paste any tokens that need to be pasted. */ + for (i = 0, j = 0; i < count; i++, j++) + { + const cpp_token *token = first[i]; + + if (token->flags & PASTE_LEFT) + { + location_t virt_loc = pfile->invocation_location; + const cpp_token *rhs; + unsigned short flags = token->flags; + do + { + if (i == count) + abort (); + rhs = first[++i]; + if (!paste_tokens (pfile, virt_loc, &token, rhs)) + { + --i; + break; + } + } + while (rhs->flags & PASTE_LEFT); + if ((flags & PREV_WHITE) + && (token->flags & PREV_WHITE) == 0) + const_cast<cpp_token *>(token)->flags + |= PREV_WHITE; + } + + first[j] = token; + } + if (j != i) + { + while (i-- != j) + tokens_buff_remove_last_token (buff); + count = j; + } + + const cpp_token *t = stringify_arg (pfile, first, count); while (count--) tokens_buff_remove_last_token (buff); if (src->flags & PASTE_LEFT) --- gcc/testsuite/c-c++-common/cpp/va-opt-8.c.jj 2021-11-25 18:03:19.243595714 +0100 +++ gcc/testsuite/c-c++-common/cpp/va-opt-8.c 2021-11-25 18:02:12.712553219 +0100 @@ -0,0 +1,18 @@ +/* PR preprocessor/103415 */ +/* { dg-do run } */ +/* { dg-options "-std=gnu99" { target c } } */ +/* { dg-options "-std=c++20" { target c++ } } */ + +#define n(x, ...) = #__VA_OPT__(x##3) +#define o(x, ...) #__VA_OPT__(x##__VA_ARGS__##9) +const char *c n(1 2, 4); +const char *d = o(5 6, 7 8); + +int +main () +{ + if (__builtin_strcmp (c, "1 23") + || __builtin_strcmp (d, "5 67 89")) + __builtin_abort (); + return 0; +}