2019-10-28 Nathan Sidwell <nathan@acm.org>
* parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism.
Add tree_check_p flag, use as nested union discriminator.
(struct cp_lexer): Add saved_type & saved_keyword fields.
* parser.c (eof_token): Delete.
(cp_lexer_new_main): Always init last_token to last token of
buffer.
(cp_lexer_new_from_tokens): Overlay EOF token at end of range.
(cp_lexer_destroy): Restore token under the EOF.
(cp_lexer_previous_token_position): No check for eof_token here.
(cp_lexer_get_preprocessor_token): Clear tree_check_p.
(cp_lexer_peek_nth_token): Check CPP_EOF not eof_token.
(cp_lexer_consume_token): Assert not CPP_EOF, no check for
eof_token.
(cp_lexer_purge_token): Likewise.
(cp_lexer_purge_tokens_after): No check for EOF token.
(cp_parser_nested_name_specifier, cp_parser_decltype)
(cp_parser_template_id): Set tree_check_p.
===================================================================
@@ -53,9 +53,4 @@ along with GCC; see the file COPYING3.
and c-lex.c) and the C++ parser. */
-static cp_token eof_token =
-{
- CPP_EOF, RID_MAX, 0, false, false, false, 0, { NULL }
-};
-
/* The various kinds of non integral constant we encounter. */
enum non_integral_constant {
@@ -661,10 +656,8 @@ cp_lexer_new_main (void)
}
- lexer->last_token = lexer->buffer->address ()
+ lexer->next_token = lexer->buffer->address ();
+ lexer->last_token = lexer->next_token
+ lexer->buffer->length ()
- 1;
- lexer->next_token = lexer->buffer->length ()
- ? lexer->buffer->address ()
- : &eof_token;
/* Subsequent preprocessor diagnostics should use compiler
@@ -688,5 +681,12 @@ cp_lexer_new_from_tokens (cp_token_cache
/* We do not own the buffer. */
lexer->buffer = NULL;
- lexer->next_token = first == last ? &eof_token : first;
+
+ /* Insert an EOF token. */
+ lexer->saved_type = last->type;
+ lexer->saved_keyword = last->keyword;
+ last->type = CPP_EOF;
+ last->keyword = RID_MAX;
+
+ lexer->next_token = first;
lexer->last_token = last;
@@ -705,5 +705,12 @@ static void
cp_lexer_destroy (cp_lexer *lexer)
{
- vec_free (lexer->buffer);
+ if (lexer->buffer)
+ vec_free (lexer->buffer);
+ else
+ {
+ /* Restore the token we overwrite with EOF. */
+ lexer->last_token->type = lexer->saved_type;
+ lexer->last_token->keyword = lexer->saved_keyword;
+ }
lexer->saved_tokens.release ();
ggc_free (lexer);
@@ -732,6 +739,4 @@ static inline cp_token_position
cp_lexer_token_position (cp_lexer *lexer, bool previous_p)
{
- gcc_assert (!previous_p || lexer->next_token != &eof_token);
-
return lexer->next_token - previous_p;
}
@@ -752,8 +757,5 @@ static inline cp_token_position
cp_lexer_previous_token_position (cp_lexer *lexer)
{
- if (lexer->next_token == &eof_token)
- return lexer->last_token - 1;
- else
- return cp_lexer_token_position (lexer, true);
+ return cp_lexer_token_position (lexer, true);
}
@@ -808,4 +810,5 @@ cp_lexer_get_preprocessor_token (cp_lexe
token->purged_p = false;
token->error_reported = false;
+ token->tree_check_p = false;
/* On some systems, some header files are surrounded by an
@@ -1083,14 +1086,7 @@ cp_lexer_peek_nth_token (cp_lexer* lexer
--n;
token = lexer->next_token;
- gcc_assert (!n || token != &eof_token);
- while (n != 0)
+ while (n && token->type != CPP_EOF)
{
++token;
- if (token == lexer->last_token)
- {
- token = &eof_token;
- break;
- }
-
if (!token->purged_p)
--n;
@@ -1114,16 +1110,10 @@ cp_lexer_consume_token (cp_lexer* lexer)
cp_token *token = lexer->next_token;
- gcc_assert (token != &eof_token);
gcc_assert (!lexer->in_pragma || token->type != CPP_PRAGMA_EOL);
do
{
+ gcc_assert (token->type != CPP_EOF);
lexer->next_token++;
- if (lexer->next_token == lexer->last_token)
- {
- lexer->next_token = &eof_token;
- break;
- }
-
}
while (lexer->next_token->purged_p);
@@ -1151,5 +1141,5 @@ cp_lexer_purge_token (cp_lexer *lexer)
cp_token *tok = lexer->next_token;
- gcc_assert (tok != &eof_token);
+ gcc_assert (tok->type != CPP_EOF);
tok->purged_p = true;
tok->location = UNKNOWN_LOCATION;
@@ -1158,12 +1148,5 @@ cp_lexer_purge_token (cp_lexer *lexer)
do
- {
- tok++;
- if (tok == lexer->last_token)
- {
- tok = &eof_token;
- break;
- }
- }
+ tok++;
while (tok->purged_p);
lexer->next_token = tok;
@@ -1179,10 +1162,7 @@ cp_lexer_purge_tokens_after (cp_lexer *l
cp_token *peek = lexer->next_token;
- if (peek == &eof_token)
- peek = lexer->last_token;
-
gcc_assert (tok < peek);
- for ( tok += 1; tok != peek; tok += 1)
+ for (tok++; tok != peek; tok++)
{
tok->purged_p = true;
@@ -6617,4 +6597,5 @@ cp_parser_nested_name_specifier_opt (cp_
so the memory will not be reclaimed during token replacing below. */
token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
+ token->tree_check_p = true;
token->u.tree_check_value->value = parser->scope;
token->u.tree_check_value->checks = get_deferred_access_checks ();
@@ -14802,4 +14783,5 @@ cp_parser_decltype (cp_parser *parser)
start_token->type = CPP_DECLTYPE;
start_token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
+ start_token->tree_check_p = true;
start_token->u.tree_check_value->value = expr;
start_token->u.tree_check_value->checks = get_deferred_access_checks ();
@@ -16589,4 +16571,5 @@ cp_parser_template_id (cp_parser *parser
so the memory will not be reclaimed during token replacing below. */
token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
+ token->tree_check_p = true;
token->u.tree_check_value->value = template_id;
token->u.tree_check_value->checks = get_deferred_access_checks ();
===================================================================
@@ -42,21 +42,23 @@ struct GTY(()) tree_check {
struct GTY (()) cp_token {
/* The kind of token. */
- ENUM_BITFIELD (cpp_ttype) type : 8;
+ enum cpp_ttype type : 8;
/* If this token is a keyword, this value indicates which keyword.
Otherwise, this value is RID_MAX. */
- ENUM_BITFIELD (rid) keyword : 8;
+ enum rid keyword : 8;
/* Token flags. */
unsigned char flags;
/* True if this token is from a context where it is implicitly extern "C" */
- BOOL_BITFIELD implicit_extern_c : 1;
+ bool implicit_extern_c : 1;
/* True if an error has already been reported for this token, such as a
CPP_NAME token that is not a keyword (i.e., for which KEYWORD is
RID_MAX) iff this name was looked up and found to be ambiguous. */
- BOOL_BITFIELD error_reported : 1;
+ bool error_reported : 1;
/* True for a token that has been purged. If a token is purged,
it is no longer a valid token and it should be considered
deleted. */
- BOOL_BITFIELD purged_p : 1;
- /* 5 unused bits. */
+ bool purged_p : 1;
+ bool tree_check_p : 1;
+ /* 4 unused bits. */
+
/* The location at which this token was found. */
location_t location;
@@ -64,10 +66,8 @@ struct GTY (()) cp_token {
union cp_token_value {
/* Used for compound tokens such as CPP_NESTED_NAME_SPECIFIER. */
- struct tree_check* GTY((tag ("1"))) tree_check_value;
+ struct tree_check* GTY((tag ("true"))) tree_check_value;
/* Use for all other tokens. */
- tree GTY((tag ("0"))) value;
- } GTY((desc ("(%1.type == CPP_TEMPLATE_ID)"
- "|| (%1.type == CPP_NESTED_NAME_SPECIFIER)"
- "|| (%1.type == CPP_DECLTYPE)"))) u;
+ tree GTY((tag ("false"))) value;
+ } GTY((desc ("%1.tree_check_p"))) u;
};
@@ -100,4 +100,8 @@ struct GTY (()) cp_lexer {
vec<cp_token_position> GTY ((skip)) saved_tokens;
+ /* Saved pieces of end token we replaced with the eof token. */
+ enum cpp_ttype saved_type : 8;
+ enum rid saved_keyword : 8;
+
/* The next lexer in a linked list of lexers. */
struct cp_lexer *next;