[preprocessor/91639] #includes at EOF

This fixes 91639.
#include processing needs to rewind by one line before doing the 
inclusion, so that that happens on the #include line itself.  Except 
that the lexer doesn't increment the lineno on the last \n of a file -- 
because there's no point.

So we had code in the include path to figure out that special case and 
not rewind.  I broke that.

Rather than repair it there, I decided it simpler to make the lexer 
always advance when lexing a #include line.  I did consider not even 
making that conditional, but the unstacking code looked a bit hairy to 
correctly rewind in those cases (or we simply waste linemap space).

This ends up making the include rewinding simpler -- there are still 
cases it shouldn't.

Applying to trunk.

Hi,

On Thu, Sep 05 2019, Nathan Sidwell wrote:
> This fixes 91639.
> #include processing needs to rewind by one line before doing the 
> inclusion, so that that happens on the #include line itself.  Except 
> that the lexer doesn't increment the lineno on the last \n of a file -- 
> because there's no point.
>
> So we had code in the include path to figure out that special case and 
> not rewind.  I broke that.
>
> Rather than repair it there, I decided it simpler to make the lexer 
> always advance when lexing a #include line.  I did consider not even 
> making that conditional, but the unstacking code looked a bit hairy to 
> correctly rewind in those cases (or we simply waste linemap space).
>
> This ends up making the include rewinding simpler -- there are still 
> cases it shouldn't.
>
> Applying to trunk.
>
> -- 
> Nathan Sidwell
> 2019-09-05  Nathan Sidwell  <nathan@acm.org>
>
> 	libcpp/
> 	PR preprocessor/91639
> 	* directives.c (do_include_common): Tell lexer we're a #include.
> 	* files.c (_cpp_stack_file): Lexer will have always incremented.
> 	* internal.h (struct cpp_context): Extend in_directive's
> 	semantics.
> 	* lex.c (_cpp_lex_direct): Increment line for final \n when lexing
> 	for an ISO #include.
> 	* line-map.c (linemap_line_start): Remember if we overflowed.
>
> 	gcc/testsuite/
> 	PR preprocessor/91639
> 	* c-c++-common/cpp/pr91639.c: New.
> 	* c-c++-common/cpp/pr91639-one.h: New.
> 	* c-c++-common/cpp/pr91639-two.h: New.

unfortunately, this has caused PR 91991.

Martin

>
> Index: libcpp/directives.c
> ===================================================================
> --- libcpp/directives.c	(revision 275373)
> +++ libcpp/directives.c	(working copy)
> @@ -819,4 +819,8 @@ do_include_common (cpp_reader *pfile, en
>    pfile->state.save_comments = ! CPP_OPTION (pfile, discard_comments);
>  
> +  /* Tell the lexer this is an include directive -- we want it to
> +     increment the line number even if this is the last line of a file.  */
> +  pfile->state.in_directive = 2;
> +
>    fname = parse_include (pfile, &angle_brackets, &buf, &location);
>    if (!fname)
> Index: libcpp/files.c
> ===================================================================
> --- libcpp/files.c	(revision 275373)
> +++ libcpp/files.c	(working copy)
> @@ -939,23 +939,14 @@ _cpp_stack_file (cpp_reader *pfile, _cpp
>    pfile->mi_cmacro = 0;
>  
> -  /* Compensate for the increment in linemap_add that occurs when in
> -     do_file_change.   In the case of a normal #include, we're
> -     currently at the start of the line *following* the #include.  A
> -     separate location_t for this location makes no sense (until we do
> -     the LC_LEAVE), and complicates LAST_SOURCE_LINE_LOCATION.  This
> -     does not apply if we found a PCH file (in which case linemap_add
> -     is not called) or we were included from the command-line.  In the
> -     case that the #include is the last line in the file,
> -     highest_location still points to the current line, not the start
> -     of the next line, so we do not decrement in this case.  See
> -     plugin/location-overflow-test-pr83173.h for an example.  */
> -  bool decremented = false;
> -  if (file->pchname == NULL && file->err_no == 0 && type < IT_DIRECTIVE_HWM)
> -    {
> -      decremented = (pfile->line_table->highest_line
> -		     == pfile->line_table->highest_location);
> -      if (decremented)
> -	pfile->line_table->highest_location--;
> -    }
> +  /* In the case of a normal #include, we're now at the start of the
> +     line *following* the #include.  A separate location_t for this
> +     location makes no sense, until we do the LC_LEAVE.
> +
> +     This does not apply if we found a PCH file, we're not a regular
> +     include, or we ran out of locations.  */
> +  if (file->pchname == NULL
> +      && type < IT_DIRECTIVE_HWM
> +      && pfile->line_table->highest_location != LINE_MAP_MAX_LOCATION - 1)
> +    pfile->line_table->highest_location--;
>  
>    /* Add line map and do callbacks.  */
> Index: libcpp/internal.h
> ===================================================================
> --- libcpp/internal.h	(revision 275373)
> +++ libcpp/internal.h	(working copy)
> @@ -235,5 +235,6 @@ struct cpp_context
>  struct lexer_state
>  {
> -  /* Nonzero if first token on line is CPP_HASH.  */
> +  /* 1 if we're handling a directive.  2 if it's an include-like
> +     directive.  */
>    unsigned char in_directive;
>  
> Index: libcpp/lex.c
> ===================================================================
> --- libcpp/lex.c	(revision 275373)
> +++ libcpp/lex.c	(working copy)
> @@ -2772,5 +2772,11 @@ _cpp_lex_direct (cpp_reader *pfile)
>  
>      case '\n':
> -      if (buffer->cur < buffer->rlimit)
> +      /* Increment the line, unless this is the last line ...  */
> +      if (buffer->cur < buffer->rlimit
> +	  /* ... or this is a #include, (where _cpp_stack_file needs to
> +	     unwind by one line) ...  */
> +	  || (pfile->state.in_directive > 1
> +	      /* ... except traditional-cpp increments this elsewhere.  */
> +	      && !CPP_OPTION (pfile, traditional)))
>  	CPP_INCREMENT_LINE (pfile, 0);
>        buffer->need_line = true;
> Index: libcpp/line-map.c
> ===================================================================
> --- libcpp/line-map.c	(revision 275373)
> +++ libcpp/line-map.c	(working copy)
> @@ -765,5 +765,9 @@ linemap_line_start (line_maps *set, line
>       macro tokens.  */
>    if (r >= LINE_MAP_MAX_LOCATION)
> -    return 0;
> +    {
> +      /* Remember we overflowed.  */
> +      set->highest_line = set->highest_location = LINE_MAP_MAX_LOCATION - 1;
> +      return 0;
> +    }
>  
>    set->highest_line = r;
> Index: gcc/testsuite/c-c++-common/cpp/pr91639-one.h
> ===================================================================
> --- gcc/testsuite/c-c++-common/cpp/pr91639-one.h	(revision 0)
> +++ gcc/testsuite/c-c++-common/cpp/pr91639-one.h	(working copy)
> @@ -0,0 +1,2 @@
> +one
> +#include "pr91639-two.h"
> Index: gcc/testsuite/c-c++-common/cpp/pr91639-two.h
> ===================================================================
> --- gcc/testsuite/c-c++-common/cpp/pr91639-two.h	(revision 0)
> +++ gcc/testsuite/c-c++-common/cpp/pr91639-two.h	(working copy)
> @@ -0,0 +1 @@
> +two
> Index: gcc/testsuite/c-c++-common/cpp/pr91639.c
> ===================================================================
> --- gcc/testsuite/c-c++-common/cpp/pr91639.c	(revision 0)
> +++ gcc/testsuite/c-c++-common/cpp/pr91639.c	(working copy)
> @@ -0,0 +1,9 @@
> +/* PR91639 Line markers for an end-of-file #include   */
> +/* { dg-do preprocess } */
> +/* { dg-additional-options -Wno-pedantic } */
> +/* { dg-additional-files {pr91639-one.h pr91639-two.h} } */
> +
> +#include "pr91639-one.h"
> +main
> +
> +/* { dg-final { scan-file pr91639.i "# 1 \"\[^\n\"\]*pr91639-one.h\" 1\none\n# 1 \"\[^\n\"\]*pr91639-two.h\" 1\ntwo\n# 3 \"\[^\n\"\]*pr91639-one.h\" 2\n# 7 \"\[^\n\"\]*pr91639.c\" 2\nmain\n" } } */