[WIP] libcpp, c-family: Add concatenated string support for #emebd gnu::base64 argument

--- libcpp/internal.h.jj	2024-06-19 09:28:25.881760114 +0200
+++ libcpp/internal.h	2024-06-21 12:12:04.647654213 +0200
@@ -631,8 +631,7 @@  struct cpp_embed_params
   location_t loc;
   bool has_embed;
   cpp_num_part limit, offset;
-  const cpp_token *base64;
-  cpp_embed_params_tokens prefix, suffix, if_empty;
+  cpp_embed_params_tokens prefix, suffix, if_empty, base64;
 };
 
 /* Character classes.  Based on the more primitive macros in safe-ctype.h.
@@ -806,6 +805,7 @@  extern void _cpp_restore_pragma_names (c
 extern int _cpp_do__Pragma (cpp_reader *, location_t);
 extern void _cpp_init_directives (cpp_reader *);
 extern void _cpp_init_internal_pragmas (cpp_reader *);
+extern void _cpp_free_embed_params_tokens (cpp_embed_params_tokens *);
 extern bool _cpp_parse_embed_params (cpp_reader *, struct cpp_embed_params *);
 extern void _cpp_do_file_change (cpp_reader *, enum lc_reason, const char *,
 				 linenum_type, unsigned int);
--- libcpp/directives.cc.jj	2024-06-19 12:12:54.178141429 +0200
+++ libcpp/directives.cc	2024-06-21 12:59:44.669537045 +0200
@@ -932,6 +932,50 @@  do_include_next (cpp_reader *pfile)
   do_include_common (pfile, type);
 }
 
+/* Helper function for skip_balanced_token_seq and _cpp_parse_embed_params.
+   Save one token *TOKEN into *SAVE.  */
+
+static void
+save_token_for_embed (cpp_embed_params_tokens *save, const cpp_token *token)
+{
+  if (save->count == 0)
+    {
+      _cpp_init_tokenrun (&save->base_run, 4);
+      save->cur_run = &save->base_run;
+      save->cur_token = save->base_run.base;
+    }
+  else if (save->cur_token == save->cur_run->limit)
+    {
+      save->cur_run->next = XNEW (tokenrun);
+      save->cur_run->next->prev = save->cur_run;
+      _cpp_init_tokenrun (save->cur_run->next, 4);
+      save->cur_run = save->cur_run->next;
+      save->cur_token = save->cur_run->base;
+    }
+  *save->cur_token = *token;
+  save->cur_token->flags |= NO_EXPAND;
+  save->cur_token++;
+  save->count++;
+}
+
+/* Free memory associated with saved tokens in *SAVE.  */
+
+void
+_cpp_free_embed_params_tokens (cpp_embed_params_tokens *save)
+{
+  if (save->count == 0)
+    return;
+  tokenrun *n;
+  for (tokenrun *t = &save->base_run; t; t = n)
+    {
+      n = t->next;
+      XDELETEVEC (t->base);
+      if (t != &save->base_run)
+	XDELETE (t);
+    }
+  save->count = 0;
+}
+
 /* Skip over balanced preprocessing tokens until END is found.
    If SAVE is non-NULL, remember the parsed tokens in it.  */
 
@@ -959,26 +1003,7 @@  skip_balanced_token_seq (cpp_reader *pfi
       if (save
 	  && (token->type != CPP_PADDING || save->count)
 	  && (token->type != end || nested))
-	{
-	  if (save->count == 0)
-	    {
-	      _cpp_init_tokenrun (&save->base_run, 4);
-	      save->cur_run = &save->base_run;
-	      save->cur_token = save->base_run.base;
-	    }
-	  else if (save->cur_token == save->cur_run->limit)
-	    {
-	      save->cur_run->next = XNEW (tokenrun);
-	      save->cur_run->next->prev = save->cur_run;
-	      _cpp_init_tokenrun (save->cur_run->next, 4);
-	      save->cur_run = save->cur_run->next;
-	      save->cur_token = save->cur_run->base;
-	    }
-	  *save->cur_token = *token;
-	  save->cur_token->flags |= NO_EXPAND;
-	  save->cur_token++;
-	  save->count++;
-	}
+	save_token_for_embed (save, token);
       if (token->type == end)
 	return;
       switch (token->type)
@@ -1038,12 +1063,12 @@  _cpp_parse_embed_params (cpp_reader *pfi
 	      cpp_error (pfile, CPP_DL_ERROR, "expected parameter name");
 	      return false;
 	    }
-	  if (params->base64 && (seen & ((1 << 0) | (1 << 5))) != 0)
+	  if (params->base64.count && (seen & ((1 << 0) | (1 << 5))) != 0)
 	    {
 	      ret = false;
 	      if (!params->has_embed)
 		cpp_error_with_line (pfile, CPP_DL_ERROR,
-				     params->base64->src_loc, 0,
+				     params->base64.base_run.base->src_loc, 0,
 				     "'gnu::base64' parameter conflicts with "
 				     "'limit' or 'gnu::offset' parameters");
 	    }
@@ -1185,8 +1210,12 @@  _cpp_parse_embed_params (cpp_reader *pfi
 	  token = _cpp_get_token_no_padding (pfile);
 	  if (token->type == CPP_STRING)
 	    {
-	      params->base64 = token;
-	      token = _cpp_get_token_no_padding (pfile);
+	      do
+		{
+		  save_token_for_embed (&params->base64, token);
+		  token = _cpp_get_token_no_padding (pfile);
+		}
+	      while (token->type == CPP_STRING);
 	      if (token->type != CPP_CLOSE_PAREN)
 		cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 0,
 				     "expected ')'");
@@ -1280,26 +1309,10 @@  do_embed (cpp_reader *pfile)
   if (ok)
     _cpp_stack_embed (pfile, fname, angle_brackets, &params);
 
-  for (int i = 0; i < 3; ++i)
-    {
-      cpp_embed_params_tokens *p;
-      if (i == 0)
-	p = &params.prefix;
-      else if (i == 1)
-	p = &params.suffix;
-      else
-	p = &params.if_empty;
-      if (p->count == 0)
-	continue;
-      tokenrun *n;
-      for (tokenrun *t = &p->base_run; t; t = n)
-	{
-	  n = t->next;
-	  XDELETEVEC (t->base);
-	  if (t != &p->base_run)
-	    XDELETE (t);
-	}
-    }
+  _cpp_free_embed_params_tokens (&params.prefix);
+  _cpp_free_embed_params_tokens (&params.suffix);
+  _cpp_free_embed_params_tokens (&params.if_empty);
+  _cpp_free_embed_params_tokens (&params.base64);
 
  done:
   XDELETEVEC (fname);
--- libcpp/files.cc.jj	2024-06-20 19:16:39.940670936 +0200
+++ libcpp/files.cc	2024-06-21 12:54:32.633587454 +0200
@@ -1357,7 +1357,7 @@  finish_embed (cpp_reader *pfile, _cpp_fi
 	      tok->val.str.len
 		= limit - 1 - i > INT_MAX ? INT_MAX : limit - 1 - i;
 	      i += tok->val.str.len;
-	      if (tok->val.str.len < 32)
+	      if (tok->val.str.len < 32 && j)
 		{
 		  /* Avoid CPP_EMBED with a fewer than 32 bytes, shrink the
 		     previous CPP_EMBED by 64 and grow this one by 64.  */
@@ -1442,9 +1442,9 @@  static int
 finish_base64_embed (cpp_reader *pfile, const char *fname, bool angle,
 		     struct cpp_embed_params *params)
 {
-  size_t len, end, i, base64_len;
-  uchar *buf = NULL, *q, qbuf[3];
-  const uchar *p, *base64_str;
+  size_t len, end, i, j, base64_len = 0, cnt;
+  uchar *buf = NULL, *q, pbuf[4], qbuf[3];
+  const uchar *base64_str;
   if (angle || strcmp (fname, "."))
     {
       if (!params->has_embed)
@@ -1452,29 +1452,34 @@  finish_base64_embed (cpp_reader *pfile,
 		      "'gnu::base64' parameter can be only used with \".\"");
       return 0;
     }
-  if (params->base64->val.str.len < 2
-      || params->base64->val.str.text[0] != '"'
-      || params->base64->val.str.text[params->base64->val.str.len - 1] != '"')
-    {
-    fail:
-      cpp_error_at (pfile, CPP_DL_ERROR, params->loc,
-		    "'gnu::base64' argument not valid base64 encoded string");
-      free (buf);
-      return 0;
+  tokenrun *cur_run = &params->base64.base_run;
+  cpp_token *tend, *tok;
+  while (cur_run)
+    {
+      tend = cur_run->next ? cur_run->limit : params->base64.cur_token;
+      for (tok = cur_run->base; tok < tend; ++tok)
+	{
+	  if (tok->val.str.len < 2
+	      || tok->val.str.text[0] != '"'
+	      || tok->val.str.text[tok->val.str.len - 1] != '"')
+	    {
+	    fail:
+	      cpp_error_at (pfile, CPP_DL_ERROR, params->loc,
+			    "'gnu::base64' argument not valid base64 "
+			    "encoded string");
+	      free (buf);
+	      return 0;
+	    }
+	  if (tok->val.str.len - 2 > (~(size_t) 0) - base64_len)
+	    goto fail;
+	  base64_len += tok->val.str.len - 2;
+	}
+      cur_run = cur_run->next;
     }
-  base64_str = params->base64->val.str.text + 1;
-  base64_len = params->base64->val.str.len  - 2;
   if ((base64_len & 3) != 0)
     goto fail;
   len = base64_len / 4 * 3;
   end = len;
-  if (base64_len && base64_str[base64_len - 1] == '=')
-    {
-      end = len - 3;
-      --len;
-      if (base64_str[base64_len - 2] == '=')
-	--len;
-    }
 
   if (params->has_embed)
     q = qbuf;
@@ -1483,12 +1488,45 @@  finish_base64_embed (cpp_reader *pfile,
       buf = XNEWVEC (uchar, len ? len : 1);
       q = buf;
     }
-  for (i = 0, p = base64_str; i < end; i += 3, p += 4)
+  cur_run = &params->base64.base_run;
+  tend = cur_run->next ? cur_run->limit : params->base64.cur_token;
+  tok = cur_run->base;
+  base64_str = tok->val.str.text + 1;
+  cnt = tok->val.str.len - 2;
+  ++tok;
+  for (i = 0; i < end; i += 3)
     {
-      int a = base64_dec[p[0]];
-      int b = base64_dec[p[1]];
-      int c = base64_dec[p[2]];
-      int d = base64_dec[p[3]];
+      for (j = 0; j < 4; ++j)
+	{
+	  while (cnt == 0)
+	    {
+	      if (tok == tend)
+		{
+		  cur_run = cur_run->next;
+		  tend = (cur_run->next ? cur_run->limit
+			  : params->base64.cur_token);
+		  tok = cur_run->base;
+		}
+	      base64_str = tok->val.str.text + 1;
+	      cnt = tok->val.str.len - 2;
+	      ++tok;
+	    }
+	  pbuf[j] = *base64_str;
+	  base64_str++;
+	  --cnt;
+	}
+      if (pbuf[3] == '=' && i + 3 >= end)
+	{
+	  end = len - 3;
+	  --len;
+	  if (pbuf[2] == '=')
+	    --len;
+	  break;
+	}
+      int a = base64_dec[pbuf[0]];
+      int b = base64_dec[pbuf[1]];
+      int c = base64_dec[pbuf[2]];
+      int d = base64_dec[pbuf[3]];
       if (a == -1 || b == -1 || c == -1 || d == -1)
 	goto fail;
       q[0] = (a << 2) | (b >> 4);
@@ -1499,14 +1537,14 @@  finish_base64_embed (cpp_reader *pfile,
     }
   if (len != end)
     {
-      int a = base64_dec[p[0]];
-      int b = base64_dec[p[1]];
+      int a = base64_dec[pbuf[0]];
+      int b = base64_dec[pbuf[1]];
       if (a == -1 || b == -1)
 	goto fail;
       q[0] = (a << 2) | (b >> 4);
       if (len - end == 2)
 	{
-	  int c = base64_dec[p[2]];
+	  int c = base64_dec[pbuf[2]];
 	  if (c == -1)
 	    goto fail;
 	  q[1] = (b << 4) | (c >> 2);
@@ -1540,7 +1578,7 @@  int
 _cpp_stack_embed (cpp_reader *pfile, const char *fname, bool angle,
 		  struct cpp_embed_params *params)
 {
-  if (params->base64)
+  if (params->base64.count)
     return finish_base64_embed (pfile, fname, angle, params);
   cpp_dir *dir = search_path_head (pfile, fname, angle, IT_EMBED,
 				   params->has_embed);
--- libcpp/macro.cc.jj	2024-06-18 15:43:47.496276997 +0200
+++ libcpp/macro.cc	2024-06-21 12:11:20.470226027 +0200
@@ -505,6 +505,8 @@  builtin_has_embed (cpp_reader *pfile)
       if (ok && !pfile->state.skip_eval)
 	result = _cpp_stack_embed (pfile, fname, bracket, &params);
 
+      _cpp_free_embed_params_tokens (&params.base64);
+
       XDELETEVEC (fname);
     }
   else if (paren)
--- gcc/doc/cpp.texi.jj	2024-06-19 13:21:47.549613348 +0200
+++ gcc/doc/cpp.texi	2024-06-21 13:40:10.162051407 +0200
@@ -3982,12 +3982,12 @@  The @code{gnu::offset} parameter argumen
 which specifies how many bytes to skip from the start of the resource.
 @code{limit} is then counted from that position.
 
-The @code{gnu::base64} parameter argument is single character string
-literal with base64 encoded data.  See
+The @code{gnu::base64} parameter argument is a possibly concatenated
+character string literal with base64 encoded data.  See
 @uref{https://datatracker.ietf.org/doc/html/rfc4648#section-4}.  There
-should be no new-lines in the string literal and because this parameter
+should be no newlines in the string literal and because this parameter
 is meant namely for use by the preprocessor itself, there is no support
-for string literal concatenation in the argument.  If @code{gnu::base64}
+for any escape sequences in the string literal argument.  If @code{gnu::base64}
 parameter is specified, the @code{limit} and @code{gnu::offset} parameters
 should not be specified and the filename should be always @code{"."}.
 Instead of reading a file the directive will decode the base64 encoded
--- gcc/c-family/c-ppoutput.cc.jj	2024-06-20 18:21:55.030388341 +0200
+++ gcc/c-family/c-ppoutput.cc	2024-06-21 11:52:28.665867806 +0200
@@ -301,14 +301,21 @@  token_streamer::stream (cpp_reader *pfil
     }
   else if (token->type == CPP_EMBED)
     {
-      char buf[65];
+      char buf[76 + 6];
       maybe_print_line (token->src_loc);
-      fputs ("#embed \".\" __gnu__::__base64__(\"", print.outf);
-      print.printed = true;
-      unsigned int j = 0;
+      gcc_checking_assert (token->val.str.len != 0);
+      fputs ("#embed \".\" __gnu__::__base64__(", print.outf);
+      if (token->val.str.len > 30)
+	{
+	  fputs (" \\\n", print.outf);
+	  print.src_line++;
+	}
+      buf[0] = '"';
+      memcpy (buf + 1 + 76, "\" \\\n", 5);
+      unsigned int j = 1;
       static const char base64_enc[] =
 	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-      for (unsigned i = 0; i < token->val.str.len; i += 3)
+      for (unsigned i = 0; ; i += 3)
 	{
 	  unsigned char a = token->val.str.text[i];
 	  unsigned char b = 0, c = 0;
@@ -324,27 +331,25 @@  token_streamer::stream (cpp_reader *pfil
 	  buf[j++] = base64_enc[(v >> 12) & 63];
 	  buf[j++] = base64_enc[(v >> 6) & 63];
 	  buf[j++] = base64_enc[v & 63];
-	  if (n < 3)
-	    {
-	      buf[j - 1] = '=';
-	      if (n == 1)
-		buf[j - 2] = '=';
-	    }
-	  if (j == 64)
+	  if (j == 76 + 1 || n <= 3)
 	    {
-	      buf[64] = '\0';
+	      if (n < 3)
+		{
+		  buf[j - 1] = '=';
+		  if (n == 1)
+		    buf[j - 2] = '=';
+		}
+	      if (n <= 3)
+		memcpy (buf + j, "\")", 3);
+	      else
+		print.src_line++;
 	      fputs (buf, print.outf);
-	      j = 0;
+	      j = 1;
+	      if (n <= 3)
+		break;
 	    }
-	  if (n < 3)
-	    break;
-	}
-      if (j)
-	{
-	  buf[j] = '\0';
-	  fputs (buf, print.outf);
 	}
-      fputs ("\")", print.outf);
+      print.printed = true;
       maybe_print_line (token->src_loc);
       return;
     }
--- gcc/testsuite/c-c++-common/cpp/embed-17.c.jj	2024-06-19 12:48:52.176725071 +0200
+++ gcc/testsuite/c-c++-common/cpp/embed-17.c	2024-06-21 13:17:08.730983311 +0200
@@ -72,6 +72,18 @@  const unsigned char f[] = {
   #embed "." gnu::base64 (BONORUM_ET_MALORUM) prefix ([1] = ) suffix(, [0] = ' ')
 #endif
 };
+#if __has_embed ("." gnu::base64("TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg" \
+"c2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu" \
+"YSBhbGlxdWEuCg==")) == __STDC_EMBED_FOUND__
+const unsigned char g[] = {
+#embed "." gnu::base64("" \
+"T" "G9" "yZW" \
+"0gaX" \
+"BzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg" \
+"c2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu" \
+"YSBhbGlxdWEuCg==")
+};
+#endif
 
 #ifdef __cplusplus
 #define C "C"
@@ -95,4 +107,10 @@  main ()
   if (sizeof (f) != 1 + 747 || memcmp (f, " Non eram néscius, Brute",
 				       sizeof (" Non eram néscius, Brute") - 1))
     abort ();
+  const char ge[]
+    = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.";
+  if (sizeof (g) != sizeof (ge)
+      || memcmp (g, ge, sizeof (ge) - 1)
+      || g[sizeof (ge) - 1] != '\n')
+    abort ();
 }
--- gcc/testsuite/c-c++-common/cpp/embed-18.c.jj	2024-06-19 13:12:48.000000000 +0200
+++ gcc/testsuite/c-c++-common/cpp/embed-18.c	2024-06-21 13:14:04.889369971 +0200
@@ -10,12 +10,6 @@ 
 #embed "." gnu::base64("----") /* { dg-error "'gnu::base64' argument not valid base64 encoded string" } */
 #embed "." gnu::base64("a===") /* { dg-error "'gnu::base64' argument not valid base64 encoded string" } */
 #embed "." gnu::base64("TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg\nc2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu\nYSBhbGlxdWEuCg==") /* { dg-error "'gnu::base64' argument not valid base64 encoded string" } */
-/* No support for string concatenation in gnu::base64 argument.  */
-#embed "." gnu::base64("TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg" \
-"c2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu" \
-"YSBhbGlxdWEuCg==")
-/* { dg-error "expected '\\\)'" "" { target *-*-* } .-2 } */
-/* { dg-error "expected parameter name" "" { target *-*-* } .-2 } */
 #embed "embed-18.c" gnu::base64("SA==") /* { dg-error "'gnu::base64' parameter can be only used with \\\".\\\"" } */
 #embed <embed-18.c> gnu::base64("SA==") /* { dg-error "'gnu::base64' parameter can be only used with \\\".\\\"" } */
 #embed <.> gnu::base64("SA==") /* { dg-error "'gnu::base64' parameter can be only used with \\\".\\\"" } */
@@ -37,10 +31,3 @@ 
 #endif
 #if 1 + __has_embed ("." gnu::base64("TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg\nc2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu\nYSBhbGlxdWEuCg==")) /* { dg-error "'gnu::base64' argument not valid base64 encoded string" } */
 #endif
-#if 1 + __has_embed ("." gnu::base64("TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg" \
-"c2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu" \
-"YSBhbGlxdWEuCg=="))
-/* { dg-error "expected '\\\)'" "" { target *-*-* } .-2 } */
-/* { dg-error "expected parameter name" "" { target *-*-* } .-2 } */
-/* { dg-error "missing '\\\(' in expression" "" { target *-*-* } .-3 } */
-#endif