diff mbox

[4/4] Handle ld -r with LTO v2

Message ID 87zkxy8mcl.fsf_-_@basil.nowhere.org
State New
Headers show

Commit Message

Andi Kleen July 11, 2010, 8:16 a.m. UTC
Andi Kleen <andi@firstfloor.org> writes:

> This patch fixes ld -r handling for LTO.  Currently on a ld -r the linker
> would merge all the LTO sections, which then confuses the reader.

Here's a new version of this patch. The previous version had one
left-over debug printf in the lto-plugin that was a bit noisy.
Also fixed one incorrect comment. The other patches in the series
are still valid and needed.

-Andi

Handle ld -r with LTO

This patch fixes ld -r handling for LTO.  Currently on a ld -r the linker
would merge all the LTO sections, which then confuses the reader.

Instead postfix each LTO section (except for the file options) with a random
number: the sub id. This prevents the linker from merging the section.

Then fix the LTO reader and the gold plugin to handle different sub ids 
like different files. The knowledge of this is hidden in the early LTO
reader which creates different file_datas for each sub module. Each
file_data has an own hash table and resolution vector only containing
information for LTO sections carrying that sub id.

This also fixes an O(n^2) loop in the lto-plugin and some incorrect
comments in the functions I changed.

I have a simple test case for this, but it doesn't fit cleanly into
the gcc test suite so I didn't include it for now.

I need this to use LTO for a large project that uses ld -r in its Makefiles.
The project unfortunately still doesn't build due to other LTO problems,
but with this fix it gets a lot closer.

Requires the two earlier patches posted in the series.

Passes boot strap on x86_64-linux and testing. I have a copy right
assignment on file, but no SVN account, so someone else would need to
commit it for me.

gcc:

2010-07-10   Andi Kleen <ak@linux.intel.com>

        * lto-opts.c (lto_write_options): Add NULL file_data argument to 
	lto_get_section_name.
	* lto-section-out.c (lto_destroy_simple_output_block): Likewise.
	* lto-streamer-out.c (produce_asm): Likewise.
	(copy_function): Likewise.
	(produce_symtab): Likewise.
	(produce_asm_for_decls): Likewise.
	* lto-streamer.c (lto_get_section_name): Add file_data argument.	
	Rewrite to add random postfix to LTO sections.
	* lto-streamer.h (lto_file_decl_data): Add next, id, resolutions.
	(lto_get_section_name): Add file_data argument to prototype.
	

lto:

2010-07-10   Andi Kleen <ak@linux.intel.com>

        * lto.c: Include splay-tree.h
	(lto_resolution_read): Change to walk file_ids tree and parse
	extra file_id in resolution file.
	(lto_section_with_id): Add.
	(create_subid_section_table): Add.
	(lwstate): Add.
	(lto_create_files_from_ids): Add.
	(lto_file_read): Change to handle sub file ids and create list
	of file_datas. Add output argument for count.
	(get_section_data): Pass file_data to lto_get_section_name.
	(lto_flatten_file): Add.
	(read_cgraph_and_symbols): Handle linked lists of file_datas.
	
lto-plugin:

2010-07-10   Andi Kleen <ak@linux.intel.com>

        * lto-plugin.c (sym_aux): Add.
	(plugin_symtab): Remove slots. Add aux and id.
	(parse_table_entry): Change to use aux instead of slots.
	(LTO_SECTION_PREFIX): Add.
	(translate): Improve buffer allocation. Change to append
	symbols to existing out buffer.
	(get_section): Remove.
	(process_symtab): Add. 
	(free_2): Free symtab->aux.
	(write_resolution): Handle aux instead of slots.
	Print sub id to resolution file.
	(claim_file_handler): Clear lto_file. Replace get_symtab/translate
	calls with call to process_symtab.
diff mbox

Patch

diff --git a/gcc/lto-opts.c b/gcc/lto-opts.c
index 8405714..fa52f95 100644
--- a/gcc/lto-opts.c
+++ b/gcc/lto-opts.c
@@ -294,7 +294,7 @@  output_options (struct lto_output_stream *stream)
 void
 lto_write_options (void)
 {
-  char *const section_name = lto_get_section_name (LTO_section_opts, NULL);
+  char *const section_name = lto_get_section_name (LTO_section_opts, NULL, NULL);
   struct lto_output_stream stream;
   struct lto_simple_header header;
   struct lto_output_stream *header_stream;
@@ -356,6 +356,8 @@  lto_read_file_options (struct lto_file_decl_data *file_data)
   struct lto_input_block ib;
 
   data = lto_get_section_data (file_data, LTO_section_opts, NULL, &len);
+  if (!data)
+	  return;
   header = (const struct lto_simple_header *) data;
   opts_offset = sizeof (*header);
 
diff --git a/gcc/lto-section-out.c b/gcc/lto-section-out.c
index b5353f3..e9b7b0a 100644
--- a/gcc/lto-section-out.c
+++ b/gcc/lto-section-out.c
@@ -486,7 +486,7 @@  lto_destroy_simple_output_block (struct lto_simple_output_block *ob)
   struct lto_simple_header header;
   struct lto_output_stream *header_stream;
 
-  section_name = lto_get_section_name (ob->section_type, NULL);
+  section_name = lto_get_section_name (ob->section_type, NULL, NULL);
   lto_begin_section (section_name, !flag_wpa);
   free (section_name);
 
diff --git a/gcc/lto-streamer-out.c b/gcc/lto-streamer-out.c
index db9ce4b..b35c527 100644
--- a/gcc/lto-streamer-out.c
+++ b/gcc/lto-streamer-out.c
@@ -1791,10 +1791,10 @@  produce_asm (struct output_block *ob, tree fn)
   if (section_type == LTO_section_function_body)
     {
       const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (fn));
-      section_name = lto_get_section_name (section_type, name);
+      section_name = lto_get_section_name (section_type, name, NULL);
     }
   else
-    section_name = lto_get_section_name (section_type, NULL);
+    section_name = lto_get_section_name (section_type, NULL, NULL);
 
   lto_begin_section (section_name, !flag_wpa);
   free (section_name);
@@ -2022,7 +2022,7 @@  copy_function (struct cgraph_node *node)
   size_t len;
   const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (function));
   char *section_name =
-    lto_get_section_name (LTO_section_function_body, name);
+    lto_get_section_name (LTO_section_function_body, name, NULL);
   size_t i, j;
   struct lto_in_decl_state *in_state;
   struct lto_out_decl_state *out_state = lto_get_out_decl_state ();
@@ -2379,7 +2379,7 @@  produce_symtab (struct output_block *ob,
 	        cgraph_node_set set, varpool_node_set vset)
 {
   struct lto_streamer_cache_d *cache = ob->writer_cache;
-  char *section_name = lto_get_section_name (LTO_section_symtab, NULL);
+  char *section_name = lto_get_section_name (LTO_section_symtab, NULL, NULL);
   bitmap seen;
   struct cgraph_node *node, *alias;
   struct varpool_node *vnode, *valias;
@@ -2458,7 +2458,7 @@  produce_asm_for_decls (cgraph_node_set set, varpool_node_set vset)
 
   memset (&header, 0, sizeof (struct lto_decl_header));
 
-  section_name = lto_get_section_name (LTO_section_decls, NULL);
+  section_name = lto_get_section_name (LTO_section_decls, NULL, NULL);
   lto_begin_section (section_name, !flag_wpa);
   free (section_name);
 
diff --git a/gcc/lto-streamer.c b/gcc/lto-streamer.c
index 4060960..0536bb9 100644
--- a/gcc/lto-streamer.c
+++ b/gcc/lto-streamer.c
@@ -133,57 +133,45 @@  lto_bitmap_free (bitmap b)
 
 
 /* Get a section name for a particular type or name.  The NAME field
-   is only used if SECTION_TYPE is LTO_section_function_body or
-   LTO_static_initializer.  For all others it is ignored.  The callee
-   of this function is responcible to free the returned name.  */
+   is only used if SECTION_TYPE is LTO_section_function_body. For all
+   others it is ignored.  The callee of this function is responsible
+   to free the returned name.  */
 
 char *
-lto_get_section_name (int section_type, const char *name)
+lto_get_section_name (int section_type, const char *name, struct lto_file_decl_data *f)
 {
-  switch (section_type)
+  const char *add;
+  char post[32];
+  const char *sep;
+
+  if (section_type == LTO_section_function_body)
     {
-    case LTO_section_function_body:
       gcc_assert (name != NULL);
       if (name[0] == '*')
 	name++;
-      return concat (LTO_SECTION_NAME_PREFIX, name, NULL);
-
-    case LTO_section_static_initializer:
-      return concat (LTO_SECTION_NAME_PREFIX, ".statics", NULL);
-
-    case LTO_section_symtab:
-      return concat (LTO_SECTION_NAME_PREFIX, ".symtab", NULL);
-
-    case LTO_section_decls:
-      return concat (LTO_SECTION_NAME_PREFIX, ".decls", NULL);
-
-    case LTO_section_cgraph:
-      return concat (LTO_SECTION_NAME_PREFIX, ".cgraph", NULL);
-
-    case LTO_section_varpool:
-      return concat (LTO_SECTION_NAME_PREFIX, ".vars", NULL);
-
-    case LTO_section_refs:
-      return concat (LTO_SECTION_NAME_PREFIX, ".refs", NULL);
-
-    case LTO_section_jump_functions:
-      return concat (LTO_SECTION_NAME_PREFIX, ".jmpfuncs", NULL);
-
-    case LTO_section_ipa_pure_const:
-      return concat (LTO_SECTION_NAME_PREFIX, ".pureconst", NULL);
-
-    case LTO_section_ipa_reference:
-      return concat (LTO_SECTION_NAME_PREFIX, ".reference", NULL);
+      add = name;
+      sep = "";
+    }
+  else if (section_type < LTO_N_SECTION_TYPES)
+    {
+      add = lto_section_name[section_type];
+      sep = ".";
+    }
+  else
+    internal_error ("bytecode stream: unexpected LTO section %s", name);
 
-    case LTO_section_opts:
-      return concat (LTO_SECTION_NAME_PREFIX, ".opts", NULL);
+  /* Make the section name unique so that ld -r combining sections
+     doesn't confuse the reader with merged sections.
 
-    case LTO_section_cgraph_opt_sum:
-      return concat (LTO_SECTION_NAME_PREFIX, ".cgraphopt", NULL);
+     For options don't add a ID, the option reader cannot deal with them
+     and merging should be ok here.
 
-    default:
-      internal_error ("bytecode stream: unexpected LTO section %s", name);
-    }
+     XXX: use crc64 to minimize collisions? */
+  if (section_type == LTO_section_opts)
+    strcpy (post, "");
+  else
+    sprintf (post, ".%x", f ? f->id : crc32_string(0, get_random_seed (false)));
+  return concat (LTO_SECTION_NAME_PREFIX, sep, add, post, NULL);
 }
 
 
diff --git a/gcc/lto-streamer.h b/gcc/lto-streamer.h
index cc38b22..3304132 100644
--- a/gcc/lto-streamer.h
+++ b/gcc/lto-streamer.h
@@ -600,6 +600,15 @@  struct GTY(()) lto_file_decl_data
 
   /* Hash new name of renamed global declaration to its original name.  */
   htab_t GTY((skip)) renaming_hash_table;
+
+  /* Linked list used temporarily in reader */
+  struct lto_file_decl_data *next;
+
+  /* Sub ID for merged objects. */
+  unsigned id;
+
+  /* Symbol resolutions for this file */
+  VEC(ld_plugin_symbol_resolution_t,heap) *resolutions;
 };
 
 typedef struct lto_file_decl_data *lto_file_decl_data_ptr;
@@ -815,7 +824,7 @@  extern void lto_record_function_out_decl_state (tree,
 extern const char *lto_tag_name (enum LTO_tags);
 extern bitmap lto_bitmap_alloc (void);
 extern void lto_bitmap_free (bitmap);
-extern char *lto_get_section_name (int, const char *);
+extern char *lto_get_section_name (int, const char *, struct lto_file_decl_data *);
 extern void print_lto_report (void);
 extern bool lto_streamer_cache_insert (struct lto_streamer_cache_d *, tree,
 				       int *, unsigned *);
diff --git a/gcc/lto/lto-coff.c b/gcc/lto/lto-coff.c
index 176c322..f5aaff8 100644
--- a/gcc/lto/lto-coff.c
+++ b/gcc/lto/lto-coff.c
@@ -134,29 +134,6 @@  lto_file_init (lto_file *file, const char *filename, off_t offset)
   file->offset = offset;
 }
 
-/* Returns a hash code for P.  */
-
-static hashval_t
-hash_name (const void *p)
-{
-  const struct lto_section_slot *ds = (const struct lto_section_slot *) p;
-  return (hashval_t) htab_hash_string (ds->name);
-}
-
-/* Returns nonzero if P1 and P2 are equal.  */
-
-static int
-eq_name (const void *p1, const void *p2)
-{
-  const struct lto_section_slot *s1 =
-    (const struct lto_section_slot *) p1;
-  const struct lto_section_slot *s2 =
-    (const struct lto_section_slot *) p2;
-
-  return strcmp (s1->name, s2->name) == 0;
-}
-
-
 /* Build a hash table whose key is the section names and whose data is
    the start and size of each section in the .o file.  */
 
@@ -169,7 +146,7 @@  lto_obj_build_section_table (lto_file *lto_file)
   ssize_t strtab_size;
   char *strtab;
 
-  section_hash_table = htab_create (37, hash_name, eq_name, free);
+  section_hash_table = lto_obj_create_section_hash_table ();
 
   /* Seek to start of string table.  */
   if (coff_file->strtab_offs != lseek (coff_file->fd,
diff --git a/gcc/lto/lto.c b/gcc/lto/lto.c
index 4ac1ac1..e4ee214 100644
--- a/gcc/lto/lto.c
+++ b/gcc/lto/lto.c
@@ -43,6 +43,7 @@  along with GCC; see the file COPYING3.  If not see
 #include "lto.h"
 #include "lto-tree.h"
 #include "lto-streamer.h"
+#include "splay-tree.h"
 
 /* This needs to be included after config.h.  Otherwise, _GNU_SOURCE will not
    be defined in time to set __USE_GNU in the system headers, and strsignal
@@ -268,11 +310,10 @@  lto_parse_hex (const char *p) {
 }
 
 /* Read resolution for file named FILE_NAME. The resolution is read from
-   RESOLUTION. An array with the symbol resolution is returned. The array
-   size is written to SIZE. */
+   RESOLUTION. */
 
-static VEC(ld_plugin_symbol_resolution_t,heap) *
-lto_resolution_read (FILE *resolution, lto_file *file)
+static void
+lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
 {
   /* We require that objects in the resolution file are in the same
      order as the lto1 command line. */
@@ -280,11 +321,12 @@  lto_resolution_read (FILE *resolution, lto_file *file)
   char *obj_name;
   unsigned int num_symbols;
   unsigned int i;
-  VEC(ld_plugin_symbol_resolution_t,heap) *ret = NULL;
+  struct lto_file_decl_data *file_data;
   unsigned max_index = 0;
+  splay_tree_node nd = NULL; 
 
   if (!resolution)
-    return NULL;
+    return;
 
   name_len = strlen (file->filename);
   obj_name = XNEWVEC (char, name_len + 1);
@@ -315,15 +357,15 @@  lto_resolution_read (FILE *resolution, lto_file *file)
   for (i = 0; i < num_symbols; i++)
     {
       int t;
-      unsigned index;
+      unsigned index, id;
       char r_str[27];
       enum ld_plugin_symbol_resolution r = (enum ld_plugin_symbol_resolution) 0;
       unsigned int j;
       unsigned int lto_resolution_str_len =
 	sizeof (lto_resolution_str) / sizeof (char *);
 
-      t = fscanf (resolution, "%u %26s %*[^\n]\n", &index, r_str);
-      if (t != 2)
+      t = fscanf (resolution, "%u %x %26s %*[^\n]\n", &index, &id, r_str);
+      if (t != 3)
         internal_error ("Invalid line in the resolution file.");
       if (index > max_index)
 	max_index = index;
@@ -339,12 +381,120 @@  lto_resolution_read (FILE *resolution, lto_file *file)
       if (j == lto_resolution_str_len)
 	internal_error ("Invalid resolution in the resolution file.");
 
-      VEC_safe_grow_cleared (ld_plugin_symbol_resolution_t, heap, ret,
+      if (!(nd && nd->key == id))
+	{
+	  nd = splay_tree_lookup (file_ids, id);
+	  if (nd == NULL)
+	    internal_error ("Resolution sub id %x not in object file", id);
+	}
+
+      file_data = (struct lto_file_decl_data *)nd->value;
+      if (cgraph_dump_file)
+	fprintf (cgraph_dump_file, "Adding resolution %u %u to id %x\n",
+		 index, r, file_data->id);
+      VEC_safe_grow_cleared (ld_plugin_symbol_resolution_t, heap, 
+			     file_data->resolutions,
 			     max_index + 1);
-      VEC_replace (ld_plugin_symbol_resolution_t, ret, index, r);
+      VEC_replace (ld_plugin_symbol_resolution_t, 
+		   file_data->resolutions, index, r);
     }
+}
 
-  return ret;
+/* Is the name for a id'ed LTO section? */
+
+static int 
+lto_section_with_id (const char *name, unsigned *id)
+{
+  char *s;
+
+  if (strncmp (name, LTO_SECTION_NAME_PREFIX, strlen (LTO_SECTION_NAME_PREFIX)))
+    return 0;
+  s = strrchr (name, '.');
+  return s && sscanf (s, ".%x", id) == 1;
+}
+
+/* Create file_data of each sub file id */
+
+static int 
+create_subid_section_table (void **slot, void *data)
+{
+  struct lto_section_slot s_slot, *new_slot;
+  struct lto_section_slot *ls = *(struct lto_section_slot **)slot;
+  splay_tree file_ids = (splay_tree)data;
+  unsigned id;
+  splay_tree_node nd;
+  void **hash_slot;
+  char *new_name;
+  struct lto_file_decl_data *file_data;
+
+  if (!lto_section_with_id (ls->name, &id))
+    return 1;
+  
+  /* Find hash table of sub module id */
+  nd = splay_tree_lookup (file_ids, id);
+  if (nd != NULL)
+    {
+      file_data = (struct lto_file_decl_data *)nd->value;
+    }
+  else
+    {
+      file_data = ggc_alloc_lto_file_decl_data ();
+      memset(file_data, 0, sizeof (struct lto_file_decl_data));
+      file_data->id = id;
+      file_data->section_hash_table = lto_obj_create_section_hash_table ();;
+      splay_tree_insert (file_ids, id, (splay_tree_value)file_data);
+    }
+
+  /* Copy section into sub module hash table */
+  new_name = XDUPVEC (char, ls->name, strlen (ls->name) + 1);
+  s_slot.name = new_name;
+  hash_slot = htab_find_slot (file_data->section_hash_table, &s_slot, INSERT);
+  gcc_assert (*hash_slot == NULL);
+
+  new_slot = XDUP (struct lto_section_slot, ls);
+  new_slot->name = new_name;
+  *hash_slot = new_slot;
+  return 1;
+}
+
+/* Read declarations and other initializations for a FILE_DATA. */
+
+static void
+lto_file_finalize (struct lto_file_decl_data *file_data, lto_file *file)
+{
+  const char *data;
+  size_t len;
+
+  file_data->renaming_hash_table = lto_create_renaming_table ();
+  file_data->file_name = file->filename;
+  data = lto_get_section_data (file_data, LTO_section_decls, NULL, &len);
+  gcc_assert (data != NULL);
+  lto_read_decls (file_data, data, file_data->resolutions);
+  lto_free_section_data (file_data, LTO_section_decls, NULL, data, len);
+}
+
+struct lwstate
+{
+  lto_file *file;
+  struct lto_file_decl_data **file_data;
+  int *count;
+};
+
+/* Traverse ids and create a list of file_datas out of it. */      
+
+static int lto_create_files_from_ids (splay_tree_node node, void *data)
+{
+  struct lwstate *lw = (struct lwstate *)data;
+  struct lto_file_decl_data *file_data = (struct lto_file_decl_data *)node->value;
+
+  lto_file_finalize (file_data, lw->file);
+  if (cgraph_dump_file)
+    fprintf (cgraph_dump_file, "Creating file %s with sub id %x\n", 
+	     file_data->file_name, file_data->id);
+  file_data->next = *lw->file_data;
+  *lw->file_data = file_data;
+  (*lw->count)++;
+  return 0;
 }
 
 /* Generate a TREE representation for all types and external decls
@@ -355,23 +505,32 @@  lto_resolution_read (FILE *resolution, lto_file *file)
    the .o file to load the functions and ipa information.   */
 
 static struct lto_file_decl_data *
-lto_file_read (lto_file *file, FILE *resolution_file)
+lto_file_read (lto_file *file, FILE *resolution_file, int *count)
 {
-  struct lto_file_decl_data *file_data;
-  const char *data;
-  size_t len;
-  VEC(ld_plugin_symbol_resolution_t,heap) *resolutions;
+  struct lto_file_decl_data *file_data = NULL;
+  splay_tree file_ids;
+  htab_t section_hash_table;
+  struct lwstate state;
   
-  resolutions = lto_resolution_read (resolution_file, file);
+  section_hash_table = lto_obj_build_section_table (file);
 
-  file_data = ggc_alloc_lto_file_decl_data ();
-  file_data->file_name = file->filename;
-  file_data->section_hash_table = lto_obj_build_section_table (file);
-  file_data->renaming_hash_table = lto_create_renaming_table ();
-
-  data = lto_get_section_data (file_data, LTO_section_decls, NULL, &len);
-  lto_read_decls (file_data, data, resolutions);
-  lto_free_section_data (file_data, LTO_section_decls, NULL, data, len);
+  /* Find all sub modules in the object and put their sections into new hash
+     tables in a splay tree. */
+  file_ids = splay_tree_new (splay_tree_compare_ints, NULL, NULL);
+  htab_traverse (section_hash_table, create_subid_section_table, file_ids);
+  
+  /* Add resolutions to file ids */
+  lto_resolution_read (file_ids, resolution_file, file);
+
+  /* Finalize each lto file for each submodule in the merged object
+     and create list for returning. */
+  state.file = file;
+  state.file_data = &file_data;
+  state.count = count;
+  splay_tree_foreach (file_ids, lto_create_files_from_ids, &state);
+    
+  splay_tree_delete (file_ids);
+  htab_delete (section_hash_table);
 
   return file_data;
 }
@@ -466,7 +625,7 @@  get_section_data (struct lto_file_decl_data *file_data,
   htab_t section_hash_table = file_data->section_hash_table;
   struct lto_section_slot *f_slot;
   struct lto_section_slot s_slot;
-  const char *section_name = lto_get_section_name (section_type, name);
+  const char *section_name = lto_get_section_name (section_type, name, file_data);
   char *data = NULL;
 
   *len = 0;
@@ -1497,6 +1656,33 @@  lto_read_all_file_options (void)
 
 static GTY((length ("lto_stats.num_input_files + 1"))) struct lto_file_decl_data **all_file_decl_data;
 
+/* Turn file datas for sub files into a single array, so that they look
+   like separate files for further passes. */
+
+static void
+lto_flatten_files (struct lto_file_decl_data **orig, int count, int last_file_ix)
+{
+  struct lto_file_decl_data *n, *next;
+  int i, k;
+
+  lto_stats.num_input_files = count;
+  all_file_decl_data
+    = ggc_alloc_cleared_vec_lto_file_decl_data_ptr (count + 1);
+  /* Set the hooks so that all of the ipa passes can read in their data.  */
+  lto_set_in_hooks (all_file_decl_data, get_section_data, free_section_data);
+  for (i = 0, k = 0; i < last_file_ix; i++) 
+    {
+      for (n = orig[i]; n != NULL; n = next)
+	{
+	  all_file_decl_data[k++] = n;
+	  next = n->next;
+	  n->next = NULL;
+	}
+    }
+  all_file_decl_data[k] = NULL;
+  gcc_assert (k == count);
+}
+
 /* Read all the symbols from the input files FNAMES.  NFILES is the
    number of files requested in the command line.  Instantiate a
    global call graph by aggregating all the sub-graphs found in each
@@ -1508,16 +1694,14 @@  read_cgraph_and_symbols (unsigned nfiles, const char **fnames)
   unsigned int i, last_file_ix;
   FILE *resolution;
   struct cgraph_node *node;
+  int count = 0;
+  struct lto_file_decl_data **decl_data;
 
-  lto_stats.num_input_files = nfiles;
   init_cgraph ();
 
   timevar_push (TV_IPA_LTO_DECL_IN);
 
-  /* Set the hooks so that all of the ipa passes can read in their data.  */
-  all_file_decl_data
-    = ggc_alloc_cleared_vec_lto_file_decl_data_ptr (nfiles + 1);
-  lto_set_in_hooks (all_file_decl_data, get_section_data, free_section_data);
+  decl_data = (struct lto_file_decl_data **)xmalloc (sizeof(*decl_data) * (nfiles+1));
 
   /* Read the resolution file.  */
   resolution = NULL;
@@ -1554,11 +1738,11 @@  read_cgraph_and_symbols (unsigned nfiles, const char **fnames)
       if (!current_lto_file)
 	break;
 
-      file_data = lto_file_read (current_lto_file, resolution);
+      file_data = lto_file_read (current_lto_file, resolution, &count);
       if (!file_data)
 	break;
 
-      all_file_decl_data[last_file_ix++] = file_data;
+      decl_data[last_file_ix++] = file_data;
 
       lto_obj_file_close (current_lto_file);
       current_lto_file = NULL;
@@ -1566,11 +1750,13 @@  read_cgraph_and_symbols (unsigned nfiles, const char **fnames)
          code in gimple.c uses hashtables that are not ggc aware.  */
     }
 
+  lto_flatten_files (decl_data, count, last_file_ix);
+  lto_stats.num_input_files = count;
+  free(decl_data);
+
   if (resolution_file_name)
     fclose (resolution);
 
-  all_file_decl_data[last_file_ix] = NULL;
-
   /* Set the hooks so that all of the ipa passes can read in their data.  */
   lto_set_in_hooks (all_file_decl_data, get_section_data, free_section_data);
 
diff --git a/lto-plugin/lto-plugin.c b/lto-plugin/lto-plugin.c
index c82f50d..0cfecc9 100644
--- a/lto-plugin/lto-plugin.c
+++ b/lto-plugin/lto-plugin.c
@@ -55,11 +55,18 @@  along with this program; see the file COPYING3.  If not see
    must keep SYMS until all_symbols_read is called to give the linker time to
    copy the symbol information. */
 
+struct sym_aux
+{
+  uint32_t slot;
+  unsigned id;
+};
+
 struct plugin_symtab
 {
   int nsyms;
-  uint32_t *slots;
+  struct sym_aux *aux;
   struct ld_plugin_symbol *syms;
+  unsigned id;
 };
 
 /* All that we have to remember about a file. */
@@ -120,7 +127,8 @@  check (bool gate, enum ld_plugin_level level, const char *text)
    Returns the address of the next entry. */
 
 static char *
-parse_table_entry (char *p, struct ld_plugin_symbol *entry, uint32_t *slot)
+parse_table_entry (char *p, struct ld_plugin_symbol *entry, 
+		   struct sym_aux *aux)
 {
   unsigned char t;
   enum ld_plugin_symbol_kind translate_kind[] =
@@ -170,7 +178,7 @@  parse_table_entry (char *p, struct ld_plugin_symbol *entry, uint32_t *slot)
   entry->size = *(uint64_t *) p;
   p += 8;
 
-  *slot = *(uint32_t *) p;
+  aux->slot = *(uint32_t *) p;
   p += 4;
 
   entry->resolution = LDPR_UNKNOWN;
@@ -178,16 +186,50 @@  parse_table_entry (char *p, struct ld_plugin_symbol *entry, uint32_t *slot)
   return p;
 }
 
-/* Return the section in ELF that is named NAME. */
+#define LTO_SECTION_PREFIX ".gnu.lto_.symtab"
+
+/* Translate the IL symbol table SYMTAB. Append the slots and symbols to OUT. */
+
+static void
+translate (Elf_Data *symtab, struct plugin_symtab *out)
+{
+  struct sym_aux *aux;
+  char *data = symtab->d_buf;
+  char *end = data + symtab->d_size;
+  struct ld_plugin_symbol *syms = NULL;
+  int n, len;
+
+  /* This overestimates the output buffer sizes, but at least 
+     the algorithm is O(n) now. */
+
+  len = (end - data)/8 + out->nsyms + 1;
+  syms = xrealloc (out->syms, len * sizeof (struct ld_plugin_symbol));
+  aux = xrealloc (out->aux, len * sizeof (struct sym_aux));
+  
+  for (n = out->nsyms; data < end; n++) 
+    { 
+      aux[n].id = out->id; 
+      data = parse_table_entry (data, &syms[n], &aux[n]);
+    }
+
+  assert(n < len);
+
+  out->nsyms = n;
+  out->syms = syms;
+  out->aux = aux;
+}
+
+/* Process all lto symtabs of file ELF. */
 
-static Elf_Scn *
-get_section (Elf *elf, const char *name)
+static int
+process_symtab (Elf *elf, struct plugin_symtab *out)
 {
+  int found = 0;
   Elf_Scn *section = 0;
   GElf_Ehdr header;
   GElf_Ehdr *t = gelf_getehdr (elf, &header);
   if (t == NULL)
-    return NULL;
+    return 0;
   assert (t == &header);
 
   while ((section = elf_nextscn(elf, section)) != 0)
@@ -198,51 +240,16 @@  get_section (Elf *elf, const char *name)
       assert (tshdr == &shdr);
       t = elf_strptr (elf, header.e_shstrndx, shdr.sh_name);
       assert (t != NULL);
-      if (strcmp (t, name) == 0)
-	return section;
-    }
-  return NULL;
-}
-
-/* Returns the IL symbol table of file ELF. */
-
-static Elf_Data *
-get_symtab (Elf *elf)
-{
-  Elf_Data *data = 0;
-  Elf_Scn *section = get_section (elf, ".gnu.lto_.symtab");
-  if (!section)
-    return NULL;
-
-  data = elf_getdata (section, data);
-  assert (data);
-  return data;
-}
-
-/* Translate the IL symbol table SYMTAB. Write the slots and symbols in OUT. */
-
-static void
-translate (Elf_Data *symtab, struct plugin_symtab *out)
-{
-  uint32_t *slots = NULL;
-  char *data = symtab->d_buf;
-  char *end = data + symtab->d_size;
-  struct ld_plugin_symbol *syms = NULL;
-  int n = 0;
-
-  while (data < end)
-    {
-      n++;
-      syms = xrealloc (syms, n * sizeof (struct ld_plugin_symbol));
-      check (syms, LDPL_FATAL, "could not allocate memory");
-      slots = xrealloc (slots, n * sizeof (uint32_t));
-      check (slots, LDPL_FATAL, "could not allocate memory");
-      data = parse_table_entry (data, &syms[n - 1], &slots[n - 1]);
+      if (strncmp (t, LTO_SECTION_PREFIX, strlen (LTO_SECTION_PREFIX)) == 0) 
+	{
+	  char *s = strrchr (t, '.');
+	  if (s)
+	      sscanf (s, ".%x", &out->id);
+	  translate (elf_getdata (section, NULL), out);
+	  found = 1;
+	}
     }
-
-  out->nsyms = n;
-  out->syms = syms;
-  out->slots = slots;
+  return found;
 }
 
 /* Free all memory that is no longer needed after writing the symbol
@@ -279,7 +286,7 @@  free_2 (void)
     {
       struct plugin_file_info *info = &claimed_files[i];
       struct plugin_symtab *symtab = &info->symtab;
-      free (symtab->slots);
+      free (symtab->aux);
       free (info->name);
     }
 
@@ -323,9 +330,10 @@  write_resolution (void)
 
       for (j = 0; j < info->symtab.nsyms; j++)
 	{
-	  uint32_t slot = symtab->slots[j];
+	  uint32_t slot = symtab->aux[j].slot;
 	  unsigned int resolution = syms[j].resolution;
-	  fprintf (f, "%d %s %s\n", slot, lto_resolution_str[resolution], syms[j].name);
+	  fprintf (f, "%d %x %s %s\n", slot, symtab->aux[j].id,
+		   lto_resolution_str[resolution], syms[j].name);
 	}
     }
   fclose (f);
@@ -551,7 +559,8 @@  claim_file_handler (const struct ld_plugin_input_file *file, int *claimed)
   enum ld_plugin_status status;
   Elf *elf;
   struct plugin_file_info lto_file;
-  Elf_Data *symtab;
+
+  memset (&lto_file, 0, sizeof (struct plugin_file_info));
 
   if (file->offset != 0)
     {
@@ -588,15 +597,9 @@  claim_file_handler (const struct ld_plugin_input_file *file, int *claimed)
 
   *claimed = 0;
 
-  if (!elf)
+  if (!elf || !process_symtab (elf, &lto_file.symtab))
     goto err;
 
-  symtab = get_symtab (elf);
-  if (!symtab)
-    goto err;
-
-  translate (symtab, &lto_file.symtab);
-
   status = add_symbols (file->handle, lto_file.symtab.nsyms,
 			lto_file.symtab.syms);
   check (status == LDPS_OK, LDPL_FATAL, "could not add symbols");