Message ID | aa63ee12-6914-5bdf-3db0-c91205ba3ad1@suse.cz |
---|---|
State | New |
Headers | show |
Series | [1/3] Factor out jobserver_active_p. | expand |
On Tue, Aug 9, 2022 at 2:02 PM Martin Liška <mliska@suse.cz> wrote: > > Patch can bootstrap on x86_64-linux-gnu and survives regression tests. > > Ready to be installed? > Thanks, > Martin > > PR lto/106328 > > gcc/ChangeLog: > > * jobserver.h (struct jobserver_info): Add pipefd. > (jobserver_info::connect): New. > (jobserver_info::disconnect): Likewise. > (jobserver_info::get_token): Likewise. > (jobserver_info::return_token): Likewise. > > gcc/lto/ChangeLog: > > * lto.cc (wait_for_child): Decrement nruns once a process > finishes. > (stream_out_partitions): Use job server if active. > (do_whole_program_analysis): Likewise. > --- > gcc/jobserver.h | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ > gcc/lto/lto.cc | 55 +++++++++++++++++++++++++++++++++++++------------ > 2 files changed, 96 insertions(+), 13 deletions(-) > > diff --git a/gcc/jobserver.h b/gcc/jobserver.h > index 856e326ddfc..2a7dc9f4113 100644 > --- a/gcc/jobserver.h > +++ b/gcc/jobserver.h > @@ -31,6 +31,18 @@ struct jobserver_info > /* Default constructor. */ > jobserver_info (); > > + /* Connect to the server. */ > + void connect (); > + > + /* Disconnect from the server. */ > + void disconnect (); > + > + /* Get token from the server. */ > + bool get_token (); > + > + /* Return token to the server. */ > + void return_token (); > + > /* Error message if there is a problem. */ > string error_msg = ""; > /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ > @@ -41,6 +53,8 @@ struct jobserver_info > int wfd = -1; > /* Named pipe path. */ > string pipe_path = ""; > + /* Pipe file descriptor. */ > + int pipefd = -1; > /* Return true if jobserver is active. */ > bool is_active = false; > }; > @@ -97,4 +111,44 @@ jobserver_info::jobserver_info () > error_msg = "jobserver is not available: " + error_msg; > } > > +void > +jobserver_info::connect () > +{ > + if (!pipe_path.empty ()) > + pipefd = open (pipe_path.c_str (), O_RDWR); > +} > + > +void > +jobserver_info::disconnect () If it's not a template and the methods not inline won't we get multiple definition errors when this include file is included in multiple places from an executable? opts-common.cc might be a good place to stuff it, and opts.h to declare (if it were not for std::string, so possibly jobserver.h is good enough, or opts-jobserver.h?) > +{ > + if (!pipe_path.empty ()) > + { > + gcc_assert (close (pipefd) == 0); > + pipefd = -1; > + } > +} > + > +bool > +jobserver_info::get_token () > +{ > + int fd = pipe_path.empty () ? rfd : pipefd; > + char c; > + unsigned n = read (fd, &c, 1); > + if (n != 1) > + { > + gcc_assert (errno == EAGAIN); > + return false; > + } > + else > + return true; > +} > + > +void > +jobserver_info::return_token () > +{ > + int fd = pipe_path.empty () ? wfd : pipefd; > + char c = 'G'; > + gcc_assert (write (fd, &c, 1) == 1); > +} > + > #endif /* GCC_JOBSERVER_H */ > diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc > index 31b0c1862f7..56266195ead 100644 > --- a/gcc/lto/lto.cc > +++ b/gcc/lto/lto.cc > @@ -54,11 +54,17 @@ along with GCC; see the file COPYING3. If not see > #include "attribs.h" > #include "builtins.h" > #include "lto-common.h" > - > +#include "jobserver.h" > > /* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver. */ > static int lto_parallelism; > > +/* Number of active WPA streaming processes. */ > +static int nruns = 0; > + > +/* GNU make's jobserver info. */ > +static jobserver_info *jinfo = NULL; > + > /* Return true when NODE has a clone that is analyzed (i.e. we need > to load its body even if the node itself is not needed). */ > > @@ -205,6 +211,12 @@ wait_for_child () > "streaming subprocess was killed by signal"); > } > while (!WIFEXITED (status) && !WIFSIGNALED (status)); > + > + --nruns; > + > + /* Return token to the jobserver if active. */ > + if (jinfo != NULL && jinfo->is_active) > + jinfo->return_token (); > } > #endif > > @@ -228,25 +240,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, > bool ARG_UNUSED (last)) > { > #ifdef HAVE_WORKING_FORK > - static int nruns; > - > if (lto_parallelism <= 1) > { > stream_out_partitions_1 (temp_filename, blen, min, max); > return; > } > > - /* Do not run more than LTO_PARALLELISM streamings > - FIXME: we ignore limits on jobserver. */ > if (lto_parallelism > 0 && nruns >= lto_parallelism) > - { > - wait_for_child (); > - nruns --; > - } > + wait_for_child (); > + > /* If this is not the last parallel partition, execute new > streaming process. */ > if (!last) > { > + if (jinfo != NULL && jinfo->is_active) > + while (true) > + { > + if (jinfo->get_token ()) > + break; > + if (nruns > 0) > + wait_for_child (); > + else > + { > + /* There are no free tokens, lets do the job outselves. */ > + stream_out_partitions_1 (temp_filename, blen, min, max); > + asm_nodes_output = true; > + return; > + } > + } > + > pid_t cpid = fork (); > > if (!cpid) > @@ -264,10 +286,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, > /* Last partition; stream it and wait for all children to die. */ > else > { > - int i; > stream_out_partitions_1 (temp_filename, blen, min, max); > - for (i = 0; i < nruns; i++) > + while (nruns > 0) > wait_for_child (); > + > + if (jinfo != NULL && jinfo->is_active) > + jinfo->disconnect (); > } > asm_nodes_output = true; > #else > @@ -460,9 +484,14 @@ do_whole_program_analysis (void) > > lto_parallelism = 1; > > - /* TODO: jobserver communication is not supported, yet. */ > if (!strcmp (flag_wpa, "jobserver")) > - lto_parallelism = param_max_lto_streaming_parallelism; > + { > + jinfo = new jobserver_info (); > + if (jinfo->is_active) > + jinfo->connect (); > + > + lto_parallelism = param_max_lto_streaming_parallelism; > + } > else > { > lto_parallelism = atoi (flag_wpa); > -- > 2.37.1 >
On 8/10/22 12:52, Richard Biener wrote: > If it's not a template and the methods not inline won't we get multiple > definition errors when this include file is included in multiple places > from an executable? opts-common.cc might be a good place to > stuff it, and opts.h to declare (if it were not for std::string, so possibly > jobserver.h is good enough, or opts-jobserver.h?) Works for me, updated in v2. Martin
diff --git a/gcc/jobserver.h b/gcc/jobserver.h index 856e326ddfc..2a7dc9f4113 100644 --- a/gcc/jobserver.h +++ b/gcc/jobserver.h @@ -31,6 +31,18 @@ struct jobserver_info /* Default constructor. */ jobserver_info (); + /* Connect to the server. */ + void connect (); + + /* Disconnect from the server. */ + void disconnect (); + + /* Get token from the server. */ + bool get_token (); + + /* Return token to the server. */ + void return_token (); + /* Error message if there is a problem. */ string error_msg = ""; /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ @@ -41,6 +53,8 @@ struct jobserver_info int wfd = -1; /* Named pipe path. */ string pipe_path = ""; + /* Pipe file descriptor. */ + int pipefd = -1; /* Return true if jobserver is active. */ bool is_active = false; }; @@ -97,4 +111,44 @@ jobserver_info::jobserver_info () error_msg = "jobserver is not available: " + error_msg; } +void +jobserver_info::connect () +{ + if (!pipe_path.empty ()) + pipefd = open (pipe_path.c_str (), O_RDWR); +} + +void +jobserver_info::disconnect () +{ + if (!pipe_path.empty ()) + { + gcc_assert (close (pipefd) == 0); + pipefd = -1; + } +} + +bool +jobserver_info::get_token () +{ + int fd = pipe_path.empty () ? rfd : pipefd; + char c; + unsigned n = read (fd, &c, 1); + if (n != 1) + { + gcc_assert (errno == EAGAIN); + return false; + } + else + return true; +} + +void +jobserver_info::return_token () +{ + int fd = pipe_path.empty () ? wfd : pipefd; + char c = 'G'; + gcc_assert (write (fd, &c, 1) == 1); +} + #endif /* GCC_JOBSERVER_H */ diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc index 31b0c1862f7..56266195ead 100644 --- a/gcc/lto/lto.cc +++ b/gcc/lto/lto.cc @@ -54,11 +54,17 @@ along with GCC; see the file COPYING3. If not see #include "attribs.h" #include "builtins.h" #include "lto-common.h" - +#include "jobserver.h" /* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver. */ static int lto_parallelism; +/* Number of active WPA streaming processes. */ +static int nruns = 0; + +/* GNU make's jobserver info. */ +static jobserver_info *jinfo = NULL; + /* Return true when NODE has a clone that is analyzed (i.e. we need to load its body even if the node itself is not needed). */ @@ -205,6 +211,12 @@ wait_for_child () "streaming subprocess was killed by signal"); } while (!WIFEXITED (status) && !WIFSIGNALED (status)); + + --nruns; + + /* Return token to the jobserver if active. */ + if (jinfo != NULL && jinfo->is_active) + jinfo->return_token (); } #endif @@ -228,25 +240,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, bool ARG_UNUSED (last)) { #ifdef HAVE_WORKING_FORK - static int nruns; - if (lto_parallelism <= 1) { stream_out_partitions_1 (temp_filename, blen, min, max); return; } - /* Do not run more than LTO_PARALLELISM streamings - FIXME: we ignore limits on jobserver. */ if (lto_parallelism > 0 && nruns >= lto_parallelism) - { - wait_for_child (); - nruns --; - } + wait_for_child (); + /* If this is not the last parallel partition, execute new streaming process. */ if (!last) { + if (jinfo != NULL && jinfo->is_active) + while (true) + { + if (jinfo->get_token ()) + break; + if (nruns > 0) + wait_for_child (); + else + { + /* There are no free tokens, lets do the job outselves. */ + stream_out_partitions_1 (temp_filename, blen, min, max); + asm_nodes_output = true; + return; + } + } + pid_t cpid = fork (); if (!cpid) @@ -264,10 +286,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, /* Last partition; stream it and wait for all children to die. */ else { - int i; stream_out_partitions_1 (temp_filename, blen, min, max); - for (i = 0; i < nruns; i++) + while (nruns > 0) wait_for_child (); + + if (jinfo != NULL && jinfo->is_active) + jinfo->disconnect (); } asm_nodes_output = true; #else @@ -460,9 +484,14 @@ do_whole_program_analysis (void) lto_parallelism = 1; - /* TODO: jobserver communication is not supported, yet. */ if (!strcmp (flag_wpa, "jobserver")) - lto_parallelism = param_max_lto_streaming_parallelism; + { + jinfo = new jobserver_info (); + if (jinfo->is_active) + jinfo->connect (); + + lto_parallelism = param_max_lto_streaming_parallelism; + } else { lto_parallelism = atoi (flag_wpa);