libgo patch committed: Support NumCPU on more platforms

This patch to libgo, from Shenghou Ma, adds support for NumCPU on
additional platforms: Solaris, Irix, *BSD, Darwin.  Bootstrapped and ran
Go testsuite on x86_64-unknown-linux-gnu.  Committed to mainline.

Ian

On 07/08/12 05:43, Ian Lance Taylor wrote:
> This patch to libgo, from Shenghou Ma, adds support for NumCPU on
> additional platforms: Solaris, Irix, *BSD, Darwin.  Bootstrapped and ran
> Go testsuite on x86_64-unknown-linux-gnu.  Committed to mainline.
> 
> Ian
> 
> 

Wouldn't it be more useful on Linux to check the task's affinity
settings?  Then when a task is locked to a limited set of cores it won't
overload those cores with threads.

R.

> foo.patch
> 
> 
> diff -r 12a361bc53d0 libgo/Makefile.am
> --- a/libgo/Makefile.am	Mon Aug 06 21:39:47 2012 -0700
> +++ b/libgo/Makefile.am	Mon Aug 06 21:40:21 2012 -0700
> @@ -390,6 +390,32 @@
>  runtime_lock_files = runtime/lock_sema.c runtime/thread-sema.c
>  endif
>  
> +if LIBGO_IS_LINUX
> +runtime_getncpu_file = runtime/getncpu-linux.c
> +else
> +if LIBGO_IS_DARWIN
> +runtime_getncpu_file = runtime/getncpu-bsd.c
> +else
> +if LIBGO_IS_IRIX
> +runtime_getncpu_file = runtime/getncpu-irix.c
> +else
> +if LIBGO_IS_SOLARIS
> +runtime_getncpu_file = runtime/getncpu-solaris.c
> +else
> +if LIBGO_IS_FREEBSD
> +runtime_getncpu_file = runtime/getncpu-bsd.c
> +else
> +if LIBGO_IS_NETBSD
> +runtime_getncpu_file = runtime/getncpu-bsd.c
> +else
> +runtime_getncpu_file = runtime/getncpu-none.c
> +endif
> +endif
> +endif
> +endif
> +endif
> +endif
> +
>  runtime_files = \
>  	runtime/go-append.c \
>  	runtime/go-assert.c \
> @@ -481,7 +507,8 @@
>  	sema.c \
>  	sigqueue.c \
>  	string.c \
> -	time.c
> +	time.c \
> +	$(runtime_getncpu_file)
>  
>  goc2c.$(OBJEXT): runtime/goc2c.c
>  	$(CC_FOR_BUILD) -c $(CFLAGS_FOR_BUILD) $<
> diff -r 12a361bc53d0 libgo/runtime/getncpu-bsd.c
> --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
> +++ b/libgo/runtime/getncpu-bsd.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -0,0 +1,24 @@
> +// Copyright 2012 The Go Authors. All rights reserved.
> +// Use of this source code is governed by a BSD-style
> +// license that can be found in the LICENSE file.
> +
> +#include <sys/types.h>
> +#include <sys/sysctl.h>
> +
> +#include "runtime.h"
> +#include "defs.h"
> +
> +int32
> +getproccount(void)
> +{
> +	int mib[2], out;
> +	size_t len;
> +
> +	mib[0] = CTL_HW;
> +	mib[1] = HW_NCPU;
> +	len = sizeof(out);
> +	if(sysctl(mib, 2, &out, &len, NULL, 0) >= 0)
> +		return (int32)out;
> +	else
> +		return 0;
> +}
> diff -r 12a361bc53d0 libgo/runtime/getncpu-irix.c
> --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
> +++ b/libgo/runtime/getncpu-irix.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -0,0 +1,16 @@
> +// Copyright 2012 The Go Authors. All rights reserved.
> +// Use of this source code is governed by a BSD-style
> +// license that can be found in the LICENSE file.
> +
> +#include <unistd.h>
> +
> +#include "runtime.h"
> +#include "defs.h"
> +
> +int32
> +getproccount(void)
> +{
> +	int32 n;
> +	n = (int32)sysconf(_SC_NPROC_ONLN);
> +	return n > 1 ? n : 1;
> +}
> diff -r 12a361bc53d0 libgo/runtime/getncpu-linux.c
> --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
> +++ b/libgo/runtime/getncpu-linux.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -0,0 +1,47 @@
> +// Copyright 2012 The Go Authors. All rights reserved.
> +// Use of this source code is governed by a BSD-style
> +// license that can be found in the LICENSE file.
> +
> +#include <string.h>
> +#include <sys/types.h>
> +#include <fcntl.h>
> +#include <unistd.h>
> +
> +#include "runtime.h"
> +#include "defs.h"
> +
> +#ifndef O_CLOEXEC
> +#define O_CLOEXEC 0
> +#endif
> +
> +int32
> +getproccount(void)
> +{
> +	int32 fd, rd, cnt, cpustrlen;
> +	const char *cpustr;
> +	const byte *pos;
> +	byte *bufpos;
> +	byte buf[256];
> +
> +	fd = open("/proc/stat", O_RDONLY|O_CLOEXEC, 0);
> +	if(fd == -1)
> +		return 1;
> +	cnt = 0;
> +	bufpos = buf;
> +	cpustr = "\ncpu";
> +	cpustrlen = strlen(cpustr);
> +	for(;;) {
> +		rd = read(fd, bufpos, sizeof(buf)-cpustrlen);
> +		if(rd == -1)
> +			break;
> +		bufpos[rd] = 0;
> +		for(pos=buf; (pos=(const byte*)strstr((const char*)pos, cpustr)) != nil; cnt++, pos++) {
> +		}
> +		if(rd < cpustrlen)
> +			break;
> +		memmove(buf, bufpos+rd-cpustrlen+1, cpustrlen-1);
> +		bufpos = buf+cpustrlen-1;
> +	}
> +	close(fd);
> +	return cnt ? cnt : 1;
> +}
> diff -r 12a361bc53d0 libgo/runtime/getncpu-none.c
> --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
> +++ b/libgo/runtime/getncpu-none.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -0,0 +1,12 @@
> +// Copyright 2012 The Go Authors. All rights reserved.
> +// Use of this source code is governed by a BSD-style
> +// license that can be found in the LICENSE file.
> +
> +#include "runtime.h"
> +#include "defs.h"
> +
> +int32
> +getproccount(void)
> +{
> +	return 0;
> +}
> diff -r 12a361bc53d0 libgo/runtime/getncpu-solaris.c
> --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
> +++ b/libgo/runtime/getncpu-solaris.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -0,0 +1,16 @@
> +// Copyright 2012 The Go Authors. All rights reserved.
> +// Use of this source code is governed by a BSD-style
> +// license that can be found in the LICENSE file.
> +
> +#include <unistd.h>
> +
> +#include "runtime.h"
> +#include "defs.h"
> +
> +int32
> +getproccount(void)
> +{
> +	int32 n;
> +	n = (int32)sysconf(_SC_NPROCESSORS_ONLN);
> +	return n > 1 ? n : 1;
> +}
> diff -r 12a361bc53d0 libgo/runtime/runtime.h
> --- a/libgo/runtime/runtime.h	Mon Aug 06 21:39:47 2012 -0700
> +++ b/libgo/runtime/runtime.h	Mon Aug 06 21:40:21 2012 -0700
> @@ -518,3 +518,5 @@
>  extern uintptr runtime_stacks_sys;
>  
>  extern _Bool __go_file_line (uintptr, String*, String*, int *);
> +
> +int32 getproccount(void);
> diff -r 12a361bc53d0 libgo/runtime/thread-linux.c
> --- a/libgo/runtime/thread-linux.c	Mon Aug 06 21:39:47 2012 -0700
> +++ b/libgo/runtime/thread-linux.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -72,42 +72,6 @@
>  	*(int32*)0x1006 = 0x1006;
>  }
>  
> -#ifndef O_CLOEXEC
> -#define O_CLOEXEC 0
> -#endif
> -
> -static int32
> -getproccount(void)
> -{
> -	int32 fd, rd, cnt, cpustrlen;
> -	const char *cpustr;
> -	const byte *pos;
> -	byte *bufpos;
> -	byte buf[256];
> -
> -	fd = open("/proc/stat", O_RDONLY|O_CLOEXEC, 0);
> -	if(fd == -1)
> -		return 1;
> -	cnt = 0;
> -	bufpos = buf;
> -	cpustr = "\ncpu";
> -	cpustrlen = strlen(cpustr);
> -	for(;;) {
> -		rd = read(fd, bufpos, sizeof(buf)-cpustrlen);
> -		if(rd == -1)
> -			break;
> -		bufpos[rd] = 0;
> -		for(pos=buf; (pos=(const byte*)strstr((const char*)pos, cpustr)) != nil; cnt++, pos++) {
> -		}
> -		if(rd < cpustrlen)
> -			break;
> -		memmove(buf, bufpos+rd-cpustrlen+1, cpustrlen-1);
> -		bufpos = buf+cpustrlen-1;
> -	}
> -	close(fd);
> -	return cnt ? cnt : 1;
> -}
> -
>  void
>  runtime_osinit(void)
>  {
> diff -r 12a361bc53d0 libgo/runtime/thread-sema.c
> --- a/libgo/runtime/thread-sema.c	Mon Aug 06 21:39:47 2012 -0700
> +++ b/libgo/runtime/thread-sema.c	Mon Aug 06 21:40:21 2012 -0700
> @@ -138,6 +138,7 @@
>  void
>  runtime_osinit (void)
>  {
> +  runtime_ncpu = getproccount();
>  }
>  
>  void
>

On Tue, Aug 7, 2012 at 5:21 AM, Richard Earnshaw <rearnsha@arm.com> wrote:
>
> Wouldn't it be more useful on Linux to check the task's affinity
> settings?  Then when a task is locked to a limited set of cores it won't
> overload those cores with threads.

Good question.  I'm not sure.  This patch does not change the number
of threads that the Go runtime will use, so your particular concern is
not an issue.  This patch changes the return value of the standard
library function runtime.NumCPU.  And if the number of processors is >
1, it tweaks some of the runtime code, e.g., to use a limited time
spin lock (if the number of processors == 1, a spin lock is pointless,
and the code moves straight to a futex).  I guess that if the process
is restricted to a single processor by affinity, then it might be
better to be aware of that and skip the spin lock.  But I'm not sure
what the return value of runtime.NumCPU should be.  It's documented as
"NumCPU returns the number of logical CPUs on the local machine."

Ian