Message ID | CAMe9rOrik_jJkM+vd9n7LuPEUR=xM7T8MC1cMiFXgnPsYUiTEw@mail.gmail.com |
---|---|
State | New |
Headers | show |
On Tue, May 30, 2017 at 3:14 PM, H.J. Lu <hjl.tools@gmail.com> wrote: > On Sat, May 20, 2017 at 6:56 AM, H.J. Lu <hjl.tools@gmail.com> wrote: >> The difference between strlen and wcslen is byte vs int. We can >> replace pminub and pcmpeqb with pminud and pcmpeqd to turn strlen >> into wcslen. Tested on Ivy Bridge with benchtests/bench-wcslen.c, >> the new strlen based wcslen is as fast as the old wcslen. >> >> OK for master? >> >> H.J. >> --- >> * sysdeps/x86_64/strlen.S (PMINU): New. >> (PCMPEQ): Likewise. >> (SHIFT_RETURN): Likewise. >> (FIND_ZERO): Replace pcmpeqb with PCMPEQ. >> (strlen): Add SHIFT_RETURN before ret. Replace pcmpeqb and >> pminub with PCMPEQ and PMINU. >> * sysdeps/x86_64/wcslen.S: Define AS_WCSLEN and strlen. >> Include "strlen.S". >> * sysdeps/x86_64/wcsnlen.S: New file. > > Here is the updated patch only with SSE2 wcsnlen.S. Any > comments? > I will check it in today.
On 2017.06.05 at 07:24 -0700, H.J. Lu wrote: > On Tue, May 30, 2017 at 3:14 PM, H.J. Lu <hjl.tools@gmail.com> wrote: > > On Sat, May 20, 2017 at 6:56 AM, H.J. Lu <hjl.tools@gmail.com> wrote: > >> The difference between strlen and wcslen is byte vs int. We can > >> replace pminub and pcmpeqb with pminud and pcmpeqd to turn strlen > >> into wcslen. Tested on Ivy Bridge with benchtests/bench-wcslen.c, > >> the new strlen based wcslen is as fast as the old wcslen. > >> > >> OK for master? > >> > >> H.J. > >> --- > >> * sysdeps/x86_64/strlen.S (PMINU): New. > >> (PCMPEQ): Likewise. > >> (SHIFT_RETURN): Likewise. > >> (FIND_ZERO): Replace pcmpeqb with PCMPEQ. > >> (strlen): Add SHIFT_RETURN before ret. Replace pcmpeqb and > >> pminub with PCMPEQ and PMINU. > >> * sysdeps/x86_64/wcslen.S: Define AS_WCSLEN and strlen. > >> Include "strlen.S". > >> * sysdeps/x86_64/wcsnlen.S: New file. > > > > Here is the updated patch only with SSE2 wcsnlen.S. Any > > comments? > > > > I will check it in today. It doesn't work on old machines without SSE4.1: FAIL: stdio-common/tstdiomisc FAIL: wcsmbs/test-wcpncpy FAIL: wcsmbs/test-wcsncmp FAIL: wcsmbs/test-wcsncpy FAIL: wcsmbs/test-wcsnlen FAIL: wcsmbs/wcsatcliff markus@x4 tmp % gdb --args /var/tmp/glibc-build/elf/ld-linux-x86-64.so.2 --library-path /var/tmp/glibc-build:/var/tmp/glibc-build/math:/var/tmp/glibc-build/elf:/var/tmp/gl[31/150]d/dlfcn:/var/tmp/glibc-build/nss:/var/tmp/glibc-build/nis:/var/tmp/glibc-build/rt:/var/tmp/glibc-build/resolv:/var/tmp/glibc-build/crypt:/var/tmp/glibc-build/mathvec:/var/tmp/glib c-build/support:/var/tmp/glibc-build/nptl /var/tmp/glibc-build/stdio-common/tstdiomisc Reading symbols from /var/tmp/glibc-build/elf/ld-linux-x86-64.so.2...done. (gdb) run Starting program: /home/markus/tmp/glibc-build/elf/ld-linux-x86-64.so.2 --library-path /var/tmp/glibc-build:/var/tmp/glibc-build/math:/var/tmp/glibc-build/elf:/var/tmp/glibc-build/dlfcn:/var/tmp/glibc-build/nss:/var/tmp/glibc-build/nis:/var/tmp/glibc-build/rt:/var/tmp/glibc-build/resolv:/var/tmp/glibc-build/crypt:/var/tmp/glibc-build/mathvec:/var/tmp/glibc-build/support:/var/tmp/glibc-build/nptl /var/tmp/glibc-build/stdio-common/tstdiomisc t1: count=5 sscanf ("12345", "%ld", &x) => 1, x = 12345 sscanf ("12345", "%llllld", &x) => 0, x = -1 sscanf ("12345", "%LLLLLd", &x) => 0, x = -1 sscanf ("test ", "%*s%n", &x) => 0, x = 4 sscanf ("test ", "%2*s%n", &x) => 0, x = -1 sscanf ("12 ", "%l2d", &x) => 0, x = -1 sscanf ("12 ", "%2ld", &x) => 1, x = 12 sscanf ("1 1", "%d %Z", &n, &N) => 1, n = 1, N = -1 expected "nan NAN nan NAN nan NAN nan NAN", got "nan NAN nan NAN nan NAN nan NAN" expected "-nan -NAN -nan -NAN -nan -NAN -nan -NAN", got "-nan -NAN -nan -NAN -nan -NAN -nan -NAN" expected "nan NAN nan NAN nan NAN nan NAN", got "nan NAN nan NAN nan NAN nan NAN" expected "-nan -NAN -nan -NAN -nan -NAN -nan -NAN", got "-nan -NAN -nan -NAN -nan -NAN -nan -NAN" expected "inf INF inf INF inf INF inf INF", got "inf INF inf INF inf INF inf INF" expected "-inf -INF -inf -INF -inf -INF -inf -INF", got "-inf -INF -inf -INF -inf -INF -inf -INF" Program received signal SIGILL, Illegal instruction. wcsnlen () at ../sysdeps/x86_64/strlen.S:180 180 PMINU 16(%rax), %xmm0
From 5798d0fbf03c531dfcd8d145217a5fbfd2319d95 Mon Sep 17 00:00:00 2001 From: "H.J. Lu" <hjl.tools@gmail.com> Date: Sat, 20 May 2017 06:48:04 -0700 Subject: [PATCH] x86-64: Update strlen.S to support wcslen/wcsnlen The difference between strlen and wcslen is byte vs int. We can replace pminub and pcmpeqb with pminud and pcmpeqd to turn strlen/strnlen into wcslen/wcsnlen. The SSE2 wcsnlen is up to 4X faster on Haswell. * sysdeps/x86_64/strlen.S (PMINU): New. (PCMPEQ): Likewise. (SHIFT_RETURN): Likewise. (FIND_ZERO): Replace pcmpeqb with PCMPEQ. (strlen): Add SHIFT_RETURN before ret. Replace pcmpeqb and pminub with PCMPEQ and PMINU. * sysdeps/x86_64/wcsnlen.S: New file. --- sysdeps/x86_64/strlen.S | 61 +++++++++++++++++++++++++++++++----------------- sysdeps/x86_64/wcsnlen.S | 7 ++++++ 2 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 sysdeps/x86_64/wcsnlen.S diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S index 5896e6b9..b5ab117 100644 --- a/sysdeps/x86_64/strlen.S +++ b/sysdeps/x86_64/strlen.S @@ -1,4 +1,4 @@ -/* SSE2 version of strlen. +/* SSE2 version of strlen/wcslen. Copyright (C) 2012-2017 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -18,6 +18,16 @@ #include <sysdep.h> +#ifdef AS_WCSLEN +# define PMINU pminud +# define PCMPEQ pcmpeqd +# define SHIFT_RETURN shrq $2, %rax +#else +# define PMINU pminub +# define PCMPEQ pcmpeqb +# define SHIFT_RETURN +#endif + /* Long lived register in strlen(s), strnlen(s, n) are: %xmm3 - zero @@ -32,10 +42,10 @@ ENTRY(strlen) /* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */ #define FIND_ZERO \ - pcmpeqb (%rax), %xmm0; \ - pcmpeqb 16(%rax), %xmm1; \ - pcmpeqb 32(%rax), %xmm2; \ - pcmpeqb 48(%rax), %xmm3; \ + PCMPEQ (%rax), %xmm0; \ + PCMPEQ 16(%rax), %xmm1; \ + PCMPEQ 32(%rax), %xmm2; \ + PCMPEQ 48(%rax), %xmm3; \ pmovmskb %xmm0, %esi; \ pmovmskb %xmm1, %edx; \ pmovmskb %xmm2, %r8d; \ @@ -54,6 +64,9 @@ ENTRY(strlen) xor %rax, %rax ret L(n_nonzero): +# ifdef AS_WCSLEN + shlq $2, %rsi +# endif /* Initialize long lived registers. */ @@ -96,6 +109,7 @@ L(n_nonzero): test %rdx, %rdx; \ je L(lab); \ bsfq %rdx, %rax; \ + SHIFT_RETURN; \ ret #ifdef AS_STRNLEN @@ -104,19 +118,20 @@ L(n_nonzero): #else /* Test first 16 bytes unaligned. */ movdqu (%rax), %xmm4 - pcmpeqb %xmm0, %xmm4 + PCMPEQ %xmm0, %xmm4 pmovmskb %xmm4, %edx test %edx, %edx je L(next48_bytes) bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */ + SHIFT_RETURN ret L(next48_bytes): /* Same as FIND_ZERO except we do not check first 16 bytes. */ andq $-16, %rax - pcmpeqb 16(%rax), %xmm1 - pcmpeqb 32(%rax), %xmm2 - pcmpeqb 48(%rax), %xmm3 + PCMPEQ 16(%rax), %xmm1 + PCMPEQ 32(%rax), %xmm2 + PCMPEQ 48(%rax), %xmm3 pmovmskb %xmm1, %edx pmovmskb %xmm2, %r8d pmovmskb %xmm3, %ecx @@ -145,6 +160,7 @@ L(strnlen_ret): test %rdx, %rdx je L(loop_init) bsfq %rdx, %rax + SHIFT_RETURN ret #endif .p2align 4 @@ -161,10 +177,10 @@ L(loop): je L(exit_end) movdqa (%rax), %xmm0 - pminub 16(%rax), %xmm0 - pminub 32(%rax), %xmm0 - pminub 48(%rax), %xmm0 - pcmpeqb %xmm3, %xmm0 + PMINU 16(%rax), %xmm0 + PMINU 32(%rax), %xmm0 + PMINU 48(%rax), %xmm0 + PCMPEQ %xmm3, %xmm0 pmovmskb %xmm0, %edx testl %edx, %edx jne L(exit) @@ -182,6 +198,7 @@ L(first): bsfq %rdx, %rdx addq %rdx, %rax subq %rdi, %rax + SHIFT_RETURN ret .p2align 4 @@ -192,6 +209,7 @@ L(exit): bsfq %rdx, %rdx addq %rdx, %rax subq %rdi, %rax + SHIFT_RETURN ret #else @@ -201,10 +219,10 @@ L(exit): L(loop): movdqa 64(%rax), %xmm0 - pminub 80(%rax), %xmm0 - pminub 96(%rax), %xmm0 - pminub 112(%rax), %xmm0 - pcmpeqb %xmm3, %xmm0 + PMINU 80(%rax), %xmm0 + PMINU 96(%rax), %xmm0 + PMINU 112(%rax), %xmm0 + PCMPEQ %xmm3, %xmm0 pmovmskb %xmm0, %edx testl %edx, %edx jne L(exit64) @@ -212,10 +230,10 @@ L(loop): subq $-128, %rax movdqa (%rax), %xmm0 - pminub 16(%rax), %xmm0 - pminub 32(%rax), %xmm0 - pminub 48(%rax), %xmm0 - pcmpeqb %xmm3, %xmm0 + PMINU 16(%rax), %xmm0 + PMINU 32(%rax), %xmm0 + PMINU 48(%rax), %xmm0 + PCMPEQ %xmm3, %xmm0 pmovmskb %xmm0, %edx testl %edx, %edx jne L(exit0) @@ -231,6 +249,7 @@ L(exit0): bsfq %rdx, %rdx addq %rdx, %rax subq %rdi, %rax + SHIFT_RETURN ret #endif diff --git a/sysdeps/x86_64/wcsnlen.S b/sysdeps/x86_64/wcsnlen.S new file mode 100644 index 0000000..968bb69 --- /dev/null +++ b/sysdeps/x86_64/wcsnlen.S @@ -0,0 +1,7 @@ +#define AS_WCSLEN +#define AS_STRNLEN +#define strlen __wcsnlen + +#include "strlen.S" + +weak_alias(__wcsnlen, wcsnlen) -- 2.9.4