From patchwork Fri Jun 26 11:51:52 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Stefan Liebler X-Patchwork-Id: 488775 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from sourceware.org (server1.sourceware.org [209.132.180.131]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 8C684140271 for ; Fri, 26 Jun 2015 21:55:10 +1000 (AEST) Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=sourceware.org header.i=@sourceware.org header.b=yLMEMVwP; dkim-atps=neutral DomainKey-Signature: a=rsa-sha1; c=nofws; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id:in-reply-to :references; q=dns; s=default; b=EBDocaNw/83mRCX/n5bbAVAyl04XCLU RXK+CYZi0dV3MOP2U5B/5SS31WBtvqGXKTRK2xIakTCz11YQkuHkWV+q0CyWe0RE rO+VSxiKNjOVWGR8rH2+tsz5Nkeo2jxsg06lEjciKr8+ACjgcs/btoiJbhI3W4Sl jL3ntT7rRh8I= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id:in-reply-to :references; s=default; bh=JfmKwO2r4/2zYa9nPFuOSJbwbCg=; b=yLMEM VwPTxARjba1kY6iT2qYnHcEXIh5LE0bnZs7ZZnaaKysPYBZ/FzeL1s4wEhWO1jg8 BV9D3hDplnAfydYSddCc9JLDw7ZBE0cF0+zQq6EbCK9a3BaEyNscBpjqKTPEHQIL JduSo+gx86s3kmPBacBJc9waLHGgQeheVOqOYs= Received: (qmail 26726 invoked by alias); 26 Jun 2015 11:52:59 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 23276 invoked by uid 89); 26 Jun 2015 11:52:28 -0000 Authentication-Results: sourceware.org; auth=none X-Virus-Found: No X-Spam-SWARE-Status: No, score=-2.0 required=5.0 tests=AWL, BAYES_00, KAM_LAZY_DOMAIN_SECURITY, RP_MATCHES_RCVD autolearn=ham version=3.3.2 X-HELO: e06smtp10.uk.ibm.com X-MailFrom: stli@linux.vnet.ibm.com X-RcptTo: libc-alpha@sourceware.org From: Stefan Liebler To: libc-alpha@sourceware.org Cc: Stefan Liebler Subject: [PATCH 27/27] S390: Optimize memrchr. Date: Fri, 26 Jun 2015 13:51:52 +0200 Message-Id: <1435319512-22245-28-git-send-email-stli@linux.vnet.ibm.com> In-Reply-To: <1435319512-22245-1-git-send-email-stli@linux.vnet.ibm.com> References: <1435319512-22245-1-git-send-email-stli@linux.vnet.ibm.com> X-TM-AS-MML: disable X-Content-Scanned: Fidelis XPS MAILER x-cbid: 15062611-0041-0000-0000-000004BCB3ED This patch provides optimized version of memrchr with the z13 vector instructions. ChangeLog: * sysdeps/s390/multiarch/memrchr-c.c: New File. * sysdeps/s390/multiarch/memrchr-vx.S: Likewise. * sysdeps/s390/multiarch/memrchr.c: Likewise. * sysdeps/s390/multiarch/Makefile (sysdep_routines): Add memrchr functions. * sysdeps/s390/multiarch/ifunc-impl-list-common.c (__libc_ifunc_impl_list_common): Add ifunc test for memrchr. --- sysdeps/s390/multiarch/Makefile | 3 +- sysdeps/s390/multiarch/ifunc-impl-list.c | 2 + sysdeps/s390/multiarch/memrchr-c.c | 25 +++++++ sysdeps/s390/multiarch/memrchr-vx.S | 116 +++++++++++++++++++++++++++++++ sysdeps/s390/multiarch/memrchr.c | 28 ++++++++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 sysdeps/s390/multiarch/memrchr-c.c create mode 100644 sysdeps/s390/multiarch/memrchr-vx.S create mode 100644 sysdeps/s390/multiarch/memrchr.c diff --git a/sysdeps/s390/multiarch/Makefile b/sysdeps/s390/multiarch/Makefile index 929a545..0805b07 100644 --- a/sysdeps/s390/multiarch/Makefile +++ b/sysdeps/s390/multiarch/Makefile @@ -17,7 +17,8 @@ sysdep_routines += strlen strlen-vx strlen-c \ strcspn strcspn-vx strcspn-c \ memchr memchr-vx \ rawmemchr rawmemchr-vx rawmemchr-c \ - memccpy memccpy-vx memccpy-c + memccpy memccpy-vx memccpy-c \ + memrchr memrchr-vx memrchr-c endif ifeq ($(subdir),wcsmbs) diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c index 5ea258b..c235bdc 100644 --- a/sysdeps/s390/multiarch/ifunc-impl-list.c +++ b/sysdeps/s390/multiarch/ifunc-impl-list.c @@ -137,6 +137,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, IFUNC_VX_IMPL (wmemcmp); + IFUNC_VX_IMPL (memrchr); + #endif /* HAVE_S390_VX_ASM_SUPPORT */ return i; diff --git a/sysdeps/s390/multiarch/memrchr-c.c b/sysdeps/s390/multiarch/memrchr-c.c new file mode 100644 index 0000000..ac481fd --- /dev/null +++ b/sysdeps/s390/multiarch/memrchr-c.c @@ -0,0 +1,25 @@ +/* Default memrchr implementation for S/390. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && IS_IN (libc) +# define MEMRCHR __memrchr_c + +# include +extern __typeof (__memrchr) __memrchr_c; +# include +#endif diff --git a/sysdeps/s390/multiarch/memrchr-vx.S b/sysdeps/s390/multiarch/memrchr-vx.S new file mode 100644 index 0000000..53c44ef --- /dev/null +++ b/sysdeps/s390/multiarch/memrchr-vx.S @@ -0,0 +1,116 @@ +/* Vector optimized 32/64 bit S/390 version of memrchr. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && IS_IN (libc) + +# include "sysdep.h" +# include "asm-syntax.h" + + .text + +/* void *memrchr (const void *s, int c, size_t n) + Scans memory for character c backwards + and returns pointer to first c. + + Register usage: + -r1=tmp + -r2=s + -r3=c + -r4=n + -r5=s in loop + + -v16=part of s + -v17=index of found c + -v18=c replicated +*/ +ENTRY(__memrchr_vx) + .machine "z13" + .machinemode "zarch_nohighgprs" + +# if !defined __s390x__ + llgfr %r4,%r4 +# endif /* !defined __s390x__ */ + + vlvgb %v18,%r3,0 /* Generate vector which elements are all c. + If c > 255, c will be truncated. */ + vrepb %v18,%v18,0 + + clgijh %r4,16,.Lg16 /* Jump away if n > 16. */ + +.Ll16: + /* Process n <= 16 bytes; r2 points to begin of n-bytes. */ + clgije %r4,0,.Lnf_end + + aghi %r4,-1 /* vll needs highest index. */ + vll %v16,%r4,0(%r2) + vfeeb %v17,%v16,%v18 /* Find c. */ + vlgvb %r1,%v17,7 /* Index of c or 16 if not found. */ + clr %r1,%r4 + jle .Lfound_lt16 /* Found c within loaded bytes. */ +.Lnf_end: + lghi %r2,0 /* Return null. */ + br %r14 + +.Lfound_g16: + vlgvb %r1,%v17,7 /* Index of c or 16 if not found. */ + lgr %r2,%r5 + lghi %r4,15 /* Highest index of a full vr. */ + clr %r1,%r4 +.Lfound_lt16: + la %r2,0(%r1,%r2) /* Store current pointer to found c. */ + ber %r14 /* Return if found c is last loaded byte. */ + + /* Shift vector elements left and start search again. */ + aghi %r1,1 /* Start search after current index. */ + slr %r4,%r1 /* New highest index. */ + sll %r1,3 /* Calculate byte count for vector shift + left. */ + vlvgg %v17,%r1,0 + vslb %v16,%v16,%v17 /* Vector shift left by byte by number of bytes + specified in bits 1-4 of byte 7 in v17. */ + vfeeb %v17,%v16,%v18 /* Find c. */ + la %r5,1(%r2) /* Save start-address of shifted v16. */ + vlgvb %r1,%v17,7 /* Index of c or 16 if not found */ + clr %r1,%r4 + locgrle %r2,%r5 /* Use stored address as base if c found. */ + jle .Lfound_lt16 /* Found c within loaded bytes. */ + br %r14 /* No further c found, return last stored c. */ + +.Lfound_end: + la %r2,0(%r4,%r2) /* Return pointer to found c */ + br %r14 + +.Lg16: + /* Process 16byte blocks - n > 16. */ +.Lloop: + aghi %r4,-16 + la %r5,0(%r4,%r2) /* Get address n -16. */ + vl %v16,0(%r5) + vfeebs %v17,%v16,%v18 /* Find c. */ + jno .Lfound_g16 /* Jump away if c was found. */ + clgijle %r4,16,.Ll16 /* Process remaining. */ + + aghi %r4,-16 + la %r5,0(%r4,%r2) /* Get address n -16. */ + vl %v16,0(%r5) + vfeebs %v17,%v16,%v18 /* Find c. */ + jno .Lfound_g16 /* Jump away if c was found. */ + clgijh %r4,16,.Lloop /* Loop until n > 16. */ + j .Ll16 +END(__memrchr_vx) +#endif /* HAVE_S390_VX_ASM_SUPPORT && IS_IN (libc) */ diff --git a/sysdeps/s390/multiarch/memrchr.c b/sysdeps/s390/multiarch/memrchr.c new file mode 100644 index 0000000..8ac2f52 --- /dev/null +++ b/sysdeps/s390/multiarch/memrchr.c @@ -0,0 +1,28 @@ +/* Multiple versions of memrchr. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && IS_IN (libc) +# include +# include + +s390_vx_libc_ifunc (__memrchr) +weak_alias (__memrchr, memrchr) + +#else +# include +#endif /* !(defined HAVE_S390_VX_ASM_SUPPORT && IS_IN (libc)) */