From patchwork Fri Oct 12 11:49:25 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feng Xue X-Patchwork-Id: 983005 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=sourceware.org (client-ip=209.132.180.131; helo=sourceware.org; envelope-from=libc-alpha-return-96391-incoming=patchwork.ozlabs.org@sourceware.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=hotmail.com Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; secure) header.d=sourceware.org header.i=@sourceware.org header.b="cIP3PScu"; dkim=pass (2048-bit key; unprotected) header.d=hotmail.com header.i=@hotmail.com header.b="nGBmrtRC"; dkim-atps=neutral Received: from sourceware.org (server1.sourceware.org [209.132.180.131]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 42WmNB3NWPz9s3Z for ; Fri, 12 Oct 2018 22:49:46 +1100 (AEDT) DomainKey-Signature: a=rsa-sha1; c=nofws; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id :content-type:content-transfer-encoding:mime-version; q=dns; s= default; b=Mz9caLeUbifmL9v5T9aDRf+hW12vqQT0y37tabhWGONU5y+Fk5a6V +GH9gpOEDOetcRSjOs0QbfSO9vRiYsvB6YCrjbDKOdxOvGgA0XPNtb/x4OLgwFy+ oww4Yj9EmazuP9hYhn94WqXlN7fhlC5nK3bjMH/C4SsbEAdQFqKPiQ= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id :content-type:content-transfer-encoding:mime-version; s=default; bh=1KlsrIeCajl17R4CuJOFaxhAH4o=; b=cIP3PScucJ7CWQCphsYILtJ7o+Xj kYTEun92LSNB1rAcguVEyAnMxCKXUu5PW1SKcsEirKVpSS06UjNzDXWbe99z3tzI kJcpvhX7VPzcYI0PklKL93CHXPNK7ULI1DNkad9UOt6k/U4IuOX/VTsh01y1AgC1 D9TXItj3LYGFyWA= Received: (qmail 16616 invoked by alias); 12 Oct 2018 11:49:34 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 16544 invoked by uid 89); 12 Oct 2018 11:49:31 -0000 Authentication-Results: sourceware.org; auth=none X-Spam-SWARE-Status: No, score=-26.0 required=5.0 tests=BAYES_00, FORGED_HOTMAIL_RCVD2, FREEMAIL_FROM, GIT_PATCH_0, GIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, MIME_BASE64_BLANKS, RCVD_IN_DNSWL_NONE, SPF_HELO_PASS, SPF_PASS autolearn=ham version=3.3.2 spammy=H*c:gb2312, Hx-languages-length:2751 X-HELO: APC01-SG2-obe.outbound.protection.outlook.com DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=hotmail.com; s=selector1; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=qz8GhXcLBA1/vqWfZZzIbOdvalsFwIv49aeotpOj/V0=; b=nGBmrtRCE0deG4QTR4aHJljoSSnHqYwbkGd3kuBDnua5hpiCDr/hO4YPaiJJ2q76HAb12LqWde9sF5/wjHbvq4GB8cXYaA7Pg87WuCfIES6Z/zzpFVeoxsGtBwkM3wPo4Ka24WhFXP7XK0FP5+E5BZpFzXZ3pBgWnG51Wn45RLyL8glLNqV5zXBkBwEbH8GyJimOjyRIq7zp/WVDuXHSDZwk4h/qA0ZwiN75Sj/5iAeXuy/NjgIM1vhuM2sm9eN79ddLn0odcXDoSp+XobJdyfPHSXSHYftxvOGtnk9/B228PTvkMCcl9qSWmkOykFraC4QsnaOkMthFkJTCfZoa3w== From: Xue Feng To: "libc-alpha@sourceware.org" CC: Feng Xue Subject: [PATCH 1/3] aarch64: Add AmpereComputing skylark to tunable cpu list Date: Fri, 12 Oct 2018 11:49:25 +0000 Message-ID: MIME-Version: 1.0 I'm from AmpereComputing Corp, and working on adding support for Skylark processor to glibc. Feng --- Skylark is a 64-bit CPU core released by AmpereComputing. Add its name to cpu list, and corresponding macro as utilities for later IFUNC dispatch. * manual/tunables.texi (Tunable glibc.cpu.name): Add skylark. * sysdeps/unix/sysv/linux/aarch64/cpu-features.c (cpu_list): Add skylark. * sysdeps/unix/sysv/linux/aarch64/cpu-features.h (IS_SKYLARK): New macro. --- ChangeLog | 8 ++++++++ manual/tunables.texi | 2 +- sysdeps/unix/sysv/linux/aarch64/cpu-features.c | 1 + sysdeps/unix/sysv/linux/aarch64/cpu-features.h | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 10b38cc..2533c9d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2018-10-11 Feng Xue + + * manual/tunables.texi (Tunable glibc.cpu.name): Add skylark. + * sysdeps/unix/sysv/linux/aarch64/cpu-features.c (cpu_list): + Add skylark. + * sysdeps/unix/sysv/linux/aarch64/cpu-features.h (IS_SKYLARK): + New macro. + 2018-10-10 Joseph Myers * sysdeps/unix/sysv/linux/bits/shm.h: Include . diff --git a/manual/tunables.texi b/manual/tunables.texi index 3345a23..4fa61fc 100644 --- a/manual/tunables.texi +++ b/manual/tunables.texi @@ -333,7 +333,7 @@ This tunable is specific to powerpc, powerpc64 and powerpc64le. The @code{glibc.cpu.name=xxx} tunable allows the user to tell @theglibc{} to assume that the CPU is @code{xxx} where xxx may have one of these values: @code{generic}, @code{falkor}, @code{thunderxt88}, @code{thunderx2t99}, -@code{thunderx2t99p1}. +@code{thunderx2t99p1}, @code{skylark}. This tunable is specific to aarch64. @end deftp diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.c b/sysdeps/unix/sysv/linux/aarch64/cpu-features.c index b4f3485..6448493 100644 --- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.c +++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.c @@ -36,6 +36,7 @@ static struct cpu_list cpu_list[] = { {"thunderx2t99", 0x431F0AF0}, {"thunderx2t99p1", 0x420F5160}, {"phecda", 0x680F0000}, + {"skylark", 0x503F0001}, {"generic", 0x0} }; diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h index eb35adf..c53b65d 100644 --- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h +++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h @@ -52,6 +52,9 @@ #define IS_PHECDA(midr) (MIDR_IMPLEMENTOR(midr) == 'h' \ && MIDR_PARTNUM(midr) == 0x000) +#define IS_SKYLARK(midr) (MIDR_IMPLEMENTOR(midr) == 'P' \ + && MIDR_PARTNUM(midr) == 0x000) + struct cpu_features { uint64_t midr_el1; From patchwork Fri Oct 12 11:41:29 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feng Xue X-Patchwork-Id: 982997 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=sourceware.org (client-ip=209.132.180.131; helo=sourceware.org; envelope-from=libc-alpha-return-96389-incoming=patchwork.ozlabs.org@sourceware.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=hotmail.com Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; secure) header.d=sourceware.org header.i=@sourceware.org header.b="dP96PJ11"; dkim=pass (2048-bit key; unprotected) header.d=hotmail.com header.i=@hotmail.com header.b="RxuO7mjo"; dkim-atps=neutral Received: from sourceware.org (server1.sourceware.org [209.132.180.131]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 42WmBw1V8Mz9s47 for ; Fri, 12 Oct 2018 22:41:43 +1100 (AEDT) DomainKey-Signature: a=rsa-sha1; c=nofws; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id :content-type:content-transfer-encoding:mime-version; q=dns; s= default; b=t/M3pON+PXLL1b8SYagm/9s61oFfEwuUjOprVVesjETyLNbkYimEZ vKpcRiKsu/rxPb7KUf7pTKDY0tQ0pj+1L28W/7U+dMHuKiRcviEcaM+NmYWUW/Gl 4IotyvTsElh6mdGJ1/Lw0mHbVWEoTQejZQ0TI/RU1gxzrH55U6j2kI= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id :content-type:content-transfer-encoding:mime-version; s=default; bh=1IfhAxDdqeiJmlFYQeoqltkUgy4=; b=dP96PJ11zt28eJ9m5h+erfVhrPRb cgFeFiEFXyTymJcvMcbkwzEVHQzzA8psTygdug4dZAyReAT07bpPRcWcFpHHVs51 WxT9OOPZ1Cayx8nnOnfbdIkkgR1da3AIxsyIjQ2YE1Q+aPwlWY6e4BB28r3seklv RJ67TCKJjHdrSPQ= Received: (qmail 54078 invoked by alias); 12 Oct 2018 11:41:37 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 53357 invoked by uid 89); 12 Oct 2018 11:41:37 -0000 Authentication-Results: sourceware.org; auth=none X-Spam-SWARE-Status: No, score=-26.0 required=5.0 tests=AWL, BAYES_00, FORGED_HOTMAIL_RCVD2, FREEMAIL_FROM, GIT_PATCH_0, GIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, KAM_SHORT, MIME_BASE64_BLANKS, RCVD_IN_DNSWL_NONE, SPF_HELO_PASS, SPF_PASS autolearn=ham version=3.3.2 spammy=H*c:gb2312, H*RU:sk:HK2APC0, H*r:sk:HK2APC0, H*r:10.152.249 X-HELO: APC01-HK2-obe.outbound.protection.outlook.com DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=hotmail.com; s=selector1; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=yFoPCJdN++Tmp+mIVY+gmML/xTrz7Cq/m7GTrAKuQbU=; b=RxuO7mjoDLrnQEktvOsb0sDivxMoOxuXWOIE8WA1U9CV2IG4aL2mcE+h5z9HSy4/UzeoQBozV0srnJ9NmRaUDjlzXJDOwEzzJo4JCrEbkvVJ+wVMSzG+AIIaaDKV2ciJIgfSO3YHLi4r0AHrWYKuZhZ0gBY4v/m+4ZvuNA3zyntjSIg6RVujLU8PaIjznWre0NkY6FklUzd4/4Jn+ozozlCWmLJyXZUhhZszmiMUdr6zKgAdydrWFY7sGjOwUeXteHs0MHCw7Fdswqu+DQQSlHA4qmvCGd/bpgmt/BsOOKC2ldu139PuTin7zTOQDKzJdx6Fjj7FQlZNurEagXHtzw== From: Xue Feng To: "libc-alpha@sourceware.org" CC: "marcus.shawcroft@linaro.org" , "szabolcs.nagy@arm.com" , Feng Xue Subject: [PATCH 2/3] aarch64: Optimized memset specific to AmpereComputing skylark Date: Fri, 12 Oct 2018 11:41:29 +0000 Message-ID: MIME-Version: 1.0 This version uses general register based memory store instead of vector register based, for the former is faster than the latter in skylark. The fact that DC ZVA size in skylark is 64-byte, is used by IFUNC dispatch to select this memset, so that cost of runtime-check on DC ZVA size can be saved. * sysdeps/aarch64/multiarch/Makefile (sysdep_routines): Add memset_skylark. * sysdeps/aarch64/multiarch/ifunc-impl-list.c (__libc_ifunc_impl_list): Add __memset_skylark to memset ifunc. * sysdeps/aarch64/multiarch/memset.c (libc_ifunc): Add IS_SKYLARK check for ifunc dispatch. * sysdeps/aarch64/multiarch/memset_skylark.S: New file. --- ChangeLog | 10 ++ sysdeps/aarch64/multiarch/Makefile | 3 +- sysdeps/aarch64/multiarch/ifunc-impl-list.c | 1 + sysdeps/aarch64/multiarch/memset.c | 5 +- sysdeps/aarch64/multiarch/memset_skylark.S | 176 ++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 sysdeps/aarch64/multiarch/memset_skylark.S diff --git a/ChangeLog b/ChangeLog index 2533c9d..28370f9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,13 @@ +2018-10-12 Feng Xue + + * sysdeps/aarch64/multiarch/Makefile (sysdep_routines): + Add memset_skylark. + * sysdeps/aarch64/multiarch/ifunc-impl-list.c + (__libc_ifunc_impl_list): Add __memset_skylark to memset ifunc. + * sysdeps/aarch64/multiarch/memset.c (libc_ifunc): + Add IS_SKYLARK check for ifunc dispatch. + * sysdeps/aarch64/multiarch/memset_skylark.S: New file. + 2018-10-11 Feng Xue * manual/tunables.texi (Tunable glibc.cpu.name): Add skylark. diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch/Makefile index b1a5f59..828ce4f 100644 --- a/sysdeps/aarch64/multiarch/Makefile +++ b/sysdeps/aarch64/multiarch/Makefile @@ -1,5 +1,6 @@ ifeq ($(subdir),string) sysdep_routines += memcpy_generic memcpy_thunderx memcpy_thunderx2 \ - memcpy_falkor memmove_falkor memset_generic memset_falkor \ + memcpy_falkor memmove_falkor \ + memset_generic memset_falkor memset_skylark \ strlen_generic strlen_asimd endif diff --git a/sysdeps/aarch64/multiarch/ifunc-impl-list.c b/sysdeps/aarch64/multiarch/ifunc-impl-list.c index af44665..baf01a0 100644 --- a/sysdeps/aarch64/multiarch/ifunc-impl-list.c +++ b/sysdeps/aarch64/multiarch/ifunc-impl-list.c @@ -51,6 +51,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Enable this on non-falkor processors too so that other cores can do a comparative analysis with __memset_generic. */ IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_falkor) + IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_skylark) IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic)) IFUNC_IMPL (i, name, strlen, diff --git a/sysdeps/aarch64/multiarch/memset.c b/sysdeps/aarch64/multiarch/memset.c index d74ed3a..b732a7e 100644 --- a/sysdeps/aarch64/multiarch/memset.c +++ b/sysdeps/aarch64/multiarch/memset.c @@ -29,12 +29,15 @@ extern __typeof (__redirect_memset) __libc_memset; extern __typeof (__redirect_memset) __memset_falkor attribute_hidden; +extern __typeof (__redirect_memset) __memset_skylark attribute_hidden; extern __typeof (__redirect_memset) __memset_generic attribute_hidden; libc_ifunc (__libc_memset, ((IS_FALKOR (midr) || IS_PHECDA (midr)) && zva_size == 64 ? __memset_falkor - : __memset_generic)); + : (IS_SKYLARK (midr) && zva_size == 64 + ? __memset_skylark + : __memset_generic))); # undef memset strong_alias (__libc_memset, memset); diff --git a/sysdeps/aarch64/multiarch/memset_skylark.S b/sysdeps/aarch64/multiarch/memset_skylark.S new file mode 100644 index 0000000..22bf576 --- /dev/null +++ b/sysdeps/aarch64/multiarch/memset_skylark.S @@ -0,0 +1,176 @@ +/* Optimized memset for AmpereComputing skylark processor. + Copyright (C) 2018 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include +#include "memset-reg.h" + +#if IS_IN (libc) +# define MEMSET __memset_skylark + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses + * + */ + +ENTRY_ALIGN (MEMSET, 6) + + DELOUSE (0) + DELOUSE (2) + + bfi valw, valw, 8, 8 + bfi valw, valw, 16, 16 + bfi val, val, 32, 32 + + add dstend, dstin, count + + cmp count, 96 + b.hi L(set_long) + cmp count, 16 + b.hs L(set_medium) + + /* Set 0..15 bytes. */ + tbz count, 3, 1f + str val, [dstin] + str val, [dstend, -8] + ret + + .p2align 3 +1: tbz count, 2, 2f + str valw, [dstin] + str valw, [dstend, -4] + ret +2: cbz count, 3f + strb valw, [dstin] + tbz count, 1, 3f + strh valw, [dstend, -2] +3: ret + + .p2align 3 + /* Set 16..96 bytes. */ +L(set_medium): + stp val, val, [dstin] + tbnz count, 6, L(set96) + stp val, val, [dstend, -16] + tbz count, 5, 1f + stp val, val, [dstin, 16] + stp val, val, [dstend, -32] +1: ret + + .p2align 4 + /* Set 64..96 bytes. Write 64 bytes from the start and + 32 bytes from the end. */ +L(set96): + stp val, val, [dstin, 16] + stp val, val, [dstin, 32] + stp val, val, [dstin, 48] + stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret + + .p2align 4 +L(set_long): + stp val, val, [dstin] + cmp count, 512 + ccmp val, 0, 0, cs + bic dst, dstin, 15 + b.eq L(zva_64) + + /* Small-size or non-zero memset does not use DC ZVA. */ + sub count, dstend, dst + + /* + * Adjust count and bias for loop. By substracting extra 1 from count, + * it is easy to use tbz instruction to check whether loop tailing + * count is less than 33 bytes, so as to bypass 2 unneccesary stps. + */ + sub count, count, 64+16+1 + nop + +1: stp val, val, [dst, 16] + stp val, val, [dst, 32] + stp val, val, [dst, 48] + stp val, val, [dst, 64]! + subs count, count, 64 + b.hs 1b + + tbz count, 5, 1f /* Remaining count is less than 33 bytes? */ + stp val, val, [dst, 16] + stp val, val, [dst, 32] +1: stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret + + .p2align 3 +L(zva_64): + stp val, val, [dst, 16] + stp val, val, [dst, 32] + stp val, val, [dst, 48] + bic dst, dst, 63 + + /* + * Previous memory writes might cross cache line boundary, and cause + * cache line partially dirty. Zeroing this kind of cache line using + * DC ZVA will incur extra cost, for it requires loading untouched + * part of the line from memory before zeoring. + * + * So, write the first 64 byte aligned block using stp to force + * fully dirty cache line. + */ + stp val, val, [dst, 64] + stp val, val, [dst, 80] + stp val, val, [dst, 96] + stp val, val, [dst, 112] + + sub count, dstend, dst + /* + * Adjust count and bias for loop. By substracting extra 1 from count, + * it is easy to use tbz instruction to check whether loop tailing + * count is less than 33 bytes, so as to bypass 2 unneccesary stps. + */ + sub count, count, 128+64+64+1 + add dst, dst, 128 + nop + + /* DC ZVA sets 64 bytes each time. */ +1: dc zva, dst + add dst, dst, 64 + subs count, count, 64 + b.hs 1b + + /* + * Write the last 64 byte aligned block using stp to force fully + * dirty cache line. + */ + stp val, val, [dst, 0] + stp val, val, [dst, 16] + stp val, val, [dst, 32] + stp val, val, [dst, 48] + + tbz count, 5, 1f /* Remaining count is less than 33 bytes? */ + stp val, val, [dst, 64] + stp val, val, [dst, 80] +1: stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret + +END (MEMSET) +libc_hidden_builtin_def (MEMSET) + +#endif From patchwork Fri Oct 12 11:44:28 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feng Xue X-Patchwork-Id: 982999 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=sourceware.org (client-ip=209.132.180.131; helo=sourceware.org; envelope-from=libc-alpha-return-96390-incoming=patchwork.ozlabs.org@sourceware.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=hotmail.com Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; secure) header.d=sourceware.org header.i=@sourceware.org header.b="MXy51H8E"; dkim=pass (2048-bit key; unprotected) header.d=hotmail.com header.i=@hotmail.com header.b="Y55EzkBx"; dkim-atps=neutral Received: from sourceware.org (server1.sourceware.org [209.132.180.131]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 42WmGN4mPtz9s3Z for ; Fri, 12 Oct 2018 22:44:44 +1100 (AEDT) DomainKey-Signature: a=rsa-sha1; c=nofws; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id :content-type:content-transfer-encoding:mime-version; q=dns; s= default; b=vHazxJsLS6JmtZeGTPEGb3BV88/Z6R+olhXyhD74/txMrLc8MddW9 zDz/Iic3gL4QZrgkxOFYBS2pwNFMsCKDqvMr59u7CfjQ8pSvUG0nljKx38CxJxDr h6X3nGExE3d1xd2BN/B8bKl8b1mpG0Om9wV0D8nPegXSGjzvwQVXL0= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=sourceware.org; h=list-id :list-unsubscribe:list-subscribe:list-archive:list-post :list-help:sender:from:to:cc:subject:date:message-id :content-type:content-transfer-encoding:mime-version; s=default; bh=X5qutnywbT8fhygWRjHMgmtuo+w=; b=MXy51H8E1cGoLFpeptS87FPA//Pf /uzfU0I7yero1rs2MWIVFo8CC1aPIKwRgW0waylg1GSz6pt7G8yoZlyIQiWncviI hF7Fr52CE8TlC4RWZCI4uimUZ164NJtZvQqZd+tLkkBbbGe5JkNv0fkVxj6/jSMT ChTV2wZWdsdqqs8= Received: (qmail 130662 invoked by alias); 12 Oct 2018 11:44:38 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 130632 invoked by uid 89); 12 Oct 2018 11:44:37 -0000 Authentication-Results: sourceware.org; auth=none X-Spam-SWARE-Status: No, score=-26.0 required=5.0 tests=AWL, BAYES_00, FORGED_HOTMAIL_RCVD2, FREEMAIL_FROM, GIT_PATCH_0, GIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, KAM_SHORT, MIME_BASE64_BLANKS, RCVD_IN_DNSWL_NONE, SPF_HELO_PASS, SPF_PASS autolearn=ham version=3.3.2 spammy=H*c:gb2312 X-HELO: APC01-HK2-obe.outbound.protection.outlook.com DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=hotmail.com; s=selector1; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=/ttBaZdFJL4EHjQX7RMCsTV/RlnWp78dtW8dxVR+6bQ=; b=Y55EzkBxEbTLBEaethbghEzhws7RNlnqsR1T67z8N1vgmLrIW9YGgepY6nusz2hA8MRQRmUpti6eOKrZ+a5OK/hx7MYz+f0N0uPulxtvBfpWAPUxVVNc4P7HVogmB+K5aA8g4Oh+3Mvtht3gHkuX1DA+eyhHevhyWzDZRrJ/22oxJkaP2yr8uQ+vwwBLFqsFgl9LgF/5PpYC9TfI97/jCeH4/UJ7bknw5iqPGd5nMC/cb8hwQ6uFOgaFb3Fn7IToVbXGxvG/xwFZ85Q9oZJHNADspfd+ROOqMvr5JkHHdi4YyouyW70Pvi52xEo0EfFCNsfFshu9FbWTpv/iCrAKmA== From: Xue Feng To: "libc-alpha@sourceware.org" CC: "marcus.shawcroft@linaro.org" , "szabolcs.nagy@arm.com" , Feng Xue Subject: [PATCH 3/3] aarch64: Optimized memchr specific to AmpereComputing skylark Date: Fri, 12 Oct 2018 11:44:28 +0000 Message-ID: MIME-Version: 1.0 This version uses general register based memory instruction to load data, because vector register based is slightly slower in skylark. Character-matching is performed on 16-byte (both size and alignment) memory block in parallel each iteration. Another minor optimization is to preload next 16-byte block at the begin of each iteration, which can hide some memory access latency. * sysdeps/aarch64/memchr.S (__memchr): Rename to MEMCHR. [!MEMCHR](MEMCHR): Set to __memchr. * sysdeps/aarch64/multiarch/Makefile (sysdep_routines): Add memchr_generic and memchr_skylark. * sysdeps/aarch64/multiarch/ifunc-impl-list.c (__libc_ifunc_impl_list): Add memchr ifuncs. * sysdeps/aarch64/multiarch/memchr.c: New file. * sysdeps/aarch64/multiarch/memchr_generic.S: Likewise. * sysdeps/aarch64/multiarch/memchr_skylark.S: Likewise. --- ChangeLog | 12 ++ sysdeps/aarch64/memchr.S | 10 +- sysdeps/aarch64/multiarch/Makefile | 1 + sysdeps/aarch64/multiarch/ifunc-impl-list.c | 3 + sysdeps/aarch64/multiarch/memchr.c | 41 +++++ sysdeps/aarch64/multiarch/memchr_generic.S | 33 +++++ sysdeps/aarch64/multiarch/memchr_skylark.S | 222 ++++++++++++++++++++++++++++ 7 files changed, 319 insertions(+), 3 deletions(-) create mode 100644 sysdeps/aarch64/multiarch/memchr.c create mode 100644 sysdeps/aarch64/multiarch/memchr_generic.S create mode 100644 sysdeps/aarch64/multiarch/memchr_skylark.S diff --git a/ChangeLog b/ChangeLog index 28370f9..e64b8b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,15 @@ +2018-10-13 Feng Xue + + * sysdeps/aarch64/memchr.S (__memchr): Rename to MEMCHR. + [!MEMCHR](MEMCHR): Set to __memchr. + * sysdeps/aarch64/multiarch/Makefile (sysdep_routines): + Add memchr_generic and memchr_skylark. + * sysdeps/aarch64/multiarch/ifunc-impl-list.c + (__libc_ifunc_impl_list): Add memchr ifuncs. + * sysdeps/aarch64/multiarch/memchr.c: New file. + * sysdeps/aarch64/multiarch/memchr_generic.S: Likewise. + * sysdeps/aarch64/multiarch/memchr_skylark.S: Likewise. + 2018-10-12 Feng Xue * sysdeps/aarch64/multiarch/Makefile (sysdep_routines): diff --git a/sysdeps/aarch64/memchr.S b/sysdeps/aarch64/memchr.S index e422aef..4afebd3 100644 --- a/sysdeps/aarch64/memchr.S +++ b/sysdeps/aarch64/memchr.S @@ -26,6 +26,10 @@ * Neon Available. */ +#ifndef MEMCHR +# define MEMCHR __memchr +#endif + /* Arguments and results. */ #define srcin x0 #define chrin w1 @@ -59,7 +63,7 @@ * identify exactly which byte has matched. */ -ENTRY (__memchr) +ENTRY (MEMCHR) /* Do not dereference srcin if no bytes to compare. */ cbz cntin, L(zero_length) /* @@ -152,6 +156,6 @@ L(tail): L(zero_length): mov result, #0 ret -END (__memchr) -weak_alias (__memchr, memchr) +END (MEMCHR) +weak_alias (MEMCHR, memchr) libc_hidden_builtin_def (memchr) diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch/Makefile index 828ce4f..353ece7 100644 --- a/sysdeps/aarch64/multiarch/Makefile +++ b/sysdeps/aarch64/multiarch/Makefile @@ -2,5 +2,6 @@ ifeq ($(subdir),string) sysdep_routines += memcpy_generic memcpy_thunderx memcpy_thunderx2 \ memcpy_falkor memmove_falkor \ memset_generic memset_falkor memset_skylark \ + memchr_generic memchr_skylark \ strlen_generic strlen_asimd endif diff --git a/sysdeps/aarch64/multiarch/ifunc-impl-list.c b/sysdeps/aarch64/multiarch/ifunc-impl-list.c index baf01a0..f5014d2 100644 --- a/sysdeps/aarch64/multiarch/ifunc-impl-list.c +++ b/sysdeps/aarch64/multiarch/ifunc-impl-list.c @@ -53,6 +53,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_falkor) IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_skylark) IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic)) + IFUNC_IMPL (i, name, memchr, + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_skylark) + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_generic)) IFUNC_IMPL (i, name, strlen, IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_asimd) diff --git a/sysdeps/aarch64/multiarch/memchr.c b/sysdeps/aarch64/multiarch/memchr.c new file mode 100644 index 0000000..cbcf8b7 --- /dev/null +++ b/sysdeps/aarch64/multiarch/memchr.c @@ -0,0 +1,41 @@ +/* Multiple versions of memchr. AARCH64 version. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* Define multiple versions only for the definition in libc. */ + +#if IS_IN (libc) +/* Redefine memchr so that the compiler won't complain about the type + mismatch with the IFUNC selector in strong_alias, below. */ +# undef memchr +# define memchr __redirect_memchr +# include +# include + +extern __typeof (__redirect_memchr) __memchr; + +extern __typeof (__redirect_memchr) __memchr_generic attribute_hidden; +extern __typeof (__redirect_memchr) __memchr_skylark attribute_hidden; + +libc_ifunc (__memchr, + ((IS_SKYLARK (midr) + ? __memchr_skylark + : __memchr_generic))); + +# undef memchr +strong_alias (__memchr, memchr); +#endif diff --git a/sysdeps/aarch64/multiarch/memchr_generic.S b/sysdeps/aarch64/multiarch/memchr_generic.S new file mode 100644 index 0000000..707148b --- /dev/null +++ b/sysdeps/aarch64/multiarch/memchr_generic.S @@ -0,0 +1,33 @@ +/* Memchr for aarch64, default version for internal use. + Copyright (C) 2018 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#if IS_IN (libc) +# define MEMCHR __memchr_generic + +/* Do not hide the generic version of memchr, we use it internally. */ +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(name) + +/* Add a hidden definition for use within libc.so. */ +# ifdef SHARED + .globl __GI_memchr; __GI_memchr = __memchr_generic +# endif +#endif + +# include "../memchr.S" diff --git a/sysdeps/aarch64/multiarch/memchr_skylark.S b/sysdeps/aarch64/multiarch/memchr_skylark.S new file mode 100644 index 0000000..9277104 --- /dev/null +++ b/sysdeps/aarch64/multiarch/memchr_skylark.S @@ -0,0 +1,222 @@ +/* Optimized memchr for AmpereComputing skylark processor. + + Copyright (C) 2018 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include + +#if IS_IN (libc) +# define MEMCHR __memchr_skylark + +/* Arguments and results. */ +#define srcin x0 +#define chrin x1 +#define cntin x2 + +#define result x0 + +#define repchr x1 + +#define tmp1 x2 +#define tmp2 x3 +#define tmp3 x4 +#define tmp4 x5 + +#define src x6 +#define srcend x7 +#define srcend16 x8 + +#define zeroones x9 + +#define data1 x10 +#define data2 x11 +#define data1a x12 +#define data2a x13 + +#define has_chr1 x14 +#define has_chr2 x15 + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f + +ENTRY_ALIGN (MEMCHR, 6) + + DELOUSE (0) + DELOUSE (2) + + /* Do not dereference srcin if no bytes to compare. */ + cbz cntin, L(none_chr) + + bic src, srcin, 15 + /* + * Load the first 16-byte aligned block where memory to + * search starts. + */ + ldp data1, data2, [src] + + mov zeroones, REP8_01 + and repchr, chrin, 255 + /* Generate a qword integer as |c|c|c|c|c|c|c|c|. */ + mul repchr, repchr, zeroones + + add srcend, srcin, cntin + /* srcend16 is begin address of last 16-byte aligned block. */ + sub srcend, srcend, 1 + bic srcend16, srcend, 15 + + lsl tmp1, srcin, 3 + mov tmp2, ~0 +#ifdef __AARCH64EB__ + lsr tmp3, tmp2, tmp1 +#else + lsl tmp3, tmp2, tmp1 +#endif + /* Real starting point is in the first or the second qword? */ + tst srcin, 8 + + /* + * Transform any byte in the 16-byte block, which equals to the + * specified char, to zero using xor operation. In this way, + * searching the char becomes detecting zero in the resulting + * two qword integers. + */ + eor data1a, data1, repchr + eor data2a, data2, repchr + + /* + * Set those unused bytes(before the real starting point) to + * 0xff, so that they will not hit any zero detection. + */ + orn tmp1, data1a, tmp3 + orn tmp2, data2a, tmp3 + + csinv data1a, tmp1, xzr, eq + csel data2a, data2a, tmp2, eq + + cmp src, srcend16 + b.eq L(last) + +L(loop): + /* + * Preload the next 16-byte aligned block to hide some memory + * access latency. + */ + ldp data1, data2, [src, 16]! + + /* + * Use the following integer test to find out if any byte in a + * qword is zero. If do not contain zero-valued byte, test result + * is zero. + * + * (qword - 0x0101010101010101) & ~(qword) & 0x8080808080808080 + * = + * (qword - 0x0101010101010101) & ~(qword | 0x7f7f7f7f7f7f7f7f) + * + */ + sub tmp1, data1a, zeroones + sub tmp2, data2a, zeroones + + orr tmp3, data1a, REP8_7f + orr tmp4, data2a, REP8_7f + + bics has_chr1, tmp1, tmp3 + bic has_chr2, tmp2, tmp4 + + ccmp has_chr2, 0, 0, eq /* NZCV = 0000 */ + ccmp src, srcend16, 4, eq /* NZCV = 0100 */ + + /* + * Transform any byte in next 16-byte aligned block, which equals + * to the specified char, to zero using xor operation. + */ + eor data1a, data1, repchr + eor data2a, data2, repchr + + /* + * Here use equal-or-not comparison, instead of less/more, to + * make it work even for case of address wrap-around. + */ + b.ne L(loop) + + cbz has_chr1, 1f + sub src, src, 16 +#ifdef __AARCH64EB__ + /* + * For big-endian, can not directly use has_chr1/has_chr2 because + * two qwords has been reversed after loading from memory. + * Thus, one way is to restore two qwords in some means, and + * byte-swap them, then perform char detection on them. + */ + add data2a, tmp1, zeroones + b L(find_chr) +#else + b L(done) +#endif + +1: cbz has_chr2, L(last) + sub src, src, 8 +#ifdef __AARCH64EB__ + add data2a, tmp2, zeroones + b L(find_chr) +#else + mov has_chr1, has_chr2 + b L(done) +#endif + +L(last): +#ifdef __AARCH64EB__ + rev data1a, data1a +#endif + sub tmp1, data1a, zeroones + orr tmp3, data1a, REP8_7f + bics has_chr1, tmp1, tmp3 + b.ne L(done) + + add src, src, 8 +#ifdef __AARCH64EB__ +L(find_chr): + rev data2a, data2a +#endif + sub tmp1, data2a, zeroones + orr tmp3, data2a, REP8_7f + bics has_chr1, tmp1, tmp3 + b.eq L(none_chr) + +L(done): + /* + * If the specified char is found in a qword, the corresponding + * byte of in has_chr has value of 1, while this is only true for + * the first occurence not all. + */ + rev has_chr1, has_chr1 + bic tmp1, src, 15 + cmp tmp1, srcend16 /* At the last 16-byte aligned block? */ + clz tmp2, has_chr1 + add src, src, tmp2, lsr 3 + ccmp src, srcend, 0, eq /* NZCV = 0000 */ + csel result, src, xzr, ls + ret + +L(none_chr): + mov result, 0 + ret + +END (MEMCHR) +libc_hidden_builtin_def (MEMCHR) + +#endif