From patchwork Wed Nov 11 11:07:21 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nicholas Piggin X-Patchwork-Id: 1398171 Return-Path: X-Original-To: patchwork-incoming@ozlabs.org Delivered-To: patchwork-incoming@ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [203.11.71.2]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4CWMgw3hggz9sVk for ; Wed, 11 Nov 2020 22:19:24 +1100 (AEDT) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=gmail.com Authentication-Results: ozlabs.org; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20161025 header.b=fWEIRK3i; dkim-atps=neutral Received: from bilbo.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 4CWMgt2RvLzDqbc for ; Wed, 11 Nov 2020 22:19:22 +1100 (AEDT) X-Original-To: linuxppc-dev@lists.ozlabs.org Delivered-To: linuxppc-dev@lists.ozlabs.org Authentication-Results: lists.ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=gmail.com (client-ip=2607:f8b0:4864:20::442; helo=mail-pf1-x442.google.com; envelope-from=npiggin@gmail.com; receiver=) Authentication-Results: lists.ozlabs.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: lists.ozlabs.org; dkim=pass (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20161025 header.b=fWEIRK3i; dkim-atps=neutral Received: from mail-pf1-x442.google.com (mail-pf1-x442.google.com [IPv6:2607:f8b0:4864:20::442]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.ozlabs.org (Postfix) with ESMTPS id 4CWMQN04GnzDqb1 for ; Wed, 11 Nov 2020 22:07:39 +1100 (AEDT) Received: by mail-pf1-x442.google.com with SMTP id y7so1383987pfq.11 for ; Wed, 11 Nov 2020 03:07:39 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=YaRNVN7IQp0J42DlqO/RdFB7HR0Qo59Y85JYdPkkOEM=; b=fWEIRK3iIhUBZSJpMSOtGh80z4bLXIAmmbUdeEGioPIGs7naWD+ncx2Zcz6xhUt2wq 1TN6XwTMLl82pYCdwh3i2xLnm5s2YF2B69fp2kt8dUeARS9Vcx8/v8HdzvXFj6sVkbjc d04FYVE6FJrp63gOQ12+EpQb0OIyqStFLrB+FCAEPTye3Yk1qfpg3sQ+PUhIgzkwQFEN DldxPGBUQYE4JWUmq7gOOpc4vlXTYld7zzY4l06gugPpAf1l4+Lpn9YsTxHgcJ+kkmRg uRO1AfZ2R13T37eZYhjcm0FlkkrgVCzfqy5QyT2Wi7eD7qy1oCabrOD5VRdddzIieiiL Oy4A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=YaRNVN7IQp0J42DlqO/RdFB7HR0Qo59Y85JYdPkkOEM=; b=HRcn1HwGUwu2RTj49aSUAnyHygLlvbUCPsBtDcR7HYmf1t+yM4d6uncBUdC8GIEv6R eHrMCXg7+EDNUHLh0g8n0YZ/sOwDITCVpvdTepGBWAXObRch1DPcbUUW0CshDPK1ec3m c9zDPukIUoa30QcBneE6D+WvcD0fBIxz+8rGDtL5bfnfZ3+VZtilJbLL/mUC+hzulpdy Yr7YuPYmVQioJjeOSI33f9LfNNFWWV+o5ld6+n4nhLfujb41n8dZb7PIYLRJPb7wgy7S MxgnZ/LMjWkdp8oTSNBWR9MJqsoo3QF7VVg7IuuTsUUQNj/IbbPl2WXLd0exlurqQEND r6dg== X-Gm-Message-State: AOAM531V0xn3gxQJ4Q6Z/7AYlomfOHgrPkpwsNzNQESIqOV2GjPep3Wc pDpsVKghBw0PU364bmgybAa6LHB4A+A= X-Google-Smtp-Source: ABdhPJwP8uQQpmy1oaw81lNCVlsjzZpMF4ckneC9iNjwb10hHnlFsPcz7ZW8ntP8YCetS0IAZDYTuA== X-Received: by 2002:aa7:838b:0:b029:18c:42ad:7721 with SMTP id u11-20020aa7838b0000b029018c42ad7721mr9010716pfm.15.1605092856784; Wed, 11 Nov 2020 03:07:36 -0800 (PST) Received: from bobo.ozlabs.ibm.com (27-32-36-31.tpgi.com.au. [27.32.36.31]) by smtp.gmail.com with ESMTPSA id 9sm2154943pfp.102.2020.11.11.03.07.32 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 11 Nov 2020 03:07:36 -0800 (PST) From: Nicholas Piggin To: linuxppc-dev@lists.ozlabs.org Subject: [PATCH 1/3] asm-generic/atomic64: Add support for ARCH_ATOMIC Date: Wed, 11 Nov 2020 21:07:21 +1000 Message-Id: <20201111110723.3148665-2-npiggin@gmail.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20201111110723.3148665-1-npiggin@gmail.com> References: <20201111110723.3148665-1-npiggin@gmail.com> MIME-Version: 1.0 X-BeenThere: linuxppc-dev@lists.ozlabs.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Linux on PowerPC Developers Mail List List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Christophe Leroy , linux-arch@vger.kernel.org, Arnd Bergmann , Peter Zijlstra , Boqun Feng , linux-kernel@vger.kernel.org, Nicholas Piggin , Alexey Kardashevskiy , Will Deacon Errors-To: linuxppc-dev-bounces+patchwork-incoming=ozlabs.org@lists.ozlabs.org Sender: "Linuxppc-dev" This passes atomic64 selftest on ppc32 on qemu (uniprocessor only) both before and after powerpc is converted to use ARCH_ATOMIC. Signed-off-by: Nicholas Piggin --- include/asm-generic/atomic64.h | 70 +++++++++++++++++++++++++++------- lib/atomic64.c | 36 ++++++++--------- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h index 370f01d4450f..2b1ecb591bb9 100644 --- a/include/asm-generic/atomic64.h +++ b/include/asm-generic/atomic64.h @@ -15,19 +15,17 @@ typedef struct { #define ATOMIC64_INIT(i) { (i) } -extern s64 atomic64_read(const atomic64_t *v); -extern void atomic64_set(atomic64_t *v, s64 i); - -#define atomic64_set_release(v, i) atomic64_set((v), (i)) +extern s64 __atomic64_read(const atomic64_t *v); +extern void __atomic64_set(atomic64_t *v, s64 i); #define ATOMIC64_OP(op) \ -extern void atomic64_##op(s64 a, atomic64_t *v); +extern void __atomic64_##op(s64 a, atomic64_t *v); #define ATOMIC64_OP_RETURN(op) \ -extern s64 atomic64_##op##_return(s64 a, atomic64_t *v); +extern s64 __atomic64_##op##_return(s64 a, atomic64_t *v); #define ATOMIC64_FETCH_OP(op) \ -extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); +extern s64 __atomic64_fetch_##op(s64 a, atomic64_t *v); #define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op) @@ -46,11 +44,57 @@ ATOMIC64_OPS(xor) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -extern s64 atomic64_dec_if_positive(atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive -extern s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n); -extern s64 atomic64_xchg(atomic64_t *v, s64 new); -extern s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u); -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +extern s64 __atomic64_dec_if_positive(atomic64_t *v); +extern s64 __atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n); +extern s64 __atomic64_xchg(atomic64_t *v, s64 new); +extern s64 __atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u); + +#ifdef ARCH_ATOMIC +#define arch_atomic64_read __atomic64_read +#define arch_atomic64_set __atomic64_set +#define arch_atomic64_add __atomic64_add +#define arch_atomic64_add_return __atomic64_add_return +#define arch_atomic64_fetch_add __atomic64_fetch_add +#define arch_atomic64_sub __atomic64_sub +#define arch_atomic64_sub_return __atomic64_sub_return +#define arch_atomic64_fetch_sub __atomic64_fetch_sub +#define arch_atomic64_and __atomic64_and +#define arch_atomic64_and_return __atomic64_and_return +#define arch_atomic64_fetch_and __atomic64_fetch_and +#define arch_atomic64_or __atomic64_or +#define arch_atomic64_or_return __atomic64_or_return +#define arch_atomic64_fetch_or __atomic64_fetch_or +#define arch_atomic64_xor __atomic64_xor +#define arch_atomic64_xor_return __atomic64_xor_return +#define arch_atomic64_fetch_xor __atomic64_fetch_xor +#define arch_atomic64_xchg __atomic64_xchg +#define arch_atomic64_cmpxchg __atomic64_cmpxchg +#define arch_atomic64_set_release(v, i) __atomic64_set((v), (i)) +#define arch_atomic64_dec_if_positive __atomic64_dec_if_positive +#define arch_atomic64_fetch_add_unless __atomic64_fetch_add_unless +#else +#define atomic64_read __atomic64_read +#define atomic64_set __atomic64_set +#define atomic64_add __atomic64_add +#define atomic64_add_return __atomic64_add_return +#define atomic64_fetch_add __atomic64_fetch_add +#define atomic64_sub __atomic64_sub +#define atomic64_sub_return __atomic64_sub_return +#define atomic64_fetch_sub __atomic64_fetch_sub +#define atomic64_and __atomic64_and +#define atomic64_and_return __atomic64_and_return +#define atomic64_fetch_and __atomic64_fetch_and +#define atomic64_or __atomic64_or +#define atomic64_or_return __atomic64_or_return +#define atomic64_fetch_or __atomic64_fetch_or +#define atomic64_xor __atomic64_xor +#define atomic64_xor_return __atomic64_xor_return +#define atomic64_fetch_xor __atomic64_fetch_xor +#define atomic64_xchg __atomic64_xchg +#define atomic64_cmpxchg __atomic64_cmpxchg +#define atomic64_set_release(v, i) __atomic64_set((v), (i)) +#define atomic64_dec_if_positive __atomic64_dec_if_positive +#define atomic64_fetch_add_unless __atomic64_fetch_add_unless +#endif #endif /* _ASM_GENERIC_ATOMIC64_H */ diff --git a/lib/atomic64.c b/lib/atomic64.c index e98c85a99787..05aba5e3268f 100644 --- a/lib/atomic64.c +++ b/lib/atomic64.c @@ -42,7 +42,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v) return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; } -s64 atomic64_read(const atomic64_t *v) +s64 __atomic64_read(const atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -53,9 +53,9 @@ s64 atomic64_read(const atomic64_t *v) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_read); +EXPORT_SYMBOL(__atomic64_read); -void atomic64_set(atomic64_t *v, s64 i) +void __atomic64_set(atomic64_t *v, s64 i) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -64,10 +64,10 @@ void atomic64_set(atomic64_t *v, s64 i) v->counter = i; raw_spin_unlock_irqrestore(lock, flags); } -EXPORT_SYMBOL(atomic64_set); +EXPORT_SYMBOL(__atomic64_set); #define ATOMIC64_OP(op, c_op) \ -void atomic64_##op(s64 a, atomic64_t *v) \ +void __atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -76,10 +76,10 @@ void atomic64_##op(s64 a, atomic64_t *v) \ v->counter c_op a; \ raw_spin_unlock_irqrestore(lock, flags); \ } \ -EXPORT_SYMBOL(atomic64_##op); +EXPORT_SYMBOL(__atomic64_##op); #define ATOMIC64_OP_RETURN(op, c_op) \ -s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +s64 __atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -90,10 +90,10 @@ s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ raw_spin_unlock_irqrestore(lock, flags); \ return val; \ } \ -EXPORT_SYMBOL(atomic64_##op##_return); +EXPORT_SYMBOL(__atomic64_##op##_return); #define ATOMIC64_FETCH_OP(op, c_op) \ -s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +s64 __atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -105,7 +105,7 @@ s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ raw_spin_unlock_irqrestore(lock, flags); \ return val; \ } \ -EXPORT_SYMBOL(atomic64_fetch_##op); +EXPORT_SYMBOL(__atomic64_fetch_##op); #define ATOMIC64_OPS(op, c_op) \ ATOMIC64_OP(op, c_op) \ @@ -130,7 +130,7 @@ ATOMIC64_OPS(xor, ^=) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -s64 atomic64_dec_if_positive(atomic64_t *v) +s64 __atomic64_dec_if_positive(atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -143,9 +143,9 @@ s64 atomic64_dec_if_positive(atomic64_t *v) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_dec_if_positive); +EXPORT_SYMBOL(__atomic64_dec_if_positive); -s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) +s64 __atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -158,9 +158,9 @@ s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_cmpxchg); +EXPORT_SYMBOL(__atomic64_cmpxchg); -s64 atomic64_xchg(atomic64_t *v, s64 new) +s64 __atomic64_xchg(atomic64_t *v, s64 new) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -172,9 +172,9 @@ s64 atomic64_xchg(atomic64_t *v, s64 new) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_xchg); +EXPORT_SYMBOL(__atomic64_xchg); -s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +s64 __atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -188,4 +188,4 @@ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return val; } -EXPORT_SYMBOL(atomic64_fetch_add_unless); +EXPORT_SYMBOL(__atomic64_fetch_add_unless); From patchwork Wed Nov 11 11:07:22 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nicholas Piggin X-Patchwork-Id: 1398175 Return-Path: X-Original-To: patchwork-incoming@ozlabs.org Delivered-To: patchwork-incoming@ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [203.11.71.2]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4CWMn71BMPz9sRK for ; Wed, 11 Nov 2020 22:23:55 +1100 (AEDT) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=gmail.com Authentication-Results: ozlabs.org; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20161025 header.b=eGYwfQVY; dkim-atps=neutral Received: from bilbo.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 4CWMn66qKjzDqfk for ; Wed, 11 Nov 2020 22:23:54 +1100 (AEDT) X-Original-To: linuxppc-dev@lists.ozlabs.org Delivered-To: linuxppc-dev@lists.ozlabs.org Authentication-Results: lists.ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=gmail.com (client-ip=2607:f8b0:4864:20::1043; helo=mail-pj1-x1043.google.com; envelope-from=npiggin@gmail.com; receiver=) Authentication-Results: lists.ozlabs.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: lists.ozlabs.org; dkim=pass (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20161025 header.b=eGYwfQVY; dkim-atps=neutral Received: from mail-pj1-x1043.google.com (mail-pj1-x1043.google.com [IPv6:2607:f8b0:4864:20::1043]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.ozlabs.org (Postfix) with ESMTPS id 4CWMQT0x4nzDqb0 for ; Wed, 11 Nov 2020 22:07:45 +1100 (AEDT) Received: by mail-pj1-x1043.google.com with SMTP id gi3so566771pjb.3 for ; Wed, 11 Nov 2020 03:07:45 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=i9QuJLEjIPLpDeuvL6cggUWm+kIo+T8XFZuMVw3L5OE=; b=eGYwfQVYMWlAeg1rJiTF6BgSFNuFxWfjnStIkSCh3ghGiJ5LRQ4+acK9lN8qdE+YHi KRGeNNjgbVzznUCqy9/CLybZMT9+o5cOykz8iBxcZSJ6bDFNbL/81S8/WhKwejAjt1uR 57hjA36rzhWWwof23dU+U+Id5oDepo2753RBTq9PF2+mwbKcPhNLuTrIibXVJcqXOorH 5uNA1Iili2Rxg+H5NxyIIB6cu7EfXSSCnGIe9fminLaYZ7SL+GXp2SIQ+FJG4m+yCIBy y2HtxDSDPDayXiDd7W3Yxr8MHZAybkTVZD9PMIXbCm8svJDZSoh2j7TL9K33XfemsfOy NJUA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=i9QuJLEjIPLpDeuvL6cggUWm+kIo+T8XFZuMVw3L5OE=; b=skKl7INxXswAv1Yl4r1497CfEbKNfxT0RfKk7HJRsjoLi3kD/YV6l8qxWmP+SQ75R9 RMwIXbY3tInAjN0duGzlM99g7zZa62Gfwso/IDcq5o2JmrFFjha/2Ldu74fXlW5m2+Vl lz55tKRw5ETVsiOwTKV6U0dM9/RDCj4nzd2c7ut7YXrOaBIXx+IJGo9HftSJdnZFpHmJ akUozlcbiDlMIykP0YECNG5HGa74nI0245dkPT5U7n9H8jfFqTJFUuZp0vLsmBh5FxhO YocmBh55tFAULe9/f73pGq30Czqk6EWi0MTrnU3CIRlk+aoEXd0rkkKvhl4tEnJXUHKD +TpQ== X-Gm-Message-State: AOAM53159KZ1wZbdoRcv/PIOFNMg4tAVz/jFQOI2qLIkVaX3Ulhmzjza HUXmKcYZd8+mscVhhiooNmpn4i1KeXU= X-Google-Smtp-Source: ABdhPJwaXv990DxU17tDbiasD9ZoY0rRxQ452QK0z4AJVdpOnJ0xgah0UQo8IYOY/9879g2TvzstLQ== X-Received: by 2002:a17:902:fe81:b029:d6:8677:6741 with SMTP id x1-20020a170902fe81b02900d686776741mr20695174plm.79.1605092861472; Wed, 11 Nov 2020 03:07:41 -0800 (PST) Received: from bobo.ozlabs.ibm.com (27-32-36-31.tpgi.com.au. [27.32.36.31]) by smtp.gmail.com with ESMTPSA id 9sm2154943pfp.102.2020.11.11.03.07.37 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 11 Nov 2020 03:07:41 -0800 (PST) From: Nicholas Piggin To: linuxppc-dev@lists.ozlabs.org Subject: [PATCH 2/3] powerpc/64s/iommu: don't use atomic_ function on atomic64_t type Date: Wed, 11 Nov 2020 21:07:22 +1000 Message-Id: <20201111110723.3148665-3-npiggin@gmail.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20201111110723.3148665-1-npiggin@gmail.com> References: <20201111110723.3148665-1-npiggin@gmail.com> MIME-Version: 1.0 X-BeenThere: linuxppc-dev@lists.ozlabs.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Linux on PowerPC Developers Mail List List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Christophe Leroy , linux-arch@vger.kernel.org, Arnd Bergmann , Peter Zijlstra , Boqun Feng , linux-kernel@vger.kernel.org, Nicholas Piggin , Alexey Kardashevskiy , Will Deacon Errors-To: linuxppc-dev-bounces+patchwork-incoming=ozlabs.org@lists.ozlabs.org Sender: "Linuxppc-dev" Signed-off-by: Nicholas Piggin --- arch/powerpc/mm/book3s64/iommu_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/mm/book3s64/iommu_api.c b/arch/powerpc/mm/book3s64/iommu_api.c index 563faa10bb66..685d7bb3d26f 100644 --- a/arch/powerpc/mm/book3s64/iommu_api.c +++ b/arch/powerpc/mm/book3s64/iommu_api.c @@ -263,7 +263,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem) goto unlock_exit; /* Are there still mappings? */ - if (atomic_cmpxchg(&mem->mapped, 1, 0) != 1) { + if (atomic64_cmpxchg(&mem->mapped, 1, 0) != 1) { ++mem->used; ret = -EBUSY; goto unlock_exit; From patchwork Wed Nov 11 11:07:23 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nicholas Piggin X-Patchwork-Id: 1398176 Return-Path: X-Original-To: patchwork-incoming@ozlabs.org Delivered-To: patchwork-incoming@ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4CWMst3fpmz9sVK for ; Wed, 11 Nov 2020 22:28:02 +1100 (AEDT) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=gmail.com Authentication-Results: ozlabs.org; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20161025 header.b=hoPLSr65; dkim-atps=neutral Received: from bilbo.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 4CWMsr6QmDzDqfK for ; Wed, 11 Nov 2020 22:28:00 +1100 (AEDT) X-Original-To: linuxppc-dev@lists.ozlabs.org Delivered-To: linuxppc-dev@lists.ozlabs.org Authentication-Results: lists.ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=gmail.com (client-ip=2607:f8b0:4864:20::1043; helo=mail-pj1-x1043.google.com; envelope-from=npiggin@gmail.com; receiver=) Authentication-Results: lists.ozlabs.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: lists.ozlabs.org; dkim=pass (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20161025 header.b=hoPLSr65; dkim-atps=neutral Received: from mail-pj1-x1043.google.com (mail-pj1-x1043.google.com [IPv6:2607:f8b0:4864:20::1043]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.ozlabs.org (Postfix) with ESMTPS id 4CWMQY37yPzDqb1 for ; Wed, 11 Nov 2020 22:07:49 +1100 (AEDT) Received: by mail-pj1-x1043.google.com with SMTP id oc3so568247pjb.4 for ; Wed, 11 Nov 2020 03:07:49 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=/UIW1gAUzBEzTUOsAcWn7qW7OyNqnASsw4lDTZl0FWE=; b=hoPLSr65fhCBlKT1VNnZGlpp8320ybiNzghiUtSrUpv4/rDqBW0OKg+vyPC1uwlNSt sWE0BQKb4meLQW4OLVtf3emKnHCI+B5XOgIz2Vn2yjZ0yTg1OUtwbrcI7if+dAVk0uwp Mlsp/KMqIOjw8jQ8nskLyJlTWyk9itq0T98mbAWoft+44ItCTUVz/XrDoZsFSDqidgav N37EIWcUXMLohDADRbySWIj0HW3QvAsxH4j6A761Q/Gh9K/B1tLC/2iyNileIB8bSlYZ HcsCUJvoAM0z17ikkTSmN4fYl1CF3ijhlA2tRQKokAvkbwGo6EnpNUPRzLjfk78EwES8 UcOw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=/UIW1gAUzBEzTUOsAcWn7qW7OyNqnASsw4lDTZl0FWE=; b=dGDjIUXzmc87x5O8HZ5qUN/cmTnlosSRUqfmDFE3K1Yq3CXGeHW56fOKspyqeAc+0R /KmWc29wmzfjkGrHkH49jjWGQiwEEoNzoG3r1ESv0s1OD9KXtcnljEMa5YMx+z+muKaD VvGKImexkaZiNdgi9xd8zkqDONHUlvC3j4Ehh5LPN1BiHvpwK9wrXaYi7U1DAEM2iZ12 yRO1E7MVVyvFr0Foc5YJ5JdX4hlrm7jfGGvbHBEchjYosljS8+y6iyLWOUlibERtFuUB YJhmrmBAdp0QqPoTQxC5R47XAgyL6xw5VB6K5ddnqMQP6DDHNpMy1u9az7f/mRz+S2em 2EEQ== X-Gm-Message-State: AOAM531QF8P7VFSrLaHZlUsFGnmRsoZam028L00hO4QRm5z5dOQHmY4m 7aYI/5fFbrR8hdOwP/VN2RbIUibzlZ0= X-Google-Smtp-Source: ABdhPJwr0jUOBgYEsvk/u4lRLIWKVDdyqiy0TRizYXEn5puj5GHS6DqNpyDGCk5ZX5dL8sGMDVajEg== X-Received: by 2002:a17:902:26a:b029:d6:caca:620a with SMTP id 97-20020a170902026ab02900d6caca620amr21811870plc.46.1605092866694; Wed, 11 Nov 2020 03:07:46 -0800 (PST) Received: from bobo.ozlabs.ibm.com (27-32-36-31.tpgi.com.au. [27.32.36.31]) by smtp.gmail.com with ESMTPSA id 9sm2154943pfp.102.2020.11.11.03.07.41 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 11 Nov 2020 03:07:46 -0800 (PST) From: Nicholas Piggin To: linuxppc-dev@lists.ozlabs.org Subject: [PATCH 3/3] powerpc: rewrite atomics to use ARCH_ATOMIC Date: Wed, 11 Nov 2020 21:07:23 +1000 Message-Id: <20201111110723.3148665-4-npiggin@gmail.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20201111110723.3148665-1-npiggin@gmail.com> References: <20201111110723.3148665-1-npiggin@gmail.com> MIME-Version: 1.0 X-BeenThere: linuxppc-dev@lists.ozlabs.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Linux on PowerPC Developers Mail List List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Christophe Leroy , linux-arch@vger.kernel.org, Arnd Bergmann , Peter Zijlstra , Boqun Feng , linux-kernel@vger.kernel.org, Nicholas Piggin , Alexey Kardashevskiy , Will Deacon Errors-To: linuxppc-dev-bounces+patchwork-incoming=ozlabs.org@lists.ozlabs.org Sender: "Linuxppc-dev" All the cool kids are doing it. Signed-off-by: Nicholas Piggin Reported-by: kernel test robot Reported-by: kernel test robot --- arch/powerpc/include/asm/atomic.h | 681 ++++++++++------------------- arch/powerpc/include/asm/cmpxchg.h | 62 +-- 2 files changed, 248 insertions(+), 495 deletions(-) diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 8a55eb8cc97b..899aa2403ba7 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -11,185 +11,285 @@ #include #include +#define ARCH_ATOMIC + +#ifndef CONFIG_64BIT +#include +#endif + /* * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with * a "bne-" instruction at the end, so an isync is enough as a acquire barrier * on the platform without lwsync. */ #define __atomic_acquire_fence() \ - __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") + asm volatile(PPC_ACQUIRE_BARRIER "" : : : "memory") #define __atomic_release_fence() \ - __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") + asm volatile(PPC_RELEASE_BARRIER "" : : : "memory") -static __inline__ int atomic_read(const atomic_t *v) -{ - int t; +#define __atomic_pre_full_fence smp_mb - __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); +#define __atomic_post_full_fence smp_mb - return t; +#define arch_atomic_read(v) __READ_ONCE((v)->counter) +#define arch_atomic_set(v, i) __WRITE_ONCE(((v)->counter), (i)) +#ifdef CONFIG_64BIT +#define ATOMIC64_INIT(i) { (i) } +#define arch_atomic64_read(v) __READ_ONCE((v)->counter) +#define arch_atomic64_set(v, i) __WRITE_ONCE(((v)->counter), (i)) +#endif + +#define ATOMIC_OP(name, type, dtype, width, asm_op) \ +static inline void arch_##name(dtype a, type *v) \ +{ \ + dtype t; \ + \ + asm volatile( \ +"1: l" #width "arx %0,0,%3 # " #name "\n" \ +"\t" #asm_op " %0,%2,%0 \n" \ +" st" #width "cx. %0,0,%3 \n" \ +" bne- 1b \n" \ + : "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (&v->counter) \ + : "cr0", "xer"); \ } -static __inline__ void atomic_set(atomic_t *v, int i) -{ - __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); +#define ATOMIC_OP_IMM(name, type, dtype, width, asm_op, imm) \ +static inline void arch_##name(type *v) \ +{ \ + dtype t; \ + \ + asm volatile( \ +"1: l" #width "arx %0,0,%3 # " #name "\n" \ +"\t" #asm_op " %0,%0,%2 \n" \ +" st" #width "cx. %0,0,%3 \n" \ +" bne- 1b \n" \ + : "=&r" (t), "+m" (v->counter) \ + : "i" (imm), "r" (&v->counter) \ + : "cr0", "xer"); \ } -#define ATOMIC_OP(op, asm_op) \ -static __inline__ void atomic_##op(int a, atomic_t *v) \ +#define ATOMIC_OP_RETURN_RELAXED(name, type, dtype, width, asm_op) \ +static inline dtype arch_##name##_relaxed(dtype a, type *v) \ { \ - int t; \ + dtype t; \ \ - __asm__ __volatile__( \ -"1: lwarx %0,0,%3 # atomic_" #op "\n" \ - #asm_op " %0,%2,%0\n" \ -" stwcx. %0,0,%3 \n" \ -" bne- 1b\n" \ + asm volatile( \ +"1: l" #width "arx %0,0,%3 # " #name "\n" \ +"\t" #asm_op " %0,%2,%0 \n" \ +" st" #width "cx. %0,0,%3 \n" \ +" bne- 1b \n" \ : "=&r" (t), "+m" (v->counter) \ : "r" (a), "r" (&v->counter) \ - : "cc"); \ -} \ + : "cr0", "xer"); \ + \ + return t; \ +} -#define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ -static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ +#define ATOMIC_OP_IMM_RETURN_RELAXED(name, type, dtype, width, asm_op, imm) \ +static inline dtype arch_##name##_relaxed(type *v) \ { \ - int t; \ + dtype t; \ \ - __asm__ __volatile__( \ -"1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ - #asm_op " %0,%2,%0\n" \ -" stwcx. %0,0,%3\n" \ -" bne- 1b\n" \ + asm volatile( \ +"1: l" #width "arx %0,0,%3 # " #name "\n" \ +"\t" #asm_op " %0,%0,%2 \n" \ +" st" #width "cx. %0,0,%3 \n" \ +" bne- 1b \n" \ : "=&r" (t), "+m" (v->counter) \ - : "r" (a), "r" (&v->counter) \ - : "cc"); \ + : "i" (imm), "r" (&v->counter) \ + : "cr0", "xer"); \ \ return t; \ } -#define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ -static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ +#define ATOMIC_FETCH_OP_RELAXED(name, type, dtype, width, asm_op) \ +static inline dtype arch_##name##_relaxed(dtype a, type *v) \ { \ - int res, t; \ + dtype res, t; \ \ - __asm__ __volatile__( \ -"1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ - #asm_op " %1,%3,%0\n" \ -" stwcx. %1,0,%4\n" \ -" bne- 1b\n" \ + asm volatile( \ +"1: l" #width "arx %0,0,%4 # " #name "\n" \ +"\t" #asm_op " %1,%3,%0 \n" \ +" st" #width "cx. %1,0,%4 \n" \ +" bne- 1b \n" \ : "=&r" (res), "=&r" (t), "+m" (v->counter) \ : "r" (a), "r" (&v->counter) \ - : "cc"); \ + : "cr0", "xer"); \ \ return res; \ } +#define ATOMIC_FETCH_OP_UNLESS_RELAXED(name, type, dtype, width, asm_op) \ +static inline int arch_##name##_relaxed(type *v, dtype a, dtype u) \ +{ \ + dtype res, t; \ + \ + asm volatile( \ +"1: l" #width "arx %0,0,%5 # " #name "\n" \ +" cmp" #width " 0,%0,%3 \n" \ +" beq- 2f \n" \ +"\t" #asm_op " %1,%2,%0 \n" \ +" st" #width "cx. %1,0,%5 \n" \ +" bne- 1b \n" \ +"2: \n" \ + : "=&r" (res), "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (u), "r" (&v->counter) \ + : "cr0", "xer"); \ + \ + return res; \ +} + +#define ATOMIC_INC_NOT_ZERO_RELAXED(name, type, dtype, width) \ +static inline dtype arch_##name##_relaxed(type *v) \ +{ \ + dtype t1, t2; \ + \ + asm volatile( \ +"1: l" #width "arx %0,0,%3 # " #name "\n" \ +" cmp" #width "i 0,%0,0 \n" \ +" beq- 2f \n" \ +" addic %1,%2,1 \n" \ +" st" #width "cx. %1,0,%3 \n" \ +" bne- 1b \n" \ +"2: \n" \ + : "=&r" (t1), "=&r" (t2), "+m" (v->counter) \ + : "r" (&v->counter) \ + : "cr0", "xer"); \ + \ + return t1; \ +} + +#undef ATOMIC_OPS #define ATOMIC_OPS(op, asm_op) \ - ATOMIC_OP(op, asm_op) \ - ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ - ATOMIC_FETCH_OP_RELAXED(op, asm_op) +ATOMIC_OP(atomic_##op, atomic_t, int, w, asm_op) \ +ATOMIC_OP_RETURN_RELAXED(atomic_##op##_return, atomic_t, int, w, asm_op) \ +ATOMIC_FETCH_OP_RELAXED(atomic_fetch_##op, atomic_t, int, w, asm_op) \ +ATOMIC_FETCH_OP_UNLESS_RELAXED(atomic_fetch_##op##_unless, atomic_t, int, w, asm_op) + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, asm_op) \ +ATOMIC_OP(atomic64_##op, atomic64_t, u64, d, asm_op) \ +ATOMIC_OP_RETURN_RELAXED(atomic64_##op##_return, atomic64_t, u64, d, asm_op) \ +ATOMIC_FETCH_OP_RELAXED(atomic64_fetch_##op, atomic64_t, u64, d, asm_op) \ +ATOMIC_FETCH_OP_UNLESS_RELAXED(atomic64_fetch_##op##_unless, atomic64_t, u64, d, asm_op) ATOMIC_OPS(add, add) +#define arch_atomic_add arch_atomic_add +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_add_unless_relaxed arch_atomic_fetch_add_unless_relaxed + ATOMIC_OPS(sub, subf) +#define arch_atomic_sub arch_atomic_sub +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed +/* skip atomic_fetch_sub_unless_relaxed */ -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#ifdef CONFIG_64BIT +ATOMIC64_OPS(add, add) +#define arch_atomic64_add arch_atomic64_add +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_add_unless_relaxed arch_atomic64_fetch_add_unless_relaxed -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +ATOMIC64_OPS(sub, subf) +#define arch_atomic64_sub arch_atomic64_sub +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed +/* skip atomic64_fetch_sub_unless_relaxed */ +#endif #undef ATOMIC_OPS #define ATOMIC_OPS(op, asm_op) \ - ATOMIC_OP(op, asm_op) \ - ATOMIC_FETCH_OP_RELAXED(op, asm_op) +ATOMIC_OP(atomic_##op, atomic_t, int, w, asm_op) \ +ATOMIC_FETCH_OP_RELAXED(atomic_fetch_##op, atomic_t, int, w, asm_op) + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, asm_op) \ +ATOMIC_OP(atomic64_##op, atomic64_t, u64, d, asm_op) \ +ATOMIC_FETCH_OP_RELAXED(atomic64_fetch_##op, atomic64_t, u64, d, asm_op) ATOMIC_OPS(and, and) +#define arch_atomic_and arch_atomic_and +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed + ATOMIC_OPS(or, or) +#define arch_atomic_or arch_atomic_or +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed + ATOMIC_OPS(xor, xor) +#define arch_atomic_xor arch_atomic_xor +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed + +#ifdef CONFIG_64BIT +ATOMIC64_OPS(and, and) +#define arch_atomic64_and arch_atomic64_and +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +ATOMIC64_OPS(or, or) +#define arch_atomic64_or arch_atomic64_or +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed + +ATOMIC64_OPS(xor, xor) +#define arch_atomic64_xor arch_atomic64_xor +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed +#endif #undef ATOMIC_OPS +#define ATOMIC_OPS(op, asm_op, imm) \ +ATOMIC_OP_IMM(atomic_##op, atomic_t, int, w, asm_op, imm) \ +ATOMIC_OP_IMM_RETURN_RELAXED(atomic_##op##_return, atomic_t, int, w, asm_op, imm) + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, asm_op, imm) \ +ATOMIC_OP_IMM(atomic64_##op, atomic64_t, u64, d, asm_op, imm) \ +ATOMIC_OP_IMM_RETURN_RELAXED(atomic64_##op##_return, atomic64_t, u64, d, asm_op, imm) + +ATOMIC_OPS(inc, addic, 1) +#define arch_atomic_inc arch_atomic_inc +#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed + +ATOMIC_OPS(dec, addic, -1) +#define arch_atomic_dec arch_atomic_dec +#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed + +#ifdef CONFIG_64BIT +ATOMIC64_OPS(inc, addic, 1) +#define arch_atomic64_inc arch_atomic64_inc +#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed + +ATOMIC64_OPS(dec, addic, -1) +#define arch_atomic64_dec arch_atomic64_dec +#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed +#endif + +ATOMIC_INC_NOT_ZERO_RELAXED(atomic_inc_not_zero, atomic_t, int, w) +#define arch_atomic_inc_not_zero_relaxed(v) arch_atomic_inc_not_zero_relaxed(v) + +#ifdef CONFIG_64BIT +ATOMIC_INC_NOT_ZERO_RELAXED(atomic64_inc_not_zero, atomic64_t, u64, d) +#define arch_atomic64_inc_not_zero_relaxed(v) arch_atomic64_inc_not_zero_relaxed(v) +#endif + +#undef ATOMIC_INC_NOT_ZERO_RELAXED +#undef ATOMIC_FETCH_OP_UNLESS_RELAXED #undef ATOMIC_FETCH_OP_RELAXED +#undef ATOMIC_OP_IMM_RETURN_RELAXED #undef ATOMIC_OP_RETURN_RELAXED +#undef ATOMIC_OP_IMM #undef ATOMIC_OP +#undef ATOMIC_OPS +#undef ATOMIC64_OPS -static __inline__ void atomic_inc(atomic_t *v) -{ - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%2 # atomic_inc\n\ - addic %0,%0,1\n" -" stwcx. %0,0,%2 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); -} -#define atomic_inc atomic_inc - -static __inline__ int atomic_inc_return_relaxed(atomic_t *v) -{ - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" -" addic %0,%0,1\n" -" stwcx. %0,0,%2\n" -" bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); - - return t; -} - -static __inline__ void atomic_dec(atomic_t *v) -{ - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%2 # atomic_dec\n\ - addic %0,%0,-1\n" -" stwcx. %0,0,%2\n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); -} -#define atomic_dec atomic_dec - -static __inline__ int atomic_dec_return_relaxed(atomic_t *v) -{ - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" -" addic %0,%0,-1\n" -" stwcx. %0,0,%2\n" -" bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); - - return t; -} - -#define atomic_inc_return_relaxed atomic_inc_return_relaxed -#define atomic_dec_return_relaxed atomic_dec_return_relaxed - -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic_cmpxchg_relaxed(v, o, n) \ - cmpxchg_relaxed(&((v)->counter), (o), (n)) -#define atomic_cmpxchg_acquire(v, o, n) \ - cmpxchg_acquire(&((v)->counter), (o), (n)) +#define arch_atomic_cmpxchg_relaxed(v, o, n) arch_cmpxchg_relaxed(&((v)->counter), (o), (n)) +#define arch_atomic_xchg_relaxed(v, new) arch_xchg_relaxed(&((v)->counter), (new)) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) +#ifdef CONFIG_64BIT +#define arch_atomic64_cmpxchg_relaxed(v, o, n) arch_cmpxchg_relaxed(&((v)->counter), (o), (n)) +#define arch_atomic64_xchg_relaxed(v, new) arch_xchg_relaxed(&((v)->counter), (new)) +#endif /* * Don't want to override the generic atomic_try_cmpxchg_acquire, because @@ -203,7 +303,7 @@ atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) int r, o = *old; __asm__ __volatile__ ( -"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n" +"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_lock \n" " cmpw 0,%0,%3 \n" " bne- 2f \n" " stwcx. %4,0,%2 \n" @@ -219,270 +319,41 @@ atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) return likely(r == o); } -/** - * atomic_fetch_add_unless - add unless the number is a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) -{ - int t; - - __asm__ __volatile__ ( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ - cmpw 0,%0,%3 \n\ - beq 2f \n\ - add %0,%2,%0 \n" -" stwcx. %0,0,%1 \n\ - bne- 1b \n" - PPC_ATOMIC_EXIT_BARRIER -" subf %0,%2,%0 \n\ -2:" - : "=&r" (t) - : "r" (&v->counter), "r" (a), "r" (u) - : "cc", "memory"); - - return t; -} -#define atomic_fetch_add_unless atomic_fetch_add_unless - -/** - * atomic_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1, so long as @v is non-zero. - * Returns non-zero if @v was non-zero, and zero otherwise. - */ -static __inline__ int atomic_inc_not_zero(atomic_t *v) -{ - int t1, t2; - - __asm__ __volatile__ ( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ - cmpwi 0,%0,0\n\ - beq- 2f\n\ - addic %1,%0,1\n" -" stwcx. %1,0,%2\n\ - bne- 1b\n" - PPC_ATOMIC_EXIT_BARRIER - "\n\ -2:" - : "=&r" (t1), "=&r" (t2) - : "r" (&v->counter) - : "cc", "xer", "memory"); - - return t1; -} -#define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) - /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1, even if * the atomic variable, v, was not decremented. */ -static __inline__ int atomic_dec_if_positive(atomic_t *v) +static inline int atomic_dec_if_positive_relaxed(atomic_t *v) { int t; - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ - cmpwi %0,1\n\ - addi %0,%0,-1\n\ - blt- 2f\n" -" stwcx. %0,0,%1\n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - "\n\ -2:" : "=&b" (t) + asm volatile( +"1: lwarx %0,0,%1 # atomic_dec_if_positive \n" +" cmpwi %0,1 \n" +" addi %0,%0,-1 \n" +" blt- 2f \n" +" stwcx. %0,0,%1 \n" +" bne- 1b \n" +"2: \n" + : "=&b" (t) : "r" (&v->counter) : "cc", "memory"); return t; } -#define atomic_dec_if_positive atomic_dec_if_positive - -#ifdef __powerpc64__ - -#define ATOMIC64_INIT(i) { (i) } - -static __inline__ s64 atomic64_read(const atomic64_t *v) -{ - s64 t; - - __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); - - return t; -} - -static __inline__ void atomic64_set(atomic64_t *v, s64 i) -{ - __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); -} - -#define ATOMIC64_OP(op, asm_op) \ -static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ -{ \ - s64 t; \ - \ - __asm__ __volatile__( \ -"1: ldarx %0,0,%3 # atomic64_" #op "\n" \ - #asm_op " %0,%2,%0\n" \ -" stdcx. %0,0,%3 \n" \ -" bne- 1b\n" \ - : "=&r" (t), "+m" (v->counter) \ - : "r" (a), "r" (&v->counter) \ - : "cc"); \ -} - -#define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ -static inline s64 \ -atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ -{ \ - s64 t; \ - \ - __asm__ __volatile__( \ -"1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ - #asm_op " %0,%2,%0\n" \ -" stdcx. %0,0,%3\n" \ -" bne- 1b\n" \ - : "=&r" (t), "+m" (v->counter) \ - : "r" (a), "r" (&v->counter) \ - : "cc"); \ - \ - return t; \ -} - -#define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ -static inline s64 \ -atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ -{ \ - s64 res, t; \ - \ - __asm__ __volatile__( \ -"1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ - #asm_op " %1,%3,%0\n" \ -" stdcx. %1,0,%4\n" \ -" bne- 1b\n" \ - : "=&r" (res), "=&r" (t), "+m" (v->counter) \ - : "r" (a), "r" (&v->counter) \ - : "cc"); \ - \ - return res; \ -} - -#define ATOMIC64_OPS(op, asm_op) \ - ATOMIC64_OP(op, asm_op) \ - ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ - ATOMIC64_FETCH_OP_RELAXED(op, asm_op) - -ATOMIC64_OPS(add, add) -ATOMIC64_OPS(sub, subf) - -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed - -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed - -#undef ATOMIC64_OPS -#define ATOMIC64_OPS(op, asm_op) \ - ATOMIC64_OP(op, asm_op) \ - ATOMIC64_FETCH_OP_RELAXED(op, asm_op) - -ATOMIC64_OPS(and, and) -ATOMIC64_OPS(or, or) -ATOMIC64_OPS(xor, xor) - -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed - -#undef ATOPIC64_OPS -#undef ATOMIC64_FETCH_OP_RELAXED -#undef ATOMIC64_OP_RETURN_RELAXED -#undef ATOMIC64_OP - -static __inline__ void atomic64_inc(atomic64_t *v) -{ - s64 t; - - __asm__ __volatile__( -"1: ldarx %0,0,%2 # atomic64_inc\n\ - addic %0,%0,1\n\ - stdcx. %0,0,%2 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); -} -#define atomic64_inc atomic64_inc - -static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) -{ - s64 t; - - __asm__ __volatile__( -"1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" -" addic %0,%0,1\n" -" stdcx. %0,0,%2\n" -" bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); - - return t; -} - -static __inline__ void atomic64_dec(atomic64_t *v) -{ - s64 t; - - __asm__ __volatile__( -"1: ldarx %0,0,%2 # atomic64_dec\n\ - addic %0,%0,-1\n\ - stdcx. %0,0,%2\n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); -} -#define atomic64_dec atomic64_dec - -static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) -{ - s64 t; - - __asm__ __volatile__( -"1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" -" addic %0,%0,-1\n" -" stdcx. %0,0,%2\n" -" bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); - - return t; -} - -#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed -#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed +#define atomic_dec_if_positive_relaxed atomic_dec_if_positive_relaxed +#ifdef CONFIG_64BIT /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1. */ -static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) +static inline s64 atomic64_dec_if_positive_relaxed(atomic64_t *v) { s64 t; - __asm__ __volatile__( + asm volatile( PPC_ATOMIC_ENTRY_BARRIER "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ addic. %0,%0,-1\n\ @@ -497,80 +368,8 @@ static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) return t; } -#define atomic64_dec_if_positive atomic64_dec_if_positive - -#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic64_cmpxchg_relaxed(v, o, n) \ - cmpxchg_relaxed(&((v)->counter), (o), (n)) -#define atomic64_cmpxchg_acquire(v, o, n) \ - cmpxchg_acquire(&((v)->counter), (o), (n)) - -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) - -/** - * atomic64_fetch_add_unless - add unless the number is a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) -{ - s64 t; - - __asm__ __volatile__ ( - PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ - cmpd 0,%0,%3 \n\ - beq 2f \n\ - add %0,%2,%0 \n" -" stdcx. %0,0,%1 \n\ - bne- 1b \n" - PPC_ATOMIC_EXIT_BARRIER -" subf %0,%2,%0 \n\ -2:" - : "=&r" (t) - : "r" (&v->counter), "r" (a), "r" (u) - : "cc", "memory"); - - return t; -} -#define atomic64_fetch_add_unless atomic64_fetch_add_unless - -/** - * atomic_inc64_not_zero - increment unless the number is zero - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1, so long as @v is non-zero. - * Returns non-zero if @v was non-zero, and zero otherwise. - */ -static __inline__ int atomic64_inc_not_zero(atomic64_t *v) -{ - s64 t1, t2; - - __asm__ __volatile__ ( - PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ - cmpdi 0,%0,0\n\ - beq- 2f\n\ - addic %1,%0,1\n\ - stdcx. %1,0,%2\n\ - bne- 1b\n" - PPC_ATOMIC_EXIT_BARRIER - "\n\ -2:" - : "=&r" (t1), "=&r" (t2) - : "r" (&v->counter) - : "cc", "xer", "memory"); - - return t1 != 0; -} -#define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) - -#endif /* __powerpc64__ */ +#define atomic64_dec_if_positive_relaxed atomic64_dec_if_positive_relaxed +#endif #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_ATOMIC_H_ */ diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h index cf091c4c22e5..181f7e8b3281 100644 --- a/arch/powerpc/include/asm/cmpxchg.h +++ b/arch/powerpc/include/asm/cmpxchg.h @@ -192,7 +192,7 @@ __xchg_relaxed(void *ptr, unsigned long x, unsigned int size) (unsigned long)_x_, sizeof(*(ptr))); \ }) -#define xchg_relaxed(ptr, x) \ +#define arch_xchg_relaxed(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ @@ -448,35 +448,7 @@ __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new, return old; } -static __always_inline unsigned long -__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, - unsigned int size) -{ - switch (size) { - case 1: - return __cmpxchg_u8_acquire(ptr, old, new); - case 2: - return __cmpxchg_u16_acquire(ptr, old, new); - case 4: - return __cmpxchg_u32_acquire(ptr, old, new); -#ifdef CONFIG_PPC64 - case 8: - return __cmpxchg_u64_acquire(ptr, old, new); -#endif - } - BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire"); - return old; -} -#define cmpxchg(ptr, o, n) \ - ({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ - (unsigned long)_n_, sizeof(*(ptr))); \ - }) - - -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -484,7 +456,7 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, (unsigned long)_n_, sizeof(*(ptr))); \ }) -#define cmpxchg_relaxed(ptr, o, n) \ +#define arch_cmpxchg_relaxed(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -493,38 +465,20 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, sizeof(*(ptr))); \ }) -#define cmpxchg_acquire(ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ - (unsigned long)_o_, (unsigned long)_n_, \ - sizeof(*(ptr))); \ -}) #ifdef CONFIG_PPC64 -#define cmpxchg64(ptr, o, n) \ - ({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ - }) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ + arch_cmpxchg_local((ptr), (o), (n)); \ }) -#define cmpxchg64_relaxed(ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_relaxed((ptr), (o), (n)); \ -}) -#define cmpxchg64_acquire(ptr, o, n) \ +#define arch_cmpxchg64_relaxed(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_acquire((ptr), (o), (n)); \ + arch_cmpxchg_relaxed((ptr), (o), (n)); \ }) #else #include -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) #endif #endif /* __KERNEL__ */