From patchwork Fri Aug 12 16:55:54 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christophe Leroy X-Patchwork-Id: 658728 Return-Path: X-Original-To: patchwork-incoming@ozlabs.org Delivered-To: patchwork-incoming@ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [103.22.144.68]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 3s9rn51TsSz9t0M for ; Sat, 13 Aug 2016 03:03:21 +1000 (AEST) Received: from ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 3s9rn50k2RzDrgk for ; Sat, 13 Aug 2016 03:03:21 +1000 (AEST) X-Original-To: linuxppc-dev@lists.ozlabs.org Delivered-To: linuxppc-dev@lists.ozlabs.org Received: from pegase1.c-s.fr (pegase1.c-s.fr [93.17.236.30]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by lists.ozlabs.org (Postfix) with ESMTPS id 3s9rcZ11gXzDr2W for ; Sat, 13 Aug 2016 02:55:58 +1000 (AEST) Received: from localhost (unknown [192.168.12.234]) by localhost (Postfix) with ESMTP id 3s9rcV0Ds8z9ttFr; Fri, 12 Aug 2016 18:55:54 +0200 (CEST) X-Virus-Scanned: Debian amavisd-new at c-s.fr Received: from pegase1.c-s.fr ([192.168.12.234]) by localhost (pegase1.c-s.fr [192.168.12.234]) (amavisd-new, port 10024) with ESMTP id iUcR25cj7Gmr; Fri, 12 Aug 2016 18:55:53 +0200 (CEST) Received: from messagerie.si.c-s.fr (messagerie.si.c-s.fr [192.168.25.192]) by pegase1.c-s.fr (Postfix) with ESMTP id 3s9rcT6lqWz9ttFP; Fri, 12 Aug 2016 18:55:53 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by messagerie.si.c-s.fr (Postfix) with ESMTP id 0E7E38B966; Fri, 12 Aug 2016 18:55:55 +0200 (CEST) X-Virus-Scanned: amavisd-new at c-s.fr Received: from messagerie.si.c-s.fr ([127.0.0.1]) by localhost (messagerie.si.c-s.fr [127.0.0.1]) (amavisd-new, port 10023) with ESMTP id 51f5E0zx9rB9; Fri, 12 Aug 2016 18:55:54 +0200 (CEST) Received: from PO10863.localdomain (po10863.idsi0.si.c-s.fr [172.25.231.6]) by messagerie.si.c-s.fr (Postfix) with ESMTP id C62698B8BC; Fri, 12 Aug 2016 18:55:54 +0200 (CEST) Received: by localhost.localdomain (Postfix, from userid 0) id CDF441A2456; Fri, 12 Aug 2016 18:55:54 +0200 (CEST) Message-Id: <4d466533adfd270c232bff42cd33743e9ef7e4bf.1471020647.git.christophe.leroy@c-s.fr> In-Reply-To: References: From: Christophe Leroy Subject: [PATCH 5/6] powerpc/8xx: make user addr DTLB miss the short path To: Benjamin Herrenschmidt , Paul Mackerras , Michael Ellerman , Scott Wood Date: Fri, 12 Aug 2016 18:55:54 +0200 (CEST) X-BeenThere: linuxppc-dev@lists.ozlabs.org X-Mailman-Version: 2.1.22 Precedence: list List-Id: Linux on PowerPC Developers Mail List List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: linuxppc-dev@lists.ozlabs.org, linux-kernel@vger.kernel.org Errors-To: linuxppc-dev-bounces+patchwork-incoming=ozlabs.org@lists.ozlabs.org Sender: "Linuxppc-dev" User space DTLB miss represent approximatly 90% of TLB misses so make it the shortest path. Also remove an unneccessary double jump in FixupDAR Before this patch, we spend 3.3 TB ticks in the handler for each user address miss and 3.4 TB ticks for each kernel address miss After this patch, we send 3.0 TB ticks in the handler for each user address miss and 3.9 TB ticks for each kernel address miss Taking into account that user misses represent 90% of the total, this patch provides an improvement of approx. 9% Signed-off-by: Christophe Leroy --- arch/powerpc/kernel/head_8xx.S | 53 ++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 5f122e6..5ce67f2 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -383,30 +383,31 @@ InstructionTLBMiss: . = 0x1200 DataStoreTLBMiss: + mtspr SPRN_SPRG_SCRATCH2, r3 EXCEPTION_PROLOG_0 - mfcr r10 + mfcr r3 /* If we are faulting a kernel address, we have to use the * kernel page tables. */ - mfspr r11, SPRN_MD_EPN - rlwinm r11, r11, 16, 0xfff8 + mfspr r10, SPRN_MD_EPN + rlwinm r10, r10, 16, 0xfff8 + cmpli cr0, r10, PAGE_OFFSET@h + mfspr r11, SPRN_M_TW /* Get level 1 table */ + blt+ 3f #ifndef CONFIG_PIN_TLB_IMMR - cmpli cr0, r11, VIRT_IMMR_BASE@h + cmpli cr0, r10, VIRT_IMMR_BASE@h #endif - cmpli cr7, r11, PAGE_OFFSET@h +_ENTRY(DTLBMiss_cmp) + cmpli cr7, r10, (PAGE_OFFSET + 0x1800000)@h + lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha #ifndef CONFIG_PIN_TLB_IMMR _ENTRY(DTLBMiss_jmp) beq- DTLBMissIMMR #endif - bge- cr7, DTLBMissLinear - - mfspr r11, SPRN_M_TW /* Get level 1 table */ + blt cr7, DTLBMissLinear 3: - mtcr r10 -#ifdef CONFIG_8xx_CPU6 - mtspr SPRN_SPRG_SCRATCH2, r3 -#endif + mtcr r3 mfspr r10, SPRN_MD_EPN /* Insert level 1 index */ @@ -459,9 +460,7 @@ _ENTRY(DTLBMiss_jmp) MTSPR_CPU6(SPRN_MD_RPN, r10, r3) /* Update TLB entry */ /* Restore registers */ -#ifdef CONFIG_8xx_CPU6 mfspr r3, SPRN_SPRG_SCRATCH2 -#endif mtspr SPRN_DAR, r11 /* Tag DAR */ EXCEPTION_EPILOG_0 rfi @@ -532,7 +531,7 @@ DARFixed:/* Return from dcbx instruction bug workaround */ * not enough space in the DataStoreTLBMiss area. */ DTLBMissIMMR: - mtcr r10 + mtcr r3 /* Set 512k byte guarded page and mark it valid */ li r10, MD_PS512K | MD_GUARDED | MD_SVALID MTSPR_CPU6(SPRN_MD_TWC, r10, r11) @@ -544,27 +543,23 @@ DTLBMissIMMR: li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r3, SPRN_SPRG_SCRATCH2 EXCEPTION_EPILOG_0 rfi DTLBMissLinear: -_ENTRY(DTLBMiss_cmp) - cmpli cr0, r11, (PAGE_OFFSET + 0x1800000)@h - lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha - bge- 3b - - mtcr r10 + mtcr r3 /* Set 8M byte page and mark it valid */ - li r10, MD_PS8MEG | MD_SVALID - MTSPR_CPU6(SPRN_MD_TWC, r10, r11) - mfspr r10, SPRN_MD_EPN - rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ + li r11, MD_PS8MEG | MD_SVALID + MTSPR_CPU6(SPRN_MD_TWC, r11, r3) + rlwinm r10, r10, 16, 0x0f800000 /* 8xx supports max 256Mb RAM */ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ _PAGE_PRESENT MTSPR_CPU6(SPRN_MD_RPN, r10, r11) /* Update TLB entry */ li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r3, SPRN_SPRG_SCRATCH2 EXCEPTION_EPILOG_0 rfi @@ -584,7 +579,9 @@ FixupDAR:/* Entry point for dcbx workaround. */ rlwinm r11, r10, 16, 0xfff8 _ENTRY(FixupDAR_cmp) cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h - blt- cr7, 200f + /* create physical page address from effective address */ + tophys(r11, r10) + blt- cr7, 201f lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha /* Insert level 1 index */ 3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29 @@ -614,10 +611,6 @@ _ENTRY(FixupDAR_cmp) 141: mfspr r10,SPRN_SPRG_SCRATCH2 b DARFixed /* Nope, go back to normal TLB processing */ - /* create physical page address from effective address */ -200: tophys(r11, r10) - b 201b - 144: mfspr r10, SPRN_DSISR rlwinm r10, r10,0,7,5 /* Clear store bit for buggy dcbst insn */ mtspr SPRN_DSISR, r10