[3/4] powerpc/mm/radix: Avoid flushing the PWC on every flush_tlb_range

Submitted by Benjamin Herrenschmidt on July 14, 2017, 1:52 a.m.

Details

Message ID 20170714015258.7933-3-benh@kernel.crashing.org
State Superseded
Headers show

Commit Message

Benjamin Herrenschmidt July 14, 2017, 1:52 a.m.
We do that because it's used by THP pmd collapsing, so use
instead a dedicated flush function.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---
 .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  1 +
 arch/powerpc/mm/pgtable-radix.c                    |  5 ++-
 arch/powerpc/mm/tlb-radix.c                        | 41 +++++++++++++++++++---
 3 files changed, 41 insertions(+), 6 deletions(-)

Comments

Aneesh Kumar K.V July 14, 2017, 5:44 a.m.
Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:

> We do that because it's used by THP pmd collapsing, so use
> instead a dedicated flush function.
>
> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> ---
>  .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  1 +
>  arch/powerpc/mm/pgtable-radix.c                    |  5 ++-
>  arch/powerpc/mm/tlb-radix.c                        | 41 +++++++++++++++++++---
>  3 files changed, 41 insertions(+), 6 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> index 7196999..9b433a6 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> @@ -36,6 +36,7 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
>  #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
>  #endif
>  extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
> +extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
>  extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
>  				     unsigned long page_size);
>  extern void radix__flush_tlb_lpid(unsigned long lpid);
> diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
> index c28165d..83d70f7 100644
> --- a/arch/powerpc/mm/pgtable-radix.c
> +++ b/arch/powerpc/mm/pgtable-radix.c
> @@ -706,9 +706,12 @@ pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre
>  	 */
>  	pmd = *pmdp;
>  	pmd_clear(pmdp);
> +
>  	/*FIXME!!  Verify whether we need this kick below */
>  	kick_all_cpus_sync();
> -	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
> +
> +	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
> +
>  	return pmd;
>  }
>
> diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> index f65c928..73d3fbf 100644
> --- a/arch/powerpc/mm/tlb-radix.c
> +++ b/arch/powerpc/mm/tlb-radix.c
> @@ -267,11 +267,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
>  {
>  	struct mm_struct *mm = vma->vm_mm;
>
> -	/*
> -	 * This is currently used when collapsing THPs so we need to
> -	 * flush the PWC. We should fix this.
> -	 */
> -	radix__flush_all_mm(mm);
> +	radix__flush_tlb_mm(mm);
>  }
>  EXPORT_SYMBOL(radix__flush_tlb_range);
>
> @@ -350,6 +346,41 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
>  	preempt_enable();
>  }
>
> +void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
> +{
> +	int local = mm_is_thread_local(mm);
> +	unsigned long ap = mmu_get_ap(mmu_virtual_psize);
> +	unsigned long pid, end;
> +
> +
> +	pid = mm ? mm->context.id : 0;
> +	if (unlikely(pid == MMU_NO_CONTEXT))
> +		goto no_context;
> +
> +	/* 4k page size, just blow the world */
> +	if (PAGE_SIZE == 0x1000) {
> +		radix__flush_all_mm(mm);
> +		return;
> +	}

What is special here ? because it is 4096 tlb flushes ?


> +
> +	/* Otherwise first do the PWC */
> +	if (local)
> +		_tlbiel_pid(pid, RIC_FLUSH_PWC);
> +	else
> +		_tlbie_pid(pid, RIC_FLUSH_PWC);
> +
> +	/* Then iterate the pages */
> +	end = addr + HPAGE_PMD_SIZE;
> +	for (; addr < end; addr += PAGE_SIZE) {
> +		if (local)
> +			_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
> +		else
> +			_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
> +	}
> +no_context:
> +	preempt_enable();
> +}
> +
>  void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
>  			      unsigned long page_size)
>  {
> -- 
> 2.9.4
Benjamin Herrenschmidt July 14, 2017, 6:22 a.m.
On Fri, 2017-07-14 at 11:14 +0530, Aneesh Kumar K.V wrote:
> > +     pid = mm ? mm->context.id : 0;
> > +     if (unlikely(pid == MMU_NO_CONTEXT))
> > +             goto no_context;
> > +
> > +     /* 4k page size, just blow the world */
> > +     if (PAGE_SIZE == 0x1000) {
> > +             radix__flush_all_mm(mm);
> > +             return;
> > +     }
> 
> What is special here ? because it is 4096 tlb flushes ?

Yes, keeping in line with the existing logic in flush_tlb_pages(), more
than 32, do a full flush.

Cheers,
Ben.
kbuild test robot July 17, 2017, 12:12 p.m.
Hi Benjamin,

[auto build test ERROR on powerpc/next]
[also build test ERROR on v4.13-rc1 next-20170717]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Benjamin-Herrenschmidt/powerpc-mm-radix-Don-t-iterate-all-sets-when-flushing-the-PWC/20170715-044140
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-pasemi_defconfig (attached as .config)
compiler: powerpc64-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
        wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=powerpc 

All error/warnings (new ones prefixed by >>):

   In file included from include/asm-generic/bug.h:4:0,
                    from arch/powerpc/include/asm/bug.h:127,
                    from include/linux/bug.h:4,
                    from include/linux/mmdebug.h:4,
                    from include/linux/mm.h:8,
                    from arch/powerpc/mm/tlb-radix.c:12:
   arch/powerpc/mm/tlb-radix.c: In function 'radix__flush_tlb_collapsed_pmd':
>> include/linux/compiler.h:529:38: error: call to '__compiletime_assert_378' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:512:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:529:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/bug.h:54:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/bug.h:88:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
>> include/linux/huge_mm.h:218:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:378:15: note: in expansion of macro 'HPAGE_PMD_SIZE'
     end = addr + HPAGE_PMD_SIZE;
                  ^~~~~~~~~~~~~~

vim +/HPAGE_PMD_SIZE +378 arch/powerpc/mm/tlb-radix.c

   353	
   354	void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
   355	{
   356		int local = mm_is_thread_local(mm);
   357		unsigned long ap = mmu_get_ap(mmu_virtual_psize);
   358		unsigned long pid, end;
   359	
   360	
   361		pid = mm ? mm->context.id : 0;
   362		if (unlikely(pid == MMU_NO_CONTEXT))
   363			goto no_context;
   364	
   365		/* 4k page size, just blow the world */
   366		if (PAGE_SIZE == 0x1000) {
   367			radix__flush_all_mm(mm);
   368			return;
   369		}
   370	
   371		/* Otherwise first do the PWC */
   372		if (local)
   373			_tlbiel_pid(pid, RIC_FLUSH_PWC);
   374		else
   375			_tlbie_pid(pid, RIC_FLUSH_PWC);
   376	
   377		/* Then iterate the pages */
 > 378		end = addr + HPAGE_PMD_SIZE;
   379		for (; addr < end; addr += PAGE_SIZE) {
   380			if (local)
   381				_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
   382			else
   383				_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
   384		}
   385	no_context:
   386		preempt_enable();
   387	}
   388	

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

Patch hide | download patch | download mbox

diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 7196999..9b433a6 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -36,6 +36,7 @@  extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
 #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
 #endif
 extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
+extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
 extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
 				     unsigned long page_size);
 extern void radix__flush_tlb_lpid(unsigned long lpid);
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index c28165d..83d70f7 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -706,9 +706,12 @@  pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre
 	 */
 	pmd = *pmdp;
 	pmd_clear(pmdp);
+
 	/*FIXME!!  Verify whether we need this kick below */
 	kick_all_cpus_sync();
-	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+
+	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
+
 	return pmd;
 }
 
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index f65c928..73d3fbf 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -267,11 +267,7 @@  void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 {
 	struct mm_struct *mm = vma->vm_mm;
 
-	/*
-	 * This is currently used when collapsing THPs so we need to
-	 * flush the PWC. We should fix this.
-	 */
-	radix__flush_all_mm(mm);
+	radix__flush_tlb_mm(mm);
 }
 EXPORT_SYMBOL(radix__flush_tlb_range);
 
@@ -350,6 +346,41 @@  void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
 	preempt_enable();
 }
 
+void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
+{
+	int local = mm_is_thread_local(mm);
+	unsigned long ap = mmu_get_ap(mmu_virtual_psize);
+	unsigned long pid, end;
+
+
+	pid = mm ? mm->context.id : 0;
+	if (unlikely(pid == MMU_NO_CONTEXT))
+		goto no_context;
+
+	/* 4k page size, just blow the world */
+	if (PAGE_SIZE == 0x1000) {
+		radix__flush_all_mm(mm);
+		return;
+	}
+
+	/* Otherwise first do the PWC */
+	if (local)
+		_tlbiel_pid(pid, RIC_FLUSH_PWC);
+	else
+		_tlbie_pid(pid, RIC_FLUSH_PWC);
+
+	/* Then iterate the pages */
+	end = addr + HPAGE_PMD_SIZE;
+	for (; addr < end; addr += PAGE_SIZE) {
+		if (local)
+			_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
+		else
+			_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
+	}
+no_context:
+	preempt_enable();
+}
+
 void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
 			      unsigned long page_size)
 {