powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP

Message ID 20180613095858.31078-1-npiggin@gmail.com
State Superseded
Headers show
Series
  • powerpc/64s/radix: Fix MADV_[FREE|DONTNEED] TLB flush miss problem with THP
Related show

Commit Message

Nicholas Piggin June 13, 2018, 9:58 a.m.
The patch 99baac21e4 ("mm: fix MADV_[FREE|DONTNEED] TLB flush miss
problem") added a force flush mode to the mmu_gather flush, which
unconditionally flushes the entire address range being invalidated
(even if actual ptes only covered a smaller range), to solve a problem
with concurrent threads invalidating the same PTEs causing them to
miss TLBs that need flushing.

This does not work with powerpc that invalidates mmu_gather batches
according to page size. Have powerpc flush all possible page sizes in
the range if it encounters the concurrency condition.

Hash does not have a problem because it invalidates TLBs inside the
page table locks.

Reported-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
Since RFC:
- Account for hugetlb pages that can be mixed with the tlb_flush
  range.

 arch/powerpc/mm/tlb-radix.c | 85 ++++++++++++++++++++++++++++---------
 1 file changed, 66 insertions(+), 19 deletions(-)

Comments

kbuild test robot June 14, 2018, 12:35 a.m. | #1
Hi Nicholas,

I love your patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on next-20180613]
[cannot apply to v4.17]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-64s-radix-Fix-MADV_-FREE-DONTNEED-TLB-flush-miss-problem-with-THP/20180613-180928
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-cell_defconfig (attached as .config)
compiler: powerpc64-linux-gnu-gcc (Debian 7.2.0-11) 7.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        GCC_VERSION=7.2.0 make.cross ARCH=powerpc 

All error/warnings (new ones prefixed by >>):

   In file included from include/asm-generic/bug.h:5:0,
                    from arch/powerpc/include/asm/bug.h:128,
                    from include/linux/bug.h:5,
                    from include/linux/mmdebug.h:5,
                    from include/linux/mm.h:9,
                    from arch/powerpc/mm/tlb-radix.c:12:
   In function '__radix__flush_tlb_range',
       inlined from 'radix__tlb_flush' at arch/powerpc/mm/tlb-radix.c:898:3:
>> include/linux/compiler.h:339:38: error: call to '__compiletime_assert_745' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
>> include/linux/huge_mm.h:251:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:745:22: note: in expansion of macro 'HPAGE_PMD_SIZE'
       hstart = (start + HPAGE_PMD_SIZE - 1) & HPAGE_PMD_MASK;
                         ^~~~~~~~~~~~~~
>> include/linux/compiler.h:339:38: error: call to '__compiletime_assert_745' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:250:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:745:44: note: in expansion of macro 'HPAGE_PMD_MASK'
       hstart = (start + HPAGE_PMD_SIZE - 1) & HPAGE_PMD_MASK;
                                               ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_746' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:250:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
   arch/powerpc/mm/tlb-radix.c:746:17: note: in expansion of macro 'HPAGE_PMD_MASK'
       hend = end & HPAGE_PMD_MASK;
                    ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_752' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:255:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:752:22: note: in expansion of macro 'HPAGE_PUD_SIZE'
       gstart = (start + HPAGE_PUD_SIZE - 1) & HPAGE_PUD_MASK;
                         ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_752' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:254:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:752:44: note: in expansion of macro 'HPAGE_PUD_MASK'
       gstart = (start + HPAGE_PUD_SIZE - 1) & HPAGE_PUD_MASK;
                                               ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_753' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:254:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
   arch/powerpc/mm/tlb-radix.c:753:17: note: in expansion of macro 'HPAGE_PUD_MASK'
       gend = end & HPAGE_PUD_MASK;
                    ^~~~~~~~~~~~~~
>> include/linux/compiler.h:339:38: error: call to '__compiletime_assert_745' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
>> include/linux/huge_mm.h:251:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:745:22: note: in expansion of macro 'HPAGE_PMD_SIZE'
       hstart = (start + HPAGE_PMD_SIZE - 1) & HPAGE_PMD_MASK;
                         ^~~~~~~~~~~~~~
>> include/linux/compiler.h:339:38: error: call to '__compiletime_assert_745' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:250:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:745:44: note: in expansion of macro 'HPAGE_PMD_MASK'
       hstart = (start + HPAGE_PMD_SIZE - 1) & HPAGE_PMD_MASK;
                                               ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_746' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:250:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
   arch/powerpc/mm/tlb-radix.c:746:17: note: in expansion of macro 'HPAGE_PMD_MASK'
       hend = end & HPAGE_PMD_MASK;
                    ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_752' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:255:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:752:22: note: in expansion of macro 'HPAGE_PUD_SIZE'
       gstart = (start + HPAGE_PUD_SIZE - 1) & HPAGE_PUD_MASK;
                         ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_752' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:254:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:752:44: note: in expansion of macro 'HPAGE_PUD_MASK'
       gstart = (start + HPAGE_PUD_SIZE - 1) & HPAGE_PUD_MASK;
                                               ^~~~~~~~~~~~~~
   include/linux/compiler.h:339:38: error: call to '__compiletime_assert_753' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:319:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:339:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:45:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/build_bug.h:79:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
   include/linux/huge_mm.h:254:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
   arch/powerpc/mm/tlb-radix.c:753:17: note: in expansion of macro 'HPAGE_PUD_MASK'
       gend = end & HPAGE_PUD_MASK;
                    ^~~~~~~~~~~~~~

vim +/HPAGE_PMD_SIZE +745 arch/powerpc/mm/tlb-radix.c

   691	
   692	static inline void __radix__flush_tlb_range(struct mm_struct *mm,
   693						unsigned long start, unsigned long end,
   694						bool flush_all_sizes)
   695	
   696	{
   697		unsigned long pid;
   698		unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift;
   699		unsigned long page_size = 1UL << page_shift;
   700		unsigned long nr_pages = (end - start) >> page_shift;
   701		bool local, full;
   702	
   703		pid = mm->context.id;
   704		if (unlikely(pid == MMU_NO_CONTEXT))
   705			return;
   706	
   707		preempt_disable();
   708		smp_mb(); /* see radix__flush_tlb_mm */
   709		if (!mm_is_thread_local(mm)) {
   710			if (unlikely(mm_is_singlethreaded(mm))) {
   711				if (end != TLB_FLUSH_ALL) {
   712					exit_flush_lazy_tlbs(mm);
   713					goto is_local;
   714				}
   715			}
   716			local = false;
   717			full = (end == TLB_FLUSH_ALL ||
   718					nr_pages > tlb_single_page_flush_ceiling);
   719		} else {
   720	is_local:
   721			local = true;
   722			full = (end == TLB_FLUSH_ALL ||
   723					nr_pages > tlb_local_single_page_flush_ceiling);
   724		}
   725	
   726		if (full) {
   727			if (local) {
   728				_tlbiel_pid(pid, RIC_FLUSH_TLB);
   729			} else {
   730				if (mm_needs_flush_escalation(mm))
   731					_tlbie_pid(pid, RIC_FLUSH_ALL);
   732				else
   733					_tlbie_pid(pid, RIC_FLUSH_TLB);
   734			}
   735		} else {
   736			bool hflush = flush_all_sizes;
   737			bool gflush = flush_all_sizes;
   738			unsigned long hstart, hend;
   739			unsigned long gstart, gend;
   740	
   741			if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
   742				hflush = true;
   743	
   744			if (hflush) {
 > 745				hstart = (start + HPAGE_PMD_SIZE - 1) & HPAGE_PMD_MASK;
   746				hend = end & HPAGE_PMD_MASK;
   747				if (hstart == hend)
   748					hflush = false;
   749			}
   750	
   751			if (gflush) {
 > 752				gstart = (start + HPAGE_PUD_SIZE - 1) & HPAGE_PUD_MASK;
   753				gend = end & HPAGE_PUD_MASK;
   754				if (gstart == gend)
   755					gflush = false;
   756			}
   757	
   758			asm volatile("ptesync": : :"memory");
   759			if (local) {
   760				__tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize);
   761				if (hflush)
   762					__tlbiel_va_range(hstart, hend, pid,
   763							HPAGE_PMD_SIZE, MMU_PAGE_2M);
   764				if (gflush)
   765					__tlbiel_va_range(gstart, gend, pid,
   766							HPAGE_PUD_SIZE, MMU_PAGE_1G);
   767				asm volatile("ptesync": : :"memory");
   768			} else {
   769				__tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize);
   770				if (hflush)
   771					__tlbie_va_range(hstart, hend, pid,
   772							HPAGE_PMD_SIZE, MMU_PAGE_2M);
   773				if (gflush)
   774					__tlbie_va_range(gstart, gend, pid,
   775							HPAGE_PUD_SIZE, MMU_PAGE_1G);
   776				fixup_tlbie();
   777				asm volatile("eieio; tlbsync; ptesync": : :"memory");
   778			}
   779		}
   780		preempt_enable();
   781	}
   782	

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

Patch

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 67a6e86d3e7e..9dbccda651d7 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -689,22 +689,17 @@  EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
 static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2;
 
-void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
-		     unsigned long end)
+static inline void __radix__flush_tlb_range(struct mm_struct *mm,
+					unsigned long start, unsigned long end,
+					bool flush_all_sizes)
 
 {
-	struct mm_struct *mm = vma->vm_mm;
 	unsigned long pid;
 	unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift;
 	unsigned long page_size = 1UL << page_shift;
 	unsigned long nr_pages = (end - start) >> page_shift;
 	bool local, full;
 
-#ifdef CONFIG_HUGETLB_PAGE
-	if (is_vm_hugetlb_page(vma))
-		return radix__flush_hugetlb_tlb_range(vma, start, end);
-#endif
-
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
 		return;
@@ -738,18 +733,27 @@  void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 				_tlbie_pid(pid, RIC_FLUSH_TLB);
 		}
 	} else {
-		bool hflush = false;
+		bool hflush = flush_all_sizes;
+		bool gflush = flush_all_sizes;
 		unsigned long hstart, hend;
+		unsigned long gstart, gend;
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-		hstart = (start + HPAGE_PMD_SIZE - 1) >> HPAGE_PMD_SHIFT;
-		hend = end >> HPAGE_PMD_SHIFT;
-		if (hstart < hend) {
-			hstart <<= HPAGE_PMD_SHIFT;
-			hend <<= HPAGE_PMD_SHIFT;
+		if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
 			hflush = true;
+
+		if (hflush) {
+			hstart = (start + HPAGE_PMD_SIZE - 1) & HPAGE_PMD_MASK;
+			hend = end & HPAGE_PMD_MASK;
+			if (hstart == hend)
+				hflush = false;
+		}
+
+		if (gflush) {
+			gstart = (start + HPAGE_PUD_SIZE - 1) & HPAGE_PUD_MASK;
+			gend = end & HPAGE_PUD_MASK;
+			if (gstart == gend)
+				gflush = false;
 		}
-#endif
 
 		asm volatile("ptesync": : :"memory");
 		if (local) {
@@ -757,18 +761,36 @@  void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 			if (hflush)
 				__tlbiel_va_range(hstart, hend, pid,
 						HPAGE_PMD_SIZE, MMU_PAGE_2M);
+			if (gflush)
+				__tlbiel_va_range(gstart, gend, pid,
+						HPAGE_PUD_SIZE, MMU_PAGE_1G);
 			asm volatile("ptesync": : :"memory");
 		} else {
 			__tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize);
 			if (hflush)
 				__tlbie_va_range(hstart, hend, pid,
 						HPAGE_PMD_SIZE, MMU_PAGE_2M);
+			if (gflush)
+				__tlbie_va_range(gstart, gend, pid,
+						HPAGE_PUD_SIZE, MMU_PAGE_1G);
 			fixup_tlbie();
 			asm volatile("eieio; tlbsync; ptesync": : :"memory");
 		}
 	}
 	preempt_enable();
 }
+
+void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+		     unsigned long end)
+
+{
+#ifdef CONFIG_HUGETLB_PAGE
+	if (is_vm_hugetlb_page(vma))
+		return radix__flush_hugetlb_tlb_range(vma, start, end);
+#endif
+
+	__radix__flush_tlb_range(vma->vm_mm, start, end, false);
+}
 EXPORT_SYMBOL(radix__flush_tlb_range);
 
 static int radix_get_mmu_psize(int page_size)
@@ -837,6 +859,8 @@  void radix__tlb_flush(struct mmu_gather *tlb)
 	int psize = 0;
 	struct mm_struct *mm = tlb->mm;
 	int page_size = tlb->page_size;
+	unsigned long start = tlb->start;
+	unsigned long end = tlb->end;
 
 	/*
 	 * if page size is not something we understand, do a full mm flush
@@ -847,15 +871,38 @@  void radix__tlb_flush(struct mmu_gather *tlb)
 	 */
 	if (tlb->fullmm) {
 		__flush_all_mm(mm, true);
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
+	} else if (mm_tlb_flush_pending(mm)) {
+		/*
+		 * If there is a concurrent invalidation that is clearing ptes,
+		 * then it's possible this invalidation will miss one of those
+		 * cleared ptes and miss flushing the TLB. If this invalidate
+		 * returns before the other one flushes TLBs, that can result
+		 * in it returning while there are still valid TLBs inside the
+		 * range to be invalidated.
+		 *
+		 * See mm/memory.c:tlb_finish_mmu() for more details.
+		 *
+		 * The solution to this is ensure the entire range is always
+		 * flushed here. The problem for powerpc is that the flushes
+		 * are page size specific, so this "forced flush" would not
+		 * do the right thing if there are a mix of page sizes in
+		 * the range to be invalidated. So use __flush_tlb_range
+		 * which invalidates all possible page sizes in the range.
+		 *
+		 * PWC flush probably is not be required because the core code
+		 * shouldn't free page tables in this path, but accounting
+		 * for the possibility makes us a bit more robust.
+		 */
+		WARN_ON_ONCE(tlb->need_flush_all);
+		__radix__flush_tlb_range(mm, start, end, true);
+#endif
 	} else if ( (psize = radix_get_mmu_psize(page_size)) == -1) {
 		if (!tlb->need_flush_all)
 			radix__flush_tlb_mm(mm);
 		else
 			radix__flush_all_mm(mm);
 	} else {
-		unsigned long start = tlb->start;
-		unsigned long end = tlb->end;
-
 		if (!tlb->need_flush_all)
 			radix__flush_tlb_range_psize(mm, start, end, psize);
 		else