| Message ID | 20170907145148.24398-9-npiggin@gmail.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Further radix TLB flush optimisations | expand |
On Fri, 2017-09-08 at 00:51 +1000, Nicholas Piggin wrote: > When permissiveness is relaxed, or found to have been relaxed by > another thread, we flush that address out of the TLB to avoid a > future fault or micro-fault due to a stale TLB entry. > > Currently for processes with TLBs on other CPUs, this flush is always > done with a global tlbie. Although that could reduce faults on remote > CPUs, a broadcast operation seems to be wasteful for something that > can be handled in-core by the remote CPU if it comes to it. > > This is not benchmarked yet. It does seem cut some tlbie operations > from the bus. What happens with the nest MMU here ? > Signed-off-by: Nicholas Piggin <npiggin@gmail.com> > --- > .../powerpc/include/asm/book3s/64/tlbflush-radix.h | 5 ++++ > arch/powerpc/include/asm/book3s/64/tlbflush.h | 11 +++++++++ > arch/powerpc/mm/pgtable-book3s64.c | 5 +++- > arch/powerpc/mm/pgtable.c | 2 +- > arch/powerpc/mm/tlb-radix.c | 27 ++++++++++++++++++++++ > 5 files changed, 48 insertions(+), 2 deletions(-) > > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > index b12460b306a7..34cd864b8fc1 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > @@ -16,6 +16,8 @@ extern bool radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long sta > unsigned long end, int psize); > extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, > unsigned long start, unsigned long end); > +extern void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma, > + unsigned long start, unsigned long end); > extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end); > extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); > @@ -24,6 +26,9 @@ extern void radix__local_flush_tlb_mm(struct mm_struct *mm); > extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); > extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, > int psize); > +extern void radix__local_flush_tlb_range_psize(struct mm_struct *mm, > + unsigned long start, unsigned long end, > + int psize); > extern void radix__tlb_flush(struct mmu_gather *tlb); > #ifdef CONFIG_SMP > extern void radix__flush_tlb_mm(struct mm_struct *mm); > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h > index 72b925f97bab..8a8b3e11a28e 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h > @@ -83,6 +83,17 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, > #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) > #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) > #endif /* CONFIG_SMP */ > + > +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault > +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, > + unsigned long address) > +{ > + if (radix_enabled()) > + radix__local_flush_tlb_page(vma, address); > + else > + flush_tlb_page(vma, address); > +} > + > /* > * flush the page walk cache for the address > */ > diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c > index 3b65917785a5..e46f346388d6 100644 > --- a/arch/powerpc/mm/pgtable-book3s64.c > +++ b/arch/powerpc/mm/pgtable-book3s64.c > @@ -40,7 +40,10 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, > if (changed) { > __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp), > pmd_pte(entry), address); > - flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); > + if (radix_enabled()) > + radix__local_flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); > + else > + flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); > } > return changed; > } > diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c > index a03ff3d99e0c..acd6ae8062ce 100644 > --- a/arch/powerpc/mm/pgtable.c > +++ b/arch/powerpc/mm/pgtable.c > @@ -223,7 +223,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, > if (!is_vm_hugetlb_page(vma)) > assert_pte_locked(vma->vm_mm, address); > __ptep_set_access_flags(vma->vm_mm, ptep, entry, address); > - flush_tlb_page(vma, address); > + flush_tlb_fix_spurious_fault(vma, address); > } > return changed; > } > diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c > index 7452e1f4aa3c..bcb41d037593 100644 > --- a/arch/powerpc/mm/tlb-radix.c > +++ b/arch/powerpc/mm/tlb-radix.c > @@ -396,6 +396,27 @@ void radix__tlb_flush(struct mmu_gather *tlb) > static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; > static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; > > +void radix__local_flush_tlb_range_psize(struct mm_struct *mm, > + unsigned long start, unsigned long end, > + int psize) > +{ > + unsigned long pid; > + unsigned int page_shift = mmu_psize_defs[psize].shift; > + unsigned long page_size = 1UL << page_shift; > + > + pid = mm ? mm->context.id : 0; > + if (unlikely(pid == MMU_NO_CONTEXT)) > + return; > + > + preempt_disable(); > + if (end == TLB_FLUSH_ALL || ((end - start) >> page_shift) > > + tlb_local_single_page_flush_ceiling) > + _tlbiel_pid(pid, RIC_FLUSH_TLB); > + else > + _tlbiel_va_range(start, end, pid, page_size, psize); > + preempt_enable(); > +} > + > static bool __radix__flush_tlb_range_psize(struct mm_struct *mm, > unsigned long start, unsigned long end, > int psize, bool also_pwc) > @@ -518,6 +539,12 @@ void radix__flush_tlb_lpid(unsigned long lpid) > } > EXPORT_SYMBOL(radix__flush_tlb_lpid); > > +void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma, > + unsigned long start, unsigned long end) > +{ > + radix__local_flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); > +} > + > void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, > unsigned long start, unsigned long end) > {
On Fri, 08 Sep 2017 08:05:38 +1000 Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote: > On Fri, 2017-09-08 at 00:51 +1000, Nicholas Piggin wrote: > > When permissiveness is relaxed, or found to have been relaxed by > > another thread, we flush that address out of the TLB to avoid a > > future fault or micro-fault due to a stale TLB entry. > > > > Currently for processes with TLBs on other CPUs, this flush is always > > done with a global tlbie. Although that could reduce faults on remote > > CPUs, a broadcast operation seems to be wasteful for something that > > can be handled in-core by the remote CPU if it comes to it. > > > > This is not benchmarked yet. It does seem cut some tlbie operations > > from the bus. > > What happens with the nest MMU here ? Good question, I'm not sure. I can't tell from the UM or not if the agent and NMMU must discard cached translations if there is a translation cached but it has a permission fault. It's not clear from that I've read that if it's relying on the host to send back a tlbie. I'll keep digging. Thanks, Nick
Nicholas Piggin <npiggin@gmail.com> writes: > When permissiveness is relaxed, or found to have been relaxed by > another thread, we flush that address out of the TLB to avoid a > future fault or micro-fault due to a stale TLB entry. > > Currently for processes with TLBs on other CPUs, this flush is always > done with a global tlbie. Although that could reduce faults on remote > CPUs, a broadcast operation seems to be wasteful for something that > can be handled in-core by the remote CPU if it comes to it. > > This is not benchmarked yet. It does seem cut some tlbie operations > from the bus. > > Signed-off-by: Nicholas Piggin <npiggin@gmail.com> > --- > .../powerpc/include/asm/book3s/64/tlbflush-radix.h | 5 ++++ > arch/powerpc/include/asm/book3s/64/tlbflush.h | 11 +++++++++ > arch/powerpc/mm/pgtable-book3s64.c | 5 +++- > arch/powerpc/mm/pgtable.c | 2 +- > arch/powerpc/mm/tlb-radix.c | 27 ++++++++++++++++++++++ > 5 files changed, 48 insertions(+), 2 deletions(-) > > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > index b12460b306a7..34cd864b8fc1 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > @@ -16,6 +16,8 @@ extern bool radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long sta > unsigned long end, int psize); > extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, > unsigned long start, unsigned long end); > +extern void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma, > + unsigned long start, unsigned long end); > extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end); > extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); > @@ -24,6 +26,9 @@ extern void radix__local_flush_tlb_mm(struct mm_struct *mm); > extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); > extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, > int psize); > +extern void radix__local_flush_tlb_range_psize(struct mm_struct *mm, > + unsigned long start, unsigned long end, > + int psize); > extern void radix__tlb_flush(struct mmu_gather *tlb); > #ifdef CONFIG_SMP > extern void radix__flush_tlb_mm(struct mm_struct *mm); > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h > index 72b925f97bab..8a8b3e11a28e 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h > @@ -83,6 +83,17 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, > #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) > #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) > #endif /* CONFIG_SMP */ > + > +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault > +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, > + unsigned long address) > +{ > + if (radix_enabled()) > + radix__local_flush_tlb_page(vma, address); > + else > + flush_tlb_page(vma, address); > +} > + > /* > * flush the page walk cache for the address > */ > diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c > index 3b65917785a5..e46f346388d6 100644 > --- a/arch/powerpc/mm/pgtable-book3s64.c > +++ b/arch/powerpc/mm/pgtable-book3s64.c > @@ -40,7 +40,10 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, > if (changed) { > __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp), > pmd_pte(entry), address); > - flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); > + if (radix_enabled()) > + radix__local_flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); > + else > + flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); ^^^^ this is no-op for hash. > } > return changed; > } > diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c > index a03ff3d99e0c..acd6ae8062ce 100644 -aneesh
On Fri, 2017-09-08 at 14:44 +1000, Nicholas Piggin wrote: > On Fri, 08 Sep 2017 08:05:38 +1000 > Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote: > > > On Fri, 2017-09-08 at 00:51 +1000, Nicholas Piggin wrote: > > > When permissiveness is relaxed, or found to have been relaxed by > > > another thread, we flush that address out of the TLB to avoid a > > > future fault or micro-fault due to a stale TLB entry. > > > > > > Currently for processes with TLBs on other CPUs, this flush is always > > > done with a global tlbie. Although that could reduce faults on remote > > > CPUs, a broadcast operation seems to be wasteful for something that > > > can be handled in-core by the remote CPU if it comes to it. > > > > > > This is not benchmarked yet. It does seem cut some tlbie operations > > > from the bus. > > > > What happens with the nest MMU here ? > > Good question, I'm not sure. I can't tell from the UM or not if the > agent and NMMU must discard cached translations if there is a > translation cached but it has a permission fault. It's not clear > from that I've read that if it's relying on the host to send back a > tlbie. I think it's supposed to re-do a tablewalk. > I'll keep digging. > > Thanks, > Nick
On Fri, 8 Sep 2017 14:44:37 +1000 Nicholas Piggin <npiggin@gmail.com> wrote: > On Fri, 08 Sep 2017 08:05:38 +1000 > Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote: > > > On Fri, 2017-09-08 at 00:51 +1000, Nicholas Piggin wrote: > > > When permissiveness is relaxed, or found to have been relaxed by > > > another thread, we flush that address out of the TLB to avoid a > > > future fault or micro-fault due to a stale TLB entry. > > > > > > Currently for processes with TLBs on other CPUs, this flush is always > > > done with a global tlbie. Although that could reduce faults on remote > > > CPUs, a broadcast operation seems to be wasteful for something that > > > can be handled in-core by the remote CPU if it comes to it. > > > > > > This is not benchmarked yet. It does seem cut some tlbie operations > > > from the bus. > > > > What happens with the nest MMU here ? > > Good question, I'm not sure. I can't tell from the UM or not if the > agent and NMMU must discard cached translations if there is a > translation cached but it has a permission fault. It's not clear > from that I've read that if it's relying on the host to send back a > tlbie. > > I'll keep digging. Okay, talking to Alistair and for NPU/CAPI, this is a concern because the NMMU may not walk page tables if it has a cached translation. Have to confirm that with hardware. So we've got them wanting to hook into the core code and make it give them tibies. For now I'll put this on hold until NPU and CAPI and VAS get their patches in and working, then maybe revisit. Might be good to get them all moved to using a common nmmu.c driver that gives them a page fault handler and uses mmu notifiers to do their nmmu invalidations. Thanks, Nick
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index b12460b306a7..34cd864b8fc1 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -16,6 +16,8 @@ extern bool radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long sta unsigned long end, int psize); extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); @@ -24,6 +26,9 @@ extern void radix__local_flush_tlb_mm(struct mm_struct *mm); extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize); +extern void radix__local_flush_tlb_range_psize(struct mm_struct *mm, + unsigned long start, unsigned long end, + int psize); extern void radix__tlb_flush(struct mmu_gather *tlb); #ifdef CONFIG_SMP extern void radix__flush_tlb_mm(struct mm_struct *mm); diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 72b925f97bab..8a8b3e11a28e 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -83,6 +83,17 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) #endif /* CONFIG_SMP */ + +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, + unsigned long address) +{ + if (radix_enabled()) + radix__local_flush_tlb_page(vma, address); + else + flush_tlb_page(vma, address); +} + /* * flush the page walk cache for the address */ diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c index 3b65917785a5..e46f346388d6 100644 --- a/arch/powerpc/mm/pgtable-book3s64.c +++ b/arch/powerpc/mm/pgtable-book3s64.c @@ -40,7 +40,10 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, if (changed) { __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp), pmd_pte(entry), address); - flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + if (radix_enabled()) + radix__local_flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + else + flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); } return changed; } diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index a03ff3d99e0c..acd6ae8062ce 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -223,7 +223,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, if (!is_vm_hugetlb_page(vma)) assert_pte_locked(vma->vm_mm, address); __ptep_set_access_flags(vma->vm_mm, ptep, entry, address); - flush_tlb_page(vma, address); + flush_tlb_fix_spurious_fault(vma, address); } return changed; } diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c index 7452e1f4aa3c..bcb41d037593 100644 --- a/arch/powerpc/mm/tlb-radix.c +++ b/arch/powerpc/mm/tlb-radix.c @@ -396,6 +396,27 @@ void radix__tlb_flush(struct mmu_gather *tlb) static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; +void radix__local_flush_tlb_range_psize(struct mm_struct *mm, + unsigned long start, unsigned long end, + int psize) +{ + unsigned long pid; + unsigned int page_shift = mmu_psize_defs[psize].shift; + unsigned long page_size = 1UL << page_shift; + + pid = mm ? mm->context.id : 0; + if (unlikely(pid == MMU_NO_CONTEXT)) + return; + + preempt_disable(); + if (end == TLB_FLUSH_ALL || ((end - start) >> page_shift) > + tlb_local_single_page_flush_ceiling) + _tlbiel_pid(pid, RIC_FLUSH_TLB); + else + _tlbiel_va_range(start, end, pid, page_size, psize); + preempt_enable(); +} + static bool __radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, unsigned long end, int psize, bool also_pwc) @@ -518,6 +539,12 @@ void radix__flush_tlb_lpid(unsigned long lpid) } EXPORT_SYMBOL(radix__flush_tlb_lpid); +void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + radix__local_flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); +} + void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) {
When permissiveness is relaxed, or found to have been relaxed by another thread, we flush that address out of the TLB to avoid a future fault or micro-fault due to a stale TLB entry. Currently for processes with TLBs on other CPUs, this flush is always done with a global tlbie. Although that could reduce faults on remote CPUs, a broadcast operation seems to be wasteful for something that can be handled in-core by the remote CPU if it comes to it. This is not benchmarked yet. It does seem cut some tlbie operations from the bus. Signed-off-by: Nicholas Piggin <npiggin@gmail.com> --- .../powerpc/include/asm/book3s/64/tlbflush-radix.h | 5 ++++ arch/powerpc/include/asm/book3s/64/tlbflush.h | 11 +++++++++ arch/powerpc/mm/pgtable-book3s64.c | 5 +++- arch/powerpc/mm/pgtable.c | 2 +- arch/powerpc/mm/tlb-radix.c | 27 ++++++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-)