| Message ID | 20161208081956.26221-7-aik@ozlabs.ru (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
On Thu, Dec 08, 2016 at 07:19:53PM +1100, Alexey Kardashevskiy wrote: > In real mode, TCE tables are invalidated using special > cache-inhibited store instructions which are not available in > virtual mode > > This defines and implements exchange_rm() callback. This does not > define set_rm/clear_rm/flush_rm callbacks as there is no user for those - > exchange/exchange_rm are only to be used by KVM for VFIO. > > The exchange_rm callback is defined for IODA1/IODA2 powernv platforms. > > This replaces list_for_each_entry_rcu with its lockless version as > from now on pnv_pci_ioda2_tce_invalidate() can be called in > the real mode too. > > Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> > --- > arch/powerpc/include/asm/iommu.h | 7 +++++++ > arch/powerpc/kernel/iommu.c | 23 +++++++++++++++++++++++ > arch/powerpc/platforms/powernv/pci-ioda.c | 26 +++++++++++++++++++++++++- > 3 files changed, 55 insertions(+), 1 deletion(-) > > diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h > index 9de8bad1fdf9..82e77ebf85f4 100644 > --- a/arch/powerpc/include/asm/iommu.h > +++ b/arch/powerpc/include/asm/iommu.h > @@ -64,6 +64,11 @@ struct iommu_table_ops { > long index, > unsigned long *hpa, > enum dma_data_direction *direction); > + /* Real mode */ > + int (*exchange_rm)(struct iommu_table *tbl, > + long index, > + unsigned long *hpa, > + enum dma_data_direction *direction); > #endif > void (*clear)(struct iommu_table *tbl, > long index, long npages); > @@ -209,6 +214,8 @@ extern void iommu_del_device(struct device *dev); > extern int __init tce_iommu_bus_notifier_init(void); > extern long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, > unsigned long *hpa, enum dma_data_direction *direction); > +extern long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry, > + unsigned long *hpa, enum dma_data_direction *direction); > #else > static inline void iommu_register_group(struct iommu_table_group *table_group, > int pci_domain_number, > diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c > index d12496889ce9..d02b8d22fb50 100644 > --- a/arch/powerpc/kernel/iommu.c > +++ b/arch/powerpc/kernel/iommu.c > @@ -1022,6 +1022,29 @@ long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, > } > EXPORT_SYMBOL_GPL(iommu_tce_xchg); > > +long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry, > + unsigned long *hpa, enum dma_data_direction *direction) > +{ > + long ret; > + > + ret = tbl->it_ops->exchange_rm(tbl, entry, hpa, direction); > + > + if (!ret && ((*direction == DMA_FROM_DEVICE) || > + (*direction == DMA_BIDIRECTIONAL))) { > + struct page *pg = realmode_pfn_to_page(*hpa >> PAGE_SHIFT); > + > + if (likely(pg)) { > + SetPageDirty(pg); > + } else { > + tbl->it_ops->exchange_rm(tbl, entry, hpa, direction); > + ret = -EFAULT; > + } > + } > + > + return ret; > +} > +EXPORT_SYMBOL_GPL(iommu_tce_xchg_rm); > + > int iommu_take_ownership(struct iommu_table *tbl) > { > unsigned long flags, i, sz = (tbl->it_size + 7) >> 3; > diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c > index ea181f02bebd..f2c2ab8fbb3e 100644 > --- a/arch/powerpc/platforms/powernv/pci-ioda.c > +++ b/arch/powerpc/platforms/powernv/pci-ioda.c > @@ -1855,6 +1855,17 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index, > > return ret; > } > + > +static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index, > + unsigned long *hpa, enum dma_data_direction *direction) > +{ > + long ret = pnv_tce_xchg(tbl, index, hpa, direction); > + > + if (!ret) > + pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true); > + > + return ret; > +} > #endif > > static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index, > @@ -1869,6 +1880,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = { > .set = pnv_ioda1_tce_build, > #ifdef CONFIG_IOMMU_API > .exchange = pnv_ioda1_tce_xchg, > + .exchange_rm = pnv_ioda1_tce_xchg_rm, > #endif > .clear = pnv_ioda1_tce_free, > .get = pnv_tce_get, > @@ -1943,7 +1955,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl, > { > struct iommu_table_group_link *tgl; > > - list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { > + list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) { > struct pnv_ioda_pe *pe = container_of(tgl->table_group, > struct pnv_ioda_pe, table_group); > struct pnv_phb *phb = pe->phb; > @@ -1999,6 +2011,17 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index, > > return ret; > } > + > +static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index, > + unsigned long *hpa, enum dma_data_direction *direction) > +{ > + long ret = pnv_tce_xchg(tbl, index, hpa, direction); > + > + if (!ret) > + pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true); > + > + return ret; > +} > #endif > > static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index, > @@ -2018,6 +2041,7 @@ static struct iommu_table_ops pnv_ioda2_iommu_ops = { > .set = pnv_ioda2_tce_build, > #ifdef CONFIG_IOMMU_API > .exchange = pnv_ioda2_tce_xchg, > + .exchange_rm = pnv_ioda2_tce_xchg_rm, > #endif > .clear = pnv_ioda2_tce_free, > .get = pnv_tce_get,
diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index 9de8bad1fdf9..82e77ebf85f4 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -64,6 +64,11 @@ struct iommu_table_ops { long index, unsigned long *hpa, enum dma_data_direction *direction); + /* Real mode */ + int (*exchange_rm)(struct iommu_table *tbl, + long index, + unsigned long *hpa, + enum dma_data_direction *direction); #endif void (*clear)(struct iommu_table *tbl, long index, long npages); @@ -209,6 +214,8 @@ extern void iommu_del_device(struct device *dev); extern int __init tce_iommu_bus_notifier_init(void); extern long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, unsigned long *hpa, enum dma_data_direction *direction); +extern long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry, + unsigned long *hpa, enum dma_data_direction *direction); #else static inline void iommu_register_group(struct iommu_table_group *table_group, int pci_domain_number, diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index d12496889ce9..d02b8d22fb50 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1022,6 +1022,29 @@ long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, } EXPORT_SYMBOL_GPL(iommu_tce_xchg); +long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry, + unsigned long *hpa, enum dma_data_direction *direction) +{ + long ret; + + ret = tbl->it_ops->exchange_rm(tbl, entry, hpa, direction); + + if (!ret && ((*direction == DMA_FROM_DEVICE) || + (*direction == DMA_BIDIRECTIONAL))) { + struct page *pg = realmode_pfn_to_page(*hpa >> PAGE_SHIFT); + + if (likely(pg)) { + SetPageDirty(pg); + } else { + tbl->it_ops->exchange_rm(tbl, entry, hpa, direction); + ret = -EFAULT; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(iommu_tce_xchg_rm); + int iommu_take_ownership(struct iommu_table *tbl) { unsigned long flags, i, sz = (tbl->it_size + 7) >> 3; diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index ea181f02bebd..f2c2ab8fbb3e 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1855,6 +1855,17 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index, return ret; } + +static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index, + unsigned long *hpa, enum dma_data_direction *direction) +{ + long ret = pnv_tce_xchg(tbl, index, hpa, direction); + + if (!ret) + pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true); + + return ret; +} #endif static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index, @@ -1869,6 +1880,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = { .set = pnv_ioda1_tce_build, #ifdef CONFIG_IOMMU_API .exchange = pnv_ioda1_tce_xchg, + .exchange_rm = pnv_ioda1_tce_xchg_rm, #endif .clear = pnv_ioda1_tce_free, .get = pnv_tce_get, @@ -1943,7 +1955,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl, { struct iommu_table_group_link *tgl; - list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { + list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) { struct pnv_ioda_pe *pe = container_of(tgl->table_group, struct pnv_ioda_pe, table_group); struct pnv_phb *phb = pe->phb; @@ -1999,6 +2011,17 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index, return ret; } + +static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index, + unsigned long *hpa, enum dma_data_direction *direction) +{ + long ret = pnv_tce_xchg(tbl, index, hpa, direction); + + if (!ret) + pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true); + + return ret; +} #endif static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index, @@ -2018,6 +2041,7 @@ static struct iommu_table_ops pnv_ioda2_iommu_ops = { .set = pnv_ioda2_tce_build, #ifdef CONFIG_IOMMU_API .exchange = pnv_ioda2_tce_xchg, + .exchange_rm = pnv_ioda2_tce_xchg_rm, #endif .clear = pnv_ioda2_tce_free, .get = pnv_tce_get,
In real mode, TCE tables are invalidated using special cache-inhibited store instructions which are not available in virtual mode This defines and implements exchange_rm() callback. This does not define set_rm/clear_rm/flush_rm callbacks as there is no user for those - exchange/exchange_rm are only to be used by KVM for VFIO. The exchange_rm callback is defined for IODA1/IODA2 powernv platforms. This replaces list_for_each_entry_rcu with its lockless version as from now on pnv_pci_ioda2_tce_invalidate() can be called in the real mode too. Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru> --- arch/powerpc/include/asm/iommu.h | 7 +++++++ arch/powerpc/kernel/iommu.c | 23 +++++++++++++++++++++++ arch/powerpc/platforms/powernv/pci-ioda.c | 26 +++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 1 deletion(-)