@@ -471,6 +471,10 @@ DECLARE_STATS_COUNTER(cnt_map_single);
DECLARE_STATS_COUNTER(cnt_unmap_single);
DECLARE_STATS_COUNTER(cnt_map_sg);
DECLARE_STATS_COUNTER(cnt_unmap_sg);
+#ifdef CONFIG_HAS_DMA_P2P
+DECLARE_STATS_COUNTER(cnt_map_peer_resource);
+DECLARE_STATS_COUNTER(cnt_unmap_peer_resource);
+#endif
DECLARE_STATS_COUNTER(cnt_alloc_coherent);
DECLARE_STATS_COUNTER(cnt_free_coherent);
DECLARE_STATS_COUNTER(cross_page);
@@ -509,6 +513,10 @@ static void amd_iommu_stats_init(void)
amd_iommu_stats_add(&cnt_unmap_single);
amd_iommu_stats_add(&cnt_map_sg);
amd_iommu_stats_add(&cnt_unmap_sg);
+#ifdef CONFIG_HAS_DMA_P2P
+ amd_iommu_stats_add(&cnt_map_peer_resource);
+ amd_iommu_stats_add(&cnt_unmap_peer_resource);
+#endif
amd_iommu_stats_add(&cnt_alloc_coherent);
amd_iommu_stats_add(&cnt_free_coherent);
amd_iommu_stats_add(&cross_page);
@@ -2585,20 +2593,16 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
}
/*
- * The exported map_single function for dma_ops.
+ * Wrapper function that contains code common to mapping a physical address
+ * range from a page or a resource.
*/
-static dma_addr_t map_page(struct device *dev, struct page *page,
- unsigned long offset, size_t size,
- enum dma_data_direction dir,
- struct dma_attrs *attrs)
+static dma_addr_t __map_phys(struct device *dev, phys_addr_t paddr,
+ size_t size, enum dma_data_direction dir)
{
unsigned long flags;
struct protection_domain *domain;
dma_addr_t addr;
u64 dma_mask;
- phys_addr_t paddr = page_to_phys(page) + offset;
-
- INC_STATS_COUNTER(cnt_map_single);
domain = get_domain(dev);
if (PTR_ERR(domain) == -EINVAL)
@@ -2624,16 +2628,15 @@ out:
}
/*
- * The exported unmap_single function for dma_ops.
+ * Wrapper function that contains code common to unmapping a physical address
+ * range from a page or a resource.
*/
-static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
- enum dma_data_direction dir, struct dma_attrs *attrs)
+static void __unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir)
{
unsigned long flags;
struct protection_domain *domain;
- INC_STATS_COUNTER(cnt_unmap_single);
-
domain = get_domain(dev);
if (IS_ERR(domain))
return;
@@ -2707,6 +2710,72 @@ unmap:
}
/*
+ * The exported map_single function for dma_ops.
+ */
+static dma_addr_t map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ INC_STATS_COUNTER(cnt_map_single);
+
+ return __map_phys(dev, page_to_phys(page) + offset, size, dir);
+}
+
+/*
+ * The exported unmap_single function for dma_ops.
+ */
+static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+ INC_STATS_COUNTER(cnt_unmap_single);
+
+ __unmap_phys(dev, dma_addr, size, dir);
+}
+
+#ifdef CONFIG_HAS_DMA_P2P
+/*
+ * The exported map_peer_resource function for dma_ops.
+ */
+static dma_peer_addr_t map_peer_resource(struct device *dev,
+ struct device *peer,
+ struct resource *res,
+ unsigned long offset,
+ size_t size,
+ enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ struct pci_dev *pdev;
+ struct pci_dev *ppeer;
+
+ INC_STATS_COUNTER(cnt_map_peer_resource);
+
+ if (!dev_is_pci(dev) || !dev_is_pci(peer))
+ return DMA_ERROR_CODE;
+
+ pdev = to_pci_dev(dev);
+ ppeer = to_pci_dev(peer);
+
+ if (!pci_peer_traffic_supported(pdev, ppeer))
+ return DMA_ERROR_CODE;
+
+ return __map_phys(dev, res->start + offset, size, dir);
+}
+
+/*
+ * The exported unmap_peer_resource function for dma_ops.
+ */
+static void unmap_peer_resource(struct device *dev, dma_peer_addr_t dma_addr,
+ size_t size, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ INC_STATS_COUNTER(cnt_unmap_peer_resource);
+
+ __unmap_phys(dev, dma_addr, size, dir);
+}
+#endif
+
+/*
* The exported map_sg function for dma_ops (handles scatter-gather
* lists).
*/
@@ -2852,6 +2921,10 @@ static struct dma_map_ops amd_iommu_dma_ops = {
.unmap_page = unmap_page,
.map_sg = map_sg,
.unmap_sg = unmap_sg,
+#ifdef CONFIG_HAS_DMA_P2P
+ .map_peer_resource = map_peer_resource,
+ .unmap_peer_resource = unmap_peer_resource,
+#endif
.dma_supported = amd_iommu_dma_supported,
};