@@ -528,12 +528,8 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
WARN_ON_ONCE(!iod->nents);
- if (is_pci_p2pdma_page(sg_page(iod->sg)))
- pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
- rq_dma_dir(req));
- else
- dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
-
+ dma_unmap_sg_attrs(dev->dev, iod->sg, iod->nents, rq_dma_dir(req),
+ DMA_ATTR_P2PDMA);
if (iod->npages == 0)
dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
@@ -570,7 +566,7 @@ static void nvme_print_sgl(struct scatterlist *sgl, int nents)
pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
"dma_address:%pad dma_length:%d\n",
i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
- sg_dma_len(sg));
+ sg_dma_p2pdma_len(sg));
}
}
@@ -581,7 +577,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
struct dma_pool *pool;
int length = blk_rq_payload_bytes(req);
struct scatterlist *sg = iod->sg;
- int dma_len = sg_dma_len(sg);
+ int dma_len = sg_dma_p2pdma_len(sg);
u64 dma_addr = sg_dma_address(sg);
int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
__le64 *prp_list;
@@ -601,7 +597,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
} else {
sg = sg_next(sg);
dma_addr = sg_dma_address(sg);
- dma_len = sg_dma_len(sg);
+ dma_len = sg_dma_p2pdma_len(sg);
}
if (length <= NVME_CTRL_PAGE_SIZE) {
@@ -650,7 +646,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
goto bad_sgl;
sg = sg_next(sg);
dma_addr = sg_dma_address(sg);
- dma_len = sg_dma_len(sg);
+ dma_len = sg_dma_p2pdma_len(sg);
}
done:
@@ -670,7 +666,7 @@ static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
struct scatterlist *sg)
{
sge->addr = cpu_to_le64(sg_dma_address(sg));
- sge->length = cpu_to_le32(sg_dma_len(sg));
+ sge->length = cpu_to_le32(sg_dma_p2pdma_len(sg));
sge->type = NVME_SGL_FMT_DATA_DESC << 4;
}
@@ -814,14 +810,12 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
if (!iod->nents)
goto out;
- if (is_pci_p2pdma_page(sg_page(iod->sg)))
- nr_mapped = pci_p2pdma_map_sg_attrs(dev->dev, iod->sg,
- iod->nents, rq_dma_dir(req), DMA_ATTR_NO_WARN);
- else
- nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents,
- rq_dma_dir(req), DMA_ATTR_NO_WARN);
- if (!nr_mapped)
+ nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents,
+ rq_dma_dir(req), DMA_ATTR_NO_WARN | DMA_ATTR_P2PDMA);
+ if (!nr_mapped) {
+ ret = BLK_STS_IOERR;
goto out;
+ }
iod->use_sgl = nvme_pci_use_sgls(dev, req);
if (iod->use_sgl)
Switch to using sg_dma_p2pdma_len() in places where sg_dma_len() is used. Then replace the calls to pci_p2pdma_[un]map_sg() with calls to dma_[un]map_sg() with DMA_ATTR_P2PDMA. This should be equivalent, though support will be somewhat less (only dma-direct and dma-iommu are currently supported). Using DMA_ATTR_P2PDMA is safe because the block layer restricts requests to be much less than 2GB, thus there's no way for a segment to be greater than 2GB. Signed-off-by: Logan Gunthorpe <logang@deltatee.com> --- drivers/nvme/host/pci.c | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-)