@@ -98,8 +98,8 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
* DMA buffers.
*/
if (is_isa_arcv2() && ioc_enable && coherent)
- dev->dma_coherent = true;
+ dev_set_dma_coherent(dev);
dev_info(dev, "use %scoherent DMA ops\n",
- dev->dma_coherent ? "" : "non");
+ dev_dma_coherent(dev) ? "" : "non");
}
@@ -98,7 +98,7 @@ static int highbank_platform_notifier(struct notifier_block *nb,
if (of_property_read_bool(dev->of_node, "dma-coherent")) {
val = readl(sregs_base + reg);
writel(val | 0xff01, sregs_base + reg);
- dev->dma_coherent = true;
+ dev_set_dma_coherent(dev);
}
return NOTIFY_OK;
@@ -95,7 +95,7 @@ static int mvebu_hwcc_notifier(struct notifier_block *nb,
if (event != BUS_NOTIFY_ADD_DEVICE)
return NOTIFY_DONE;
- dev->dma_coherent = true;
+ dev_set_dma_coherent(dev);
return NOTIFY_OK;
}
@@ -42,11 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
* enough to check if MPU is in use or not since in absence of
* MPU system memory map is used.
*/
- dev->dma_coherent = cacheid ? coherent : true;
+ dev_assign_dma_coherent(dev, cacheid ? coherent : true);
} else {
/*
* Assume coherent DMA in case MMU/MPU has not been set up.
*/
- dev->dma_coherent = (get_cr() & CR_M) ? coherent : true;
+ dev_assign_dma_coherent(dev, (get_cr() & CR_M) ? coherent : true);
}
}
@@ -1076,7 +1076,7 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
struct page **pages;
void *addr = NULL;
- int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL;
+ int coherent_flag = dev_dma_coherent(dev) ? COHERENT : NORMAL;
*handle = DMA_MAPPING_ERROR;
size = PAGE_ALIGN(size);
@@ -1124,7 +1124,7 @@ static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
if (vma->vm_pgoff >= nr_pages)
return -ENXIO;
- if (!dev->dma_coherent)
+ if (!dev_dma_coherent(dev))
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
err = vm_map_pages(vma, pages, nr_pages);
@@ -1141,7 +1141,7 @@ static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t handle, unsigned long attrs)
{
- int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL;
+ int coherent_flag = dev_dma_coherent(dev) ? COHERENT : NORMAL;
struct page **pages;
size = PAGE_ALIGN(size);
@@ -1202,7 +1202,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
phys_addr_t phys = page_to_phys(sg_page(s));
unsigned int len = PAGE_ALIGN(s->offset + s->length);
- if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+ if (!dev_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
arch_sync_dma_for_device(sg_phys(s), s->length, dir);
prot = __dma_info_to_prot(dir, attrs);
@@ -1304,7 +1304,7 @@ static void arm_iommu_unmap_sg(struct device *dev,
if (sg_dma_len(s))
__iommu_remove_mapping(dev, sg_dma_address(s),
sg_dma_len(s));
- if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+ if (!dev_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
arch_sync_dma_for_cpu(sg_phys(s), s->length, dir);
}
}
@@ -1323,7 +1323,7 @@ static void arm_iommu_sync_sg_for_cpu(struct device *dev,
struct scatterlist *s;
int i;
- if (dev->dma_coherent)
+ if (dev_dma_coherent(dev))
return;
for_each_sg(sg, s, nents, i)
@@ -1345,7 +1345,7 @@ static void arm_iommu_sync_sg_for_device(struct device *dev,
struct scatterlist *s;
int i;
- if (dev->dma_coherent)
+ if (dev_dma_coherent(dev))
return;
for_each_sg(sg, s, nents, i)
@@ -1371,7 +1371,7 @@ static dma_addr_t arm_iommu_map_phys(struct device *dev, phys_addr_t phys,
dma_addr_t dma_addr;
int ret, prot;
- if (!dev->dma_coherent &&
+ if (!dev_dma_coherent(dev) &&
!(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))
arch_sync_dma_for_device(phys, size, dir);
@@ -1412,7 +1412,7 @@ static void arm_iommu_unmap_phys(struct device *dev, dma_addr_t handle,
if (!iova)
return;
- if (!dev->dma_coherent &&
+ if (!dev_dma_coherent(dev) &&
!(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) {
phys_addr_t phys = iommu_iova_to_phys(mapping->domain, iova);
@@ -1431,7 +1431,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
unsigned int offset = handle & ~PAGE_MASK;
phys_addr_t phys;
- if (dev->dma_coherent || !iova)
+ if (dev_dma_coherent(dev) || !iova)
return;
phys = iommu_iova_to_phys(mapping->domain, iova);
@@ -1446,7 +1446,7 @@ static void arm_iommu_sync_single_for_device(struct device *dev,
unsigned int offset = handle & ~PAGE_MASK;
phys_addr_t phys;
- if (dev->dma_coherent || !iova)
+ if (dev_dma_coherent(dev) || !iova)
return;
phys = iommu_iova_to_phys(mapping->domain, iova);
@@ -1701,13 +1701,13 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) { }
void arch_setup_dma_ops(struct device *dev, bool coherent)
{
/*
- * Due to legacy code that sets the ->dma_coherent flag from a bus
- * notifier we can't just assign coherent to the ->dma_coherent flag
+ * Due to legacy code that sets the dma_coherent flag from a bus
+ * notifier we can't just assign coherent to the dma_coherent flag
* here, but instead have to make sure we only set but never clear it
* for now.
*/
if (coherent)
- dev->dma_coherent = true;
+ dev_set_dma_coherent(dev);
/*
* Don't override the dma_ops if they have already been set. Ideally
@@ -48,7 +48,7 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
dev_driver_string(dev), dev_name(dev),
ARCH_DMA_MINALIGN, cls);
- dev->dma_coherent = coherent;
+ dev_assign_dma_coherent(dev, coherent);
xen_setup_dma_ops(dev);
}
@@ -139,6 +139,6 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
void arch_setup_dma_ops(struct device *dev, bool coherent)
{
- dev->dma_coherent = coherent;
+ dev_assign_dma_coherent(dev, coherent);
}
#endif
@@ -140,7 +140,7 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
"%s %s: device non-coherent but no non-coherent operations supported",
dev_driver_string(dev), dev_name(dev));
- dev->dma_coherent = coherent;
+ dev_assign_dma_coherent(dev, coherent);
}
void riscv_noncoherent_supported(void)
@@ -3173,7 +3173,7 @@ void device_initialize(struct device *dev)
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
- dev->dma_coherent = dma_default_coherent;
+ dev_assign_dma_coherent(dev, dma_default_coherent);
#endif
swiotlb_dev_init(dev);
}
@@ -312,7 +312,7 @@ k3_udma_glue_request_tx_chn_common(struct device *dev,
if (xudma_is_pktdma(tx_chn->common.udmax)) {
/* prepare the channel device as coherent */
- tx_chn->common.chan_dev.dma_coherent = true;
+ dev_set_dma_coherent(&tx_chn->common.chan_dev);
dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
DMA_BIT_MASK(48));
}
@@ -1003,7 +1003,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
if (xudma_is_pktdma(rx_chn->common.udmax)) {
/* prepare the channel device as coherent */
- rx_chn->common.chan_dev.dma_coherent = true;
+ dev_set_dma_coherent(&rx_chn->common.chan_dev);
dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
DMA_BIT_MASK(48));
}
@@ -1104,7 +1104,7 @@ k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn
if (xudma_is_pktdma(rx_chn->common.udmax)) {
/* prepare the channel device as coherent */
- rx_chn->common.chan_dev.dma_coherent = true;
+ dev_set_dma_coherent(&rx_chn->common.chan_dev);
dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
DMA_BIT_MASK(48));
rx_chn->single_fdq = false;
@@ -428,18 +428,18 @@ static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)
/* No special handling for the channel */
chan->dev->chan_dma_dev = false;
- chan_dev->dma_coherent = false;
+ dev_clear_dma_coherent(chan_dev);
chan_dev->dma_parms = NULL;
} else if (asel == 14 || asel == 15) {
chan->dev->chan_dma_dev = true;
- chan_dev->dma_coherent = true;
+ dev_set_dma_coherent(chan_dev);
dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));
chan_dev->dma_parms = chan_dev->parent->dma_parms;
} else {
dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);
- chan_dev->dma_coherent = false;
+ dev_clear_dma_coherent(chan_dev);
chan_dev->dma_parms = NULL;
}
}
@@ -481,6 +481,8 @@ struct device_physical_location {
* @DEV_FLAG_STATE_SYNCED: The hardware state of this device has been synced to
* match the software state of this device by calling the
* driver/bus sync_state() callback.
+ * @DEV_FLAG_DMA_COHERENT: This particular device is dma coherent, even if the
+ * architecture supports non-coherent devices.
*/
enum struct_device_flags {
DEV_FLAG_READY_TO_PROBE = 0,
@@ -489,6 +491,7 @@ enum struct_device_flags {
DEV_FLAG_DMA_SKIP_SYNC = 3,
DEV_FLAG_DMA_OPS_BYPASS = 4,
DEV_FLAG_STATE_SYNCED = 5,
+ DEV_FLAG_DMA_COHERENT = 6,
DEV_FLAG_COUNT
};
@@ -572,8 +575,6 @@ enum struct_device_flags {
* @offline: Set after successful invocation of bus type's .offline().
* @of_node_reused: Set if the device-tree node is shared with an ancestor
* device.
- * @dma_coherent: this particular device is dma coherent, even if the
- * architecture supports non-coherent devices.
* @flags: DEV_FLAG_XXX flags. Use atomic bitfield operations to modify.
*
* At the lowest level, every device in a Linux system is represented by an
@@ -681,11 +682,6 @@ struct device {
bool offline_disabled:1;
bool offline:1;
bool of_node_reused:1;
-#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
- defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
- defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
- bool dma_coherent:1;
-#endif
DECLARE_BITMAP(flags, DEV_FLAG_COUNT);
};
@@ -718,6 +714,7 @@ __create_dev_flag_accessors(dma_iommu, DEV_FLAG_DMA_IOMMU);
__create_dev_flag_accessors(dma_skip_sync, DEV_FLAG_DMA_SKIP_SYNC);
__create_dev_flag_accessors(dma_ops_bypass, DEV_FLAG_DMA_OPS_BYPASS);
__create_dev_flag_accessors(state_synced, DEV_FLAG_STATE_SYNCED);
+__create_dev_flag_accessors(dma_coherent, DEV_FLAG_DMA_COHERENT);
/**
* struct device_link - Device link representation.
@@ -230,7 +230,7 @@ int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
extern bool dma_default_coherent;
static inline bool dev_is_dma_coherent(struct device *dev)
{
- return dev->dma_coherent;
+ return dev_dma_coherent(dev);
}
#else
#define dma_default_coherent true
In C, bitfields are not necessarily safe to modify from multiple threads without locking. Switch "dma_coherent" over to the "flags" field so modifications are safe. Cc: Christoph Hellwig <hch@lst.de> Signed-off-by: Douglas Anderson <dianders@chromium.org> --- Not fixing any known bugs; problem is theoretical and found by code inspection. Change is done somewhat manually and only lightly tested (mostly compile-time tested). NOTE: even though previously we only took up a bit if CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE, CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU, or CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL, in this change I reserve the bit unconditionally. While we could get the "dynamic" behavior by changing the flags definition to be an "enum", it doesn't seem worth it at this point. Changes in v4: - Use accessor functions for flags Changes in v3: - New arch/arc/mm/dma.c | 4 ++-- arch/arm/mach-highbank/highbank.c | 2 +- arch/arm/mach-mvebu/coherency.c | 2 +- arch/arm/mm/dma-mapping-nommu.c | 4 ++-- arch/arm/mm/dma-mapping.c | 28 ++++++++++++++-------------- arch/arm64/mm/dma-mapping.c | 2 +- arch/mips/mm/dma-noncoherent.c | 2 +- arch/riscv/mm/dma-noncoherent.c | 2 +- drivers/base/core.c | 2 +- drivers/dma/ti/k3-udma-glue.c | 6 +++--- drivers/dma/ti/k3-udma.c | 6 +++--- include/linux/device.h | 11 ++++------- include/linux/dma-map-ops.h | 2 +- 13 files changed, 35 insertions(+), 38 deletions(-)