diff --git a/hw/ppc/spapr_iommu.c b/hw/ppc/spapr_iommu.c
index c945dba..7af2700 100644
--- a/hw/ppc/spapr_iommu.c
+++ b/hw/ppc/spapr_iommu.c
@@ -155,6 +155,16 @@ static uint64_t spapr_tce_get_page_sizes(MemoryRegion *iommu)
     return 1ULL << tcet->page_shift;
 }
 
+static void spapr_tce_vfio_start(MemoryRegion *iommu)
+{
+    spapr_tce_set_need_vfio(container_of(iommu, sPAPRTCETable, iommu), true);
+}
+
+static void spapr_tce_vfio_stop(MemoryRegion *iommu)
+{
+    spapr_tce_set_need_vfio(container_of(iommu, sPAPRTCETable, iommu), false);
+}
+
 static void spapr_tce_table_do_enable(sPAPRTCETable *tcet);
 static void spapr_tce_table_do_disable(sPAPRTCETable *tcet);
 
@@ -239,6 +249,8 @@ static const VMStateDescription vmstate_spapr_tce_table = {
 static MemoryRegionIOMMUOps spapr_iommu_ops = {
     .translate = spapr_tce_translate_iommu,
     .get_page_sizes = spapr_tce_get_page_sizes,
+    .vfio_start = spapr_tce_vfio_start,
+    .vfio_stop = spapr_tce_vfio_stop,
 };
 
 static int spapr_tce_table_realize(DeviceState *dev)
diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
index 5b9ccff..51e7d56 100644
--- a/hw/ppc/spapr_pci.c
+++ b/hw/ppc/spapr_pci.c
@@ -1086,12 +1086,6 @@ static void spapr_phb_add_pci_device(sPAPRDRConnector *drc,
     void *fdt = NULL;
     int fdt_start_offset = 0, fdt_size;
 
-    if (object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
-        sPAPRTCETable *tcet = spapr_tce_find_by_liobn(phb->dma_liobn);
-
-        spapr_tce_set_need_vfio(tcet, true);
-    }
-
     if (dev->hotplugged) {
         fdt = create_device_tree(&fdt_size);
         fdt_start_offset = spapr_create_pci_child_dt(phb, pdev, fdt, 0);
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 3f2fb23..03daf88 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -421,6 +421,26 @@ static void vfio_listener_region_add(MemoryListener *listener,
         QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
 
         memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
+
+        if (section->mr->iommu_ops && section->mr->iommu_ops->vfio_start) {
+            VFIOIOMMUMR *iommumr;
+            bool found = false;
+
+            QLIST_FOREACH(iommumr, &container->space->iommumrs, iommumr_next) {
+                if (iommumr->iommu == section->mr) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                iommumr = g_malloc0(sizeof(*iommumr));
+                iommumr->iommu = section->mr;
+                section->mr->iommu_ops->vfio_start(section->mr);
+                QLIST_INSERT_HEAD(&container->space->iommumrs, iommumr,
+                                  iommumr_next);
+            }
+            ++iommumr->users;
+        }
         memory_region_iommu_replay(giommu->iommu, &giommu->n,
                                    false);
 
@@ -470,6 +490,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
     hwaddr iova, end;
     Int128 llend, llsize;
     int ret;
+    MemoryRegion *iommu = NULL;
 
     if (vfio_listener_skipped_section(section)) {
         trace_vfio_listener_region_del_skip(
@@ -490,13 +511,30 @@ static void vfio_listener_region_del(MemoryListener *listener,
 
         QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
             if (giommu->iommu == section->mr) {
+                VFIOIOMMUMR *iommumr;
+
                 memory_region_unregister_iommu_notifier(&giommu->n);
+
+                QLIST_FOREACH(iommumr, &container->space->iommumrs,
+                              iommumr_next) {
+                    if (iommumr->iommu != section->mr) {
+                        continue;
+                    }
+                    --iommumr->users;
+                    if (iommumr->users) {
+                        break;
+                    }
+                    QLIST_REMOVE(iommumr, iommumr_next);
+                    g_free(iommumr);
+                    iommu = giommu->iommu;
+                    break;
+                }
+
                 QLIST_REMOVE(giommu, giommu_next);
                 g_free(giommu);
                 break;
             }
         }
-
         /*
          * FIXME: We assume the one big unmap below is adequate to
          * remove any individual page mappings in the IOMMU which
@@ -527,6 +565,10 @@ static void vfio_listener_region_del(MemoryListener *listener,
                      "0x%"HWADDR_PRIx") = %d (%m)",
                      container, iova, int128_get64(llsize), ret);
     }
+
+    if (iommu && iommu->iommu_ops && iommu->iommu_ops->vfio_stop) {
+        iommu->iommu_ops->vfio_stop(section->mr);
+    }
 }
 
 static const MemoryListener vfio_memory_listener = {
@@ -787,6 +829,7 @@ static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
     space = g_malloc0(sizeof(*space));
     space->as = as;
     QLIST_INIT(&space->containers);
+    QLIST_INIT(&space->iommumrs);
 
     QLIST_INSERT_HEAD(&vfio_address_spaces, space, list);
 
diff --git a/include/exec/memory.h b/include/exec/memory.h
index a3a1703..52d2c70 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -151,6 +151,10 @@ struct MemoryRegionIOMMUOps {
     IOMMUTLBEntry (*translate)(MemoryRegion *iommu, hwaddr addr, bool is_write);
     /* Returns supported page sizes */
     uint64_t (*get_page_sizes)(MemoryRegion *iommu);
+    /* Called when VFIO starts using this */
+    void (*vfio_start)(MemoryRegion *iommu);
+    /* Called when VFIO stops using this */
+    void (*vfio_stop)(MemoryRegion *iommu);
 };
 
 typedef struct CoalescedMemoryRange CoalescedMemoryRange;
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 808263b..a9e6e33 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -64,9 +64,16 @@ typedef struct VFIORegion {
 typedef struct VFIOAddressSpace {
     AddressSpace *as;
     QLIST_HEAD(, VFIOContainer) containers;
+    QLIST_HEAD(, VFIOIOMMUMR) iommumrs;
     QLIST_ENTRY(VFIOAddressSpace) list;
 } VFIOAddressSpace;
 
+typedef struct VFIOIOMMUMR {
+    MemoryRegion *iommu;
+    int users;
+    QLIST_ENTRY(VFIOIOMMUMR) iommumr_next;
+} VFIOIOMMUMR;
+
 struct VFIOGroup;
 
 typedef struct VFIOContainer {
