[qemu,v13,05/16] spapr_iommu: Add root memory region

We are going to have multiple DMA windows at different offsets on
a PCI bus. For the sake of migration, we will have as many TCE table
objects pre-created as many windows supported.
So we need a way to map windows dynamically onto a PCI bus
when migration of a table is completed but at this stage a TCE table
object does not have access to a PHB to ask it to map a DMA window
backed by just migrated TCE table.

This adds a "root" memory region (UINT64_MAX long) to the TCE object.
This new region is mapped on a PCI bus with enabled overlapping as
there will be one root MR per TCE table, each of them mapped at 0.
The actual IOMMU memory region is a subregion of the root region and
a TCE table enables/disables this subregion and maps it at
the specific offset inside the root MR which is 1:1 mapping of
a PCI address space.

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Reviewed-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Thomas Huth <thuth@redhat.com>
---
 hw/ppc/spapr_iommu.c   | 13 ++++++++++---
 hw/ppc/spapr_pci.c     |  5 +++--
 include/hw/ppc/spapr.h |  2 +-
 3 files changed, 14 insertions(+), 6 deletions(-)

On Tue, Mar 01, 2016 at 08:10:30PM +1100, Alexey Kardashevskiy wrote:
> We are going to have multiple DMA windows at different offsets on
> a PCI bus. For the sake of migration, we will have as many TCE table
> objects pre-created as many windows supported.
> So we need a way to map windows dynamically onto a PCI bus
> when migration of a table is completed but at this stage a TCE table
> object does not have access to a PHB to ask it to map a DMA window
> backed by just migrated TCE table.
> 
> This adds a "root" memory region (UINT64_MAX long) to the TCE object.
> This new region is mapped on a PCI bus with enabled overlapping as
> there will be one root MR per TCE table, each of them mapped at 0.
> The actual IOMMU memory region is a subregion of the root region and
> a TCE table enables/disables this subregion and maps it at
> the specific offset inside the root MR which is 1:1 mapping of
> a PCI address space.
> 
> Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
> Reviewed-by: David Gibson <david@gibson.dropbear.id.au>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> ---
>  hw/ppc/spapr_iommu.c   | 13 ++++++++++---
>  hw/ppc/spapr_pci.c     |  5 +++--
>  include/hw/ppc/spapr.h |  2 +-
>  3 files changed, 14 insertions(+), 6 deletions(-)
> 
> diff --git a/hw/ppc/spapr_iommu.c b/hw/ppc/spapr_iommu.c
> index e66e128..ba9ddbb 100644
> --- a/hw/ppc/spapr_iommu.c
> +++ b/hw/ppc/spapr_iommu.c
> @@ -172,10 +172,15 @@ static MemoryRegionIOMMUOps spapr_iommu_ops = {
>  static int spapr_tce_table_realize(DeviceState *dev)
>  {
>      sPAPRTCETable *tcet = SPAPR_TCE_TABLE(dev);
> +    Object *tcetobj = OBJECT(tcet);
> +    char tmp[32];
>  
>      tcet->fd = -1;
> -    memory_region_init_iommu(&tcet->iommu, OBJECT(dev), &spapr_iommu_ops,
> -                             "iommu-spapr", 0);
> +    snprintf(tmp, sizeof(tmp), "tce-root-%x", tcet->liobn);
> +    memory_region_init(&tcet->root, tcetobj, tmp, UINT64_MAX);
> +
> +    snprintf(tmp, sizeof(tmp), "tce-iommu-%x", tcet->liobn);
> +    memory_region_init_iommu(&tcet->iommu, tcetobj, &spapr_iommu_ops, tmp, 0);
>  
>      QLIST_INSERT_HEAD(&spapr_tce_tables, tcet, list);
>  
> @@ -253,6 +258,7 @@ static void spapr_tce_table_do_enable(sPAPRTCETable *tcet)
>  
>      memory_region_set_size(&tcet->iommu,
>                             (uint64_t)tcet->nb_table << tcet->page_shift);
> +    memory_region_add_subregion(&tcet->root, tcet->bus_offset, &tcet->iommu);
>  
>      tcet->enabled = true;
>  }
> @@ -279,6 +285,7 @@ static void spapr_tce_table_disable(sPAPRTCETable *tcet)
>          return;
>      }
>  
> +    memory_region_del_subregion(&tcet->root, &tcet->iommu);
>      memory_region_set_size(&tcet->iommu, 0);
>  
>      spapr_tce_free_table(tcet->table, tcet->fd, tcet->nb_table);
> @@ -302,7 +309,7 @@ static void spapr_tce_table_unrealize(DeviceState *dev, Error **errp)
>  
>  MemoryRegion *spapr_tce_get_iommu(sPAPRTCETable *tcet)
>  {
> -    return &tcet->iommu;
> +    return &tcet->root;
>  }
>  
>  static void spapr_tce_reset(DeviceState *dev)
> diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
> index c34a906..7b40687 100644
> --- a/hw/ppc/spapr_pci.c
> +++ b/hw/ppc/spapr_pci.c
> @@ -822,8 +822,6 @@ static int spapr_phb_dma_window_enable(sPAPRPHBState *sphb,
>  
>      spapr_tce_table_enable(tcet, page_shift, window_addr, nb_table, false);
>  
> -    memory_region_add_subregion(&sphb->iommu_root, tcet->bus_offset,
> -                                spapr_tce_get_iommu(tcet));
>      return 0;
>  }
>  
> @@ -1411,6 +1409,9 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
>          return;
>      }
>  
> +    memory_region_add_subregion(&sphb->iommu_root, 0,
> +                                spapr_tce_get_iommu(tcet));
> +

Logically this patch should add the _overlap() option rather than a
later one, yes?


>      /* Register default 32bit DMA window */
>      spapr_phb_dma_window_enable(sphb, sphb->dma_liobn,
>                                  SPAPR_TCE_PAGE_SHIFT,
> diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
> index 3e6bb84..bdf27ec 100644
> --- a/include/hw/ppc/spapr.h
> +++ b/include/hw/ppc/spapr.h
> @@ -548,7 +548,7 @@ struct sPAPRTCETable {
>      bool bypass;
>      bool need_vfio;
>      int fd;
> -    MemoryRegion iommu;
> +    MemoryRegion root, iommu;
>      struct VIOsPAPRDevice *vdev; /* for @bypass migration compatibility only */
>      QLIST_ENTRY(sPAPRTCETable) list;
>  };