[qemu,v5,08/12] spapr_pci: Rework reset to reset DMA configuration

On a system reset, DMA configuration has to reset too. At the moment
it clears the table content. This is enough for the single table case
but with DDW, we will also have to disable all DMA windows except
the default one. Furthermore according to sPAPR, if the guest removed
the default window and created a huge one at the same zero offset on
a PCI bus, the reset handler has to recreate the default window with
the default properties (2GB big, 4K pages).

This reworks SPAPR PHB code to disable the existing DMA window on reset
and then configure and enable the default window.
Without DDW that means that the same window will be disabled and then
enabled with no other change in behaviour.

This changes the table creation to do it in one place in PHB (VFIO PHB
just inherits the behaviour from PHB). The actual table allocation is
done from the reset handler and this is where finish_realize() is called.

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
---
 hw/ppc/spapr_pci.c          | 42 ++++++++++++++++++++++++------------------
 hw/ppc/spapr_pci_vfio.c     | 17 +----------------
 include/hw/pci-host/spapr.h |  1 +
 3 files changed, 26 insertions(+), 34 deletions(-)

On Tue, Mar 31, 2015 at 04:28:43PM +1100, Alexey Kardashevskiy wrote:
> On a system reset, DMA configuration has to reset too. At the moment
> it clears the table content. This is enough for the single table case
> but with DDW, we will also have to disable all DMA windows except
> the default one. Furthermore according to sPAPR, if the guest removed
> the default window and created a huge one at the same zero offset on
> a PCI bus, the reset handler has to recreate the default window with
> the default properties (2GB big, 4K pages).
> 
> This reworks SPAPR PHB code to disable the existing DMA window on reset
> and then configure and enable the default window.
> Without DDW that means that the same window will be disabled and then
> enabled with no other change in behaviour.
> 
> This changes the table creation to do it in one place in PHB (VFIO PHB
> just inherits the behaviour from PHB). The actual table allocation is
> done from the reset handler and this is where finish_realize() is called.
> 
> Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
> ---
>  hw/ppc/spapr_pci.c          | 42 ++++++++++++++++++++++++------------------
>  hw/ppc/spapr_pci_vfio.c     | 17 +----------------
>  include/hw/pci-host/spapr.h |  1 +
>  3 files changed, 26 insertions(+), 34 deletions(-)
> 
> diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
> index acfdbe5..57bbc82 100644
> --- a/hw/ppc/spapr_pci.c
> +++ b/hw/ppc/spapr_pci.c
> @@ -722,6 +722,22 @@ static const MemoryRegionOps spapr_msi_ops = {
>  };
>  
>  /*
> + * DMA windows
> + */
> +int spapr_phb_dma_reset(sPAPRPHBState *sphb)
> +{
> +    const uint32_t liobn = SPAPR_PCI_LIOBN(sphb->index, 0);
> +    sPAPRTCETable *tcet = spapr_tce_find_by_liobn(liobn);
> +    sPAPRPHBClass *spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
> +    Error *err = NULL;
> +
> +    spapr_tce_table_disable(tcet);
> +    spc->finish_realize(sphb, &err);

Looks like "finish_realize" is no longer the right name for this hook.

> +
> +    return 0;
> +}
> +
> +/*
>   * PHB PCI device
>   */
>  static AddressSpace *spapr_pci_dma_iommu(PCIBus *bus, void *opaque, int devfn)
> @@ -736,11 +752,11 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
>      SysBusDevice *s = SYS_BUS_DEVICE(dev);
>      sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(s);
>      PCIHostState *phb = PCI_HOST_BRIDGE(s);
> -    sPAPRPHBClass *info = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(s);
>      char *namebuf;
>      int i;
>      PCIBus *bus;
>      uint64_t msi_window_size = 4096;
> +    sPAPRTCETable *tcet;
>  
>      if (sphb->index != (uint32_t)-1) {
>          hwaddr windows_base;
> @@ -880,12 +896,10 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
>          sphb->lsi_table[i].irq = irq;
>      }
>  
> -    if (!info->finish_realize) {
> -        error_setg(errp, "finish_realize not defined");
> -        return;
> -    }
> -
> -    info->finish_realize(sphb, errp);
> +    /* Create default DMA window */
> +    tcet = spapr_tce_new_table(DEVICE(sphb), sphb->dma_liobn);
> +    memory_region_add_subregion_overlap(&sphb->iommu_root, 0,
> +                                        spapr_tce_get_iommu(tcet), 0);

Why is this done explicitly, rather than using the set_props and
_enable() functions from the last patch?

Also, you shouldn't need this here - even on the first boot, the reset
hook will be called between realize and actually starting the guest,
so you should be able to delay the table allocation to there.

The reason for changing from add_subregion() to
add_subregion_overlap() also isn't clear.


>      sphb->msi = g_hash_table_new_full(g_int_hash, g_int_equal, g_free, g_free);
>  }
> @@ -895,20 +909,10 @@ static void spapr_phb_finish_realize(sPAPRPHBState *sphb, Error **errp)
>      sPAPRTCETable *tcet;
>      uint32_t nb_table;
>  
> -    tcet = spapr_tce_new_table(DEVICE(sphb), sphb->dma_liobn);
> -    if (!tcet) {
> -        error_setg(errp, "Unable to create TCE table for %s",
> -                   sphb->dtbusname);
> -        return ;
> -    }
> -
> +    tcet = spapr_tce_find_by_liobn(sphb->dma_liobn);
>      nb_table = SPAPR_PCI_DMA32_SIZE >> SPAPR_TCE_PAGE_SHIFT;
>      spapr_tce_set_props(tcet, 0, SPAPR_TCE_PAGE_SHIFT, nb_table, false);
>      spapr_tce_table_enable(tcet);
> -
> -    /* Register default 32bit DMA window */
> -    memory_region_add_subregion(&sphb->iommu_root, 0,
> -                                spapr_tce_get_iommu(tcet));
>  }
>  
>  static int spapr_phb_children_reset(Object *child, void *opaque)
> @@ -924,6 +928,8 @@ static int spapr_phb_children_reset(Object *child, void *opaque)
>  
>  static void spapr_phb_reset(DeviceState *qdev)
>  {
> +    spapr_phb_dma_reset(SPAPR_PCI_HOST_BRIDGE(qdev));
> +
>      /* Reset the IOMMU state */
>      object_child_foreach(OBJECT(qdev), spapr_phb_children_reset, NULL);
>  }
> diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
> index 6c9adb5..1657f6b 100644
> --- a/hw/ppc/spapr_pci_vfio.c
> +++ b/hw/ppc/spapr_pci_vfio.c
> @@ -53,25 +53,11 @@ static void spapr_phb_vfio_finish_realize(sPAPRPHBState *sphb, Error **errp)
>          return;
>      }
>  
> -    tcet = spapr_tce_new_table(DEVICE(sphb), liobn);
> -    if (!tcet) {
> -        error_setg(errp, "spapr-vfio: failed to create VFIO TCE table");
> -        return;
> -    }
> -
> -    /* Register default 32bit DMA window */
> +    tcet = spapr_tce_find_by_liobn(liobn);
>      nb_table = info.dma32_window_size >> SPAPR_TCE_PAGE_SHIFT;
>      spapr_tce_set_props(tcet, info.dma32_window_start, SPAPR_TCE_PAGE_SHIFT,
>                          nb_table, true);
>      spapr_tce_table_enable(tcet);
> -
> -    memory_region_add_subregion(&sphb->iommu_root, tcet->bus_offset,
> -                                spapr_tce_get_iommu(tcet));
> -}
> -
> -static void spapr_phb_vfio_reset(DeviceState *qdev)
> -{
> -    /* Do nothing */
>  }
>  
>  static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
> @@ -191,7 +177,6 @@ static void spapr_phb_vfio_class_init(ObjectClass *klass, void *data)
>      sPAPRPHBClass *spc = SPAPR_PCI_HOST_BRIDGE_CLASS(klass);
>  
>      dc->props = spapr_phb_vfio_properties;
> -    dc->reset = spapr_phb_vfio_reset;
>      spc->finish_realize = spapr_phb_vfio_finish_realize;
>      spc->eeh_set_option = spapr_phb_vfio_eeh_set_option;
>      spc->eeh_get_state = spapr_phb_vfio_eeh_get_state;
> diff --git a/include/hw/pci-host/spapr.h b/include/hw/pci-host/spapr.h
> index 5b497ce..f592276 100644
> --- a/include/hw/pci-host/spapr.h
> +++ b/include/hw/pci-host/spapr.h
> @@ -134,5 +134,6 @@ void spapr_pci_rtas_init(void);
>  sPAPRPHBState *spapr_pci_find_phb(sPAPREnvironment *spapr, uint64_t buid);
>  PCIDevice *spapr_pci_find_dev(sPAPREnvironment *spapr, uint64_t buid,
>                                uint32_t config_addr);
> +int spapr_phb_dma_reset(sPAPRPHBState *sphb);
>  
>  #endif /* __HW_SPAPR_PCI_H__ */