diff mbox

[v5,1/1] s390x: pci infrastructure modeling

Message ID 1440062544-13960-2-git-send-email-zyimin@linux.vnet.ibm.com
State New
Headers show

Commit Message

Yi Min Zhao Aug. 20, 2015, 9:22 a.m. UTC
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>

This patch introduce a new facility(and bus)
to hold devices representing information actually
provided by s390 firmware and I/O configuration.
usage example:
-device s390-pcihost,index=1
-device zpci,fid=2,uid=5,pci_id=vpci1,id=zpci1
-device vfio-pci,host=0000:00:00.0,id=vpci1

The first line will create a s390 pci host bridge
and init the root bus. And user must assign a
unique value to index which is a new property of
s390-pcihost device.
The second line will create a s390 pci device to
store s390 specific information, and references
the corresponding vfio pci device via device id.
We create a s390 pci facility bus to hold all the
zpci devices.
The third line will create a standard vfio pci
device, and attach it to the root bus. These are
similiar to the standard process to define a pci
device on other platform.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
---
 hw/s390x/s390-pci-bus.c    |  379 ++++++++++++++++++++++++++++++++++----------
 hw/s390x/s390-pci-bus.h    |   54 ++++++-
 hw/s390x/s390-pci-inst.c   |   80 ++++++----
 hw/s390x/s390-virtio-ccw.c |    5 +-
 4 files changed, 392 insertions(+), 126 deletions(-)
diff mbox

Patch

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 560b66a..524dd79 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -32,16 +32,16 @@  int chsc_sei_nt2_get_event(void *res)
     PciCcdfErr *eccdf;
     int rc = 1;
     SeiContainer *sei_cont;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    if (!s) {
+    if (!f) {
         return rc;
     }
 
-    sei_cont = QTAILQ_FIRST(&s->pending_sei);
+    sei_cont = QTAILQ_FIRST(&f->pending_sei);
     if (sei_cont) {
-        QTAILQ_REMOVE(&s->pending_sei, sei_cont, link);
+        QTAILQ_REMOVE(&f->pending_sei, sei_cont, link);
         nt2_res->nt = 2;
         nt2_res->cc = sei_cont->cc;
         nt2_res->length = cpu_to_be16(sizeof(ChscSeiNt2Res));
@@ -72,30 +72,41 @@  int chsc_sei_nt2_get_event(void *res)
 
 int chsc_sei_nt2_have_event(void)
 {
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    if (!s) {
+    if (!f) {
         return 0;
     }
 
-    return !QTAILQ_EMPTY(&s->pending_sei);
+    return !QTAILQ_EMPTY(&f->pending_sei);
+}
+
+void s390_pci_device_enable(S390PCIBusDevice *zpci)
+{
+    zpci->fh = zpci->fh | 1 << ENABLE_BIT_OFFSET;
+}
+
+void s390_pci_device_disable(S390PCIBusDevice *zpci)
+{
+    zpci->fh = zpci->fh & ~(1 << ENABLE_BIT_OFFSET);
+    if (zpci->is_unplugged) {
+        object_unparent(OBJECT(zpci));
+    }
 }
 
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid)
 {
     S390PCIBusDevice *pbdev;
-    int i;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    if (!s) {
+    if (!f) {
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = &s->pbdev[i];
-        if ((pbdev->fh != 0) && (pbdev->fid == fid)) {
+    QTAILQ_FOREACH(pbdev, &f->zpci_list, next) {
+        if (pbdev->fid == fid) {
             return pbdev;
         }
     }
@@ -126,39 +137,20 @@  void s390_pci_sclp_configure(int configure, SCCB *sccb)
     return;
 }
 
-static uint32_t s390_pci_get_pfid(PCIDevice *pdev)
-{
-    return PCI_SLOT(pdev->devfn);
-}
-
-static uint32_t s390_pci_get_pfh(PCIDevice *pdev)
+S390PCIBusDevice *s390_pci_find_dev_by_idx(uint64_t idx)
 {
-    return PCI_SLOT(pdev->devfn) | FH_VIRT;
-}
-
-S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
-{
-    S390PCIBusDevice *pbdev;
-    int i;
-    int j = 0;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIBusDevice *tmp;
+    S390PCIFacility *fac = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    if (!s) {
+    if (!fac) {
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = &s->pbdev[i];
-
-        if (pbdev->fh == 0) {
-            continue;
-        }
-
-        if (j == idx) {
-            return pbdev;
+    QTAILQ_FOREACH(tmp, &fac->zpci_list, next) {
+        if (idx == tmp->idx && tmp->available) {
+            return tmp;
         }
-        j++;
     }
 
     return NULL;
@@ -167,16 +159,14 @@  S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh)
 {
     S390PCIBusDevice *pbdev;
-    int i;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    if (!s || !fh) {
+    if (!f || !fh) {
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = &s->pbdev[i];
+    QTAILQ_FOREACH(pbdev, &f->zpci_list, next) {
         if (pbdev->fh == fh) {
             return pbdev;
         }
@@ -189,10 +179,10 @@  static void s390_pci_generate_event(uint8_t cc, uint16_t pec, uint32_t fh,
                                     uint32_t fid, uint64_t faddr, uint32_t e)
 {
     SeiContainer *sei_cont;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    if (!s) {
+    if (!f) {
         return;
     }
 
@@ -204,7 +194,7 @@  static void s390_pci_generate_event(uint8_t cc, uint16_t pec, uint32_t fh,
     sei_cont->faddr = faddr;
     sei_cont->e = e;
 
-    QTAILQ_INSERT_TAIL(&s->pending_sei, sei_cont, link);
+    QTAILQ_INSERT_TAIL(&f->pending_sei, sei_cont, link);
     css_generate_css_crws(0);
 }
 
@@ -308,7 +298,10 @@  static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
 {
     uint64_t pte;
     uint32_t flags;
-    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, mr);
+    S390PCIDeviceConn *conn = container_of(iommu, S390PCIDeviceConn,
+                                           iommu_mr);
+    S390PCIBusDevice *pbdev = conn->zpci;
+
     S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)
                                            ->qbus.parent);
     IOMMUTLBEntry ret = {
@@ -382,7 +375,7 @@  static AddressSpace *s390_pci_dma_iommu(PCIBus *bus, void *opaque, int devfn)
 {
     S390pciState *s = opaque;
 
-    return &s->pbdev[PCI_SLOT(devfn)].as;
+    return &s->conn[PCI_SLOT(devfn)].iommu_as;
 }
 
 static uint8_t set_ind_atomic(uint64_t ind_loc, uint8_t to_be_set)
@@ -455,9 +448,10 @@  static void s390_pcihost_init_as(S390pciState *s)
     int i;
 
     for (i = 0; i < PCI_SLOT_MAX; i++) {
-        memory_region_init_iommu(&s->pbdev[i].mr, OBJECT(s),
+        memory_region_init_iommu(&s->conn[i].iommu_mr, OBJECT(s),
                                  &s390_iommu_ops, "iommu-s390", UINT64_MAX);
-        address_space_init(&s->pbdev[i].as, &s->pbdev[i].mr, "iommu-pci");
+        address_space_init(&s->conn[i].iommu_as, &s->conn[i].iommu_mr,
+                           "iommu-pci");
     }
 
     memory_region_init_io(&s->msix_notify_mr, OBJECT(s),
@@ -465,27 +459,57 @@  static void s390_pcihost_init_as(S390pciState *s)
     address_space_init(&s->msix_notify_as, &s->msix_notify_mr, "msix-pci");
 }
 
-static int s390_pcihost_init(SysBusDevice *dev)
+static S390pciState *s390_find_pcihost_by_idx(uint32_t idx)
+{
+    S390pciState *s;
+    S390PCIFacility *fac = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    QTAILQ_FOREACH(s, &fac->zpcistate_list, next) {
+        if (s->idx == idx) {
+            return s;
+        }
+    }
+
+    return NULL;
+}
+
+static void s390_pcihost_realize(DeviceState *dev, Error **errp)
 {
     PCIBus *b;
     BusState *bus;
     PCIHostState *phb = PCI_HOST_BRIDGE(dev);
     S390pciState *s = S390_PCI_HOST_BRIDGE(dev);
+    S390PCIFacility *fac = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     DPRINTF("host_init\n");
 
+    if (s->idx == (uint32_t)-1) {
+        error_setg(errp, "s390 pci host bridge index not specified");
+        return;
+    }
+
+    if (s390_find_pcihost_by_idx(s->idx)) {
+        error_setg(errp, "s390 pci host bridge must have unique index");
+        return;
+    }
+
     b = pci_register_bus(DEVICE(dev), NULL,
                          s390_pci_set_irq, s390_pci_map_irq, NULL,
                          get_system_memory(), get_system_io(), 0, 64,
                          TYPE_PCI_BUS);
+
     s390_pcihost_init_as(s);
     pci_setup_iommu(b, s390_pci_dma_iommu, s);
 
     bus = BUS(b);
     qbus_set_hotplug_handler(bus, DEVICE(dev), NULL);
     phb->bus = b;
-    QTAILQ_INIT(&s->pending_sei);
-    return 0;
+
+    QTAILQ_INSERT_HEAD(&fac->zpcistate_list, s, next);
+
+    return;
 }
 
 static int s390_pcihost_setup_msix(S390PCIBusDevice *pbdev)
@@ -519,25 +543,37 @@  static int s390_pcihost_setup_msix(S390PCIBusDevice *pbdev)
 static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
                                   DeviceState *dev, Error **errp)
 {
-    PCIDevice *pci_dev = PCI_DEVICE(dev);
-    S390PCIBusDevice *pbdev;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pci_dev)
-                                           ->qbus.parent);
+    PCIDevice *pdev = PCI_DEVICE(dev);
+    S390PCIBusDevice *zpci;
+    S390pciState *s;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    pbdev = &s->pbdev[PCI_SLOT(pci_dev->devfn)];
+    QTAILQ_FOREACH(zpci, &f->zpci_list, next) {
+        if (!strcmp(zpci->pci_id, pdev->qdev.id)) {
+            break;
+        }
+    }
 
-    pbdev->fid = s390_pci_get_pfid(pci_dev);
-    pbdev->pdev = pci_dev;
-    pbdev->configured = true;
-    pbdev->fh = s390_pci_get_pfh(pci_dev);
+    if (!zpci) {
+        error_setg(errp, "zpci device %s not found", pdev->qdev.id);
+        return;
+    }
 
-    s390_pcihost_setup_msix(pbdev);
+    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pdev)->qbus.parent);
+    s->conn[PCI_SLOT(pdev->devfn)].zpci = zpci;
+    zpci->available = true;
+    zpci->configured = true;
+    zpci->pdev = pdev;
+    zpci->fh = zpci->fid | FH_VIRT;
+    zpci->idx = S390_ZPCI_IDX(s->idx, PCI_SLOT(pdev->devfn));
+    s390_pcihost_setup_msix(zpci);
 
     if (dev->hotplugged) {
         s390_pci_generate_plug_event(HP_EVENT_RESERVED_TO_STANDBY,
-                                     pbdev->fh, pbdev->fid);
+                                     zpci->fh, zpci->fid);
         s390_pci_generate_plug_event(HP_EVENT_TO_CONFIGURED,
-                                     pbdev->fh, pbdev->fid);
+                                     zpci->fh, zpci->fid);
     }
     return;
 }
@@ -546,34 +582,44 @@  static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
                                     DeviceState *dev, Error **errp)
 {
     PCIDevice *pci_dev = PCI_DEVICE(dev);
-    S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pci_dev)
-                                           ->qbus.parent);
-    S390PCIBusDevice *pbdev = &s->pbdev[PCI_SLOT(pci_dev->devfn)];
+    S390PCIBusDevice *pbdev;
+    HotplugHandler *hotplug_ctrl;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_GET_CLASS(f);
+    HotplugHandlerClass *hdc = HOTPLUG_HANDLER_CLASS(k);
+
+    /* unplug corresponding zpci device */
+    QTAILQ_FOREACH(pbdev, &f->zpci_list, next) {
+        if (!strcmp(pbdev->pci_id, pci_dev->qdev.id)) {
+            break;
+        }
+    }
 
-    if (pbdev->configured) {
-        pbdev->configured = false;
-        s390_pci_generate_plug_event(HP_EVENT_CONFIGURED_TO_STBRES,
-                                     pbdev->fh, pbdev->fid);
+    if (pbdev) {
+        hotplug_ctrl = pbdev->qdev.parent_bus->hotplug_handler;
+        if (hdc->unplug_request) {
+            hdc->unplug_request(hotplug_ctrl, &pbdev->qdev, errp);
+        }
     }
 
-    s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
-                                 pbdev->fh, pbdev->fid);
-    pbdev->fh = 0;
-    pbdev->fid = 0;
-    pbdev->pdev = NULL;
     object_unparent(OBJECT(pci_dev));
 }
 
+static Property s390_pcihost_properties[] = {
+    DEFINE_PROP_UINT32("index", S390pciState, idx, -1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void s390_pcihost_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
-    dc->cannot_instantiate_with_device_add_yet = true;
-    k->init = s390_pcihost_init;
+    dc->realize = s390_pcihost_realize;
     hc->plug = s390_pcihost_hot_plug;
     hc->unplug = s390_pcihost_hot_unplug;
+    dc->props = s390_pcihost_properties;
     msi_supported = true;
 }
 
@@ -588,9 +634,168 @@  static const TypeInfo s390_pcihost_info = {
     }
 };
 
+static void s390_pci_device_hot_plug(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
+{
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    QTAILQ_INSERT_HEAD(&f->zpci_list, zpci, next);
+}
+
+static void s390_pci_device_hot_unplug_request(HotplugHandler *hotplug_dev,
+                                       DeviceState *dev, Error **errp)
+{
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+
+    if (!zpci->available) {
+        object_unparent(OBJECT(zpci));
+        return;
+    }
+
+    zpci->is_unplugged = true;
+    if (zpci->configured) {
+        zpci->configured = false;
+        s390_pci_generate_plug_event(HP_EVENT_CONFIGURED_TO_STBRES,
+                                     zpci->fh, zpci->fid);
+    }
+
+    s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
+                                 zpci->fh, zpci->fid);
+}
+
+static const TypeInfo s390_pci_fac_bus_info = {
+    .name = TYPE_S390_PCI_FAC_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(S390PCIFacBus),
+};
+
+static int s390_pci_facility_init(SysBusDevice *dev)
+{
+    S390PCIFacility *f = S390_PCI_FACILITY(dev);
+    DeviceState *s = DEVICE(f);
+
+    QTAILQ_INIT(&f->zpcistate_list);
+    QTAILQ_INIT(&f->zpci_list);
+    QTAILQ_INIT(&f->pending_sei);
+    msi_supported = true;
+    f->fbus = S390_PCI_FAC_BUS(qbus_create(TYPE_S390_PCI_FAC_BUS, s, NULL));
+    qbus_set_hotplug_handler(BUS(&f->fbus->qbus), DEVICE(s), NULL);
+
+    return 0;
+}
+
+static void s390_pci_facility_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(k);
+
+    k->init = s390_pci_facility_init;
+    hc->plug = s390_pci_device_hot_plug;
+    hc->unplug_request = s390_pci_device_hot_unplug_request;
+}
+
+static const TypeInfo s390_pci_facility_info = {
+    .name          = TYPE_S390_PCI_FACILITY,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(S390PCIFacility),
+    .class_init    = s390_pci_facility_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
+};
+
+static void s390_pci_device_realize(DeviceState *dev, Error **errp)
+{
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIBusDevice *tmp;
+    PCIDevice *pdev;
+    int ret;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    ret = pci_qdev_find_device(zpci->pci_id, &pdev);
+    if (!ret) {
+        error_setg(errp, "there is already vfio pci device %s", zpci->pci_id);
+        return;
+    }
+
+    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
+        if (tmp->fid == zpci->fid || tmp->uid == zpci->uid ||
+            !strcmp(tmp->pci_id, zpci->pci_id)) {
+            error_setg(errp, "zpci needs unique fid, uid and pci_id");
+            return;
+        }
+    }
+}
+
+static void s390_pci_device_unrealize(DeviceState *dev, Error **errp)
+{
+    S390pciState *s;
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+    PCIDevice *pdev;
+    HotplugHandler *hotplug_ctrl;
+    SysBusDeviceClass *k;
+    HotplugHandlerClass *hdc;
+    int ret;
+
+    if (zpci->available) {
+        s = s390_find_pcihost_by_idx(S390_PCISTATE_IDX(zpci->idx));
+        s->conn[S390_ZPCI_SLOT(zpci->idx)].zpci = NULL;
+    }
+
+    QTAILQ_REMOVE(&f->zpci_list, zpci, next);
+    zpci->fh = 0;
+    zpci->fid = 0;
+    zpci->idx = 0;
+    zpci->pdev = NULL;
+
+    ret = pci_qdev_find_device(zpci->pci_id, &pdev);
+    if (!ret) {
+        hotplug_ctrl = pdev->qdev.parent_bus->hotplug_handler;
+        k = SYS_BUS_DEVICE_GET_CLASS(s);
+        hdc = HOTPLUG_HANDLER_CLASS(k);
+        if (hdc->unplug) {
+            hdc->unplug(hotplug_ctrl, &pdev->qdev, errp);
+        }
+    }
+}
+
+static Property s390_pci_device_properties[] = {
+    DEFINE_PROP_UINT32("fid", S390PCIBusDevice, fid, 0),
+    DEFINE_PROP_UINT32("uid", S390PCIBusDevice, uid, 0),
+    DEFINE_PROP_STRING("pci_id", S390PCIBusDevice, pci_id),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_pci_device_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->desc = "s390 pci device";
+    dc->bus_type = TYPE_S390_PCI_FAC_BUS;
+    dc->realize = s390_pci_device_realize;
+    dc->unrealize = s390_pci_device_unrealize;
+    dc->props = s390_pci_device_properties;
+}
+
+static const TypeInfo s390_pci_device_type_info = {
+    .name = TYPE_S390_PCI_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(S390PCIBusDevice),
+    .class_init = s390_pci_device_class_init,
+};
+
 static void s390_pci_register_types(void)
 {
     type_register_static(&s390_pcihost_info);
+    type_register_static(&s390_pci_facility_info);
+    type_register_static(&s390_pci_fac_bus_info);
+    type_register_static(&s390_pci_device_type_info);
 }
 
 type_init(s390_pci_register_types)
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 464a92e..fef537f 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -25,6 +25,13 @@ 
 #define ENABLE_BIT_OFFSET 31
 #define S390_PCIPT_ADAPTER 2
 
+#define S390_ZPCI_IDX(sidx, slot) \
+    ((((uint64_t)sidx) << 5) | ((uint64_t)slot))
+#define S390_ZPCI_SLOT(zpci_idx) \
+    (zpci_idx & 0x1f)
+#define S390_PCISTATE_IDX(zpci_idx) \
+    (((zpci_idx) & ~0x1f) >> 5)
+
 #define S390_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(S390pciState, (obj), TYPE_S390_PCI_HOST_BRIDGE)
 
@@ -149,6 +156,17 @@  enum ZpciIoatDtype {
 #define ZPCI_TABLE_VALID_MASK           0x20
 #define ZPCI_TABLE_PROT_MASK            0x200
 
+#define TYPE_S390_PCI_FACILITY "s390-pci-facility"
+#define TYPE_S390_PCI_FAC_BUS "s390-pci-fac-bus"
+#define TYPE_S390_PCI_DEVICE "zpci"
+
+#define S390_PCI_FACILITY(obj) \
+    OBJECT_CHECK(S390PCIFacility, (obj), TYPE_S390_PCI_FACILITY)
+#define S390_PCI_FAC_BUS(obj) \
+    OBJECT_CHECK(S390PCIFacBus, (obj), TYPE_S390_PCI_FAC_BUS)
+#define S390_PCI_DEVICE(obj) \
+    OBJECT_CHECK(S390PCIBusDevice, (obj), TYPE_S390_PCI_DEVICE)
+
 typedef struct SeiContainer {
     QTAILQ_ENTRY(SeiContainer) link;
     uint32_t fid;
@@ -214,12 +232,18 @@  typedef struct S390MsixInfo {
 } S390MsixInfo;
 
 typedef struct S390PCIBusDevice {
+    DeviceState qdev;
     PCIDevice *pdev;
     bool configured;
+    bool available;
+    bool is_unplugged;
     bool error_state;
     bool lgstg_blocked;
     uint32_t fh;
     uint32_t fid;
+    uint32_t uid;
+    uint64_t idx;
+    char *pci_id;
     uint64_t g_iota;
     uint64_t pba;
     uint64_t pal;
@@ -229,22 +253,42 @@  typedef struct S390PCIBusDevice {
     uint8_t sum;
     S390MsixInfo msix;
     AdapterRoutes routes;
-    AddressSpace as;
-    MemoryRegion mr;
+    QTAILQ_ENTRY(S390PCIBusDevice) next;
 } S390PCIBusDevice;
 
+typedef struct S390PCIDeviceConn {
+    S390PCIBusDevice *zpci;
+    AddressSpace iommu_as;
+    MemoryRegion iommu_mr;
+} S390PCIDeviceConn;
+
 typedef struct S390pciState {
     PCIHostState parent_obj;
-    S390PCIBusDevice pbdev[PCI_SLOT_MAX];
+    S390PCIDeviceConn conn[PCI_SLOT_MAX];
     AddressSpace msix_notify_as;
     MemoryRegion msix_notify_mr;
-    QTAILQ_HEAD(, SeiContainer) pending_sei;
+    uint32_t idx;
+    QTAILQ_ENTRY(S390pciState) next;
 } S390pciState;
 
+typedef struct S390PCIFacBus {
+    BusState qbus;
+} S390PCIFacBus;
+
+typedef struct S390PCIFacility {
+    SysBusDevice parent_obj;
+    S390PCIFacBus *fbus;
+    QTAILQ_HEAD(, S390pciState) zpcistate_list;
+    QTAILQ_HEAD(, S390PCIBusDevice) zpci_list;
+    QTAILQ_HEAD(, SeiContainer) pending_sei;
+} S390PCIFacility;
+
 int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(int configure, SCCB *sccb);
-S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
+void s390_pci_device_enable(S390PCIBusDevice *zpci);
+void s390_pci_device_disable(S390PCIBusDevice *zpci);
+S390PCIBusDevice *s390_pci_find_dev_by_idx(uint64_t idx);
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);
 
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index f9151a9..ddeec61 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -34,10 +34,13 @@  static void s390_set_status_code(CPUS390XState *env,
 
 static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
 {
-    S390PCIBusDevice *pbdev;
+    S390PCIBusDevice *pbdev = NULL;
+    S390pciState *s = NULL;
     uint32_t res_code, initial_l2, g_l2, finish;
-    int rc, idx;
-    uint64_t resume_token;
+    int rc, i = 0;
+    uint64_t idx, resume_token;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     rc = 0;
     if (lduw_p(&rrb->request.hdr.len) != 32) {
@@ -62,15 +65,6 @@  static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
 
     resume_token = ldq_p(&rrb->request.resume_token);
 
-    if (resume_token) {
-        pbdev = s390_pci_find_dev_by_idx(resume_token);
-        if (!pbdev) {
-            res_code = CLP_RC_LISTPCI_BADRT;
-            rc = -EINVAL;
-            goto out;
-        }
-    }
-
     if (lduw_p(&rrb->response.hdr.len) < 48) {
         res_code = CLP_RC_8K;
         rc = -EINVAL;
@@ -95,29 +89,51 @@  static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
     finish = 0;
     idx = resume_token;
     g_l2 = LIST_PCI_HDR_LEN;
+
+    if (idx != 0) {
+        QTAILQ_FOREACH(s, &f->zpcistate_list, next) {
+            if (s->idx == S390_PCISTATE_IDX(idx)) {
+                break;
+            }
+        }
+    } else {
+        s = QTAILQ_FIRST(&f->zpcistate_list);
+        idx = S390_ZPCI_IDX(s->idx, 0);
+    }
+
     do {
         pbdev = s390_pci_find_dev_by_idx(idx);
-        if (!pbdev) {
+        if (pbdev) {
+            stw_p(&rrb->response.fh_list[i].device_id,
+                pci_get_word(pbdev->pdev->config + PCI_DEVICE_ID));
+            stw_p(&rrb->response.fh_list[i].vendor_id,
+                pci_get_word(pbdev->pdev->config + PCI_VENDOR_ID));
+            stl_p(&rrb->response.fh_list[i].config, 0x80000000);
+            stl_p(&rrb->response.fh_list[i].fid, pbdev->fid);
+            stl_p(&rrb->response.fh_list[i].fh, pbdev->fh);
+
+            g_l2 += sizeof(ClpFhListEntry);
+            i++;
+            /* Add endian check for DPRINTF? */
+            DPRINTF("g_l2 %d vendor id 0x%x device id 0x%x fid 0x%x fh 0x%x\n",
+                g_l2,
+                lduw_p(&rrb->response.fh_list[i].vendor_id),
+                lduw_p(&rrb->response.fh_list[i].device_id),
+                ldl_p(&rrb->response.fh_list[i].fid),
+                ldl_p(&rrb->response.fh_list[i].fh));
+        }
+
+        idx++;
+        if (s->idx == S390_PCISTATE_IDX(idx)) {
+            continue;
+        }
+
+        s = QTAILQ_NEXT(s, next);
+        if (!s) {
             finish = 1;
             break;
         }
-        stw_p(&rrb->response.fh_list[idx - resume_token].device_id,
-            pci_get_word(pbdev->pdev->config + PCI_DEVICE_ID));
-        stw_p(&rrb->response.fh_list[idx - resume_token].vendor_id,
-            pci_get_word(pbdev->pdev->config + PCI_VENDOR_ID));
-        stl_p(&rrb->response.fh_list[idx - resume_token].config, 0x80000000);
-        stl_p(&rrb->response.fh_list[idx - resume_token].fid, pbdev->fid);
-        stl_p(&rrb->response.fh_list[idx - resume_token].fh, pbdev->fh);
-
-        g_l2 += sizeof(ClpFhListEntry);
-        /* Add endian check for DPRINTF? */
-        DPRINTF("g_l2 %d vendor id 0x%x device id 0x%x fid 0x%x fh 0x%x\n",
-            g_l2,
-            lduw_p(&rrb->response.fh_list[idx - resume_token].vendor_id),
-            lduw_p(&rrb->response.fh_list[idx - resume_token].device_id),
-            ldl_p(&rrb->response.fh_list[idx - resume_token].fid),
-            ldl_p(&rrb->response.fh_list[idx - resume_token].fh));
-        idx++;
+        idx = S390_ZPCI_IDX((uint64_t)s->idx, 0);
     } while (g_l2 < initial_l2);
 
     if (finish == 1) {
@@ -208,12 +224,12 @@  int clp_service_call(S390CPU *cpu, uint8_t r2)
 
         switch (reqsetpci->oc) {
         case CLP_SET_ENABLE_PCI_FN:
-            pbdev->fh = pbdev->fh | 1 << ENABLE_BIT_OFFSET;
+            s390_pci_device_enable(pbdev);
             stl_p(&ressetpci->fh, pbdev->fh);
             stw_p(&ressetpci->hdr.rsp, CLP_RC_OK);
             break;
         case CLP_SET_DISABLE_PCI_FN:
-            pbdev->fh = pbdev->fh & ~(1 << ENABLE_BIT_OFFSET);
+            s390_pci_device_disable(pbdev);
             pbdev->error_state = false;
             pbdev->lgstg_blocked = false;
             stl_p(&ressetpci->fh, pbdev->fh);
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index b370b96..aa8b7c0 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -130,8 +130,8 @@  static void ccw_init(MachineState *machine)
                       machine->initrd_filename, "s390-ccw.img", true);
     s390_flic_init();
 
-    dev = qdev_create(NULL, TYPE_S390_PCI_HOST_BRIDGE);
-    object_property_add_child(qdev_get_machine(), TYPE_S390_PCI_HOST_BRIDGE,
+    dev = qdev_create(NULL, TYPE_S390_PCI_FACILITY);
+    object_property_add_child(qdev_get_machine(), TYPE_S390_PCI_FACILITY,
                               OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 
@@ -174,6 +174,7 @@  static void ccw_machine_class_init(ObjectClass *oc, void *data)
     mc->use_sclp = 1;
     mc->max_cpus = 255;
     mc->hot_add_cpu = ccw_hot_add_cpu;
+    mc->has_dynamic_sysbus = true;
     nc->nmi_monitor_handler = s390_nmi;
 }