diff mbox

[v4,1/1] s390 pci infrastructure modelling

Message ID 1436334295-6012-2-git-send-email-zyimin@linux.vnet.ibm.com
State New
Headers show

Commit Message

Yi Min Zhao July 8, 2015, 5:44 a.m. UTC 
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>

This patch introduce a new facility(and bus)
to hold devices representing information actually
provided by s390 firmware and I/O configuration.
usage example:
-device s390-pcihost
-device zpci,fid=2,uid=5,pci_id=vpci1,id=zpci1
-device vfio-pci,host=0000:00:00.0,id=vpci1

The first line will create a s390 pci host bridge
and init the root bus.
The second line will create a s390 pci device to
store s390 specific information, and references
the corresponding vfio pci device via device id.
We create a s390 pci facility bus to hold all the
zpci devices.
The third line will create a standard vfio pci
evice, and attach it to the root bus. These are
similiar to the standard process to define a pci
device on other platform.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
---
 hw/s390x/s390-pci-bus.c    |  323 ++++++++++++++++++++++++++++++++++----------
 hw/s390x/s390-pci-bus.h    |   41 +++++-
 hw/s390x/s390-pci-inst.c   |   12 ++-
 hw/s390x/s390-virtio-ccw.c |    5 +-
 4 files changed, 302 insertions(+), 79 deletions(-)

Comments

Michael S. Tsirkin July 9, 2015, 7:48 a.m. UTC | #1 
On Wed, Jul 08, 2015 at 01:44:55PM +0800, Yi Min Zhao wrote:
> @@ -588,9 +606,172 @@ static const TypeInfo s390_pcihost_info = {
>      }
>  };
>  
> +static void s390_pci_device_hot_plug(HotplugHandler *hotplug_dev,
> +                                     DeviceState *dev, Error **errp)
> +{
> +    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
> +    S390PCIBusDevice *tmp;
> +    S390PCIFacility *f = S390_PCI_FACILITY(
> +        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
> +
> +    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
> +        /* for now, we use fid to sort the list, need to use uid instead
> +         * when uid is ready.

What does ready mean in this context?

> +         */
> +        if (tmp->fid > zpci->fid) {
> +            break;
> +        }
> +    }
> +
> +    if (tmp) {
> +        QTAILQ_INSERT_BEFORE(tmp, zpci, next);
> +    } else {
> +        QTAILQ_INSERT_TAIL(&f->zpci_list, zpci, next);
> +    }
> +    f->token_valid = false;
> +}

This still means hotplug will change the index.
How about just using an explicit property to set the index?
Harder to use but keeps code simple.
Yi Min Zhao July 9, 2015, 9:30 a.m. UTC | #2 
On 7/9/2015 3:48 PM, Michael S. Tsirkin wrote:
> On Wed, Jul 08, 2015 at 01:44:55PM +0800, Yi Min Zhao wrote:
>> @@ -588,9 +606,172 @@ static const TypeInfo s390_pcihost_info = {
>>       }
>>   };
>>   
>> +static void s390_pci_device_hot_plug(HotplugHandler *hotplug_dev,
>> +                                     DeviceState *dev, Error **errp)
>> +{
>> +    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
>> +    S390PCIBusDevice *tmp;
>> +    S390PCIFacility *f = S390_PCI_FACILITY(
>> +        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
>> +
>> +    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
>> +        /* for now, we use fid to sort the list, need to use uid instead
>> +         * when uid is ready.
> What does ready mean in this context?
uid is a new feature on s390 arch for pci card management.
But support of uid on OS level has not been ready.
>> +         */
>> +        if (tmp->fid > zpci->fid) {
>> +            break;
>> +        }
>> +    }
>> +
>> +    if (tmp) {
>> +        QTAILQ_INSERT_BEFORE(tmp, zpci, next);
>> +    } else {
>> +        QTAILQ_INSERT_TAIL(&f->zpci_list, zpci, next);
>> +    }
>> +    f->token_valid = false;
>> +}
> This still means hotplug will change the index.
> How about just using an explicit property to set the index?
> Harder to use but keeps code simple.
>
These code were written by Hong Bo. I think he wants to use zpci->fid to 
sort pci cards.
"index" what you talked is the order of hotplug? right? If it is, I 
think this code actually
can change the index.
Michael S. Tsirkin July 9, 2015, 9:57 a.m. UTC | #3 
On Thu, Jul 09, 2015 at 05:30:08PM +0800, zyimin wrote:
> 
> 
> On 7/9/2015 3:48 PM, Michael S. Tsirkin wrote:
> >On Wed, Jul 08, 2015 at 01:44:55PM +0800, Yi Min Zhao wrote:
> >>@@ -588,9 +606,172 @@ static const TypeInfo s390_pcihost_info = {
> >>      }
> >>  };
> >>+static void s390_pci_device_hot_plug(HotplugHandler *hotplug_dev,
> >>+                                     DeviceState *dev, Error **errp)
> >>+{
> >>+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
> >>+    S390PCIBusDevice *tmp;
> >>+    S390PCIFacility *f = S390_PCI_FACILITY(
> >>+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
> >>+
> >>+    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
> >>+        /* for now, we use fid to sort the list, need to use uid instead
> >>+         * when uid is ready.
> >What does ready mean in this context?
> uid is a new feature on s390 arch for pci card management.
> But support of uid on OS level has not been ready.

Does this matter here? I thought it's an arbitrary order -
just making it stable is enough?


> >>+         */
> >>+        if (tmp->fid > zpci->fid) {
> >>+            break;
> >>+        }
> >>+    }
> >>+
> >>+    if (tmp) {
> >>+        QTAILQ_INSERT_BEFORE(tmp, zpci, next);
> >>+    } else {
> >>+        QTAILQ_INSERT_TAIL(&f->zpci_list, zpci, next);
> >>+    }
> >>+    f->token_valid = false;
> >>+}
> >This still means hotplug will change the index.
> >How about just using an explicit property to set the index?
> >Harder to use but keeps code simple.
> >
> These code were written by Hong Bo. I think he wants to use zpci->fid to
> sort pci cards.
> "index" what you talked is the order of hotplug? right? If it is, I think
> this code actually
> can change the index.

I don't really understand what you are saying. id is how guest looks
up the device, right? it shouldn't change really.
Yi Min Zhao July 13, 2015, 2:49 a.m. UTC | #4 
On 7/9/2015 5:57 PM, Michael S. Tsirkin wrote:
> On Thu, Jul 09, 2015 at 05:30:08PM +0800, zyimin wrote:
>>
>> On 7/9/2015 3:48 PM, Michael S. Tsirkin wrote:
>>> On Wed, Jul 08, 2015 at 01:44:55PM +0800, Yi Min Zhao wrote:
>>>> @@ -588,9 +606,172 @@ static const TypeInfo s390_pcihost_info = {
>>>>       }
>>>>   };
>>>> +static void s390_pci_device_hot_plug(HotplugHandler *hotplug_dev,
>>>> +                                     DeviceState *dev, Error **errp)
>>>> +{
>>>> +    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
>>>> +    S390PCIBusDevice *tmp;
>>>> +    S390PCIFacility *f = S390_PCI_FACILITY(
>>>> +        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
>>>> +
>>>> +    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
>>>> +        /* for now, we use fid to sort the list, need to use uid instead
>>>> +         * when uid is ready.
>>> What does ready mean in this context?
>> uid is a new feature on s390 arch for pci card management.
>> But support of uid on OS level has not been ready.
> Does this matter here? I thought it's an arbitrary order -
> just making it stable is enough?
I think there i sno matter here. Because uid is just for zpci management 
of firmware
and both of fid and uid are unique for all zpci devices attached to one 
guest. FID maybe
change after s390 machine restart. If machine reboot, host will scan PCI 
devices again
and will obtain new FID of every zPCI devices. For guest, it is isolated.
>
>>>> +         */
>>>> +        if (tmp->fid > zpci->fid) {
>>>> +            break;
>>>> +        }
>>>> +    }
>>>> +
>>>> +    if (tmp) {
>>>> +        QTAILQ_INSERT_BEFORE(tmp, zpci, next);
>>>> +    } else {
>>>> +        QTAILQ_INSERT_TAIL(&f->zpci_list, zpci, next);
>>>> +    }
>>>> +    f->token_valid = false;
>>>> +}
>>> This still means hotplug will change the index.
>>> How about just using an explicit property to set the index?
>>> Harder to use but keeps code simple.
>>>
>> These code were written by Hong Bo. I think he wants to use zpci->fid to
>> sort pci cards.
>> "index" what you talked is the order of hotplug? right? If it is, I think
>> this code actually
>> can change the index.
> I don't really understand what you are saying. id is how guest looks
> up the device, right? it shouldn't change really.
For guest, it will use a special instruction to get pci info of all 
attached pci devices.
For example, there are two pci devices which already are attached to the 
guest.
PCI1: fid=1, uid=1
PCI2: fid=3, uid=3
Then, you hotplug a new pci device to the guest:
PCI3: fid=2, uid=2
Although PCI2 is at the 3rd position of zpci_list, it will not effect 
guest looks up the device.
Because guest uses fid and fh to get zpci info and do pci operations. I 
think fid and uid is just
the explicit property what you said.
diff mbox

Patch

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 560b66a..c92fa25 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -32,8 +32,8 @@  int chsc_sei_nt2_get_event(void *res)
     PciCcdfErr *eccdf;
     int rc = 1;
     SeiContainer *sei_cont;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *s = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     if (!s) {
         return rc;
@@ -72,8 +72,8 @@  int chsc_sei_nt2_get_event(void *res)
 
 int chsc_sei_nt2_have_event(void)
 {
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *s = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     if (!s) {
         return 0;
@@ -82,20 +82,31 @@  int chsc_sei_nt2_have_event(void)
     return !QTAILQ_EMPTY(&s->pending_sei);
 }
 
+void s390_pci_device_enable(S390PCIBusDevice *zpci)
+{
+    zpci->fh = zpci->fh | 1 << ENABLE_BIT_OFFSET;
+}
+
+void s390_pci_device_disable(S390PCIBusDevice *zpci)
+{
+    zpci->fh = zpci->fh & ~(1 << ENABLE_BIT_OFFSET);
+    if (zpci->is_unplugged) {
+        object_unparent(OBJECT(zpci));
+    }
+}
+
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid)
 {
     S390PCIBusDevice *pbdev;
-    int i;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *s = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     if (!s) {
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = &s->pbdev[i];
-        if ((pbdev->fh != 0) && (pbdev->fid == fid)) {
+    QTAILQ_FOREACH(pbdev, &s->zpci_list, next) {
+        if (pbdev->fid == fid) {
             return pbdev;
         }
     }
@@ -126,39 +137,24 @@  void s390_pci_sclp_configure(int configure, SCCB *sccb)
     return;
 }
 
-static uint32_t s390_pci_get_pfid(PCIDevice *pdev)
-{
-    return PCI_SLOT(pdev->devfn);
-}
-
-static uint32_t s390_pci_get_pfh(PCIDevice *pdev)
-{
-    return PCI_SLOT(pdev->devfn) | FH_VIRT;
-}
-
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
 {
     S390PCIBusDevice *pbdev;
-    int i;
-    int j = 0;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    int i = 0;
+    S390PCIFacility *s = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     if (!s) {
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = &s->pbdev[i];
-
-        if (pbdev->fh == 0) {
-            continue;
-        }
-
-        if (j == idx) {
+    QTAILQ_FOREACH(pbdev, &s->zpci_list, next) {
+        if (i == idx) {
             return pbdev;
         }
-        j++;
+        if (pbdev->configured) {
+            i++;
+        }
     }
 
     return NULL;
@@ -167,16 +163,14 @@  S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh)
 {
     S390PCIBusDevice *pbdev;
-    int i;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *s = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     if (!s || !fh) {
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = &s->pbdev[i];
+    QTAILQ_FOREACH(pbdev, &s->zpci_list, next) {
         if (pbdev->fh == fh) {
             return pbdev;
         }
@@ -189,8 +183,8 @@  static void s390_pci_generate_event(uint8_t cc, uint16_t pec, uint32_t fh,
                                     uint32_t fid, uint64_t faddr, uint32_t e)
 {
     SeiContainer *sei_cont;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(
-        object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+    S390PCIFacility *s = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     if (!s) {
         return;
@@ -308,7 +302,10 @@  static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
 {
     uint64_t pte;
     uint32_t flags;
-    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, mr);
+    S390PCIDeviceConn *conn = container_of(iommu, S390PCIDeviceConn,
+                                           iommu_mr);
+    S390PCIBusDevice *pbdev = conn->zpci;
+
     S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)
                                            ->qbus.parent);
     IOMMUTLBEntry ret = {
@@ -319,8 +316,14 @@  static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
         .perm = IOMMU_NONE,
     };
 
+    if (!pbdev) {
+        return ret;
+    }
+
     DPRINTF("iommu trans addr 0x%" PRIx64 "\n", addr);
 
+    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)->qbus.parent);
+
     /* s390 does not have an APIC mapped to main storage so we use
      * a separate AddressSpace only for msix notifications
      */
@@ -382,7 +385,7 @@  static AddressSpace *s390_pci_dma_iommu(PCIBus *bus, void *opaque, int devfn)
 {
     S390pciState *s = opaque;
 
-    return &s->pbdev[PCI_SLOT(devfn)].as;
+    return &s->conn[PCI_SLOT(devfn)].iommu_as;
 }
 
 static uint8_t set_ind_atomic(uint64_t ind_loc, uint8_t to_be_set)
@@ -455,9 +458,10 @@  static void s390_pcihost_init_as(S390pciState *s)
     int i;
 
     for (i = 0; i < PCI_SLOT_MAX; i++) {
-        memory_region_init_iommu(&s->pbdev[i].mr, OBJECT(s),
+        memory_region_init_iommu(&s->conn[i].iommu_mr, OBJECT(s),
                                  &s390_iommu_ops, "iommu-s390", UINT64_MAX);
-        address_space_init(&s->pbdev[i].as, &s->pbdev[i].mr, "iommu-pci");
+        address_space_init(&s->conn[i].iommu_as, &s->conn[i].iommu_mr,
+                           "iommu-pci");
     }
 
     memory_region_init_io(&s->msix_notify_mr, OBJECT(s),
@@ -484,7 +488,7 @@  static int s390_pcihost_init(SysBusDevice *dev)
     bus = BUS(b);
     qbus_set_hotplug_handler(bus, DEVICE(dev), NULL);
     phb->bus = b;
-    QTAILQ_INIT(&s->pending_sei);
+
     return 0;
 }
 
@@ -519,25 +523,35 @@  static int s390_pcihost_setup_msix(S390PCIBusDevice *pbdev)
 static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
                                   DeviceState *dev, Error **errp)
 {
-    PCIDevice *pci_dev = PCI_DEVICE(dev);
-    S390PCIBusDevice *pbdev;
-    S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pci_dev)
-                                           ->qbus.parent);
+    PCIDevice *pdev = PCI_DEVICE(dev);
+    S390PCIBusDevice *zpci;
+    S390pciState *s;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
-    pbdev = &s->pbdev[PCI_SLOT(pci_dev->devfn)];
+    QTAILQ_FOREACH(zpci, &f->zpci_list, next) {
+        if (!strcmp(zpci->pci_id, pdev->qdev.id)) {
+            break;
+        }
+    }
 
-    pbdev->fid = s390_pci_get_pfid(pci_dev);
-    pbdev->pdev = pci_dev;
-    pbdev->configured = true;
-    pbdev->fh = s390_pci_get_pfh(pci_dev);
+    if (!zpci) {
+        error_setg(errp, "zpci device %s not found", pdev->qdev.id);
+        return;
+    }
 
-    s390_pcihost_setup_msix(pbdev);
+    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pdev)->qbus.parent);
+    s->conn[PCI_SLOT(pdev->devfn)].zpci = zpci;
+    zpci->configured = true;
+    zpci->pdev = pdev;
+    zpci->fh = zpci->fid | FH_VIRT;
+    s390_pcihost_setup_msix(zpci);
 
     if (dev->hotplugged) {
         s390_pci_generate_plug_event(HP_EVENT_RESERVED_TO_STANDBY,
-                                     pbdev->fh, pbdev->fid);
+                                     zpci->fh, zpci->fid);
         s390_pci_generate_plug_event(HP_EVENT_TO_CONFIGURED,
-                                     pbdev->fh, pbdev->fid);
+                                     zpci->fh, zpci->fid);
     }
     return;
 }
@@ -546,31 +560,35 @@  static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
                                     DeviceState *dev, Error **errp)
 {
     PCIDevice *pci_dev = PCI_DEVICE(dev);
-    S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pci_dev)
-                                           ->qbus.parent);
-    S390PCIBusDevice *pbdev = &s->pbdev[PCI_SLOT(pci_dev->devfn)];
+    S390PCIBusDevice *pbdev;
+    HotplugHandler *hotplug_ctrl;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_GET_CLASS(f);
+    HotplugHandlerClass *hdc = HOTPLUG_HANDLER_CLASS(k);
+
+    /* unplug corresponding zpci device */
+    QTAILQ_FOREACH(pbdev, &f->zpci_list, next) {
+        if (!strcmp(pbdev->pci_id, pci_dev->qdev.id)) {
+            break;
+        }
+    }
 
-    if (pbdev->configured) {
-        pbdev->configured = false;
-        s390_pci_generate_plug_event(HP_EVENT_CONFIGURED_TO_STBRES,
-                                     pbdev->fh, pbdev->fid);
+    if (pbdev) {
+        hotplug_ctrl = pbdev->qdev.parent_bus->hotplug_handler;
+        if (hdc->unplug_request) {
+            hdc->unplug_request(hotplug_ctrl, &pbdev->qdev, errp);
+        }
     }
 
-    s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
-                                 pbdev->fh, pbdev->fid);
-    pbdev->fh = 0;
-    pbdev->fid = 0;
-    pbdev->pdev = NULL;
     object_unparent(OBJECT(pci_dev));
 }
 
 static void s390_pcihost_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-    DeviceClass *dc = DEVICE_CLASS(klass);
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
-    dc->cannot_instantiate_with_device_add_yet = true;
     k->init = s390_pcihost_init;
     hc->plug = s390_pcihost_hot_plug;
     hc->unplug = s390_pcihost_hot_unplug;
@@ -588,9 +606,172 @@  static const TypeInfo s390_pcihost_info = {
     }
 };
 
+static void s390_pci_device_hot_plug(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
+{
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIBusDevice *tmp;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
+        /* for now, we use fid to sort the list, need to use uid instead
+         * when uid is ready.
+         */
+        if (tmp->fid > zpci->fid) {
+            break;
+        }
+    }
+
+    if (tmp) {
+        QTAILQ_INSERT_BEFORE(tmp, zpci, next);
+    } else {
+        QTAILQ_INSERT_TAIL(&f->zpci_list, zpci, next);
+    }
+    f->token_valid = false;
+}
+
+static void s390_pci_device_hot_unplug_request(HotplugHandler *hotplug_dev,
+                                       DeviceState *dev, Error **errp)
+{
+    int ret;
+    PCIDevice *pdev;
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    ret = pci_qdev_find_device(zpci->pci_id, &pdev);
+    if (!ret) {
+        object_unparent(OBJECT(pdev));
+    }
+
+    zpci->is_unplugged = true;
+    f->token_valid = false;
+    if (zpci->configured) {
+        zpci->configured = false;
+        s390_pci_generate_plug_event(HP_EVENT_CONFIGURED_TO_STBRES,
+                                     zpci->fh, zpci->fid);
+    }
+
+    s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
+                                 zpci->fh, zpci->fid);
+}
+
+static const TypeInfo s390_pci_fac_bus_info = {
+    .name = TYPE_S390_PCI_FAC_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(S390PCIFacBus),
+};
+
+static int s390_pci_facility_init(SysBusDevice *dev)
+{
+    S390PCIFacility *f = S390_PCI_FACILITY(dev);
+    DeviceState *s = DEVICE(f);
+
+    QTAILQ_INIT(&f->zpci_list);
+    QTAILQ_INIT(&f->pending_sei);
+    msi_supported = true;
+    f->token_valid = true;
+    f->fbus = S390_PCI_FAC_BUS(qbus_create(TYPE_S390_PCI_FAC_BUS, s, NULL));
+    qbus_set_hotplug_handler(BUS(&f->fbus->qbus), DEVICE(s), NULL);
+
+    return 0;
+}
+
+static void s390_pci_facility_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(k);
+
+    k->init = s390_pci_facility_init;
+    hc->plug = s390_pci_device_hot_plug;
+    hc->unplug_request = s390_pci_device_hot_unplug_request;
+}
+
+static const TypeInfo s390_pci_facility_info = {
+    .name          = TYPE_S390_PCI_FACILITY,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(S390PCIFacility),
+    .class_init    = s390_pci_facility_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
+};
+
+static void s390_pci_device_realize(DeviceState *dev, Error **errp)
+{
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIBusDevice *tmp;
+    PCIDevice *pdev;
+    int ret;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    ret = pci_qdev_find_device(zpci->pci_id, &pdev);
+    if (!ret) {
+        error_setg(errp, "there is already vfio pci device %s", zpci->pci_id);
+        return;
+    }
+
+    QTAILQ_FOREACH(tmp, &f->zpci_list, next) {
+        if (tmp->fid == zpci->fid || tmp->uid == zpci->uid ||
+            !strcmp(tmp->pci_id, zpci->pci_id)) {
+            error_setg(errp, "zpci needs unique fid, uid and pci_id");
+            return;
+        }
+    }
+}
+
+static void s390_pci_device_unrealize(DeviceState *dev, Error **errp)
+{
+    S390pciState *s;
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
+
+    if (zpci->pdev) {
+        s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(zpci->pdev)->qbus.parent);
+        s->conn[PCI_SLOT(zpci->pdev->devfn)].zpci = NULL;
+    }
+    QTAILQ_REMOVE(&f->zpci_list, zpci, next);
+    zpci->fh = 0;
+    zpci->fid = 0;
+    zpci->pdev = NULL;
+}
+
+static Property s390_pci_device_properties[] = {
+    DEFINE_PROP_UINT32("fid", S390PCIBusDevice, fid, 0),
+    DEFINE_PROP_UINT32("uid", S390PCIBusDevice, uid, 0),
+    DEFINE_PROP_STRING("pci_id", S390PCIBusDevice, pci_id),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_pci_device_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->desc = "s390 pci device";
+    dc->bus_type = TYPE_S390_PCI_FAC_BUS;
+    dc->realize = s390_pci_device_realize;
+    dc->unrealize = s390_pci_device_unrealize;
+    dc->props = s390_pci_device_properties;
+}
+
+static const TypeInfo s390_pci_device_type_info = {
+    .name = TYPE_S390_PCI_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(S390PCIBusDevice),
+    .class_init = s390_pci_device_class_init,
+};
+
 static void s390_pci_register_types(void)
 {
+
     type_register_static(&s390_pcihost_info);
+    type_register_static(&s390_pci_facility_info);
+    type_register_static(&s390_pci_fac_bus_info);
+    type_register_static(&s390_pci_device_type_info);
 }
 
 type_init(s390_pci_register_types)
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 464a92e..7778d2a 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -149,6 +149,17 @@  enum ZpciIoatDtype {
 #define ZPCI_TABLE_VALID_MASK           0x20
 #define ZPCI_TABLE_PROT_MASK            0x200
 
+#define TYPE_S390_PCI_FACILITY "s390-pci-facility"
+#define TYPE_S390_PCI_FAC_BUS "s390-pci-fac-bus"
+#define TYPE_S390_PCI_DEVICE "zpci"
+
+#define S390_PCI_FACILITY(obj) \
+    OBJECT_CHECK(S390PCIFacility, (obj), TYPE_S390_PCI_FACILITY)
+#define S390_PCI_FAC_BUS(obj) \
+    OBJECT_CHECK(S390PCIFacBus, (obj), TYPE_S390_PCI_FAC_BUS)
+#define S390_PCI_DEVICE(obj) \
+    OBJECT_CHECK(S390PCIBusDevice, (obj), TYPE_S390_PCI_DEVICE)
+
 typedef struct SeiContainer {
     QTAILQ_ENTRY(SeiContainer) link;
     uint32_t fid;
@@ -214,12 +225,16 @@  typedef struct S390MsixInfo {
 } S390MsixInfo;
 
 typedef struct S390PCIBusDevice {
+    DeviceState qdev;
     PCIDevice *pdev;
     bool configured;
+    bool is_unplugged;
     bool error_state;
     bool lgstg_blocked;
     uint32_t fh;
     uint32_t fid;
+    uint32_t uid;
+    char *pci_id;
     uint64_t g_iota;
     uint64_t pba;
     uint64_t pal;
@@ -229,21 +244,39 @@  typedef struct S390PCIBusDevice {
     uint8_t sum;
     S390MsixInfo msix;
     AdapterRoutes routes;
-    AddressSpace as;
-    MemoryRegion mr;
+    QTAILQ_ENTRY(S390PCIBusDevice) next;
 } S390PCIBusDevice;
 
+typedef struct S390PCIDeviceConn {
+    S390PCIBusDevice *zpci;
+    AddressSpace iommu_as;
+    MemoryRegion iommu_mr;
+} S390PCIDeviceConn;
+
 typedef struct S390pciState {
     PCIHostState parent_obj;
-    S390PCIBusDevice pbdev[PCI_SLOT_MAX];
+    S390PCIDeviceConn conn[PCI_SLOT_MAX];
     AddressSpace msix_notify_as;
     MemoryRegion msix_notify_mr;
-    QTAILQ_HEAD(, SeiContainer) pending_sei;
 } S390pciState;
 
+typedef struct S390PCIFacBus {
+    BusState qbus;
+} S390PCIFacBus;
+
+typedef struct S390PCIFacility {
+    SysBusDevice parent_obj;
+    S390PCIFacBus *fbus;
+    bool token_valid;
+    QTAILQ_HEAD(, S390PCIBusDevice) zpci_list;
+    QTAILQ_HEAD(, SeiContainer) pending_sei;
+} S390PCIFacility;
+
 int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(int configure, SCCB *sccb);
+void s390_pci_device_enable(S390PCIBusDevice *zpci);
+void s390_pci_device_disable(S390PCIBusDevice *zpci);
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index f9151a9..c9f4230 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -38,6 +38,8 @@  static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
     uint32_t res_code, initial_l2, g_l2, finish;
     int rc, idx;
     uint64_t resume_token;
+    S390PCIFacility *f = S390_PCI_FACILITY(
+        object_resolve_path(TYPE_S390_PCI_FACILITY, NULL));
 
     rc = 0;
     if (lduw_p(&rrb->request.hdr.len) != 32) {
@@ -63,6 +65,11 @@  static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
     resume_token = ldq_p(&rrb->request.resume_token);
 
     if (resume_token) {
+        if (!f->token_valid) {
+            res_code = CLP_RC_LISTPCI_BADRT;
+            rc = -EINVAL;
+            goto out;
+        }
         pbdev = s390_pci_find_dev_by_idx(resume_token);
         if (!pbdev) {
             res_code = CLP_RC_LISTPCI_BADRT;
@@ -71,6 +78,7 @@  static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
         }
     }
 
+    f->token_valid = true;
     if (lduw_p(&rrb->response.hdr.len) < 48) {
         res_code = CLP_RC_8K;
         rc = -EINVAL;
@@ -208,12 +216,12 @@  int clp_service_call(S390CPU *cpu, uint8_t r2)
 
         switch (reqsetpci->oc) {
         case CLP_SET_ENABLE_PCI_FN:
-            pbdev->fh = pbdev->fh | 1 << ENABLE_BIT_OFFSET;
+            s390_pci_device_enable(pbdev);
             stl_p(&ressetpci->fh, pbdev->fh);
             stw_p(&ressetpci->hdr.rsp, CLP_RC_OK);
             break;
         case CLP_SET_DISABLE_PCI_FN:
-            pbdev->fh = pbdev->fh & ~(1 << ENABLE_BIT_OFFSET);
+            s390_pci_device_disable(pbdev);
             pbdev->error_state = false;
             pbdev->lgstg_blocked = false;
             stl_p(&ressetpci->fh, pbdev->fh);
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index c706590..eb668d7 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -129,8 +129,8 @@  static void ccw_init(MachineState *machine)
                       machine->initrd_filename, "s390-ccw.img", true);
     s390_flic_init();
 
-    dev = qdev_create(NULL, TYPE_S390_PCI_HOST_BRIDGE);
-    object_property_add_child(qdev_get_machine(), TYPE_S390_PCI_HOST_BRIDGE,
+    dev = qdev_create(NULL, TYPE_S390_PCI_FACILITY);
+    object_property_add_child(qdev_get_machine(), TYPE_S390_PCI_FACILITY,
                               OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 
@@ -173,6 +173,7 @@  static void ccw_machine_class_init(ObjectClass *oc, void *data)
     mc->use_sclp = 1;
     mc->max_cpus = 255;
     mc->hot_add_cpu = ccw_hot_add_cpu;
+    mc->has_dynamic_sysbus = true;
     nc->nmi_monitor_handler = s390_nmi;
 }