[v5,4/4] s390x/virtio-ccw: Wire up irq routing and irqfds.

Make use of the new s390 adapter irq routing support to enable real
in-kernel irqfds for virtio-ccw with adapter interrupts.

Note that s390 doesn't provide the common KVM_CAP_IRQCHIP capability, but
rather needs KVM_CAP_S390_IRQCHIP to be enabled. This is to ensure backward
compatibility.

Reviewed-by: Thomas Huth <thuth@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/intc/s390_flic.c          |   21 ++++++
 hw/intc/s390_flic_kvm.c      |   66 +++++++++++++++++
 hw/s390x/virtio-ccw.c        |  165 ++++++++++++++++++++++++++++++++++++++----
 hw/s390x/virtio-ccw.h        |    4 +-
 include/hw/s390x/adapter.h   |   23 ++++++
 include/hw/s390x/s390_flic.h |   12 +++
 include/qemu/typedefs.h      |    1 +
 include/sysemu/kvm.h         |    2 +
 kvm-all.c                    |   38 +++++++++-
 kvm-stub.c                   |    5 ++
 target-s390x/kvm.c           |    5 ++
 11 files changed, 324 insertions(+), 18 deletions(-)
 create mode 100644 include/hw/s390x/adapter.h

On 08/05/14 15:03, Cornelia Huck wrote:
> Make use of the new s390 adapter irq routing support to enable real
> in-kernel irqfds for virtio-ccw with adapter interrupts.
> 
> Note that s390 doesn't provide the common KVM_CAP_IRQCHIP capability, but
> rather needs KVM_CAP_S390_IRQCHIP to be enabled. This is to ensure backward
> compatibility.
> 
> Reviewed-by: Thomas Huth <thuth@linux.vnet.ibm.com>
> Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>

Small things, otherwise

Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>


[...]
> --- /dev/null
> +++ b/include/hw/s390x/adapter.h
> @@ -0,0 +1,23 @@
> +/*
> + * s390 adapter definitions
> + *
> + * Copyright 2013 IBM Corp.

2014 as well

[...]
> --- a/include/sysemu/kvm.h
> +++ b/include/sysemu/kvm.h
> @@ -365,6 +365,8 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
>  int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
>  void kvm_irqchip_release_virq(KVMState *s, int virq);
> 
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter);
> +
>  int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
>                                     EventNotifier *rn, int virq);
>  int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
> diff --git a/kvm-all.c b/kvm-all.c
> index 5cb7f26..51983bf 100644
> --- a/kvm-all.c
> +++ b/kvm-all.c
> @@ -27,6 +27,7 @@
>  #include "sysemu/sysemu.h"
>  #include "hw/hw.h"
>  #include "hw/pci/msi.h"
> +#include "hw/s390x/adapter.h"
>  #include "exec/gdbstub.h"
>  #include "sysemu/kvm.h"
>  #include "qemu/bswap.h"
> @@ -1247,6 +1248,35 @@ static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
>      return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
>  }
> 
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> +{
> +    struct kvm_irq_routing_entry kroute;
> +    int virq;
> +
> +    if (!kvm_gsi_routing_enabled()) {
> +        return -ENOSYS;
> +    }
> +
> +    virq = kvm_irqchip_get_virq(s);
> +    if (virq < 0) {
> +        return virq;
> +    }
> +
> +    kroute.gsi = virq;
> +    kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
> +    kroute.flags = 0;
> +    kroute.u.adapter.summary_addr = adapter->summary_addr;
> +    kroute.u.adapter.ind_addr = adapter->ind_addr;
> +    kroute.u.adapter.summary_offset = adapter->summary_offset;
> +    kroute.u.adapter.ind_offset = adapter->ind_offset;
> +    kroute.u.adapter.adapter_id = adapter->adapter_id;
> +
> +    kvm_add_routing_entry(s, &kroute);
> +    kvm_irqchip_commit_routes(s);
> +
> +    return virq;
> +}
> +
>  #else /* !KVM_CAP_IRQ_ROUTING */
> 
>  void kvm_init_irq_routing(KVMState *s)
> @@ -1267,6 +1297,11 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
>      return -ENOSYS;
>  }
> 
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> +{
> +    return -ENOSYS;
> +}
> +
>  static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
>  {
>      abort();
> @@ -1296,7 +1331,8 @@ static int kvm_irqchip_create(KVMState *s)
>      int ret;
> 
>      if (!qemu_opt_get_bool(qemu_get_machine_opts(), "kernel_irqchip", true) ||
> -        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
> +        (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
> +         (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
>          return 0;
>      }

Do we need a comment here and/or in target-s390x/kvm.c  why kvm_halt_in_kernel_allowed does not work without breaking backward compatibility  on s390?


> 
> diff --git a/kvm-stub.c b/kvm-stub.c
> index 8acda86..ac33d86 100644
> --- a/kvm-stub.c
> +++ b/kvm-stub.c
> @@ -136,6 +136,11 @@ int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
>      return -ENOSYS;
>  }
> 
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> +{
> +    return -ENOSYS;
> +}
> +
>  int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
>                                     EventNotifier *rn, int virq)
>  {
> diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
> index b7b0edc..43859e9 100644
> --- a/target-s390x/kvm.c
> +++ b/target-s390x/kvm.c
> @@ -938,6 +938,11 @@ void kvm_s390_enable_css_support(S390CPU *cpu)
> 
>  void kvm_arch_init_irq_routing(KVMState *s)
>  {
> +    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
> +        kvm_irqfds_allowed = true;
> +        kvm_gsi_routing_allowed = true;
> +        kvm_halt_in_kernel_allowed = false;
> +    }

Do we need a comment here and/or in kvm-all why kvm_halt_in_kernel_allowed does not work without breaking backward compatibility on s390?

[...]


Should Paolo ack the common kvm changes? (CCed)

Christian

On Mon, 12 May 2014 10:58:46 +0200
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> On 08/05/14 15:03, Cornelia Huck wrote:
> > Make use of the new s390 adapter irq routing support to enable real
> > in-kernel irqfds for virtio-ccw with adapter interrupts.
> > 
> > Note that s390 doesn't provide the common KVM_CAP_IRQCHIP capability, but
> > rather needs KVM_CAP_S390_IRQCHIP to be enabled. This is to ensure backward
> > compatibility.
> > 
> > Reviewed-by: Thomas Huth <thuth@linux.vnet.ibm.com>
> > Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> 
> Small things, otherwise
> 
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> 
> [...]
> > --- /dev/null
> > +++ b/include/hw/s390x/adapter.h
> > @@ -0,0 +1,23 @@
> > +/*
> > + * s390 adapter definitions
> > + *
> > + * Copyright 2013 IBM Corp.
> 
> 2014 as well

ok

> 
> [...]
> > diff --git a/kvm-all.c b/kvm-all.c
> > index 5cb7f26..51983bf 100644
> > --- a/kvm-all.c
> > +++ b/kvm-all.c

> > @@ -1267,6 +1297,11 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
> >      return -ENOSYS;
> >  }
> > 
> > +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> > +{
> > +    return -ENOSYS;
> > +}
> > +
> >  static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
> >  {
> >      abort();
> > @@ -1296,7 +1331,8 @@ static int kvm_irqchip_create(KVMState *s)
> >      int ret;
> > 
> >      if (!qemu_opt_get_bool(qemu_get_machine_opts(), "kernel_irqchip", true) ||
> > -        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
> > +        (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
> > +         (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
> >          return 0;
> >      }
> 
> Do we need a comment here and/or in target-s390x/kvm.c  why kvm_halt_in_kernel_allowed does not work without breaking backward compatibility  on s390?

Probably not here but...

> 
> 
> > 

> > diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
> > index b7b0edc..43859e9 100644
> > --- a/target-s390x/kvm.c
> > +++ b/target-s390x/kvm.c
> > @@ -938,6 +938,11 @@ void kvm_s390_enable_css_support(S390CPU *cpu)
> > 
> >  void kvm_arch_init_irq_routing(KVMState *s)
> >  {
> > +    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
> > +        kvm_irqfds_allowed = true;
> > +        kvm_gsi_routing_allowed = true;
> > +        kvm_halt_in_kernel_allowed = false;
> > +    }
> 
> Do we need a comment here and/or in kvm-all why kvm_halt_in_kernel_allowed does not work without breaking backward compatibility on s390?

...I'll add the following here:

/*
 * Note that while irqchip capabilities generally imply that cpustates
 * are handled in-kernel, it is not true for s390 (yet); therefore, we
 * have to override the common code kvm_halt_in_kernel_allowed setting.
 */

> 
> [...]
> 
> 
> Should Paolo ack the common kvm changes? (CCed)
> 
> Christian

Il 08/05/2014 15:03, Cornelia Huck ha scritto:
> Make use of the new s390 adapter irq routing support to enable real
> in-kernel irqfds for virtio-ccw with adapter interrupts.
>
> Note that s390 doesn't provide the common KVM_CAP_IRQCHIP capability, but
> rather needs KVM_CAP_S390_IRQCHIP to be enabled. This is to ensure backward
> compatibility.
>
> Reviewed-by: Thomas Huth <thuth@linux.vnet.ibm.com>
> Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> ---
>  hw/intc/s390_flic.c          |   21 ++++++
>  hw/intc/s390_flic_kvm.c      |   66 +++++++++++++++++
>  hw/s390x/virtio-ccw.c        |  165 ++++++++++++++++++++++++++++++++++++++----
>  hw/s390x/virtio-ccw.h        |    4 +-
>  include/hw/s390x/adapter.h   |   23 ++++++
>  include/hw/s390x/s390_flic.h |   12 +++
>  include/qemu/typedefs.h      |    1 +
>  include/sysemu/kvm.h         |    2 +
>  kvm-all.c                    |   38 +++++++++-
>  kvm-stub.c                   |    5 ++
>  target-s390x/kvm.c           |    5 ++
>  11 files changed, 324 insertions(+), 18 deletions(-)
>  create mode 100644 include/hw/s390x/adapter.h
>
> diff --git a/hw/intc/s390_flic.c b/hw/intc/s390_flic.c
> index 2b56835..03c5e89 100644
> --- a/hw/intc/s390_flic.c
> +++ b/hw/intc/s390_flic.c
> @@ -52,11 +52,32 @@ static int qemu_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
>      return 0;
>  }
>
> +static int qemu_s390_io_adapter_map(S390FLICState *fs, uint32_t id,
> +                                    uint64_t map_addr, bool do_map)
> +{
> +    /* nothing to do */
> +    return 0;
> +}
> +
> +static int qemu_s390_add_adapter_routes(S390FLICState *fs,
> +                                        AdapterRoutes *routes)
> +{
> +    return -ENOSYS;
> +}
> +
> +static void qemu_s390_release_adapter_routes(S390FLICState *fs,
> +                                             AdapterRoutes *routes)
> +{
> +}
> +
>  static void qemu_s390_flic_class_init(ObjectClass *oc, void *data)
>  {
>      S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
>
>      fsc->register_io_adapter = qemu_s390_register_io_adapter;
> +    fsc->io_adapter_map = qemu_s390_io_adapter_map;
> +    fsc->add_adapter_routes = qemu_s390_add_adapter_routes;
> +    fsc->release_adapter_routes = qemu_s390_release_adapter_routes;
>  }
>
>  static const TypeInfo qemu_s390_flic_info = {
> diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
> index cc4072e..46c9e61 100644
> --- a/hw/intc/s390_flic_kvm.c
> +++ b/hw/intc/s390_flic_kvm.c
> @@ -16,6 +16,7 @@
>  #include "sysemu/kvm.h"
>  #include "migration/qemu-file.h"
>  #include "hw/s390x/s390_flic.h"
> +#include "hw/s390x/adapter.h"
>  #include "trace.h"
>
>  #define FLIC_SAVE_INITIAL_SIZE getpagesize()
> @@ -178,6 +179,68 @@ static int kvm_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
>      return ret;
>  }
>
> +static int kvm_s390_io_adapter_map(S390FLICState *fs, uint32_t id,
> +                                   uint64_t map_addr, bool do_map)
> +{
> +    struct kvm_s390_io_adapter_req req = {
> +        .id = id,
> +        .type = do_map ? KVM_S390_IO_ADAPTER_MAP : KVM_S390_IO_ADAPTER_UNMAP,
> +        .addr = map_addr,
> +    };
> +    struct kvm_device_attr attr = {
> +        .group = KVM_DEV_FLIC_ADAPTER_MODIFY,
> +        .addr = (uint64_t)&req,
> +    };
> +    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
> +    int r;
> +
> +    if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
> +        return -ENOSYS;
> +    }
> +
> +    r = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
> +    return r ? -errno : 0;
> +}
> +
> +static int kvm_s390_add_adapter_routes(S390FLICState *fs,
> +                                       AdapterRoutes *routes)
> +{
> +    int ret, i;
> +    uint64_t ind_offset = routes->adapter.ind_offset;
> +
> +    for (i = 0; i < routes->num_routes; i++) {
> +        ret = kvm_irqchip_add_adapter_route(kvm_state, &routes->adapter);
> +        if (ret < 0) {
> +            goto out_undo;
> +        }
> +        routes->gsi[i] = ret;
> +        routes->adapter.ind_offset++;
> +    }
> +    /* Restore passed-in structure to original state. */
> +    routes->adapter.ind_offset = ind_offset;
> +    return 0;
> +out_undo:
> +    while (--i >= 0) {
> +        kvm_irqchip_release_virq(kvm_state, routes->gsi[i]);
> +        routes->gsi[i] = -1;
> +    }
> +    routes->adapter.ind_offset = ind_offset;
> +    return ret;
> +}
> +
> +static void kvm_s390_release_adapter_routes(S390FLICState *fs,
> +                                            AdapterRoutes *routes)
> +{
> +    int i;
> +
> +    for (i = 0; i < routes->num_routes; i++) {
> +        if (routes->gsi[i] >= 0) {
> +            kvm_irqchip_release_virq(kvm_state, routes->gsi[i]);
> +            routes->gsi[i] = -1;
> +        }
> +    }
> +}
> +
>  /**
>   * kvm_flic_save - Save pending floating interrupts
>   * @f: QEMUFile containing migration state
> @@ -337,6 +400,9 @@ static void kvm_s390_flic_class_init(ObjectClass *oc, void *data)
>      dc->unrealize = kvm_s390_flic_unrealize;
>      dc->reset = kvm_s390_flic_reset;
>      fsc->register_io_adapter = kvm_s390_register_io_adapter;
> +    fsc->io_adapter_map = kvm_s390_io_adapter_map;
> +    fsc->add_adapter_routes = kvm_s390_add_adapter_routes;
> +    fsc->release_adapter_routes = kvm_s390_release_adapter_routes;
>  }
>
>  static const TypeInfo kvm_s390_flic_info = {
> diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
> index d11a783..c4f21d3 100644
> --- a/hw/s390x/virtio-ccw.c
> +++ b/hw/s390x/virtio-ccw.c
> @@ -21,6 +21,8 @@
>  #include "hw/sysbus.h"
>  #include "qemu/bitops.h"
>  #include "hw/virtio/virtio-bus.h"
> +#include "hw/s390x/adapter.h"
> +#include "hw/s390x/s390_flic.h"
>
>  #include "ioinst.h"
>  #include "css.h"
> @@ -48,7 +50,16 @@ static IndAddr *get_indicator(hwaddr ind_addr, int len)
>      return indicator;
>  }
>
> -static void release_indicator(IndAddr *indicator)
> +static int s390_io_adapter_map(AdapterInfo *adapter, uint64_t map_addr,
> +                               bool do_map)
> +{
> +    S390FLICState *fs = s390_get_flic();
> +    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
> +
> +    return fsc->io_adapter_map(fs, adapter->adapter_id, map_addr, do_map);
> +}
> +
> +static void release_indicator(AdapterInfo *adapter, IndAddr *indicator)
>  {
>      assert(indicator->refcnt > 0);
>      indicator->refcnt--;
> @@ -56,9 +67,31 @@ static void release_indicator(IndAddr *indicator)
>          return;
>      }
>      QTAILQ_REMOVE(&indicator_addresses, indicator, sibling);
> +    if (indicator->map) {
> +        s390_io_adapter_map(adapter, indicator->map, false);
> +    }
>      g_free(indicator);
>  }
>
> +static int map_indicator(AdapterInfo *adapter, IndAddr *indicator)
> +{
> +    int ret;
> +
> +    if (indicator->map) {
> +        return 0; /* already mapped is not an error */
> +    }
> +    indicator->map = indicator->addr;
> +    ret = s390_io_adapter_map(adapter, indicator->map, true);
> +    if ((ret != 0) && (ret != -ENOSYS)) {
> +        goto out_err;
> +    }
> +    return 0;
> +
> +out_err:
> +    indicator->map = 0;
> +    return ret;
> +}
> +
>  static void virtio_ccw_bus_new(VirtioBusState *bus, size_t bus_size,
>                                 VirtioCcwDevice *dev);
>
> @@ -554,11 +587,12 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
>                  dev->indicators = get_indicator(thinint->device_indicator,
>                                                  thinint->ind_bit / 8 + 1);
>                  dev->thinint_isc = thinint->isc;
> -                dev->ind_bit = thinint->ind_bit;
> +                dev->routes.adapter.ind_offset = thinint->ind_bit;
> +                dev->routes.adapter.summary_offset = 7;
>                  cpu_physical_memory_unmap(thinint, hw_len, 0, hw_len);
>                  ret = css_register_io_adapter(CSS_IO_ADAPTER_VIRTIO,
>                                                dev->thinint_isc, true, false,
> -                                              &dev->adapter_id);
> +                                              &dev->routes.adapter.adapter_id);
>                  assert(ret == 0);
>                  sch->thinint_active = ((dev->indicators != NULL) &&
>                                         (dev->summary_indicator != NULL));
> @@ -732,7 +766,7 @@ static int virtio_ccw_exit(VirtioCcwDevice *dev)
>          g_free(sch);
>      }
>      if (dev->indicators) {
> -        release_indicator(dev->indicators);
> +        release_indicator(&dev->routes.adapter, dev->indicators);
>          dev->indicators = NULL;
>      }
>      return 0;
> @@ -991,9 +1025,11 @@ static void virtio_ccw_notify(DeviceState *d, uint16_t vector)
>               * ind_bit indicates the start of the indicators in a big
>               * endian notation.
>               */
> +            uint64_t ind_bit = dev->routes.adapter.ind_offset;
> +
>              virtio_set_ind_atomic(sch, dev->indicators->addr +
> -                                  (dev->ind_bit + vector) / 8,
> -                                  0x80 >> ((dev->ind_bit + vector) % 8));
> +                                  (ind_bit + vector) / 8,
> +                                  0x80 >> ((ind_bit + vector) % 8));
>              if (!virtio_set_ind_atomic(sch, dev->summary_indicator->addr,
>                                         0x01)) {
>                  css_adapter_interrupt(dev->thinint_isc);
> @@ -1033,15 +1069,15 @@ static void virtio_ccw_reset(DeviceState *d)
>      virtio_reset(vdev);
>      css_reset_sch(dev->sch);
>      if (dev->indicators) {
> -        release_indicator(dev->indicators);
> +        release_indicator(&dev->routes.adapter, dev->indicators);
>          dev->indicators = NULL;
>      }
>      if (dev->indicators2) {
> -        release_indicator(dev->indicators2);
> +        release_indicator(&dev->routes.adapter, dev->indicators2);
>          dev->indicators2 = NULL;
>      }
>      if (dev->summary_indicator) {
> -        release_indicator(dev->summary_indicator);
> +        release_indicator(&dev->routes.adapter, dev->summary_indicator);
>          dev->summary_indicator = NULL;
>      }
>  }
> @@ -1077,6 +1113,79 @@ static int virtio_ccw_set_host_notifier(DeviceState *d, int n, bool assign)
>      return virtio_ccw_set_guest2host_notifier(dev, n, assign, false);
>  }
>
> +static int virtio_ccw_get_mappings(VirtioCcwDevice *dev)
> +{
> +    int r;
> +
> +    if (!dev->sch->thinint_active) {
> +        return -EINVAL;
> +    }
> +
> +    r = map_indicator(&dev->routes.adapter, dev->summary_indicator);
> +    if (r) {
> +        return r;
> +    }
> +    r = map_indicator(&dev->routes.adapter, dev->indicators);
> +    if (r) {
> +        return r;
> +    }
> +    dev->routes.adapter.summary_addr = dev->summary_indicator->map;
> +    dev->routes.adapter.ind_addr = dev->indicators->map;
> +
> +    return 0;
> +}
> +
> +static int virtio_ccw_setup_irqroutes(VirtioCcwDevice *dev, int nvqs)
> +{
> +    int i;
> +    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
> +    int ret;
> +    S390FLICState *fs = s390_get_flic();
> +    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
> +
> +    ret = virtio_ccw_get_mappings(dev);
> +    if (ret) {
> +        return ret;
> +    }
> +    for (i = 0; i < nvqs; i++) {
> +        if (!virtio_queue_get_num(vdev, i)) {
> +            break;
> +        }
> +    }
> +    dev->routes.num_routes = i;
> +    return fsc->add_adapter_routes(fs, &dev->routes);
> +}
> +
> +static void virtio_ccw_release_irqroutes(VirtioCcwDevice *dev, int nvqs)
> +{
> +    S390FLICState *fs = s390_get_flic();
> +    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
> +
> +    fsc->release_adapter_routes(fs, &dev->routes);
> +}
> +
> +static int virtio_ccw_add_irqfd(VirtioCcwDevice *dev, int n)
> +{
> +    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
> +    VirtQueue *vq = virtio_get_queue(vdev, n);
> +    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
> +
> +    return kvm_irqchip_add_irqfd_notifier(kvm_state, notifier, NULL,
> +                                          dev->routes.gsi[n]);
> +}
> +
> +static void virtio_ccw_remove_irqfd(VirtioCcwDevice *dev, int n)
> +{
> +    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
> +    VirtQueue *vq = virtio_get_queue(vdev, n);
> +    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
> +    int ret;
> +
> +    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, notifier,
> +                                            dev->routes.gsi[n]);
> +    assert(ret == 0);
> +}
> +
>  static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n,
>                                           bool assign, bool with_irqfd)
>  {
> @@ -1092,11 +1201,17 @@ static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n,
>              return r;
>          }
>          virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
> -        /* We do not support irqfd for classic I/O interrupts, because the
> -         * classic interrupts are intermixed with the subchannel status, that
> -         * is queried with test subchannel. We want to use vhost, though.
> -         * Lets make sure to have vhost running and wire up the irq fd to
> -         * land in qemu (and only the irq fd) in this code.
> +        if (with_irqfd) {
> +            r = virtio_ccw_add_irqfd(dev, n);
> +            if (r) {
> +                virtio_queue_set_guest_notifier_fd_handler(vq, false,
> +                                                           with_irqfd);
> +                return r;
> +            }
> +        }
> +        /*
> +         * We do not support individual masking for channel devices, so we
> +         * need to manually trigger any guest masking callbacks here.
>           */
>          if (k->guest_notifier_mask) {
>              k->guest_notifier_mask(vdev, n, false);
> @@ -1110,6 +1225,9 @@ static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n,
>          if (k->guest_notifier_mask) {
>              k->guest_notifier_mask(vdev, n, true);
>          }
> +        if (with_irqfd) {
> +            virtio_ccw_remove_irqfd(dev, n);
> +        }
>          virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
>          event_notifier_cleanup(notifier);
>      }
> @@ -1121,24 +1239,39 @@ static int virtio_ccw_set_guest_notifiers(DeviceState *d, int nvqs,
>  {
>      VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
>      VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
> +    bool with_irqfd = dev->sch->thinint_active && kvm_irqfds_enabled();
>      int r, n;
>
> +    if (with_irqfd && assigned) {
> +        /* irq routes need to be set up before assigning irqfds */
> +        r = virtio_ccw_setup_irqroutes(dev, nvqs);
> +        if (r < 0) {
> +            goto irqroute_error;
> +        }
> +    }
>      for (n = 0; n < nvqs; n++) {
>          if (!virtio_queue_get_num(vdev, n)) {
>              break;
>          }
> -        /* false -> true, as soon as irqfd works */
> -        r = virtio_ccw_set_guest_notifier(dev, n, assigned, false);
> +        r = virtio_ccw_set_guest_notifier(dev, n, assigned, with_irqfd);
>          if (r < 0) {
>              goto assign_error;
>          }
>      }
> +    if (with_irqfd && !assigned) {
> +        /* release irq routes after irqfds have been released */
> +        virtio_ccw_release_irqroutes(dev, nvqs);
> +    }
>      return 0;
>
>  assign_error:
>      while (--n >= 0) {
>          virtio_ccw_set_guest_notifier(dev, n, !assigned, false);
>      }
> +irqroute_error:
> +    if (with_irqfd && assigned) {
> +        virtio_ccw_release_irqroutes(dev, nvqs);
> +    }
>      return r;
>  }
>
> diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
> index d340bf4..b8b8a8a 100644
> --- a/hw/s390x/virtio-ccw.h
> +++ b/hw/s390x/virtio-ccw.h
> @@ -22,6 +22,7 @@
>  #include <hw/virtio/virtio-balloon.h>
>  #include <hw/virtio/virtio-rng.h>
>  #include <hw/virtio/virtio-bus.h>
> +#include <hw/s390x/s390_flic.h>
>
>  #define VIRTUAL_CSSID 0xfe
>
> @@ -77,6 +78,7 @@ typedef struct VirtIOCCWDeviceClass {
>
>  typedef struct IndAddr {
>      hwaddr addr;
> +    uint64_t map;
>      unsigned long refcnt;
>      int len;
>      QTAILQ_ENTRY(IndAddr) sibling;
> @@ -92,7 +94,7 @@ struct VirtioCcwDevice {
>      bool ioeventfd_disabled;
>      uint32_t flags;
>      uint8_t thinint_isc;
> -    uint32_t adapter_id;
> +    AdapterRoutes routes;
>      /* Guest provided values: */
>      IndAddr *indicators;
>      IndAddr *indicators2;
> diff --git a/include/hw/s390x/adapter.h b/include/hw/s390x/adapter.h
> new file mode 100644
> index 0000000..7e56724
> --- /dev/null
> +++ b/include/hw/s390x/adapter.h
> @@ -0,0 +1,23 @@
> +/*
> + * s390 adapter definitions
> + *
> + * Copyright 2013 IBM Corp.
> + * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +
> +#ifndef S390X_ADAPTER_H
> +#define S390X_ADAPTER_H
> +
> +struct AdapterInfo {
> +    uint64_t ind_addr;
> +    uint64_t summary_addr;
> +    uint64_t ind_offset;
> +    uint32_t summary_offset;
> +    uint32_t adapter_id;
> +};
> +
> +#endif
> diff --git a/include/hw/s390x/s390_flic.h b/include/hw/s390x/s390_flic.h
> index 83913ec..489d73b 100644
> --- a/include/hw/s390x/s390_flic.h
> +++ b/include/hw/s390x/s390_flic.h
> @@ -14,6 +14,14 @@
>  #define __HW_S390_FLIC_H
>
>  #include "hw/sysbus.h"
> +#include "hw/s390x/adapter.h"
> +#include "hw/virtio/virtio.h"
> +
> +typedef struct AdapterRoutes {
> +    AdapterInfo adapter;
> +    int num_routes;
> +    int gsi[VIRTIO_PCI_QUEUE_MAX];
> +} AdapterRoutes;
>
>  #define TYPE_S390_FLIC_COMMON "s390-flic"
>  #define S390_FLIC_COMMON(obj) \
> @@ -34,6 +42,10 @@ typedef struct S390FLICStateClass {
>
>      int (*register_io_adapter)(S390FLICState *fs, uint32_t id, uint8_t isc,
>                                 bool swap, bool maskable);
> +    int (*io_adapter_map)(S390FLICState *fs, uint32_t id, uint64_t map_addr,
> +                          bool do_map);
> +    int (*add_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
> +    void (*release_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
>  } S390FLICStateClass;
>
>  #define TYPE_KVM_S390_FLIC "s390-flic-kvm"
> diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
> index 86bab12..5f20b0e 100644
> --- a/include/qemu/typedefs.h
> +++ b/include/qemu/typedefs.h
> @@ -74,5 +74,6 @@ typedef struct SHPCDevice SHPCDevice;
>  typedef struct FWCfgState FWCfgState;
>  typedef struct PcGuestInfo PcGuestInfo;
>  typedef struct Range Range;
> +typedef struct AdapterInfo AdapterInfo;
>
>  #endif /* QEMU_TYPEDEFS_H */
> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
> index 5ad4e0e..d219246 100644
> --- a/include/sysemu/kvm.h
> +++ b/include/sysemu/kvm.h
> @@ -365,6 +365,8 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
>  int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
>  void kvm_irqchip_release_virq(KVMState *s, int virq);
>
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter);
> +
>  int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
>                                     EventNotifier *rn, int virq);
>  int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
> diff --git a/kvm-all.c b/kvm-all.c
> index 5cb7f26..51983bf 100644
> --- a/kvm-all.c
> +++ b/kvm-all.c
> @@ -27,6 +27,7 @@
>  #include "sysemu/sysemu.h"
>  #include "hw/hw.h"
>  #include "hw/pci/msi.h"
> +#include "hw/s390x/adapter.h"
>  #include "exec/gdbstub.h"
>  #include "sysemu/kvm.h"
>  #include "qemu/bswap.h"
> @@ -1247,6 +1248,35 @@ static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
>      return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
>  }
>
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> +{
> +    struct kvm_irq_routing_entry kroute;
> +    int virq;
> +
> +    if (!kvm_gsi_routing_enabled()) {
> +        return -ENOSYS;
> +    }
> +
> +    virq = kvm_irqchip_get_virq(s);
> +    if (virq < 0) {
> +        return virq;
> +    }
> +
> +    kroute.gsi = virq;
> +    kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
> +    kroute.flags = 0;
> +    kroute.u.adapter.summary_addr = adapter->summary_addr;
> +    kroute.u.adapter.ind_addr = adapter->ind_addr;
> +    kroute.u.adapter.summary_offset = adapter->summary_offset;
> +    kroute.u.adapter.ind_offset = adapter->ind_offset;
> +    kroute.u.adapter.adapter_id = adapter->adapter_id;
> +
> +    kvm_add_routing_entry(s, &kroute);
> +    kvm_irqchip_commit_routes(s);
> +
> +    return virq;
> +}
> +
>  #else /* !KVM_CAP_IRQ_ROUTING */
>
>  void kvm_init_irq_routing(KVMState *s)
> @@ -1267,6 +1297,11 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
>      return -ENOSYS;
>  }
>
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> +{
> +    return -ENOSYS;
> +}
> +
>  static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
>  {
>      abort();
> @@ -1296,7 +1331,8 @@ static int kvm_irqchip_create(KVMState *s)
>      int ret;
>
>      if (!qemu_opt_get_bool(qemu_get_machine_opts(), "kernel_irqchip", true) ||
> -        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
> +        (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
> +         (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
>          return 0;
>      }
>
> diff --git a/kvm-stub.c b/kvm-stub.c
> index 8acda86..ac33d86 100644
> --- a/kvm-stub.c
> +++ b/kvm-stub.c
> @@ -136,6 +136,11 @@ int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
>      return -ENOSYS;
>  }
>
> +int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
> +{
> +    return -ENOSYS;
> +}
> +
>  int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
>                                     EventNotifier *rn, int virq)
>  {
> diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
> index b7b0edc..43859e9 100644
> --- a/target-s390x/kvm.c
> +++ b/target-s390x/kvm.c
> @@ -938,6 +938,11 @@ void kvm_s390_enable_css_support(S390CPU *cpu)
>
>  void kvm_arch_init_irq_routing(KVMState *s)
>  {
> +    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
> +        kvm_irqfds_allowed = true;
> +        kvm_gsi_routing_allowed = true;
> +        kvm_halt_in_kernel_allowed = false;
> +    }
>  }
>
>  int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
>

Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Paolo