ARM: KVM: Enable in-kernel timers with user space gic
diff mbox

Message ID 1498512656-75289-1-git-send-email-agraf@suse.de
State New
Headers show

Commit Message

Alexander Graf June 26, 2017, 9:30 p.m. UTC
When running with KVM enabled, you can choose between emulating the
gic in kernel or user space. If the kernel supports in-kernel virtualization
of the interrupt controller, it will default to that. If not, if will
default to user space emulation.

Unfortunately when running in user mode gic emulation, we miss out on
interrupt events which are only available from kernel space, such as the timer.
This patch leverages the new kernel/user space pending line synchronization for
timer events. It does not handle PMU events yet.

Signed-off-by: Alexander Graf <agraf@suse.de>
---
 accel/kvm/kvm-all.c    |  5 +++++
 accel/stubs/kvm-stub.c |  5 +++++
 hw/intc/arm_gic.c      |  7 +++++++
 include/sysemu/kvm.h   | 11 +++++++++++
 target/arm/cpu.h       |  3 +++
 target/arm/kvm.c       | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 80 insertions(+)

Comments

Andrew Jones June 27, 2017, 12:37 p.m. UTC | #1
On Mon, Jun 26, 2017 at 11:30:56PM +0200, Alexander Graf wrote:
> When running with KVM enabled, you can choose between emulating the
> gic in kernel or user space. If the kernel supports in-kernel virtualization
> of the interrupt controller, it will default to that. If not, if will
> default to user space emulation.
> 
> Unfortunately when running in user mode gic emulation, we miss out on
> interrupt events which are only available from kernel space, such as the timer.
> This patch leverages the new kernel/user space pending line synchronization for
> timer events. It does not handle PMU events yet.
> 
> Signed-off-by: Alexander Graf <agraf@suse.de>
> ---
>  accel/kvm/kvm-all.c    |  5 +++++
>  accel/stubs/kvm-stub.c |  5 +++++
>  hw/intc/arm_gic.c      |  7 +++++++
>  include/sysemu/kvm.h   | 11 +++++++++++
>  target/arm/cpu.h       |  3 +++
>  target/arm/kvm.c       | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
>  6 files changed, 80 insertions(+)
> 
> diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
> index 75feffa..ade32ea 100644
> --- a/accel/kvm/kvm-all.c
> +++ b/accel/kvm/kvm-all.c
> @@ -2285,6 +2285,11 @@ int kvm_has_intx_set_mask(void)
>      return kvm_state->intx_set_mask;
>  }
>  
> +bool kvm_arm_supports_user_irq(void)
> +{
> +    return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
> +}
> +
>  #ifdef KVM_CAP_SET_GUEST_DEBUG
>  struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
>                                                   target_ulong pc)
> diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c
> index ef0c734..3965c52 100644
> --- a/accel/stubs/kvm-stub.c
> +++ b/accel/stubs/kvm-stub.c
> @@ -155,4 +155,9 @@ void kvm_init_cpu_signals(CPUState *cpu)
>  {
>      abort();
>  }
> +
> +bool kvm_arm_supports_user_irq(void)
> +{
> +    return false;
> +}
>  #endif
> diff --git a/hw/intc/arm_gic.c b/hw/intc/arm_gic.c
> index b305d90..4ffd905 100644
> --- a/hw/intc/arm_gic.c
> +++ b/hw/intc/arm_gic.c
> @@ -25,6 +25,7 @@
>  #include "qom/cpu.h"
>  #include "qemu/log.h"
>  #include "trace.h"
> +#include "sysemu/kvm.h"
>  
>  /* #define DEBUG_GIC */
>  
> @@ -1412,6 +1413,12 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
>          return;
>      }
>  
> +    if (kvm_enabled() && !kvm_arm_supports_user_irq()) {
> +            error_setg(errp, "KVM with user space irqchip only works when the "
> +                             "host kernel supports KVM_CAP_ARM_USER_IRQ");
> +            return;

extra spaces here

> +    }
> +
>      /* This creates distributor and main CPU interface (s->cpuiomem[0]) */
>      gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
>  
> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
> index 1e91613..9f11fc0 100644
> --- a/include/sysemu/kvm.h
> +++ b/include/sysemu/kvm.h
> @@ -227,6 +227,17 @@ int kvm_init_vcpu(CPUState *cpu);
>  int kvm_cpu_exec(CPUState *cpu);
>  int kvm_destroy_vcpu(CPUState *cpu);
>  
> +/**
> + * kvm_arm_supports_user_irq
> + *
> + * Not all KVM implementations support notifications for kernel generated
> + * interrupt events to user space. This function indicates whether the current
> + * KVM implementation does support them.
> + *
> + * Returns: true if KVM supports using kernel generated IRQs from user space
> + */
> +bool kvm_arm_supports_user_irq(void);
> +
>  #ifdef NEED_CPU_H
>  #include "cpu.h"
>  
> diff --git a/target/arm/cpu.h b/target/arm/cpu.h
> index 16a1e59..88ec3ee 100644
> --- a/target/arm/cpu.h
> +++ b/target/arm/cpu.h
> @@ -706,6 +706,9 @@ struct ARMCPU {
>      void *el_change_hook_opaque;
>  
>      int32_t node_id; /* NUMA node this CPU belongs to */
> +
> +    /* Used to synchronize KVM and QEMU timer levels */

Eventually it won't only be for timer levels

> +    uint8_t device_irq_level;
>  };
>  
>  static inline ARMCPU *arm_env_get_cpu(CPUARMState *env)
> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
> index 4555468..29ee72d 100644
> --- a/target/arm/kvm.c
> +++ b/target/arm/kvm.c
> @@ -174,6 +174,12 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>       */
>      kvm_async_interrupts_allowed = true;
>  
> +    /*
> +     * PSCI wakes up secondary cores, so we always need to
> +     * have vCPUs waiting in kernel space
> +     */
> +    kvm_halt_in_kernel_allowed = true;
> +
>      cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
>  
>      type_register_static(&host_arm_cpu_type_info);
> @@ -528,6 +534,49 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
>  
>  MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
>  {
> +    ARMCPU *cpu;
> +    uint32_t switched_level;
> +
> +    if (kvm_irqchip_in_kernel()) {
> +        /*
> +         * We only need to sync timer states with user-space interrupt
> +         * controllers, so return early and save cycles if we don't.
> +         */
> +        return MEMTXATTRS_UNSPECIFIED;
> +    }
> +
> +    cpu = ARM_CPU(cs);
> +
> +    /* Synchronize our internal timer irq lines with the kvm ones */
> +    if (run->s.regs.device_irq_level != cpu->device_irq_level) {
> +        switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
> +
> +        qemu_mutex_lock_iothread();
> +
> +        if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
> +            qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
> +                         run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER);

Should be !!(run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER)

> +            switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
> +        }
> +
> +        if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
> +            qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
> +                         run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_PTIMER);

Also need the !!

> +            switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
> +        }
> +
> +        /* XXX PMU IRQ is missing */

So I was wondering why you chose not to wire up the PMU in order to finish
this off, since I knew KVM has commit 3dbbdf78636e "KVM: arm/arm64: Report
PMU overflow interrupts to userspace irqchip", which I presume should
allow the PMU to work too. However, attempting to start a guest with
-machine virt,accel=kvm,kernel-irqchip=off -cpu host,pmu=on doesn't provide
it a PMU. AFAICT, that's only because KVM still has

        /*
         * We currently require an in-kernel VGIC to use the PMU emulation,
         * because we do not support forwarding PMU overflow interrupts to
         * userspace yet.
         */
        if (!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))
                return -ENODEV;

in kvm_arm_pmu_v3_init(). Anyone know why that wasn't removed with
3dbbdf78636e?

> +
> +        if (switched_level) {
> +            qemu_log_mask(LOG_UNIMP, "%s: unhandled device IRQ %x\n",

Misleading message here. It's not a device IRQ that switched_level
contains.

> +                          __func__, switched_level);
> +        }
> +
> +        /* We also mark unknown levels as processed to not waste cycles */
> +        cpu->device_irq_level = run->s.regs.device_irq_level;
> +        qemu_mutex_unlock_iothread();
> +    }
> +
>      return MEMTXATTRS_UNSPECIFIED;
>  }
>  
> -- 
> 1.8.5.6
> 
>

Thanks,
drew
Alexander Graf June 27, 2017, 1 p.m. UTC | #2
On 27.06.17 14:37, Andrew Jones wrote:
> On Mon, Jun 26, 2017 at 11:30:56PM +0200, Alexander Graf wrote:
>> When running with KVM enabled, you can choose between emulating the
>> gic in kernel or user space. If the kernel supports in-kernel virtualization
>> of the interrupt controller, it will default to that. If not, if will
>> default to user space emulation.
>>
>> Unfortunately when running in user mode gic emulation, we miss out on
>> interrupt events which are only available from kernel space, such as the timer.
>> This patch leverages the new kernel/user space pending line synchronization for
>> timer events. It does not handle PMU events yet.
>>
>> Signed-off-by: Alexander Graf <agraf@suse.de>
>> ---
>>   accel/kvm/kvm-all.c    |  5 +++++
>>   accel/stubs/kvm-stub.c |  5 +++++
>>   hw/intc/arm_gic.c      |  7 +++++++
>>   include/sysemu/kvm.h   | 11 +++++++++++
>>   target/arm/cpu.h       |  3 +++
>>   target/arm/kvm.c       | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
>>   6 files changed, 80 insertions(+)
>>
>> diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
>> index 75feffa..ade32ea 100644
>> --- a/accel/kvm/kvm-all.c
>> +++ b/accel/kvm/kvm-all.c
>> @@ -2285,6 +2285,11 @@ int kvm_has_intx_set_mask(void)
>>       return kvm_state->intx_set_mask;
>>   }
>>   
>> +bool kvm_arm_supports_user_irq(void)
>> +{
>> +    return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
>> +}
>> +
>>   #ifdef KVM_CAP_SET_GUEST_DEBUG
>>   struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
>>                                                    target_ulong pc)
>> diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c
>> index ef0c734..3965c52 100644
>> --- a/accel/stubs/kvm-stub.c
>> +++ b/accel/stubs/kvm-stub.c
>> @@ -155,4 +155,9 @@ void kvm_init_cpu_signals(CPUState *cpu)
>>   {
>>       abort();
>>   }
>> +
>> +bool kvm_arm_supports_user_irq(void)
>> +{
>> +    return false;
>> +}
>>   #endif
>> diff --git a/hw/intc/arm_gic.c b/hw/intc/arm_gic.c
>> index b305d90..4ffd905 100644
>> --- a/hw/intc/arm_gic.c
>> +++ b/hw/intc/arm_gic.c
>> @@ -25,6 +25,7 @@
>>   #include "qom/cpu.h"
>>   #include "qemu/log.h"
>>   #include "trace.h"
>> +#include "sysemu/kvm.h"
>>   
>>   /* #define DEBUG_GIC */
>>   
>> @@ -1412,6 +1413,12 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
>>           return;
>>       }
>>   
>> +    if (kvm_enabled() && !kvm_arm_supports_user_irq()) {
>> +            error_setg(errp, "KVM with user space irqchip only works when the "
>> +                             "host kernel supports KVM_CAP_ARM_USER_IRQ");
>> +            return;
> 
> extra spaces here

Fixed

> 
>> +    }
>> +
>>       /* This creates distributor and main CPU interface (s->cpuiomem[0]) */
>>       gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
>>   
>> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
>> index 1e91613..9f11fc0 100644
>> --- a/include/sysemu/kvm.h
>> +++ b/include/sysemu/kvm.h
>> @@ -227,6 +227,17 @@ int kvm_init_vcpu(CPUState *cpu);
>>   int kvm_cpu_exec(CPUState *cpu);
>>   int kvm_destroy_vcpu(CPUState *cpu);
>>   
>> +/**
>> + * kvm_arm_supports_user_irq
>> + *
>> + * Not all KVM implementations support notifications for kernel generated
>> + * interrupt events to user space. This function indicates whether the current
>> + * KVM implementation does support them.
>> + *
>> + * Returns: true if KVM supports using kernel generated IRQs from user space
>> + */
>> +bool kvm_arm_supports_user_irq(void);
>> +
>>   #ifdef NEED_CPU_H
>>   #include "cpu.h"
>>   
>> diff --git a/target/arm/cpu.h b/target/arm/cpu.h
>> index 16a1e59..88ec3ee 100644
>> --- a/target/arm/cpu.h
>> +++ b/target/arm/cpu.h
>> @@ -706,6 +706,9 @@ struct ARMCPU {
>>       void *el_change_hook_opaque;
>>   
>>       int32_t node_id; /* NUMA node this CPU belongs to */
>> +
>> +    /* Used to synchronize KVM and QEMU timer levels */
> 
> Eventually it won't only be for timer levels

Ah, one place I forgot to fix up :). Fixed.

> 
>> +    uint8_t device_irq_level;
>>   };
>>   
>>   static inline ARMCPU *arm_env_get_cpu(CPUARMState *env)
>> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
>> index 4555468..29ee72d 100644
>> --- a/target/arm/kvm.c
>> +++ b/target/arm/kvm.c
>> @@ -174,6 +174,12 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>>        */
>>       kvm_async_interrupts_allowed = true;
>>   
>> +    /*
>> +     * PSCI wakes up secondary cores, so we always need to
>> +     * have vCPUs waiting in kernel space
>> +     */
>> +    kvm_halt_in_kernel_allowed = true;
>> +
>>       cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
>>   
>>       type_register_static(&host_arm_cpu_type_info);
>> @@ -528,6 +534,49 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
>>   
>>   MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
>>   {
>> +    ARMCPU *cpu;
>> +    uint32_t switched_level;
>> +
>> +    if (kvm_irqchip_in_kernel()) {
>> +        /*
>> +         * We only need to sync timer states with user-space interrupt
>> +         * controllers, so return early and save cycles if we don't.
>> +         */
>> +        return MEMTXATTRS_UNSPECIFIED;
>> +    }
>> +
>> +    cpu = ARM_CPU(cs);
>> +
>> +    /* Synchronize our internal timer irq lines with the kvm ones */
>> +    if (run->s.regs.device_irq_level != cpu->device_irq_level) {
>> +        switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
>> +
>> +        qemu_mutex_lock_iothread();
>> +
>> +        if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
>> +            qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
>> +                         run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER);
> 
> Should be !!(run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER)

Yup.

> 
>> +            switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
>> +        }
>> +
>> +        if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
>> +            qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
>> +                         run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_PTIMER);
> 
> Also need the !!

Yup.

> 
>> +            switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
>> +        }
>> +
>> +        /* XXX PMU IRQ is missing */
> 
> So I was wondering why you chose not to wire up the PMU in order to finish

Because the PMU IRQ is only available from the machine file, so it'll 
need some more plumbing to get the qemu_irq object available from kvm.c.

> this off, since I knew KVM has commit 3dbbdf78636e "KVM: arm/arm64: Report
> PMU overflow interrupts to userspace irqchip", which I presume should
> allow the PMU to work too. However, attempting to start a guest with
> -machine virt,accel=kvm,kernel-irqchip=off -cpu host,pmu=on doesn't provide
> it a PMU. AFAICT, that's only because KVM still has
> 
>          /*
>           * We currently require an in-kernel VGIC to use the PMU emulation,
>           * because we do not support forwarding PMU overflow interrupts to
>           * userspace yet.
>           */
>          if (!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))
>                  return -ENODEV;
> 
> in kvm_arm_pmu_v3_init(). Anyone know why that wasn't removed with
> 3dbbdf78636e?

I guess it should get removed when the PMU sync works?

> 
>> +
>> +        if (switched_level) {
>> +            qemu_log_mask(LOG_UNIMP, "%s: unhandled device IRQ %x\n",
> 
> Misleading message here. It's not a device IRQ that switched_level
> contains.

Ok, changed to "in-kernel device" as that really is what it is about.


Alex
Andrew Jones June 27, 2017, 1:59 p.m. UTC | #3
On Tue, Jun 27, 2017 at 03:00:10PM +0200, Alexander Graf wrote:
> > > +        /* XXX PMU IRQ is missing */
> > 
> > So I was wondering why you chose not to wire up the PMU in order to finish
> 
> Because the PMU IRQ is only available from the machine file, so it'll need
> some more plumbing to get the qemu_irq object available from kvm.c.

OK, I just wrote a patch based on this patch for the PMU and will post it
in a second.

> 
> > this off, since I knew KVM has commit 3dbbdf78636e "KVM: arm/arm64: Report
> > PMU overflow interrupts to userspace irqchip", which I presume should
> > allow the PMU to work too. However, attempting to start a guest with
> > -machine virt,accel=kvm,kernel-irqchip=off -cpu host,pmu=on doesn't provide
> > it a PMU. AFAICT, that's only because KVM still has
> > 
> >          /*
> >           * We currently require an in-kernel VGIC to use the PMU emulation,
> >           * because we do not support forwarding PMU overflow interrupts to
> >           * userspace yet.
> >           */
> >          if (!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))
> >                  return -ENODEV;
> > 
> > in kvm_arm_pmu_v3_init(). Anyone know why that wasn't removed with
> > 3dbbdf78636e?
> 
> I guess it should get removed when the PMU sync works?

OK, I'll post the KVM patch at the same time.


I noticed another thing while testing with a virtio-scsi disk; I get
this on output when booting with userspace irqchip

 virtio-scsi: Failed to set guest notifiers (-11), ensure -enable-kvm is set
 qemu-system-aarch64: virtio_bus_start_ioeventfd: failed. Fallback to userspace (slower).

Just a warning, but I guess it should be removed, at least when the
user has explicitly asked for a userspace irqchip with kernel-irqchip=no.

Thanks,
drew
Andrew Jones June 27, 2017, 2:30 p.m. UTC | #4
On Tue, Jun 27, 2017 at 03:59:51PM +0200, Andrew Jones wrote:
> On Tue, Jun 27, 2017 at 03:00:10PM +0200, Alexander Graf wrote:
> > > > +        /* XXX PMU IRQ is missing */
> > > 
> > > So I was wondering why you chose not to wire up the PMU in order to finish
> > 
> > Because the PMU IRQ is only available from the machine file, so it'll need
> > some more plumbing to get the qemu_irq object available from kvm.c.
> 
> OK, I just wrote a patch based on this patch for the PMU and will post it
> in a second.
> 
> > 
> > > this off, since I knew KVM has commit 3dbbdf78636e "KVM: arm/arm64: Report
> > > PMU overflow interrupts to userspace irqchip", which I presume should
> > > allow the PMU to work too. However, attempting to start a guest with
> > > -machine virt,accel=kvm,kernel-irqchip=off -cpu host,pmu=on doesn't provide
> > > it a PMU. AFAICT, that's only because KVM still has
> > > 
> > >          /*
> > >           * We currently require an in-kernel VGIC to use the PMU emulation,
> > >           * because we do not support forwarding PMU overflow interrupts to
> > >           * userspace yet.
> > >           */
> > >          if (!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))
> > >                  return -ENODEV;
> > > 
> > > in kvm_arm_pmu_v3_init(). Anyone know why that wasn't removed with
> > > 3dbbdf78636e?

Eh, I missed a2befacf5094 "KVM: arm64: Allow creating the PMU without the
in-kernel GIC", which is still in kvmarm/queue, not yet in Linus master.
I'll have to revisit the QEMU patch I wrote to make sure it still makes
sense.

drew

Patch
diff mbox

diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index 75feffa..ade32ea 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -2285,6 +2285,11 @@  int kvm_has_intx_set_mask(void)
     return kvm_state->intx_set_mask;
 }
 
+bool kvm_arm_supports_user_irq(void)
+{
+    return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
+}
+
 #ifdef KVM_CAP_SET_GUEST_DEBUG
 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
                                                  target_ulong pc)
diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c
index ef0c734..3965c52 100644
--- a/accel/stubs/kvm-stub.c
+++ b/accel/stubs/kvm-stub.c
@@ -155,4 +155,9 @@  void kvm_init_cpu_signals(CPUState *cpu)
 {
     abort();
 }
+
+bool kvm_arm_supports_user_irq(void)
+{
+    return false;
+}
 #endif
diff --git a/hw/intc/arm_gic.c b/hw/intc/arm_gic.c
index b305d90..4ffd905 100644
--- a/hw/intc/arm_gic.c
+++ b/hw/intc/arm_gic.c
@@ -25,6 +25,7 @@ 
 #include "qom/cpu.h"
 #include "qemu/log.h"
 #include "trace.h"
+#include "sysemu/kvm.h"
 
 /* #define DEBUG_GIC */
 
@@ -1412,6 +1413,12 @@  static void arm_gic_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    if (kvm_enabled() && !kvm_arm_supports_user_irq()) {
+            error_setg(errp, "KVM with user space irqchip only works when the "
+                             "host kernel supports KVM_CAP_ARM_USER_IRQ");
+            return;
+    }
+
     /* This creates distributor and main CPU interface (s->cpuiomem[0]) */
     gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 1e91613..9f11fc0 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -227,6 +227,17 @@  int kvm_init_vcpu(CPUState *cpu);
 int kvm_cpu_exec(CPUState *cpu);
 int kvm_destroy_vcpu(CPUState *cpu);
 
+/**
+ * kvm_arm_supports_user_irq
+ *
+ * Not all KVM implementations support notifications for kernel generated
+ * interrupt events to user space. This function indicates whether the current
+ * KVM implementation does support them.
+ *
+ * Returns: true if KVM supports using kernel generated IRQs from user space
+ */
+bool kvm_arm_supports_user_irq(void);
+
 #ifdef NEED_CPU_H
 #include "cpu.h"
 
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 16a1e59..88ec3ee 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -706,6 +706,9 @@  struct ARMCPU {
     void *el_change_hook_opaque;
 
     int32_t node_id; /* NUMA node this CPU belongs to */
+
+    /* Used to synchronize KVM and QEMU timer levels */
+    uint8_t device_irq_level;
 };
 
 static inline ARMCPU *arm_env_get_cpu(CPUARMState *env)
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 4555468..29ee72d 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -174,6 +174,12 @@  int kvm_arch_init(MachineState *ms, KVMState *s)
      */
     kvm_async_interrupts_allowed = true;
 
+    /*
+     * PSCI wakes up secondary cores, so we always need to
+     * have vCPUs waiting in kernel space
+     */
+    kvm_halt_in_kernel_allowed = true;
+
     cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
 
     type_register_static(&host_arm_cpu_type_info);
@@ -528,6 +534,49 @@  void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 
 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
+    ARMCPU *cpu;
+    uint32_t switched_level;
+
+    if (kvm_irqchip_in_kernel()) {
+        /*
+         * We only need to sync timer states with user-space interrupt
+         * controllers, so return early and save cycles if we don't.
+         */
+        return MEMTXATTRS_UNSPECIFIED;
+    }
+
+    cpu = ARM_CPU(cs);
+
+    /* Synchronize our internal timer irq lines with the kvm ones */
+    if (run->s.regs.device_irq_level != cpu->device_irq_level) {
+        switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
+
+        qemu_mutex_lock_iothread();
+
+        if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
+            qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
+                         run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER);
+            switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
+        }
+
+        if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
+            qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
+                         run->s.regs.device_irq_level & KVM_ARM_DEV_EL1_PTIMER);
+            switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
+        }
+
+        /* XXX PMU IRQ is missing */
+
+        if (switched_level) {
+            qemu_log_mask(LOG_UNIMP, "%s: unhandled device IRQ %x\n",
+                          __func__, switched_level);
+        }
+
+        /* We also mark unknown levels as processed to not waste cycles */
+        cpu->device_irq_level = run->s.regs.device_irq_level;
+        qemu_mutex_unlock_iothread();
+    }
+
     return MEMTXATTRS_UNSPECIFIED;
 }