[v4,4/5] Initial implementation of vGICv3

Get/put routines are missing, live migration is not possible.

Signed-off-by: Pavel Fedin <p.fedin@samsung.com>
---
 hw/intc/Makefile.objs   |   3 +
 hw/intc/arm_gicv3_kvm.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 195 insertions(+)
 create mode 100644 hw/intc/arm_gicv3_kvm.c

Hi,

On 07/14/2015 10:54 AM, Pavel Fedin wrote:
> Get/put routines are missing, live migration is not possible.
>
> Signed-off-by: Pavel Fedin <p.fedin@samsung.com>
> ---
>   hw/intc/Makefile.objs   |   3 +
>   hw/intc/arm_gicv3_kvm.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 195 insertions(+)
>   create mode 100644 hw/intc/arm_gicv3_kvm.c
>
> diff --git a/hw/intc/Makefile.objs b/hw/intc/Makefile.objs
> index 1317e5a..e2525a8 100644
> --- a/hw/intc/Makefile.objs
> +++ b/hw/intc/Makefile.objs
> @@ -17,6 +17,9 @@ common-obj-$(CONFIG_OPENPIC) += openpic.o
>   
>   obj-$(CONFIG_APIC) += apic.o apic_common.o
>   obj-$(CONFIG_ARM_GIC_KVM) += arm_gic_kvm.o
> +ifeq ($(ARCH), aarch64) # Only 64-bit KVM can use these
> +obj-$(CONFIG_ARM_GIC_KVM) += arm_gicv3_kvm.o
> +endif
>   obj-$(CONFIG_STELLARIS) += armv7m_nvic.o
>   obj-$(CONFIG_EXYNOS4) += exynos4210_gic.o exynos4210_combiner.o
>   obj-$(CONFIG_GRLIB) += grlib_irqmp.o
> diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
> new file mode 100644
> index 0000000..4bc8f25
> --- /dev/null
> +++ b/hw/intc/arm_gicv3_kvm.c
> @@ -0,0 +1,192 @@
> +/*
> + * ARM Generic Interrupt Controller using KVM in-kernel support
> + *
> + * Copyright (c) 2015 Samsung Electronics Co., Ltd.
> + * Written by Pavel Fedin
> + * Based on vGICv2 code by Peter Maydell
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation, either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include "hw/sysbus.h"
> +#include "sysemu/kvm.h"
> +#include "kvm_arm.h"
> +#include "gicv3_internal.h"
> +#include "vgic_common.h"
> +
> +#ifdef DEBUG_GICV3_KVM
> +static const int debug_gicv3_kvm = 1;
> +#else
> +static const int debug_gicv3_kvm = 0;
> +#endif
> +
> +#define DPRINTF(fmt, ...) do { \
> +        if (debug_gicv3_kvm) { \
> +            printf("kvm_gicv3: " fmt , ## __VA_ARGS__); \
> +        } \
> +    } while (0)
> +
> +#define TYPE_KVM_ARM_GICV3 "kvm-arm-gicv3"
> +#define KVM_ARM_GICV3(obj) \
> +     OBJECT_CHECK(GICv3State, (obj), TYPE_KVM_ARM_GICV3)
> +#define KVM_ARM_GICV3_CLASS(klass) \
> +     OBJECT_CLASS_CHECK(KVMARMGICv3Class, (klass), TYPE_KVM_ARM_GICV3)
> +#define KVM_ARM_GICV3_GET_CLASS(obj) \
> +     OBJECT_GET_CLASS(KVMARMGICv3Class, (obj), TYPE_KVM_ARM_GICV3)
> +
> +typedef struct KVMARMGICv3Class {
> +    ARMGICv3CommonClass parent_class;
> +    DeviceRealize parent_realize;
> +    void (*parent_reset)(DeviceState *dev);
> +} KVMARMGICv3Class;
> +
> +static void kvm_arm_gicv3_set_irq(void *opaque, int irq, int level)
> +{
> +    GICv3State *s = (GICv3State *)opaque;
> +
> +    kvm_arm_gic_set_irq(s->num_irq, irq, level);
> +}
> +
> +static void kvm_arm_gicv3_put(GICv3State *s)
> +{
> +    /* TODO */
> +    DPRINTF("Cannot put kernel gic state, no kernel interface\n");
> +}
> +
> +static void kvm_arm_gicv3_get(GICv3State *s)
> +{
> +    /* TODO */
> +    DPRINTF("Cannot get kernel gic state, no kernel interface\n");
> +}
> +
> +static void kvm_arm_gicv3_reset(DeviceState *dev)
> +{
> +    GICv3State *s = ARM_GICV3_COMMON(dev);
> +    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
> +
> +    DPRINTF("Reset\n");
> +
> +    kgc->parent_reset(dev);
> +    kvm_arm_gicv3_put(s);
> +}
> +
> +static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
> +{
> +    int i;
> +    GICv3State *s = KVM_ARM_GICV3(dev);
> +    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
> +    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
> +    Error *local_err = NULL;
> +    int ret;
> +
> +    DPRINTF("kvm_arm_gicv3_realize\n");
> +
> +    kgc->parent_realize(dev, &local_err);
> +    if (local_err) {
> +        error_propagate(errp, local_err);
> +        return;
> +    }
> +
> +    i = s->num_irq - GICV3_INTERNAL;
> +    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
> +     * GPIO array layout is thus:
> +     *  [0..N-1] SPIs
> +     *  [N..N+31] PPIs for CPU 0
> +     *  [N+32..N+63] PPIs for CPU 1
> +     *   ...
> +     */
> +    i += (GICV3_INTERNAL * s->num_cpu);

Here GICV3_INTERNAL is used, but kvm_arch_gicv3_set_irq relies on 
kvm_arm_gic_set_irq which is using GIC_INTERNAL, so the GICv3 related 
code is using both defines.

> +    qdev_init_gpio_in(dev, kvm_arm_gicv3_set_irq, i);
> +    /* We never use our outbound IRQ lines but provide them so that
> +     * we maintain the same interface as the non-KVM GIC.
> +     */
> +    for (i = 0; i < s->num_cpu; i++) {
> +        sysbus_init_irq(sbd, &s->parent_irq[i]);
> +    }
> +    for (i = 0; i < s->num_cpu; i++) {
> +        sysbus_init_irq(sbd, &s->parent_fiq[i]);
> +    }
> +
> +    /* Try to create the device via the device control API */
> +    s->dev_fd = -1;
> +    ret = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V3, false);
> +    if (ret >= 0) {
> +        s->dev_fd = ret;
> +    } else if (ret != -ENODEV && ret != -ENOTSUP) {
> +        error_setg_errno(errp, -ret, "error creating in-kernel VGIC");
> +        return;
> +    }
> +
> +    if (kvm_gic_supports_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0)) {
> +        uint32_t numirqs = s->num_irq;
> +        DPRINTF("KVM_DEV_ARM_VGIC_GRP_NR_IRQS = %u\n", numirqs);
> +        kvm_gic_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
> +                       0, 0, &numirqs, 1);
> +    }
> +
> +    /* Tell the kernel to complete VGIC initialization now */
> +    if (kvm_gic_supports_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
> +                              KVM_DEV_ARM_VGIC_CTRL_INIT)) {
> +	DPRINTF("KVM_DEV_ARM_VGIC_CTRL_INIT\n");
> +        kvm_gic_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
> +                          KVM_DEV_ARM_VGIC_CTRL_INIT, 0, 0, 1);
> +    }
> +
> +    /* Distributor */
> +    memory_region_init_reservation(&s->iomem_dist, OBJECT(s),
> +                                   "kvm-gicv3_dist", 0x10000);
> +    sysbus_init_mmio(sbd, &s->iomem_dist);
> +    kvm_arm_register_device(&s->iomem_dist, -1,
> +                            KVM_DEV_ARM_VGIC_GRP_ADDR,
> +                            KVM_VGIC_V3_ADDR_TYPE_DIST,
> +                            s->dev_fd);
> +
> +    /* Redistributor */
> +    memory_region_init_reservation(&s->iomem_lpi, OBJECT(s),
> +                                   "kvm-gicv3_lpi", 0x800000);
> +    sysbus_init_mmio(sbd, &s->iomem_lpi);
> +    kvm_arm_register_device(&s->iomem_lpi, -1,
> +                            KVM_DEV_ARM_VGIC_GRP_ADDR,
> +                            KVM_VGIC_V3_ADDR_TYPE_REDIST,
> +                            s->dev_fd);
> +}
> +
> +static void kvm_arm_gicv3_class_init(ObjectClass *klass, void *data)
> +{
> +    DeviceClass *dc = DEVICE_CLASS(klass);
> +    ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
> +    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass);
> +
> +    agcc->pre_save = kvm_arm_gicv3_get;
> +    agcc->post_load = kvm_arm_gicv3_put;
> +    kgc->parent_realize = dc->realize;
> +    kgc->parent_reset = dc->reset;
> +    dc->realize = kvm_arm_gicv3_realize;
> +    dc->reset = kvm_arm_gicv3_reset;
> +}
> +
> +static const TypeInfo kvm_arm_gicv3_info = {
> +    .name = TYPE_KVM_ARM_GICV3,
> +    .parent = TYPE_ARM_GICV3_COMMON,
> +    .instance_size = sizeof(GICv3State),
> +    .class_init = kvm_arm_gicv3_class_init,
> +    .class_size = sizeof(KVMARMGICv3Class),
> +};
> +
> +static void kvm_arm_gicv3_register_types(void)
> +{
> +    type_register_static(&kvm_arm_gicv3_info);
> +}
> +
> +type_init(kvm_arm_gicv3_register_types)

Diana

Hello!

> Here GICV3_INTERNAL is used, but kvm_arch_gicv3_set_irq relies on
> kvm_arm_gic_set_irq which is using GIC_INTERNAL, so the GICv3 related
> code is using both defines.

 The change relies on these definitions being equal. Ok, i agree this is a kind of hack and will
redo this in v5.

> As we do not have yet support for ITS implementation in QEMU the device
> tree for a GICv3 guest will have a msi-parent property in the device
> tree which will point to nothing:
> 
> "msi-parent = <0x0>"

 Oops, thank you, i forgot about this. Will fix.

Kind regards,
Pavel Fedin
Expert Engineer
Samsung Electronics Research center Russia