[v9,10/11] target-arm: Provide '-cpu host' when running KVM

Implement '-cpu host' for ARM when we're using KVM, broadly
in line with other KVM-supporting architectures.

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
---
 target-arm/helper.c  |    6 ++
 target-arm/kvm.c     |  224 ++++++++++++++++++++++++++++++++++++++++++++++++++
 target-arm/kvm_arm.h |   55 +++++++++++++
 3 files changed, 285 insertions(+)

On Fri, Nov 22, 2013 at 05:17:17PM +0000, Peter Maydell wrote:
> Implement '-cpu host' for ARM when we're using KVM, broadly
> in line with other KVM-supporting architectures.
> 
> Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
> ---
>  target-arm/helper.c  |    6 ++
>  target-arm/kvm.c     |  224 ++++++++++++++++++++++++++++++++++++++++++++++++++
>  target-arm/kvm_arm.h |   55 +++++++++++++
>  3 files changed, 285 insertions(+)
> 
> diff --git a/target-arm/helper.c b/target-arm/helper.c
> index 3445813..263dbbf 100644
> --- a/target-arm/helper.c
> +++ b/target-arm/helper.c
> @@ -1842,6 +1842,12 @@ void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
>      (*cpu_fprintf)(f, "Available CPUs:\n");
>      g_slist_foreach(list, arm_cpu_list_entry, &s);
>      g_slist_free(list);
> +#ifdef CONFIG_KVM
> +    /* The 'host' CPU type is dynamically registered only if KVM is
> +     * enabled, so we have to special-case it here:
> +     */
> +    (*cpu_fprintf)(f, "  host (only available in KVM mode)\n");
> +#endif
>  }
>  
>  static void arm_cpu_add_definition(gpointer data, gpointer user_data)
> diff --git a/target-arm/kvm.c b/target-arm/kvm.c
> index 182db85..f865dac 100644
> --- a/target-arm/kvm.c
> +++ b/target-arm/kvm.c
> @@ -27,12 +27,236 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
>      KVM_CAP_LAST_INFO
>  };
>  
> +bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
> +                                      int *fdarray,
> +                                      struct kvm_vcpu_init *init)
> +{
> +    int ret, kvmfd = -1, vmfd = -1, cpufd = -1;
> +
> +    kvmfd = qemu_open("/dev/kvm", O_RDWR);
> +    if (kvmfd < 0) {
> +        goto err;
> +    }
> +    vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
> +    if (vmfd < 0) {
> +        goto err;
> +    }
> +    cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
> +    if (cpufd < 0) {
> +        goto err;
> +    }
> +
> +    ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, init);
> +    if (ret >= 0) {
> +        ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
> +        if (ret < 0) {
> +            goto err;
> +        }
> +    } else {
> +        /* Old kernel which doesn't know about the
> +         * PREFERRED_TARGET ioctl: we know it will only support
> +         * creating one kind of guest CPU which is its preferred
> +         * CPU type.
> +         */
> +        while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
> +            init->target = *cpus_to_try++;
> +            memset(init->features, 0, sizeof(init->features));
> +            ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
> +            if (ret >= 0) {
> +                break;
> +            }
> +        }
> +        if (ret < 0) {
> +            goto err;
> +        }
> +    }
> +
> +    fdarray[0] = kvmfd;
> +    fdarray[1] = vmfd;
> +    fdarray[2] = cpufd;

you could consider using a define/enum/struct for this instead of an
array, but bah, not important.

> +
> +    return true;
> +
> +err:
> +    if (cpufd >= 0) {
> +        close(cpufd);
> +    }
> +    if (vmfd >= 0) {
> +        close(vmfd);
> +    }
> +    if (kvmfd >= 0) {
> +        close(kvmfd);
> +    }
> +
> +    return false;
> +}
> +
> +void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
> +{
> +    int i;
> +
> +    for (i = 2; i >= 0; i--) {
> +        close(fdarray[i]);
> +    }
> +}
> +
> +static inline void set_feature(uint64_t *features, int feature)
> +{
> +    *features |= 1ULL << feature;
> +}
> +
> +bool kvm_arm_get_host_cpu_features(ARMHostCPUClass *ahcc)
> +{
> +    /* Identify the feature bits corresponding to the host CPU, and
> +     * fill out the ARMHostCPUClass fields accordingly. To do this
> +     * we have to create a scratch VM, create a single CPU inside it,
> +     * and then query that CPU for the relevant ID registers.
> +     */
> +    int i, ret, fdarray[3];
> +    uint32_t midr, id_pfr0, id_isar0, mvfr1;
> +    uint64_t features = 0;
> +    /* Old kernels may not know about the PREFERRED_TARGET ioctl: however
> +     * we know these will only support creating one kind of guest CPU,
> +     * which is its preferred CPU type.
> +     */
> +    static const uint32_t cpus_to_try[] = {
> +        QEMU_KVM_ARM_TARGET_CORTEX_A15,
> +        QEMU_KVM_ARM_TARGET_NONE
> +    };
> +    struct kvm_vcpu_init init;
> +    struct kvm_one_reg idregs[] = {
> +        {
> +            .id = KVM_REG_ARM | KVM_REG_SIZE_U32
> +            | ENCODE_CP_REG(15, 0, 0, 0, 0, 0),
> +            .addr = (uintptr_t)&midr,
> +        },
> +        {
> +            .id = KVM_REG_ARM | KVM_REG_SIZE_U32
> +            | ENCODE_CP_REG(15, 0, 0, 1, 0, 0),
> +            .addr = (uintptr_t)&id_pfr0,
> +        },
> +        {
> +            .id = KVM_REG_ARM | KVM_REG_SIZE_U32
> +            | ENCODE_CP_REG(15, 0, 0, 2, 0, 0),
> +            .addr = (uintptr_t)&id_isar0,
> +        },
> +        {
> +            .id = KVM_REG_ARM | KVM_REG_SIZE_U32
> +            | KVM_REG_ARM_VFP | KVM_REG_ARM_VFP_MVFR1,
> +            .addr = (uintptr_t)&mvfr1,
> +        },
> +    };
> +
> +    if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) {
> +        return false;
> +    }
> +
> +    ahcc->target = init.target;
> +
> +    /* This is not strictly blessed by the device tree binding docs yet,
> +     * but in practice the kernel does not care about this string so
> +     * there is no point maintaining an KVM_ARM_TARGET_* -> string table.
> +     */
> +    ahcc->dtb_compatible = "arm,arm-v7";
> +
> +    for (i = 0; i < ARRAY_SIZE(idregs); i++) {
> +        ret = ioctl(fdarray[2], KVM_GET_ONE_REG, &idregs[i]);
> +        if (ret) {
> +            break;
> +        }
> +    }
> +
> +    kvm_arm_destroy_scratch_host_vcpu(fdarray);
> +
> +    if (ret) {
> +        return false;
> +    }
> +
> +    /* Now we've retrieved all the register information we can
> +     * set the feature bits based on the ID register fields.
> +     * We can assume any KVM supporting CPU is at least a v7
> +     * with VFPv3, LPAE and the generic timers; this in turn implies
> +     * most of the other feature bits, but a few must be tested.
> +     */
> +    set_feature(&features, ARM_FEATURE_V7);
> +    set_feature(&features, ARM_FEATURE_VFP3);
> +    set_feature(&features, ARM_FEATURE_LPAE);
> +    set_feature(&features, ARM_FEATURE_GENERIC_TIMER);
> +
> +    switch (extract32(id_isar0, 24, 4)) {
> +    case 1:
> +        set_feature(&features, ARM_FEATURE_THUMB_DIV);
> +        break;
> +    case 2:
> +        set_feature(&features, ARM_FEATURE_ARM_DIV);
> +        set_feature(&features, ARM_FEATURE_THUMB_DIV);
> +        break;
> +    default:
> +        break;
> +    }
> +
> +    if (extract32(id_pfr0, 12, 4) == 1) {
> +        set_feature(&features, ARM_FEATURE_THUMB2EE);
> +    }
> +    if (extract32(mvfr1, 20, 4) == 1) {
> +        set_feature(&features, ARM_FEATURE_VFP_FP16);
> +    }
> +    if (extract32(mvfr1, 12, 4) == 1) {
> +        set_feature(&features, ARM_FEATURE_NEON);
> +    }
> +    if (extract32(mvfr1, 28, 4) == 1) {
> +        /* FMAC support implies VFPv4 */
> +        set_feature(&features, ARM_FEATURE_VFP4);
> +    }
> +
> +    ahcc->features = features;
> +
> +    return true;
> +}
> +
> +static void kvm_arm_host_cpu_class_init(ObjectClass *oc, void *data)
> +{
> +    ARMHostCPUClass *ahcc = ARM_HOST_CPU_CLASS(oc);
> +
> +    /* All we really need to set up for the 'host' CPU
> +     * is the feature bits -- we rely on the fact that the
> +     * various ID register values in ARMCPU are only used for
> +     * TCG CPUs.
> +     */
> +    if (!kvm_arm_get_host_cpu_features(ahcc)) {
> +        fprintf(stderr, "Failed to retrieve host CPU features!\n");
> +        abort();
> +    }
> +}
> +
> +static void kvm_arm_host_cpu_initfn(Object *obj)
> +{
> +    ARMHostCPUClass *ahcc = ARM_HOST_CPU_GET_CLASS(obj);
> +    ARMCPU *cpu = ARM_CPU(obj);
> +    CPUARMState *env = &cpu->env;
> +
> +    cpu->kvm_target = ahcc->target;
> +    cpu->dtb_compatible = ahcc->dtb_compatible;
> +    env->features = ahcc->features;
> +}
> +
> +static const TypeInfo host_arm_cpu_type_info = {
> +    .name = TYPE_ARM_HOST_CPU,
> +    .parent = TYPE_ARM_CPU,
> +    .instance_init = kvm_arm_host_cpu_initfn,
> +    .class_init = kvm_arm_host_cpu_class_init,
> +    .class_size = sizeof(ARMHostCPUClass),
> +};
> +
>  int kvm_arch_init(KVMState *s)
>  {
>      /* For ARM interrupt delivery is always asynchronous,
>       * whether we are using an in-kernel VGIC or not.
>       */
>      kvm_async_interrupts_allowed = true;
> +
> +    type_register_static(&host_arm_cpu_type_info);
> +
>      return 0;
>  }
>  
> diff --git a/target-arm/kvm_arm.h b/target-arm/kvm_arm.h
> index 5d14887..cd3d13c 100644
> --- a/target-arm/kvm_arm.h
> +++ b/target-arm/kvm_arm.h
> @@ -62,4 +62,59 @@ bool write_list_to_kvmstate(ARMCPU *cpu);
>   */
>  bool write_kvmstate_to_list(ARMCPU *cpu);
>  
> +#ifdef CONFIG_KVM
> +/**
> + * kvm_arm_create_scratch_host_vcpu:
> + * @cpus_to_try: array of QEMU_KVM_ARM_TARGET_* values (terminated with
> + * QEMU_KVM_ARM_TARGET_NONE) to try as fallback if the kernel does not
> + * know the PREFERRED_TARGET ioctl
> + * @fdarray: filled in with kvmfd, vmfd, cpufd file descriptors in that order
> + * @init: filled in with the necessary values for creating a host vcpu
> + *
> + * Create a scratch vcpu in its own VM of the type preferred by the host
> + * kernel (as would be used for '-cpu host'), for purposes of probing it
> + * for capabilities.
> + *
> + * Returns: true on success (and fdarray and init are filled in),
> + * false on failure (and fdarray and init are not valid).
> + */
> +bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
> +                                      int *fdarray,
> +                                      struct kvm_vcpu_init *init);

why do we need to export this at all?

> +
> +/**
> + * kvm_arm_destroy_scratch_host_vcpu:
> + * @fdarray: array of fds as set up by kvm_arm_create_scratch_host_vcpu
> + *
> + * Tear down the scratch vcpu created by kvm_arm_create_scratch_host_vcpu.
> + */
> +void kvm_arm_destroy_scratch_host_vcpu(int *fdarray);
> +
> +#define TYPE_ARM_HOST_CPU "host-" TYPE_ARM_CPU
> +#define ARM_HOST_CPU_CLASS(klass) \
> +    OBJECT_CLASS_CHECK(ARMHostCPUClass, (klass), TYPE_ARM_HOST_CPU)
> +#define ARM_HOST_CPU_GET_CLASS(obj) \
> +    OBJECT_GET_CLASS(ARMHostCPUClass, (obj), TYPE_ARM_HOST_CPU)
> +
> +typedef struct ARMHostCPUClass {
> +    /*< private >*/
> +    ARMCPUClass parent_class;
> +    /*< public >*/
> +
> +    uint64_t features;
> +    uint32_t target;
> +    const char *dtb_compatible;
> +} ARMHostCPUClass;
> +
> +/**
> + * kvm_arm_get_host_cpu_features:
> + * @ahcc: ARMHostCPUClass to fill in
> + *
> + * Probe the capabilities of the host kernel's preferred CPU and fill
> + * in the ARMHostCPUClass struct accordingly.
> + */
> +bool kvm_arm_get_host_cpu_features(ARMHostCPUClass *ahcc);
> +
> +#endif
> +
>  #endif
> -- 
> 1.7.9.5
> 
> _______________________________________________
> kvmarm mailing list
> kvmarm@lists.cs.columbia.edu
> https://lists.cs.columbia.edu/cucslists/listinfo/kvmarm

On 22 November 2013 18:25, Christoffer Dall <christoffer.dall@linaro.org> wrote:
> On Fri, Nov 22, 2013 at 05:17:17PM +0000, Peter Maydell wrote:
>> +bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
>> +                                      int *fdarray,
>> +                                      struct kvm_vcpu_init *init);
>
> why do we need to export this at all?

For 64 bit KVM support the 32 bit code moves into kvm32.c
and then we have two users of this function, one in
kvm32.c and one in kvm64.c. It seemed easiest to just
make the function public with the proper doc comment
from the start, rather than starting it private and
then having a patch which makes it public.

thanks
-- PMM

On Fri, Nov 22, 2013 at 06:50:47PM +0000, Peter Maydell wrote:
> On 22 November 2013 18:25, Christoffer Dall <christoffer.dall@linaro.org> wrote:
> > On Fri, Nov 22, 2013 at 05:17:17PM +0000, Peter Maydell wrote:
> >> +bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
> >> +                                      int *fdarray,
> >> +                                      struct kvm_vcpu_init *init);
> >
> > why do we need to export this at all?
> 
> For 64 bit KVM support the 32 bit code moves into kvm32.c
> and then we have two users of this function, one in
> kvm32.c and one in kvm64.c. It seemed easiest to just
> make the function public with the proper doc comment
> from the start, rather than starting it private and
> then having a patch which makes it public.
> 
Yes, that does make sense.

Thanks.