[v4,4/5] s390x/vfio: ap: Introduce VFIO AP device

Message ID 1523819244-29954-5-git-send-email-akrowiak@linux.vnet.ibm.com
State New
Headers show
Series
  • s390x: vfio-ap: guest dedicated crypto adapters
Related show

Commit Message

Tony Krowiak April 15, 2018, 7:07 p.m.
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:

    -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>

There may be only one vfio-ap device configured for a guest.

The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:

/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid

When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.

Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
---
 default-configs/s390x-softmmu.mak |    1 +
 hw/vfio/Makefile.objs             |    1 +
 hw/vfio/ap.c                      |  191 +++++++++++++++++++++++++++++++++++++
 include/hw/vfio/vfio-common.h     |    1 +
 4 files changed, 194 insertions(+), 0 deletions(-)
 create mode 100644 hw/vfio/ap.c

Comments

Pierre Morel April 18, 2018, 9:11 a.m. | #1
On 15/04/2018 21:07, Tony Krowiak wrote:
> Introduces a VFIO based AP device. The device is defined via
> the QEMU command line by specifying:
>
>      -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>
>
> There may be only one vfio-ap device configured for a guest.
>
> The mediated matrix device is created by the VFIO AP device
> driver by writing a UUID to a sysfs attribute file (see
> docs/vfio-ap.txt). The mediated matrix device will be named
> after the UUID. Symbolic links to the $uuid are created in
> many places, so the path to the mediated matrix device $uuid
> can be specified in any of the following ways:
>
> /sys/devices/vfio_ap/matrix/$uuid
> /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
> /sys/bus/mdev/devices/$uuid
> /sys/bus/mdev/drivers/vfio_mdev/$uuid
>
> When the vfio-ap device is realized, it acquires and opens the
> VFIO iommu group to which the mediated matrix device is
> bound. This causes a VFIO group notification event to be
> signaled. The vfio_ap device driver's group notification
> handler will get called at which time the device driver
> will configure the the AP devices to which the guest will
> be granted access.
>
> Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
> ---
>   default-configs/s390x-softmmu.mak |    1 +
>   hw/vfio/Makefile.objs             |    1 +
>   hw/vfio/ap.c                      |  191 +++++++++++++++++++++++++++++++++++++
>   include/hw/vfio/vfio-common.h     |    1 +
>   4 files changed, 194 insertions(+), 0 deletions(-)
>   create mode 100644 hw/vfio/ap.c
>
> diff --git a/default-configs/s390x-softmmu.mak b/default-configs/s390x-softmmu.mak
> index 2f4bfe7..0b784b6 100644
> --- a/default-configs/s390x-softmmu.mak
> +++ b/default-configs/s390x-softmmu.mak
> @@ -9,3 +9,4 @@ CONFIG_S390_FLIC=y
>   CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
>   CONFIG_VFIO_CCW=$(CONFIG_LINUX)
>   CONFIG_WDT_DIAG288=y
> +CONFIG_VFIO_AP=$(CONFIG_LINUX)
> diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
> index a2e7a0a..8b3f664 100644
> --- a/hw/vfio/Makefile.objs
> +++ b/hw/vfio/Makefile.objs
> @@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
>   obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
>   obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
>   obj-$(CONFIG_SOFTMMU) += spapr.o
> +obj-$(CONFIG_VFIO_AP) += ap.o
>   endif
> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
> new file mode 100644
> index 0000000..7b48a3a
> --- /dev/null
> +++ b/hw/vfio/ap.c
> @@ -0,0 +1,191 @@
> +/*
> + * VFIO based AP matrix device assignment
> + *
> + * Copyright 2018 IBM Corp.
> + * Author(s): Tony Krowiak <akrowiak@linux.vnet.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +
> +#include <linux/vfio.h>
> +#include <sys/ioctl.h>
> +#include "qemu/osdep.h"
> +#include "qapi/error.h"
> +#include "hw/sysbus.h"
> +#include "hw/vfio/vfio.h"
> +#include "hw/vfio/vfio-common.h"
> +#include "hw/s390x/ap-device.h"
> +#include "qemu/error-report.h"
> +#include "qemu/queue.h"
> +#include "qemu/option.h"
> +#include "qemu/config-file.h"
> +#include "cpu.h"
> +#include "kvm_s390x.h"
> +#include "sysemu/sysemu.h"
> +
> +#define VFIO_AP_DEVICE_TYPE      "vfio-ap"
> +
> +typedef struct VFIOAPDevice {
> +    APDevice apdev;
> +    VFIODevice vdev;
> +    QTAILQ_ENTRY(VFIOAPDevice) sibling;
> +} VFIOAPDevice;
> +
> +VFIOAPDevice *vfio_apdev;
> +
> +static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
> +{
> +    vdev->needs_reset = false;
> +}
> +
> +/*
> + * We don't need vfio_hot_reset_multi and vfio_eoi operations for
> + * vfio-ap-matrix device now.
> + */
> +struct VFIODeviceOps vfio_ap_ops = {
> +    .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
> +};
> +
> +static void vfio_put_device(VFIOAPDevice *vapdev)
> +{
> +    g_free(vapdev->vdev.name);
> +    vfio_put_base_device(&vapdev->vdev);
> +}
> +
> +static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
> +{
> +    char *tmp, group_path[PATH_MAX];
> +    ssize_t len;
> +    int groupid;
> +
> +    tmp = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
> +    len = readlink(tmp, group_path, sizeof(group_path));
> +    g_free(tmp);
> +
> +    if (len <= 0 || len >= sizeof(group_path)) {
> +        error_setg(errp, "%s: no iommu_group found for %s",
> +                   VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev);
> +        return NULL;
> +    }
> +
> +    group_path[len] = 0;
> +
> +    if (sscanf(basename(group_path), "%d", &groupid) != 1) {
> +        error_setg(errp, "vfio: failed to read %s", group_path);
> +        return NULL;
> +    }
> +
> +    return vfio_get_group(groupid, &address_space_memory, errp);
> +}
> +
> +static void vfio_ap_realize(DeviceState *dev, Error **errp)
> +{
> +    VFIODevice *vbasedev;
> +    VFIOGroup *vfio_group;
> +    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
> +    char *mdevid;
> +    Error *local_err = NULL;
> +    int ret;
> +
> +    /*
> +     * Since a guest's matrix is configured in its entirety by the mediated
> +     * matrix device and hot plug is not currently supported, there is no
> +     * need to have more than one vfio-ap device. Check if a vfio-ap device
> +     * has already been defined.
> +     */
> +    if (vfio_apdev) {
> +        error_setg(&local_err, "Only one %s device is allowed",
> +                   VFIO_AP_DEVICE_TYPE);
> +        goto out_err;
> +    }
> +
> +    if (!s390_has_feat(S390_FEAT_AP)) {
> +        error_setg(&local_err, "AP support not enabled");
> +        goto out_err;
> +    }
> +
> +    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
> +
> +    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
> +    if (!vfio_group) {
> +        goto out_err;
> +    }
> +
> +    vfio_apdev->vdev.ops = &vfio_ap_ops;
> +    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
> +    mdevid = basename(vfio_apdev->vdev.sysfsdev);
> +    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
> +    vfio_apdev->vdev.dev = dev;
> +    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {

can this happen if you have only one device ?

> +        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
> +            error_setg(&local_err,
> +                       "%s: AP device %s has already been realized",
> +                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
> +            goto out_device_err;
> +        }
> +    }
> +
> +
...snip...
Cornelia Huck April 19, 2018, 12:03 p.m. | #2
On Sun, 15 Apr 2018 15:07:23 -0400
Tony Krowiak <akrowiak@linux.vnet.ibm.com> wrote:

> Introduces a VFIO based AP device. The device is defined via
> the QEMU command line by specifying:
> 
>     -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>
> 
> There may be only one vfio-ap device configured for a guest.
> 
> The mediated matrix device is created by the VFIO AP device
> driver by writing a UUID to a sysfs attribute file (see
> docs/vfio-ap.txt). The mediated matrix device will be named
> after the UUID. Symbolic links to the $uuid are created in
> many places, so the path to the mediated matrix device $uuid
> can be specified in any of the following ways:
> 
> /sys/devices/vfio_ap/matrix/$uuid
> /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
> /sys/bus/mdev/devices/$uuid
> /sys/bus/mdev/drivers/vfio_mdev/$uuid
> 
> When the vfio-ap device is realized, it acquires and opens the
> VFIO iommu group to which the mediated matrix device is
> bound. This causes a VFIO group notification event to be
> signaled. The vfio_ap device driver's group notification
> handler will get called at which time the device driver
> will configure the the AP devices to which the guest will
> be granted access.
> 
> Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
> ---
>  default-configs/s390x-softmmu.mak |    1 +
>  hw/vfio/Makefile.objs             |    1 +
>  hw/vfio/ap.c                      |  191 +++++++++++++++++++++++++++++++++++++
>  include/hw/vfio/vfio-common.h     |    1 +
>  4 files changed, 194 insertions(+), 0 deletions(-)
>  create mode 100644 hw/vfio/ap.c

> +static void vfio_ap_realize(DeviceState *dev, Error **errp)
> +{
> +    VFIODevice *vbasedev;
> +    VFIOGroup *vfio_group;
> +    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
> +    char *mdevid;
> +    Error *local_err = NULL;
> +    int ret;
> +
> +    /*
> +     * Since a guest's matrix is configured in its entirety by the mediated
> +     * matrix device and hot plug is not currently supported, there is no
> +     * need to have more than one vfio-ap device. Check if a vfio-ap device
> +     * has already been defined.
> +     */
> +    if (vfio_apdev) {
> +        error_setg(&local_err, "Only one %s device is allowed",
> +                   VFIO_AP_DEVICE_TYPE);
> +        goto out_err;
> +    }
> +
> +    if (!s390_has_feat(S390_FEAT_AP)) {
> +        error_setg(&local_err, "AP support not enabled");
> +        goto out_err;
> +    }
> +
> +    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
> +
> +    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
> +    if (!vfio_group) {
> +        goto out_err;
> +    }
> +
> +    vfio_apdev->vdev.ops = &vfio_ap_ops;
> +    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
> +    mdevid = basename(vfio_apdev->vdev.sysfsdev);
> +    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
> +    vfio_apdev->vdev.dev = dev;
> +    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {
> +        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
> +            error_setg(&local_err,
> +                       "%s: AP device %s has already been realized",
> +                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
> +            goto out_device_err;
> +        }
> +    }
> +
> +    ret = vfio_get_device(vfio_group, mdevid, &vfio_apdev->vdev, &local_err);
> +    if (ret) {
> +        goto out_device_err;
> +    }

Don't you need a put somewhere to avoid memory leaks?

> +
> +    return;
> +
> +
> +out_device_err:
> +    vfio_put_group(vfio_group);
> +out_err:
> +    vfio_apdev = NULL;
> +    error_propagate(errp, local_err);
> +}
Tony Krowiak April 22, 2018, 3:55 p.m. | #3
On 04/18/2018 05:11 AM, Pierre Morel wrote:
> On 15/04/2018 21:07, Tony Krowiak wrote:
>> Introduces a VFIO based AP device. The device is defined via
>> the QEMU command line by specifying:
>>
>>      -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>
>>
>> There may be only one vfio-ap device configured for a guest.
>>
>> The mediated matrix device is created by the VFIO AP device
>> driver by writing a UUID to a sysfs attribute file (see
>> docs/vfio-ap.txt). The mediated matrix device will be named
>> after the UUID. Symbolic links to the $uuid are created in
>> many places, so the path to the mediated matrix device $uuid
>> can be specified in any of the following ways:
>>
>> /sys/devices/vfio_ap/matrix/$uuid
>> /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid 
>>
>> /sys/bus/mdev/devices/$uuid
>> /sys/bus/mdev/drivers/vfio_mdev/$uuid
>>
>> When the vfio-ap device is realized, it acquires and opens the
>> VFIO iommu group to which the mediated matrix device is
>> bound. This causes a VFIO group notification event to be
>> signaled. The vfio_ap device driver's group notification
>> handler will get called at which time the device driver
>> will configure the the AP devices to which the guest will
>> be granted access.
>>
>> Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
>> ---
>>   default-configs/s390x-softmmu.mak |    1 +
>>   hw/vfio/Makefile.objs             |    1 +
>>   hw/vfio/ap.c                      |  191 
>> +++++++++++++++++++++++++++++++++++++
>>   include/hw/vfio/vfio-common.h     |    1 +
>>   4 files changed, 194 insertions(+), 0 deletions(-)
>>   create mode 100644 hw/vfio/ap.c
>>
>> diff --git a/default-configs/s390x-softmmu.mak 
>> b/default-configs/s390x-softmmu.mak
>> index 2f4bfe7..0b784b6 100644
>> --- a/default-configs/s390x-softmmu.mak
>> +++ b/default-configs/s390x-softmmu.mak
>> @@ -9,3 +9,4 @@ CONFIG_S390_FLIC=y
>>   CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
>>   CONFIG_VFIO_CCW=$(CONFIG_LINUX)
>>   CONFIG_WDT_DIAG288=y
>> +CONFIG_VFIO_AP=$(CONFIG_LINUX)
>> diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
>> index a2e7a0a..8b3f664 100644
>> --- a/hw/vfio/Makefile.objs
>> +++ b/hw/vfio/Makefile.objs
>> @@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
>>   obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
>>   obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
>>   obj-$(CONFIG_SOFTMMU) += spapr.o
>> +obj-$(CONFIG_VFIO_AP) += ap.o
>>   endif
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> new file mode 100644
>> index 0000000..7b48a3a
>> --- /dev/null
>> +++ b/hw/vfio/ap.c
>> @@ -0,0 +1,191 @@
>> +/*
>> + * VFIO based AP matrix device assignment
>> + *
>> + * Copyright 2018 IBM Corp.
>> + * Author(s): Tony Krowiak <akrowiak@linux.vnet.ibm.com>
>> + *
>> + * This work is licensed under the terms of the GNU GPL, version 2 
>> or (at
>> + * your option) any later version. See the COPYING file in the 
>> top-level
>> + * directory.
>> + */
>> +
>> +#include <linux/vfio.h>
>> +#include <sys/ioctl.h>
>> +#include "qemu/osdep.h"
>> +#include "qapi/error.h"
>> +#include "hw/sysbus.h"
>> +#include "hw/vfio/vfio.h"
>> +#include "hw/vfio/vfio-common.h"
>> +#include "hw/s390x/ap-device.h"
>> +#include "qemu/error-report.h"
>> +#include "qemu/queue.h"
>> +#include "qemu/option.h"
>> +#include "qemu/config-file.h"
>> +#include "cpu.h"
>> +#include "kvm_s390x.h"
>> +#include "sysemu/sysemu.h"
>> +
>> +#define VFIO_AP_DEVICE_TYPE      "vfio-ap"
>> +
>> +typedef struct VFIOAPDevice {
>> +    APDevice apdev;
>> +    VFIODevice vdev;
>> +    QTAILQ_ENTRY(VFIOAPDevice) sibling;
>> +} VFIOAPDevice;
>> +
>> +VFIOAPDevice *vfio_apdev;
>> +
>> +static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>> +{
>> +    vdev->needs_reset = false;
>> +}
>> +
>> +/*
>> + * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> + * vfio-ap-matrix device now.
>> + */
>> +struct VFIODeviceOps vfio_ap_ops = {
>> +    .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> +};
>> +
>> +static void vfio_put_device(VFIOAPDevice *vapdev)
>> +{
>> +    g_free(vapdev->vdev.name);
>> +    vfio_put_base_device(&vapdev->vdev);
>> +}
>> +
>> +static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
>> +{
>> +    char *tmp, group_path[PATH_MAX];
>> +    ssize_t len;
>> +    int groupid;
>> +
>> +    tmp = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
>> +    len = readlink(tmp, group_path, sizeof(group_path));
>> +    g_free(tmp);
>> +
>> +    if (len <= 0 || len >= sizeof(group_path)) {
>> +        error_setg(errp, "%s: no iommu_group found for %s",
>> +                   VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev);
>> +        return NULL;
>> +    }
>> +
>> +    group_path[len] = 0;
>> +
>> +    if (sscanf(basename(group_path), "%d", &groupid) != 1) {
>> +        error_setg(errp, "vfio: failed to read %s", group_path);
>> +        return NULL;
>> +    }
>> +
>> +    return vfio_get_group(groupid, &address_space_memory, errp);
>> +}
>> +
>> +static void vfio_ap_realize(DeviceState *dev, Error **errp)
>> +{
>> +    VFIODevice *vbasedev;
>> +    VFIOGroup *vfio_group;
>> +    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
>> +    char *mdevid;
>> +    Error *local_err = NULL;
>> +    int ret;
>> +
>> +    /*
>> +     * Since a guest's matrix is configured in its entirety by the 
>> mediated
>> +     * matrix device and hot plug is not currently supported, there 
>> is no
>> +     * need to have more than one vfio-ap device. Check if a vfio-ap 
>> device
>> +     * has already been defined.
>> +     */
>> +    if (vfio_apdev) {
>> +        error_setg(&local_err, "Only one %s device is allowed",
>> +                   VFIO_AP_DEVICE_TYPE);
>> +        goto out_err;
>> +    }
>> +
>> +    if (!s390_has_feat(S390_FEAT_AP)) {
>> +        error_setg(&local_err, "AP support not enabled");
>> +        goto out_err;
>> +    }
>> +
>> +    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
>> +
>> +    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
>> +    if (!vfio_group) {
>> +        goto out_err;
>> +    }
>> +
>> +    vfio_apdev->vdev.ops = &vfio_ap_ops;
>> +    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
>> +    mdevid = basename(vfio_apdev->vdev.sysfsdev);
>> +    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
>> +    vfio_apdev->vdev.dev = dev;
>> +    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {
>
> can this happen if you have only one device ?

No, it is a remnant from previous versions when we didn't enforce
the single device requirement. I can remove it.

>
>
>> +        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
>> +            error_setg(&local_err,
>> +                       "%s: AP device %s has already been realized",
>> +                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
>> +            goto out_device_err;
>> +        }
>> +    }
>> +
>> +
> ...snip...
>
Tony Krowiak April 22, 2018, 4:05 p.m. | #4
On 04/19/2018 08:03 AM, Cornelia Huck wrote:
> On Sun, 15 Apr 2018 15:07:23 -0400
> Tony Krowiak <akrowiak@linux.vnet.ibm.com> wrote:
>
>> Introduces a VFIO based AP device. The device is defined via
>> the QEMU command line by specifying:
>>
>>      -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>
>>
>> There may be only one vfio-ap device configured for a guest.
>>
>> The mediated matrix device is created by the VFIO AP device
>> driver by writing a UUID to a sysfs attribute file (see
>> docs/vfio-ap.txt). The mediated matrix device will be named
>> after the UUID. Symbolic links to the $uuid are created in
>> many places, so the path to the mediated matrix device $uuid
>> can be specified in any of the following ways:
>>
>> /sys/devices/vfio_ap/matrix/$uuid
>> /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
>> /sys/bus/mdev/devices/$uuid
>> /sys/bus/mdev/drivers/vfio_mdev/$uuid
>>
>> When the vfio-ap device is realized, it acquires and opens the
>> VFIO iommu group to which the mediated matrix device is
>> bound. This causes a VFIO group notification event to be
>> signaled. The vfio_ap device driver's group notification
>> handler will get called at which time the device driver
>> will configure the the AP devices to which the guest will
>> be granted access.
>>
>> Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
>> ---
>>   default-configs/s390x-softmmu.mak |    1 +
>>   hw/vfio/Makefile.objs             |    1 +
>>   hw/vfio/ap.c                      |  191 +++++++++++++++++++++++++++++++++++++
>>   include/hw/vfio/vfio-common.h     |    1 +
>>   4 files changed, 194 insertions(+), 0 deletions(-)
>>   create mode 100644 hw/vfio/ap.c
>> +static void vfio_ap_realize(DeviceState *dev, Error **errp)
>> +{
>> +    VFIODevice *vbasedev;
>> +    VFIOGroup *vfio_group;
>> +    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
>> +    char *mdevid;
>> +    Error *local_err = NULL;
>> +    int ret;
>> +
>> +    /*
>> +     * Since a guest's matrix is configured in its entirety by the mediated
>> +     * matrix device and hot plug is not currently supported, there is no
>> +     * need to have more than one vfio-ap device. Check if a vfio-ap device
>> +     * has already been defined.
>> +     */
>> +    if (vfio_apdev) {
>> +        error_setg(&local_err, "Only one %s device is allowed",
>> +                   VFIO_AP_DEVICE_TYPE);
>> +        goto out_err;
>> +    }
>> +
>> +    if (!s390_has_feat(S390_FEAT_AP)) {
>> +        error_setg(&local_err, "AP support not enabled");
>> +        goto out_err;
>> +    }
>> +
>> +    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
>> +
>> +    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
>> +    if (!vfio_group) {
>> +        goto out_err;
>> +    }
>> +
>> +    vfio_apdev->vdev.ops = &vfio_ap_ops;
>> +    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
>> +    mdevid = basename(vfio_apdev->vdev.sysfsdev);
>> +    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
>> +    vfio_apdev->vdev.dev = dev;
>> +    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {
>> +        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
>> +            error_setg(&local_err,
>> +                       "%s: AP device %s has already been realized",
>> +                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
>> +            goto out_device_err;
>> +        }
>> +    }
>> +
>> +    ret = vfio_get_device(vfio_group, mdevid, &vfio_apdev->vdev, &local_err);
>> +    if (ret) {
>> +        goto out_device_err;
>> +    }
> Don't you need a put somewhere to avoid memory leaks?

There is a call to vfio_put_device in the unrealize function.

>
>> +
>> +    return;
>> +
>> +
>> +out_device_err:
>> +    vfio_put_group(vfio_group);
>> +out_err:
>> +    vfio_apdev = NULL;
>> +    error_propagate(errp, local_err);
>> +}
Cornelia Huck April 23, 2018, 7 a.m. | #5
On Sun, 22 Apr 2018 12:05:44 -0400
Tony Krowiak <akrowiak@linux.vnet.ibm.com> wrote:

> On 04/19/2018 08:03 AM, Cornelia Huck wrote:
> > On Sun, 15 Apr 2018 15:07:23 -0400
> > Tony Krowiak <akrowiak@linux.vnet.ibm.com> wrote:
> >  
> >> Introduces a VFIO based AP device. The device is defined via
> >> the QEMU command line by specifying:
> >>
> >>      -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>
> >>
> >> There may be only one vfio-ap device configured for a guest.
> >>
> >> The mediated matrix device is created by the VFIO AP device
> >> driver by writing a UUID to a sysfs attribute file (see
> >> docs/vfio-ap.txt). The mediated matrix device will be named
> >> after the UUID. Symbolic links to the $uuid are created in
> >> many places, so the path to the mediated matrix device $uuid
> >> can be specified in any of the following ways:
> >>
> >> /sys/devices/vfio_ap/matrix/$uuid
> >> /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
> >> /sys/bus/mdev/devices/$uuid
> >> /sys/bus/mdev/drivers/vfio_mdev/$uuid
> >>
> >> When the vfio-ap device is realized, it acquires and opens the
> >> VFIO iommu group to which the mediated matrix device is
> >> bound. This causes a VFIO group notification event to be
> >> signaled. The vfio_ap device driver's group notification
> >> handler will get called at which time the device driver
> >> will configure the the AP devices to which the guest will
> >> be granted access.
> >>
> >> Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
> >> ---
> >>   default-configs/s390x-softmmu.mak |    1 +
> >>   hw/vfio/Makefile.objs             |    1 +
> >>   hw/vfio/ap.c                      |  191 +++++++++++++++++++++++++++++++++++++
> >>   include/hw/vfio/vfio-common.h     |    1 +
> >>   4 files changed, 194 insertions(+), 0 deletions(-)
> >>   create mode 100644 hw/vfio/ap.c
> >> +static void vfio_ap_realize(DeviceState *dev, Error **errp)
> >> +{
> >> +    VFIODevice *vbasedev;
> >> +    VFIOGroup *vfio_group;
> >> +    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
> >> +    char *mdevid;
> >> +    Error *local_err = NULL;
> >> +    int ret;
> >> +
> >> +    /*
> >> +     * Since a guest's matrix is configured in its entirety by the mediated
> >> +     * matrix device and hot plug is not currently supported, there is no
> >> +     * need to have more than one vfio-ap device. Check if a vfio-ap device
> >> +     * has already been defined.
> >> +     */
> >> +    if (vfio_apdev) {
> >> +        error_setg(&local_err, "Only one %s device is allowed",
> >> +                   VFIO_AP_DEVICE_TYPE);
> >> +        goto out_err;
> >> +    }
> >> +
> >> +    if (!s390_has_feat(S390_FEAT_AP)) {
> >> +        error_setg(&local_err, "AP support not enabled");
> >> +        goto out_err;
> >> +    }
> >> +
> >> +    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
> >> +
> >> +    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
> >> +    if (!vfio_group) {
> >> +        goto out_err;
> >> +    }
> >> +
> >> +    vfio_apdev->vdev.ops = &vfio_ap_ops;
> >> +    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
> >> +    mdevid = basename(vfio_apdev->vdev.sysfsdev);
> >> +    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
> >> +    vfio_apdev->vdev.dev = dev;
> >> +    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {
> >> +        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
> >> +            error_setg(&local_err,
> >> +                       "%s: AP device %s has already been realized",
> >> +                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
> >> +            goto out_device_err;
> >> +        }
> >> +    }
> >> +
> >> +    ret = vfio_get_device(vfio_group, mdevid, &vfio_apdev->vdev, &local_err);
> >> +    if (ret) {
> >> +        goto out_device_err;
> >> +    }  
> > Don't you need a put somewhere to avoid memory leaks?  
> 
> There is a call to vfio_put_device in the unrealize function.

I don't think unrealize is called if realize failed, so you need to
clean up in the error cases?

> 
> >  
> >> +
> >> +    return;
> >> +
> >> +
> >> +out_device_err:
> >> +    vfio_put_group(vfio_group);
> >> +out_err:
> >> +    vfio_apdev = NULL;
> >> +    error_propagate(errp, local_err);
> >> +}  
> 
>
Tony Krowiak April 26, 2018, 2:50 p.m. | #6
On 04/23/2018 03:00 AM, Cornelia Huck wrote:
> On Sun, 22 Apr 2018 12:05:44 -0400
> Tony Krowiak <akrowiak@linux.vnet.ibm.com> wrote:
>
>> On 04/19/2018 08:03 AM, Cornelia Huck wrote:
>>> On Sun, 15 Apr 2018 15:07:23 -0400
>>> Tony Krowiak <akrowiak@linux.vnet.ibm.com> wrote:
>>>   
>>>> Introduces a VFIO based AP device. The device is defined via
>>>> the QEMU command line by specifying:
>>>>
>>>>       -device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>
>>>>
>>>> There may be only one vfio-ap device configured for a guest.
>>>>
>>>> The mediated matrix device is created by the VFIO AP device
>>>> driver by writing a UUID to a sysfs attribute file (see
>>>> docs/vfio-ap.txt). The mediated matrix device will be named
>>>> after the UUID. Symbolic links to the $uuid are created in
>>>> many places, so the path to the mediated matrix device $uuid
>>>> can be specified in any of the following ways:
>>>>
>>>> /sys/devices/vfio_ap/matrix/$uuid
>>>> /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
>>>> /sys/bus/mdev/devices/$uuid
>>>> /sys/bus/mdev/drivers/vfio_mdev/$uuid
>>>>
>>>> When the vfio-ap device is realized, it acquires and opens the
>>>> VFIO iommu group to which the mediated matrix device is
>>>> bound. This causes a VFIO group notification event to be
>>>> signaled. The vfio_ap device driver's group notification
>>>> handler will get called at which time the device driver
>>>> will configure the the AP devices to which the guest will
>>>> be granted access.
>>>>
>>>> Signed-off-by: Tony Krowiak <akrowiak@linux.vnet.ibm.com>
>>>> ---
>>>>    default-configs/s390x-softmmu.mak |    1 +
>>>>    hw/vfio/Makefile.objs             |    1 +
>>>>    hw/vfio/ap.c                      |  191 +++++++++++++++++++++++++++++++++++++
>>>>    include/hw/vfio/vfio-common.h     |    1 +
>>>>    4 files changed, 194 insertions(+), 0 deletions(-)
>>>>    create mode 100644 hw/vfio/ap.c
>>>> +static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>>> +{
>>>> +    VFIODevice *vbasedev;
>>>> +    VFIOGroup *vfio_group;
>>>> +    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
>>>> +    char *mdevid;
>>>> +    Error *local_err = NULL;
>>>> +    int ret;
>>>> +
>>>> +    /*
>>>> +     * Since a guest's matrix is configured in its entirety by the mediated
>>>> +     * matrix device and hot plug is not currently supported, there is no
>>>> +     * need to have more than one vfio-ap device. Check if a vfio-ap device
>>>> +     * has already been defined.
>>>> +     */
>>>> +    if (vfio_apdev) {
>>>> +        error_setg(&local_err, "Only one %s device is allowed",
>>>> +                   VFIO_AP_DEVICE_TYPE);
>>>> +        goto out_err;
>>>> +    }
>>>> +
>>>> +    if (!s390_has_feat(S390_FEAT_AP)) {
>>>> +        error_setg(&local_err, "AP support not enabled");
>>>> +        goto out_err;
>>>> +    }
>>>> +
>>>> +    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
>>>> +
>>>> +    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
>>>> +    if (!vfio_group) {
>>>> +        goto out_err;
>>>> +    }
>>>> +
>>>> +    vfio_apdev->vdev.ops = &vfio_ap_ops;
>>>> +    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
>>>> +    mdevid = basename(vfio_apdev->vdev.sysfsdev);
>>>> +    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
>>>> +    vfio_apdev->vdev.dev = dev;
>>>> +    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {
>>>> +        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
>>>> +            error_setg(&local_err,
>>>> +                       "%s: AP device %s has already been realized",
>>>> +                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
>>>> +            goto out_device_err;
>>>> +        }
>>>> +    }
>>>> +
>>>> +    ret = vfio_get_device(vfio_group, mdevid, &vfio_apdev->vdev, &local_err);
>>>> +    if (ret) {
>>>> +        goto out_device_err;
>>>> +    }
>>> Don't you need a put somewhere to avoid memory leaks?
>> There is a call to vfio_put_device in the unrealize function.
> I don't think unrealize is called if realize failed, so you need to
> clean up in the error cases?

Very true. I  misinterpreted your point. Yes, I need to clean up
if realize fails.

>
>>>   
>>>> +
>>>> +    return;
>>>> +
>>>> +
>>>> +out_device_err:
>>>> +    vfio_put_group(vfio_group);
>>>> +out_err:
>>>> +    vfio_apdev = NULL;
>>>> +    error_propagate(errp, local_err);
>>>> +}
>>

Patch

diff --git a/default-configs/s390x-softmmu.mak b/default-configs/s390x-softmmu.mak
index 2f4bfe7..0b784b6 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -9,3 +9,4 @@  CONFIG_S390_FLIC=y
 CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
 CONFIG_VFIO_CCW=$(CONFIG_LINUX)
 CONFIG_WDT_DIAG288=y
+CONFIG_VFIO_AP=$(CONFIG_LINUX)
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
index a2e7a0a..8b3f664 100644
--- a/hw/vfio/Makefile.objs
+++ b/hw/vfio/Makefile.objs
@@ -6,4 +6,5 @@  obj-$(CONFIG_SOFTMMU) += platform.o
 obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
 obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
 obj-$(CONFIG_SOFTMMU) += spapr.o
+obj-$(CONFIG_VFIO_AP) += ap.o
 endif
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
new file mode 100644
index 0000000..7b48a3a
--- /dev/null
+++ b/hw/vfio/ap.c
@@ -0,0 +1,191 @@ 
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak <akrowiak@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+
+#define VFIO_AP_DEVICE_TYPE      "vfio-ap"
+
+typedef struct VFIOAPDevice {
+    APDevice apdev;
+    VFIODevice vdev;
+    QTAILQ_ENTRY(VFIOAPDevice) sibling;
+} VFIOAPDevice;
+
+VFIOAPDevice *vfio_apdev;
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+    vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap-matrix device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+    .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_put_device(VFIOAPDevice *vapdev)
+{
+    g_free(vapdev->vdev.name);
+    vfio_put_base_device(&vapdev->vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+    char *tmp, group_path[PATH_MAX];
+    ssize_t len;
+    int groupid;
+
+    tmp = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+    len = readlink(tmp, group_path, sizeof(group_path));
+    g_free(tmp);
+
+    if (len <= 0 || len >= sizeof(group_path)) {
+        error_setg(errp, "%s: no iommu_group found for %s",
+                   VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev);
+        return NULL;
+    }
+
+    group_path[len] = 0;
+
+    if (sscanf(basename(group_path), "%d", &groupid) != 1) {
+        error_setg(errp, "vfio: failed to read %s", group_path);
+        return NULL;
+    }
+
+    return vfio_get_group(groupid, &address_space_memory, errp);
+}
+
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+    VFIODevice *vbasedev;
+    VFIOGroup *vfio_group;
+    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+    char *mdevid;
+    Error *local_err = NULL;
+    int ret;
+
+    /*
+     * Since a guest's matrix is configured in its entirety by the mediated
+     * matrix device and hot plug is not currently supported, there is no
+     * need to have more than one vfio-ap device. Check if a vfio-ap device
+     * has already been defined.
+     */
+    if (vfio_apdev) {
+        error_setg(&local_err, "Only one %s device is allowed",
+                   VFIO_AP_DEVICE_TYPE);
+        goto out_err;
+    }
+
+    if (!s390_has_feat(S390_FEAT_AP)) {
+        error_setg(&local_err, "AP support not enabled");
+        goto out_err;
+    }
+
+    vfio_apdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
+
+    vfio_group = vfio_ap_get_group(vfio_apdev, &local_err);
+    if (!vfio_group) {
+        goto out_err;
+    }
+
+    vfio_apdev->vdev.ops = &vfio_ap_ops;
+    vfio_apdev->vdev.type = VFIO_DEVICE_TYPE_AP;
+    mdevid = basename(vfio_apdev->vdev.sysfsdev);
+    vfio_apdev->vdev.name = g_strdup_printf("%s", mdevid);
+    vfio_apdev->vdev.dev = dev;
+    QLIST_FOREACH(vbasedev, &vfio_group->device_list, next) {
+        if (strcmp(vbasedev->name, vfio_apdev->vdev.name) == 0) {
+            error_setg(&local_err,
+                       "%s: AP device %s has already been realized",
+                       VFIO_AP_DEVICE_TYPE, vfio_apdev->vdev.name);
+            goto out_device_err;
+        }
+    }
+
+    ret = vfio_get_device(vfio_group, mdevid, &vfio_apdev->vdev, &local_err);
+    if (ret) {
+        goto out_device_err;
+    }
+
+    return;
+
+
+out_device_err:
+    vfio_put_group(vfio_group);
+out_err:
+    vfio_apdev = NULL;
+    error_propagate(errp, local_err);
+}
+
+static void vfio_ap_unrealize(DeviceState *dev, Error **errp)
+{
+    APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+    VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
+    VFIOGroup *group = vapdev->vdev.group;
+
+    vfio_put_device(vapdev);
+    vfio_put_group(group);
+    vfio_apdev = NULL;
+}
+
+static Property vfio_ap_properties[] = {
+    DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription vfio_ap_vmstate = {
+    .name = VFIO_AP_DEVICE_TYPE,
+    .unmigratable = 1,
+};
+
+static void vfio_ap_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = vfio_ap_properties;
+    dc->vmsd = &vfio_ap_vmstate;
+    dc->desc = "VFIO-based AP device assignment";
+    dc->realize = vfio_ap_realize;
+    dc->unrealize = vfio_ap_unrealize;
+    dc->hotpluggable = false;
+}
+
+static const TypeInfo vfio_ap_info = {
+    .name = VFIO_AP_DEVICE_TYPE,
+    .parent = AP_DEVICE_TYPE,
+    .instance_size = sizeof(VFIOAPDevice),
+    .class_init = vfio_ap_class_init,
+};
+
+static void vfio_ap_type_init(void)
+{
+    type_register_static(&vfio_ap_info);
+    vfio_apdev = NULL;
+}
+
+type_init(vfio_ap_type_init)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index d936014..f29df6e 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -47,6 +47,7 @@  enum {
     VFIO_DEVICE_TYPE_PCI = 0,
     VFIO_DEVICE_TYPE_PLATFORM = 1,
     VFIO_DEVICE_TYPE_CCW = 2,
+    VFIO_DEVICE_TYPE_AP = 3,
 };
 
 typedef struct VFIOMmap {