Patchwork [V2,WIP,2/2] vhost-scsi: new device supporting the tcm_vhost Linux kernel module

login
register
mail settings
Submitter Asias He
Date March 12, 2013, 6:29 a.m.
Message ID <1363069782-10735-3-git-send-email-asias@redhat.com>
Download mbox | patch
Permalink /patch/226807/
State New
Headers show

Comments

Asias He - March 12, 2013, 6:29 a.m.
From: Paolo Bonzini <pbonzini@redhat.com>

The WWPN specified in configfs is passed to "-device vhost-scsi-pci".
The tgpt field of the SET_ENDPOINT ioctl is obsolete now, so it is not
available from the QEMU command-line.  Instead, I hardcode it to zero.

Changes in V2:
   - Add backend guest masking support (nab)
   - Bump ABI_VERSION to 1 (nab)
   - Set up set_guest_notifiers (asias)
   - Set up vs->dev.vq_index (asias)
   - Drop vs->vs.vdev.{set,clear}_vhost_endpoint (asias)
   - Drop VIRTIO_CONFIG_S_DRIVER check in vhost_scsi_set_status (asias)

Howto:
   git clone git://git.seabios.org/seabios.git
   make menuconfig, disable virtio-scsi support in seabios
   make
   cp out/bios.bin /usr/share/qemu/bios.bin
   qemu -device vhost-scsi-pci,wwpn=naa.6001405bd4e8476d,event_idx=off ...

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
Signed-off-by: Asias He <asias@redhat.com>
---
 hw/Makefile.objs           |   5 +-
 hw/s390x/s390-virtio-bus.c |  35 +++++++
 hw/vhost-scsi.c            | 242 +++++++++++++++++++++++++++++++++++++++++++++
 hw/vhost-scsi.h            |  64 ++++++++++++
 hw/virtio-pci.c            |  59 +++++++++++
 hw/virtio-scsi.h           |   2 +
 6 files changed, 406 insertions(+), 1 deletion(-)
 create mode 100644 hw/vhost-scsi.c
 create mode 100644 hw/vhost-scsi.h
Stefan Hajnoczi - March 12, 2013, 8:20 a.m.
On Tue, Mar 12, 2013 at 02:29:42PM +0800, Asias He wrote:
> diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
> index 39c1966..4a97ca1 100644
> --- a/hw/virtio-pci.c
> +++ b/hw/virtio-pci.c

These changes break the build for non-Linux hosts.  Please introduce a
CONFIG_VHOST_SCSI and #ifdef appropriate sections in hw/virtio-pci.c.

CONFIG_VIRTFS does the same thing.

> +static Property vhost_scsi_properties[] = {
> +    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),

This flag makes QEMU's virtqueue handling use ioeventfd.  Since the
vhost-scsi.c takes over the guest/host notifiers, we never do QEMU
virtqueue processing and the "ioeventfd" flag has no real meaning.  You
can drop it.
Asias He - March 13, 2013, 2:49 a.m.
On Tue, Mar 12, 2013 at 09:20:24AM +0100, Stefan Hajnoczi wrote:
> On Tue, Mar 12, 2013 at 02:29:42PM +0800, Asias He wrote:
> > diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
> > index 39c1966..4a97ca1 100644
> > --- a/hw/virtio-pci.c
> > +++ b/hw/virtio-pci.c
> 
> These changes break the build for non-Linux hosts.  Please introduce a
> CONFIG_VHOST_SCSI and #ifdef appropriate sections in hw/virtio-pci.c.
> 
> CONFIG_VIRTFS does the same thing.

okay.

> > +static Property vhost_scsi_properties[] = {
> > +    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
> 
> This flag makes QEMU's virtqueue handling use ioeventfd.  Since the
> vhost-scsi.c takes over the guest/host notifiers, we never do QEMU
> virtqueue processing and the "ioeventfd" flag has no real meaning.  You
> can drop it.

okay.

Thanks, Stefan!

Patch

diff --git a/hw/Makefile.objs b/hw/Makefile.objs
index eb7eb31..df57e1f 100644
--- a/hw/Makefile.objs
+++ b/hw/Makefile.objs
@@ -202,8 +202,11 @@  common-obj-$(CONFIG_XEN_BACKEND) += xen_console.o xenfb.o xen_disk.o xen_nic.o
 obj-$(CONFIG_VIRTIO) += dataplane/
 obj-$(CONFIG_VIRTIO) += virtio.o virtio-blk.o virtio-balloon.o virtio-net.o
 obj-$(CONFIG_VIRTIO) += virtio-serial-bus.o virtio-scsi.o
+ifeq ($(CONFIG_VIRTIO), y)
+obj-$(CONFIG_LINUX) += vhost-scsi.o
+endif
 obj-$(CONFIG_SOFTMMU) += vhost_net.o
-obj-$(CONFIG_VHOST_NET) += vhost.o
+obj-$(CONFIG_LINUX) += vhost.o
 obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
 obj-$(CONFIG_VGA) += vga.o
 
diff --git a/hw/s390x/s390-virtio-bus.c b/hw/s390x/s390-virtio-bus.c
index d9b7f83..d86365b 100644
--- a/hw/s390x/s390-virtio-bus.c
+++ b/hw/s390x/s390-virtio-bus.c
@@ -28,6 +28,8 @@ 
 #include "hw/virtio-rng.h"
 #include "hw/virtio-serial.h"
 #include "hw/virtio-net.h"
+#include "hw/virtio-scsi.h"
+#include "hw/vhost-scsi.h"
 #include "hw/sysbus.h"
 #include "sysemu/kvm.h"
 
@@ -207,6 +209,18 @@  static int s390_virtio_scsi_init(VirtIOS390Device *dev)
     return s390_virtio_device_init(dev, vdev);
 }
 
+static int s390_vhost_scsi_init(VirtIOS390Device *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = vhost_scsi_init((DeviceState *)dev, &dev->scsi);
+    if (!vdev) {
+        return -1;
+    }
+
+    return s390_virtio_device_init(dev, vdev);
+}
+
 static int s390_virtio_rng_init(VirtIOS390Device *dev)
 {
     VirtIODevice *vdev;
@@ -534,6 +548,27 @@  static const TypeInfo virtio_s390_device_info = {
     .abstract = true,
 };
 
+static Property s390_vhost_scsi_properties[] = {
+    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOS390Device, host_features, scsi),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_vhost_scsi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
+
+    k->init = s390_vhost_scsi_init;
+    dc->props = s390_vhost_scsi_properties;
+}
+
+static const TypeInfo s390_vhost_scsi = {
+    .name          = "vhost-scsi-s390",
+    .parent        = TYPE_VIRTIO_S390_DEVICE,
+    .instance_size = sizeof(VirtIOS390Device),
+    .class_init    = s390_vhost_scsi_class_init,
+};
+
 static Property s390_virtio_scsi_properties[] = {
     DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOS390Device, host_features, scsi),
     DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/vhost-scsi.c b/hw/vhost-scsi.c
new file mode 100644
index 0000000..eabb4b6
--- /dev/null
+++ b/hw/vhost-scsi.c
@@ -0,0 +1,242 @@ 
+/*
+ * vhost_scsi host device
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *
+ * Changes for QEMU mainline + tcm_vhost kernel upstream:
+ *  Nicholas Bellinger <nab@risingtidesystems.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include <sys/ioctl.h>
+#include "config.h"
+#include "qemu/queue.h"
+#include "monitor/monitor.h"
+#include "migration/migration.h"
+#include "vhost-scsi.h"
+#include "vhost.h"
+#include "virtio-scsi.h"
+
+typedef struct VHostSCSI {
+    VirtIOSCSICommon vs;
+
+    Error *migration_blocker;
+
+    struct vhost_dev dev;
+} VHostSCSI;
+
+static int vhost_scsi_set_endpoint(VirtIODevice *vdev)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    struct vhost_scsi_target backend;
+    int ret;
+
+    memset(&backend, 0, sizeof(backend));
+    pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->vs.conf->wwpn);
+    ret = ioctl(vs->dev.control, VHOST_SCSI_SET_ENDPOINT, &backend);
+    if (ret < 0) {
+        return -errno;
+    }
+    return 0;
+}
+
+static void vhost_scsi_clear_endpoint(VirtIODevice *vdev)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    struct vhost_scsi_target backend;
+
+    memset(&backend, 0, sizeof(backend));
+    pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->vs.conf->wwpn);
+    ioctl(vs->dev.control, VHOST_SCSI_CLEAR_ENDPOINT, &backend);
+}
+
+static int vhost_scsi_start(VHostSCSI *vs, VirtIODevice *vdev)
+{
+    int ret, abi_version;
+
+    if (!vdev->binding->set_guest_notifiers) {
+        error_report("binding does not support guest notifiers");
+        return -ENOSYS;
+    }
+
+    ret = ioctl(vs->dev.control, VHOST_SCSI_GET_ABI_VERSION, &abi_version);
+    if (ret < 0) {
+        return -errno;
+    }
+    if (abi_version > VHOST_SCSI_ABI_VERSION) {
+        error_report("vhost-scsi: The running tcm_vhost kernel abi_version:"
+                     " %d is greater than vhost_scsi userspace supports: %d, please"
+                     " upgrade your version of QEMU\n", abi_version,
+                     VHOST_SCSI_ABI_VERSION);
+        return -ENOSYS;
+    }
+
+    ret = vhost_dev_enable_notifiers(&vs->dev, vdev);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_dev_start(&vs->dev, vdev);
+    if (ret < 0) {
+        error_report("Error start vhost dev");
+        goto err_notifiers;
+    }
+
+    ret = vhost_scsi_set_endpoint(vdev);
+    if (ret < 0) {
+        error_report("Error set vhost-scsi endpoint");
+        goto err_vhost_stop;
+    }
+
+    ret = vdev->binding->set_guest_notifiers(vdev->binding_opaque, vs->dev.nvqs, true);
+    if (ret < 0) {
+        error_report("Error binding guest notifier");
+        goto err_endpoint;
+    }
+    return ret;
+
+err_endpoint:
+    vhost_scsi_clear_endpoint(vdev);
+err_vhost_stop:
+    vhost_dev_stop(&vs->dev, vdev);
+err_notifiers:
+    vhost_dev_disable_notifiers(&vs->dev, vdev);
+    return ret;
+}
+
+static void vhost_scsi_stop(VHostSCSI *vs, VirtIODevice *vdev)
+{
+    int ret = 0;
+
+    if (!vdev->binding->set_guest_notifiers) {
+        ret = vdev->binding->set_guest_notifiers(vdev->binding_opaque,
+                                                 vs->dev.nvqs, false);
+        if (ret < 0) {
+                error_report("vhost guest notifier cleanup failed: %d\n", ret);
+        }
+    }
+    assert(ret >= 0);
+
+    vhost_scsi_clear_endpoint(vdev);
+    vhost_dev_stop(&vs->dev, vdev);
+    vhost_dev_disable_notifiers(&vs->dev, vdev);
+}
+
+static void vhost_scsi_set_config(VirtIODevice *vdev,
+                                  const uint8_t *config)
+{
+    VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+
+    if ((uint32_t) ldl_raw(&scsiconf->sense_size) != vs->vs.sense_size ||
+        (uint32_t) ldl_raw(&scsiconf->cdb_size) != vs->vs.cdb_size) {
+        error_report("vhost-scsi does not support changing the sense data and CDB sizes");
+        exit(1);
+    }
+}
+
+static void vhost_scsi_set_status(VirtIODevice *vdev, uint8_t val)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    bool start = (val & VIRTIO_CONFIG_S_DRIVER_OK);
+
+    if (vs->dev.started == start) {
+        return;
+    }
+
+    if (start) {
+        int ret;
+
+        ret = vhost_scsi_start(vs, vdev);
+        if (ret < 0) {
+            error_report("virtio-scsi: unable to start vhost: %s\n",
+                         strerror(-ret));
+
+            /* There is no userspace virtio-scsi fallback so exit */
+            exit(1);
+        }
+    } else {
+        vhost_scsi_stop(vs, vdev);
+    }
+}
+
+static void vhost_scsi_guest_notifier_mask(VirtIODevice *vdev, int idx,
+                                           bool mask)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+
+    vhost_virtqueue_mask(&vs->dev, vdev, idx, mask);
+}
+
+static bool vhost_scsi_guest_notifier_pending(VirtIODevice *vdev, int idx)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+
+    return vhost_virtqueue_pending(&vs->dev, idx);
+}
+
+VirtIODevice *vhost_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf)
+{
+    VHostSCSI *vs;
+    int vhostfd = -1;
+    int ret;
+
+    if (!proxyconf->wwpn) {
+        error_report("vhost-scsi: missing wwpn\n");
+        return NULL;
+    }
+
+    if (proxyconf->vhostfd) {
+        vhostfd = monitor_handle_fd_param(cur_mon, proxyconf->vhostfd);
+        if (vhostfd == -1) {
+            error_report("vhost-scsi: unable to parse vhostfd\n");
+            return NULL;
+        }
+    }
+
+    vs = (VHostSCSI *)virtio_scsi_init_common(dev, proxyconf,
+                                              sizeof(VHostSCSI));
+
+    vs->vs.vdev.set_config = vhost_scsi_set_config;
+    vs->vs.vdev.set_status = vhost_scsi_set_status;
+    vs->vs.vdev.guest_notifier_mask = vhost_scsi_guest_notifier_mask;
+    vs->vs.vdev.guest_notifier_pending = vhost_scsi_guest_notifier_pending;
+
+    vs->dev.nvqs = VHOST_SCSI_VQ_NUM_FIXED + vs->vs.conf->num_queues;
+    vs->dev.vqs = g_new(struct vhost_virtqueue, vs->dev.nvqs);
+    vs->dev.vq_index = 0;
+
+    ret = vhost_dev_init(&vs->dev, vhostfd, "/dev/vhost-scsi", true);
+    if (ret < 0) {
+        error_report("vhost-scsi: vhost initialization failed: %s\n",
+                strerror(-ret));
+        return NULL;
+    }
+    vs->dev.backend_features = 0;
+    vs->dev.acked_features = 0;
+
+    error_setg(&vs->migration_blocker,
+            "vhost-scsi does not support migration");
+    migrate_add_blocker(vs->migration_blocker);
+
+    return &vs->vs.vdev;
+}
+
+void vhost_scsi_exit(VirtIODevice *vdev)
+{
+    VHostSCSI *vs = (VHostSCSI *)vdev;
+    migrate_del_blocker(vs->migration_blocker);
+    error_free(vs->migration_blocker);
+
+    /* This will stop vhost backend. */
+    vhost_scsi_set_status(vdev, 0);
+    g_free(vs->dev.vqs);
+    virtio_cleanup(vdev);
+}
+
diff --git a/hw/vhost-scsi.h b/hw/vhost-scsi.h
new file mode 100644
index 0000000..70f4a58
--- /dev/null
+++ b/hw/vhost-scsi.h
@@ -0,0 +1,64 @@ 
+/*
+ * vhost_scsi host device
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_SCSI_H
+#define VHOST_SCSI_H
+
+#include "qemu-common.h"
+#include "qdev.h"
+#include "virtio-scsi.h"
+
+/*
+ * Used by QEMU userspace to ensure a consistent vhost-scsi ABI.
+ *
+ * ABI Rev 0: July 2012 version starting point for v3.6-rc merge candidate +
+ *            RFC-v2 vhost-scsi userspace.  Add GET_ABI_VERSION ioctl usage
+ * ABI Rev 1: January 2013. Ignore vhost_tpgt filed in struct vhost_scsi_target.
+ * 	      All the targets under vhost_wwpn can be seen and used by guest.
+ */
+
+#define VHOST_SCSI_ABI_VERSION 1
+
+/* TODO #include <linux/vhost.h> properly */
+/* For VHOST_SCSI_SET_ENDPOINT/VHOST_SCSI_CLEAR_ENDPOINT ioctl */
+struct vhost_scsi_target {
+    int abi_version;
+    char vhost_wwpn[224];
+    unsigned short vhost_tpgt;
+    unsigned short reserved;
+};
+
+enum vhost_scsi_vq_list {
+    VHOST_SCSI_VQ_CONTROL = 0,
+    VHOST_SCSI_VQ_EVENT = 1,
+    VHOST_SCSI_VQ_NUM_FIXED = 2,
+};
+
+#define VHOST_VIRTIO 0xAF
+#define VHOST_SCSI_SET_ENDPOINT _IOW(VHOST_VIRTIO, 0x40, struct vhost_scsi_target)
+#define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target)
+#define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int)
+
+#define DEFINE_VHOST_SCSI_PROPERTIES(_state, _features_field, _conf_field) \
+    DEFINE_VIRTIO_COMMON_FEATURES(_state, _features_field), \
+    DEFINE_PROP_STRING("vhostfd", _state, _conf_field.vhostfd), \
+    DEFINE_PROP_STRING("wwpn", _state, _conf_field.wwpn), \
+    DEFINE_PROP_UINT32("num_queues", _state, _conf_field.num_queues, 1), \
+    DEFINE_PROP_UINT32("max_sectors", _state, _conf_field.max_sectors, 0xFFFF), \
+    DEFINE_PROP_UINT32("cmd_per_lun", _state, _conf_field.cmd_per_lun, 128)
+
+VirtIODevice *vhost_scsi_init(DeviceState *dev, VirtIOSCSIConf *proxyconf);
+void vhost_scsi_exit(VirtIODevice *vdev);
+
+
+#endif
diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
index 39c1966..4a97ca1 100644
--- a/hw/virtio-pci.c
+++ b/hw/virtio-pci.c
@@ -22,6 +22,7 @@ 
 #include "hw/virtio-net.h"
 #include "hw/virtio-serial.h"
 #include "hw/virtio-scsi.h"
+#include "hw/vhost-scsi.h"
 #include "hw/pci/pci.h"
 #include "qemu/error-report.h"
 #include "hw/pci/msi.h"
@@ -1316,6 +1317,63 @@  static const TypeInfo virtio_scsi_info = {
     .class_init    = virtio_scsi_class_init,
 };
 
+static int vhost_scsi_init_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+    VirtIODevice *vdev;
+
+    vdev = vhost_scsi_init(&pci_dev->qdev, &proxy->scsi);
+    if (!vdev) {
+        return -EINVAL;
+    }
+
+    vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
+                                        ? proxy->scsi.num_queues + 3
+                                        : proxy->nvectors;
+    virtio_init_pci(proxy, vdev);
+
+    /* make the actual value visible */
+    proxy->nvectors = vdev->nvectors;
+    return 0;
+}
+
+static void vhost_scsi_exit_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+
+    vhost_scsi_exit(proxy->vdev);
+    virtio_exit_pci(pci_dev);
+}
+
+static Property vhost_scsi_properties[] = {
+    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
+    DEFINE_VHOST_SCSI_PROPERTIES(VirtIOPCIProxy, host_features, scsi),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void vhost_scsi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = vhost_scsi_init_pci;
+    k->exit = vhost_scsi_exit_pci;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
+    k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI;
+    k->revision = 0x00;
+    k->class_id = PCI_CLASS_STORAGE_SCSI;
+    dc->reset = virtio_pci_reset;
+    dc->props = vhost_scsi_properties;
+}
+
+static const TypeInfo vhost_scsi_info = {
+    .name          = "vhost-scsi-pci",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VirtIOPCIProxy),
+    .class_init    = vhost_scsi_class_init,
+};
+
 #ifdef CONFIG_VIRTFS
 static int virtio_9p_init_pci(PCIDevice *pci_dev)
 {
@@ -1520,6 +1578,7 @@  static void virtio_pci_register_types(void)
     type_register_static(&virtio_serial_info);
     type_register_static(&virtio_balloon_info);
     type_register_static(&virtio_scsi_info);
+    type_register_static(&vhost_scsi_info);
     type_register_static(&virtio_rng_info);
     type_register_static(&virtio_pci_bus_info);
     type_register_static(&virtio_pci_info);
diff --git a/hw/virtio-scsi.h b/hw/virtio-scsi.h
index 5acb721..bbcf206 100644
--- a/hw/virtio-scsi.h
+++ b/hw/virtio-scsi.h
@@ -141,6 +141,8 @@  struct VirtIOSCSIConf {
     uint32_t num_queues;
     uint32_t max_sectors;
     uint32_t cmd_per_lun;
+    char *vhostfd;
+    char *wwpn;
 };
 
 typedef struct VirtIOSCSICommon {