[21/30] virtiofsd: Start reading commands from queue
diff mbox series

Message ID 20191021105832.36574-22-dgilbert@redhat.com
State New
Headers show
Series
  • virtiofs daemon (base)
Related show

Commit Message

Dr. David Alan Gilbert Oct. 21, 2019, 10:58 a.m. UTC
From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>

Pop queue elements off queues, copy the data from them and
pass that to fuse.

  Note: 'out' in a VuVirtqElement is from QEMU
        'in' in libfuse is into the daemon

  So we read from the out iov's to get a fuse_in_header

When we get a kick we've got to read all the elements until the queue
is empty.

Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
 contrib/virtiofsd/fuse_i.h      |  2 +
 contrib/virtiofsd/fuse_virtio.c | 99 ++++++++++++++++++++++++++++++++-
 2 files changed, 98 insertions(+), 3 deletions(-)

Comments

Marc-André Lureau Oct. 21, 2019, 2:57 p.m. UTC | #1
On Mon, Oct 21, 2019 at 1:09 PM Dr. David Alan Gilbert (git)
<dgilbert@redhat.com> wrote:
>
> From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
>
> Pop queue elements off queues, copy the data from them and
> pass that to fuse.
>
>   Note: 'out' in a VuVirtqElement is from QEMU
>         'in' in libfuse is into the daemon
>
>   So we read from the out iov's to get a fuse_in_header
>
> When we get a kick we've got to read all the elements until the queue
> is empty.
>
> Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
> ---
>  contrib/virtiofsd/fuse_i.h      |  2 +
>  contrib/virtiofsd/fuse_virtio.c | 99 ++++++++++++++++++++++++++++++++-
>  2 files changed, 98 insertions(+), 3 deletions(-)
>
> diff --git a/contrib/virtiofsd/fuse_i.h b/contrib/virtiofsd/fuse_i.h
> index ea55f618a5..5b6ef09df5 100644
> --- a/contrib/virtiofsd/fuse_i.h
> +++ b/contrib/virtiofsd/fuse_i.h
> @@ -14,6 +14,7 @@
>  #include "fuse_lowlevel.h"
>
>  struct fv_VuDev;
> +struct fv_QueueInfo;
>
>  struct fuse_req {
>         struct fuse_session *se;
> @@ -75,6 +76,7 @@ struct fuse_chan {
>         pthread_mutex_t lock;
>         int ctr;
>         int fd;
> +       struct fv_QueueInfo *qi;
>  };
>
>  /**
> diff --git a/contrib/virtiofsd/fuse_virtio.c b/contrib/virtiofsd/fuse_virtio.c
> index 0513531c13..e189511729 100644
> --- a/contrib/virtiofsd/fuse_virtio.c
> +++ b/contrib/virtiofsd/fuse_virtio.c
> @@ -32,6 +32,7 @@
>
>  #include "contrib/libvhost-user/libvhost-user.h"
>
> +struct fv_VuDev;
>  struct fv_QueueInfo {
>      pthread_t thread;
>      struct fv_VuDev *virtio_dev;
> @@ -101,10 +102,42 @@ static void fv_panic(VuDev *dev, const char *err)
>      exit(EXIT_FAILURE);
>  }
>
> +/*
> + * Copy from an iovec into a fuse_buf (memory only)
> + * Caller must ensure there is space
> + */
> +static void copy_from_iov(struct fuse_buf *buf, size_t out_num,
> +                          const struct iovec *out_sg)
> +{
> +    void *dest = buf->mem;
> +
> +    while (out_num) {
> +        size_t onelen = out_sg->iov_len;
> +        memcpy(dest, out_sg->iov_base, onelen);
> +        dest += onelen;
> +        out_sg++;
> +        out_num--;
> +    }
> +}

Or iov_to_buf()

Just found out that vhost-user-blk.c should probably link too with
util/iov.o, instead of copying the code.

> +
>  /* Thread function for individual queues, created when a queue is 'started' */
>  static void *fv_queue_thread(void *opaque)
>  {
>      struct fv_QueueInfo *qi = opaque;
> +    struct VuDev *dev = &qi->virtio_dev->dev;
> +    struct VuVirtq *q = vu_get_queue(dev, qi->qidx);
> +    struct fuse_session *se = qi->virtio_dev->se;
> +    struct fuse_chan ch;
> +    struct fuse_buf fbuf;
> +
> +    fbuf.mem = NULL;
> +    fbuf.flags = 0;
> +
> +    fuse_mutex_init(&ch.lock);
> +    ch.fd = (int)0xdaff0d111;
> +    ch.ctr = 1;
> +    ch.qi = qi;
> +
>      fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
>               qi->qidx, qi->kick_fd);
>      while (1) {
> @@ -141,11 +174,71 @@ static void *fv_queue_thread(void *opaque)
>              fuse_log(FUSE_LOG_ERR, "Eventfd_read for queue: %m\n");
>              break;
>          }
> -        if (qi->virtio_dev->se->debug) {
> -            fprintf(stderr, "%s: Queue %d gave evalue: %zx\n", __func__,
> -                    qi->qidx, (size_t)evalue);
> +        /* out is from guest, in is too guest */
> +        unsigned int in_bytes, out_bytes;
> +        vu_queue_get_avail_bytes(dev, q, &in_bytes, &out_bytes, ~0, ~0);
> +
> +        fuse_log(FUSE_LOG_DEBUG,
> +                 "%s: Queue %d gave evalue: %zx available: in: %u out: %u\n",
> +                 __func__, qi->qidx, (size_t)evalue, in_bytes, out_bytes);
> +
> +        while (1) {
> +            /*
> +             * An element contains one request and the space to send our
> +             * response They're spread over multiple descriptors in a
> +             * scatter/gather set and we can't trust the guest to keep them
> +             * still; so copy in/out.
> +             */
> +            VuVirtqElement *elem = vu_queue_pop(dev, q, sizeof(VuVirtqElement));
> +            if (!elem) {
> +                break;
> +            }
> +
> +            if (!fbuf.mem) {
> +                fbuf.mem = malloc(se->bufsize);
> +                assert(fbuf.mem);
> +                assert(se->bufsize > sizeof(struct fuse_in_header));
> +            }
> +            /* The 'out' part of the elem is from qemu */
> +            unsigned int out_num = elem->out_num;
> +            struct iovec *out_sg = elem->out_sg;
> +            size_t out_len = iov_length(out_sg, out_num);
> +            fuse_log(FUSE_LOG_DEBUG,
> +                     "%s: elem %d: with %d out desc of length %zd\n", __func__,
> +                     elem->index, out_num, out_len);
> +
> +            /*
> +             * The elem should contain a 'fuse_in_header' (in to fuse)
> +             * plus the data based on the len in the header.
> +             */
> +            if (out_len < sizeof(struct fuse_in_header)) {
> +                fuse_log(FUSE_LOG_ERR, "%s: elem %d too short for in_header\n",
> +                         __func__, elem->index);
> +                assert(0); /* TODO */
> +            }
> +            if (out_len > se->bufsize) {
> +                fuse_log(FUSE_LOG_ERR, "%s: elem %d too large for buffer\n",
> +                         __func__, elem->index);
> +                assert(0); /* TODO */
> +            }
> +            copy_from_iov(&fbuf, out_num, out_sg);
> +            fbuf.size = out_len;
> +
> +            /* TODO! Endianness of header */
> +
> +            /* TODO: Fixup fuse_send_msg */
> +            /* TODO: Add checks for fuse_session_exited */
> +            fuse_session_process_buf_int(se, &fbuf, &ch);
> +
> +            /* TODO: vu_queue_push(dev, q, elem, qi->write_count); */
> +            vu_queue_notify(dev, q);
> +
> +            free(elem);
> +            elem = NULL;
>          }
>      }
> +    pthread_mutex_destroy(&ch.lock);
> +    free(fbuf.mem);
>
>      return NULL;
>  }
> --
> 2.23.0
>
>
Dr. David Alan Gilbert Oct. 22, 2019, 10:28 a.m. UTC | #2
* Marc-André Lureau (marcandre.lureau@gmail.com) wrote:
> On Mon, Oct 21, 2019 at 1:09 PM Dr. David Alan Gilbert (git)
> <dgilbert@redhat.com> wrote:
> >
> > From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
> >
> > Pop queue elements off queues, copy the data from them and
> > pass that to fuse.
> >
> >   Note: 'out' in a VuVirtqElement is from QEMU
> >         'in' in libfuse is into the daemon
> >
> >   So we read from the out iov's to get a fuse_in_header
> >
> > When we get a kick we've got to read all the elements until the queue
> > is empty.
> >
> > Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
> > ---
> >  contrib/virtiofsd/fuse_i.h      |  2 +
> >  contrib/virtiofsd/fuse_virtio.c | 99 ++++++++++++++++++++++++++++++++-
> >  2 files changed, 98 insertions(+), 3 deletions(-)
> >
> > diff --git a/contrib/virtiofsd/fuse_i.h b/contrib/virtiofsd/fuse_i.h
> > index ea55f618a5..5b6ef09df5 100644
> > --- a/contrib/virtiofsd/fuse_i.h
> > +++ b/contrib/virtiofsd/fuse_i.h
> > @@ -14,6 +14,7 @@
> >  #include "fuse_lowlevel.h"
> >
> >  struct fv_VuDev;
> > +struct fv_QueueInfo;
> >
> >  struct fuse_req {
> >         struct fuse_session *se;
> > @@ -75,6 +76,7 @@ struct fuse_chan {
> >         pthread_mutex_t lock;
> >         int ctr;
> >         int fd;
> > +       struct fv_QueueInfo *qi;
> >  };
> >
> >  /**
> > diff --git a/contrib/virtiofsd/fuse_virtio.c b/contrib/virtiofsd/fuse_virtio.c
> > index 0513531c13..e189511729 100644
> > --- a/contrib/virtiofsd/fuse_virtio.c
> > +++ b/contrib/virtiofsd/fuse_virtio.c
> > @@ -32,6 +32,7 @@
> >
> >  #include "contrib/libvhost-user/libvhost-user.h"
> >
> > +struct fv_VuDev;
> >  struct fv_QueueInfo {
> >      pthread_t thread;
> >      struct fv_VuDev *virtio_dev;
> > @@ -101,10 +102,42 @@ static void fv_panic(VuDev *dev, const char *err)
> >      exit(EXIT_FAILURE);
> >  }
> >
> > +/*
> > + * Copy from an iovec into a fuse_buf (memory only)
> > + * Caller must ensure there is space
> > + */
> > +static void copy_from_iov(struct fuse_buf *buf, size_t out_num,
> > +                          const struct iovec *out_sg)
> > +{
> > +    void *dest = buf->mem;
> > +
> > +    while (out_num) {
> > +        size_t onelen = out_sg->iov_len;
> > +        memcpy(dest, out_sg->iov_base, onelen);
> > +        dest += onelen;
> > +        out_sg++;
> > +        out_num--;
> > +    }
> > +}
> 
> Or iov_to_buf()

Is there a variant there that will copy the whole of an iovec?
I just tried swapping the copy_from_iov out, but it's messier to use
because it wants the bytes length; where in most cases I'm taking a
single element from the iovec, or the 'rest of the iovec' by element
count.

Dave

> Just found out that vhost-user-blk.c should probably link too with
> util/iov.o, instead of copying the code.
> 
> > +
> >  /* Thread function for individual queues, created when a queue is 'started' */
> >  static void *fv_queue_thread(void *opaque)
> >  {
> >      struct fv_QueueInfo *qi = opaque;
> > +    struct VuDev *dev = &qi->virtio_dev->dev;
> > +    struct VuVirtq *q = vu_get_queue(dev, qi->qidx);
> > +    struct fuse_session *se = qi->virtio_dev->se;
> > +    struct fuse_chan ch;
> > +    struct fuse_buf fbuf;
> > +
> > +    fbuf.mem = NULL;
> > +    fbuf.flags = 0;
> > +
> > +    fuse_mutex_init(&ch.lock);
> > +    ch.fd = (int)0xdaff0d111;
> > +    ch.ctr = 1;
> > +    ch.qi = qi;
> > +
> >      fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
> >               qi->qidx, qi->kick_fd);
> >      while (1) {
> > @@ -141,11 +174,71 @@ static void *fv_queue_thread(void *opaque)
> >              fuse_log(FUSE_LOG_ERR, "Eventfd_read for queue: %m\n");
> >              break;
> >          }
> > -        if (qi->virtio_dev->se->debug) {
> > -            fprintf(stderr, "%s: Queue %d gave evalue: %zx\n", __func__,
> > -                    qi->qidx, (size_t)evalue);
> > +        /* out is from guest, in is too guest */
> > +        unsigned int in_bytes, out_bytes;
> > +        vu_queue_get_avail_bytes(dev, q, &in_bytes, &out_bytes, ~0, ~0);
> > +
> > +        fuse_log(FUSE_LOG_DEBUG,
> > +                 "%s: Queue %d gave evalue: %zx available: in: %u out: %u\n",
> > +                 __func__, qi->qidx, (size_t)evalue, in_bytes, out_bytes);
> > +
> > +        while (1) {
> > +            /*
> > +             * An element contains one request and the space to send our
> > +             * response They're spread over multiple descriptors in a
> > +             * scatter/gather set and we can't trust the guest to keep them
> > +             * still; so copy in/out.
> > +             */
> > +            VuVirtqElement *elem = vu_queue_pop(dev, q, sizeof(VuVirtqElement));
> > +            if (!elem) {
> > +                break;
> > +            }
> > +
> > +            if (!fbuf.mem) {
> > +                fbuf.mem = malloc(se->bufsize);
> > +                assert(fbuf.mem);
> > +                assert(se->bufsize > sizeof(struct fuse_in_header));
> > +            }
> > +            /* The 'out' part of the elem is from qemu */
> > +            unsigned int out_num = elem->out_num;
> > +            struct iovec *out_sg = elem->out_sg;
> > +            size_t out_len = iov_length(out_sg, out_num);
> > +            fuse_log(FUSE_LOG_DEBUG,
> > +                     "%s: elem %d: with %d out desc of length %zd\n", __func__,
> > +                     elem->index, out_num, out_len);
> > +
> > +            /*
> > +             * The elem should contain a 'fuse_in_header' (in to fuse)
> > +             * plus the data based on the len in the header.
> > +             */
> > +            if (out_len < sizeof(struct fuse_in_header)) {
> > +                fuse_log(FUSE_LOG_ERR, "%s: elem %d too short for in_header\n",
> > +                         __func__, elem->index);
> > +                assert(0); /* TODO */
> > +            }
> > +            if (out_len > se->bufsize) {
> > +                fuse_log(FUSE_LOG_ERR, "%s: elem %d too large for buffer\n",
> > +                         __func__, elem->index);
> > +                assert(0); /* TODO */
> > +            }
> > +            copy_from_iov(&fbuf, out_num, out_sg);
> > +            fbuf.size = out_len;
> > +
> > +            /* TODO! Endianness of header */
> > +
> > +            /* TODO: Fixup fuse_send_msg */
> > +            /* TODO: Add checks for fuse_session_exited */
> > +            fuse_session_process_buf_int(se, &fbuf, &ch);
> > +
> > +            /* TODO: vu_queue_push(dev, q, elem, qi->write_count); */
> > +            vu_queue_notify(dev, q);
> > +
> > +            free(elem);
> > +            elem = NULL;
> >          }
> >      }
> > +    pthread_mutex_destroy(&ch.lock);
> > +    free(fbuf.mem);
> >
> >      return NULL;
> >  }
> > --
> > 2.23.0
> >
> >
> 
> 
> -- 
> Marc-André Lureau
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK

Patch
diff mbox series

diff --git a/contrib/virtiofsd/fuse_i.h b/contrib/virtiofsd/fuse_i.h
index ea55f618a5..5b6ef09df5 100644
--- a/contrib/virtiofsd/fuse_i.h
+++ b/contrib/virtiofsd/fuse_i.h
@@ -14,6 +14,7 @@ 
 #include "fuse_lowlevel.h"
 
 struct fv_VuDev;
+struct fv_QueueInfo;
 
 struct fuse_req {
 	struct fuse_session *se;
@@ -75,6 +76,7 @@  struct fuse_chan {
 	pthread_mutex_t lock;
 	int ctr;
 	int fd;
+	struct fv_QueueInfo *qi;
 };
 
 /**
diff --git a/contrib/virtiofsd/fuse_virtio.c b/contrib/virtiofsd/fuse_virtio.c
index 0513531c13..e189511729 100644
--- a/contrib/virtiofsd/fuse_virtio.c
+++ b/contrib/virtiofsd/fuse_virtio.c
@@ -32,6 +32,7 @@ 
 
 #include "contrib/libvhost-user/libvhost-user.h"
 
+struct fv_VuDev;
 struct fv_QueueInfo {
     pthread_t thread;
     struct fv_VuDev *virtio_dev;
@@ -101,10 +102,42 @@  static void fv_panic(VuDev *dev, const char *err)
     exit(EXIT_FAILURE);
 }
 
+/*
+ * Copy from an iovec into a fuse_buf (memory only)
+ * Caller must ensure there is space
+ */
+static void copy_from_iov(struct fuse_buf *buf, size_t out_num,
+                          const struct iovec *out_sg)
+{
+    void *dest = buf->mem;
+
+    while (out_num) {
+        size_t onelen = out_sg->iov_len;
+        memcpy(dest, out_sg->iov_base, onelen);
+        dest += onelen;
+        out_sg++;
+        out_num--;
+    }
+}
+
 /* Thread function for individual queues, created when a queue is 'started' */
 static void *fv_queue_thread(void *opaque)
 {
     struct fv_QueueInfo *qi = opaque;
+    struct VuDev *dev = &qi->virtio_dev->dev;
+    struct VuVirtq *q = vu_get_queue(dev, qi->qidx);
+    struct fuse_session *se = qi->virtio_dev->se;
+    struct fuse_chan ch;
+    struct fuse_buf fbuf;
+
+    fbuf.mem = NULL;
+    fbuf.flags = 0;
+
+    fuse_mutex_init(&ch.lock);
+    ch.fd = (int)0xdaff0d111;
+    ch.ctr = 1;
+    ch.qi = qi;
+
     fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
              qi->qidx, qi->kick_fd);
     while (1) {
@@ -141,11 +174,71 @@  static void *fv_queue_thread(void *opaque)
             fuse_log(FUSE_LOG_ERR, "Eventfd_read for queue: %m\n");
             break;
         }
-        if (qi->virtio_dev->se->debug) {
-            fprintf(stderr, "%s: Queue %d gave evalue: %zx\n", __func__,
-                    qi->qidx, (size_t)evalue);
+        /* out is from guest, in is too guest */
+        unsigned int in_bytes, out_bytes;
+        vu_queue_get_avail_bytes(dev, q, &in_bytes, &out_bytes, ~0, ~0);
+
+        fuse_log(FUSE_LOG_DEBUG,
+                 "%s: Queue %d gave evalue: %zx available: in: %u out: %u\n",
+                 __func__, qi->qidx, (size_t)evalue, in_bytes, out_bytes);
+
+        while (1) {
+            /*
+             * An element contains one request and the space to send our
+             * response They're spread over multiple descriptors in a
+             * scatter/gather set and we can't trust the guest to keep them
+             * still; so copy in/out.
+             */
+            VuVirtqElement *elem = vu_queue_pop(dev, q, sizeof(VuVirtqElement));
+            if (!elem) {
+                break;
+            }
+
+            if (!fbuf.mem) {
+                fbuf.mem = malloc(se->bufsize);
+                assert(fbuf.mem);
+                assert(se->bufsize > sizeof(struct fuse_in_header));
+            }
+            /* The 'out' part of the elem is from qemu */
+            unsigned int out_num = elem->out_num;
+            struct iovec *out_sg = elem->out_sg;
+            size_t out_len = iov_length(out_sg, out_num);
+            fuse_log(FUSE_LOG_DEBUG,
+                     "%s: elem %d: with %d out desc of length %zd\n", __func__,
+                     elem->index, out_num, out_len);
+
+            /*
+             * The elem should contain a 'fuse_in_header' (in to fuse)
+             * plus the data based on the len in the header.
+             */
+            if (out_len < sizeof(struct fuse_in_header)) {
+                fuse_log(FUSE_LOG_ERR, "%s: elem %d too short for in_header\n",
+                         __func__, elem->index);
+                assert(0); /* TODO */
+            }
+            if (out_len > se->bufsize) {
+                fuse_log(FUSE_LOG_ERR, "%s: elem %d too large for buffer\n",
+                         __func__, elem->index);
+                assert(0); /* TODO */
+            }
+            copy_from_iov(&fbuf, out_num, out_sg);
+            fbuf.size = out_len;
+
+            /* TODO! Endianness of header */
+
+            /* TODO: Fixup fuse_send_msg */
+            /* TODO: Add checks for fuse_session_exited */
+            fuse_session_process_buf_int(se, &fbuf, &ch);
+
+            /* TODO: vu_queue_push(dev, q, elem, qi->write_count); */
+            vu_queue_notify(dev, q);
+
+            free(elem);
+            elem = NULL;
         }
     }
+    pthread_mutex_destroy(&ch.lock);
+    free(fbuf.mem);
 
     return NULL;
 }