| Message ID | 20120917152620.GG6879@in.ibm.com |
|---|---|
| State | New |
| Headers | show |
Am 17.09.2012 17:26, schrieb Bharata B Rao: > block: Support GlusterFS as a QEMU block backend. > > From: Bharata B Rao <bharata@linux.vnet.ibm.com> > > This patch adds gluster as the new block backend in QEMU. This gives > QEMU the ability to boot VM images from gluster volumes. Its already > possible to boot from VM images on gluster volumes using FUSE mount, but > this patchset provides the ability to boot VM images from gluster volumes > by by-passing the FUSE layer in gluster. This is made possible by > using libgfapi routines to perform IO on gluster volumes directly. > > VM Image on gluster volume is specified like this: > > file=gluster[+transport]://[server[:port]]/volname/image[?socket=...] > > 'gluster' is the protocol. > > 'transport' specifies the transport type used to connect to gluster > management daemon (glusterd). Valid transport types are > tcp, unix and rdma. If the transport type isn't specified, then tcp > type is assumed. > > 'server' specifies the server where the volume file specification for > the given volume resides. This can be either hostname or ipv4 address > or ipv6 address. ipv6 address needs to be with in square brackets [ ]. > If transport type is 'unix', then server field is ignored, but the > 'socket' field needs to be populated with the path to unix domain > socket. > > 'port' is the port number on which glusterd is listening. This is optional > and if not specified, QEMU will send 0 which will make gluster to use the > default port. port is ignored for unix type of transport. > > 'volname' is the name of the gluster volume which contains the VM image. > > 'image' is the path to the actual VM image that resides on gluster volume. > > Examples: > > file=gluster://1.2.3.4/testvol/a.img > file=gluster+tcp://1.2.3.4/testvol/a.img > file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img > file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img > file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img > file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img > file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket > file=gluster+rdma://1.2.3.4:24007/testvol/a.img > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > --- > > block/Makefile.objs | 1 > block/gluster.c | 694 +++++++++++++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 695 insertions(+), 0 deletions(-) > create mode 100644 block/gluster.c > > > diff --git a/block/Makefile.objs b/block/Makefile.objs > index b5754d3..a1ae67f 100644 > --- a/block/Makefile.objs > +++ b/block/Makefile.objs > @@ -9,3 +9,4 @@ block-obj-$(CONFIG_POSIX) += raw-posix.o > block-obj-$(CONFIG_LIBISCSI) += iscsi.o > block-obj-$(CONFIG_CURL) += curl.o > block-obj-$(CONFIG_RBD) += rbd.o > +block-obj-$(CONFIG_GLUSTERFS) += gluster.o > diff --git a/block/gluster.c b/block/gluster.c > new file mode 100644 > index 0000000..0de3286 > --- /dev/null > +++ b/block/gluster.c > @@ -0,0 +1,694 @@ > +/* > + * GlusterFS backend for QEMU > + * > + * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com> > + * > + * Pipe handling mechanism in AIO implementation is derived from > + * block/rbd.c. Hence, > + * > + * Copyright (C) 2010-2011 Christian Brunner <chb@muc.de>, > + * Josh Durgin <josh.durgin@dreamhost.com> > + * > + * This work is licensed under the terms of the GNU GPL, version 2. See > + * the COPYING file in the top-level directory. > + * > + * Contributions after 2012-01-13 are licensed under the terms of the > + * GNU GPL, version 2 or (at your option) any later version. > + */ > +#include <glusterfs/api/glfs.h> > +#include "block_int.h" > +#include "qemu_socket.h" > + > +typedef struct GlusterAIOCB { > + BlockDriverAIOCB common; > + int64_t size; > + int ret; > + bool *finished; > + QEMUBH *bh; > +} GlusterAIOCB; > + > +typedef struct BDRVGlusterState { > + struct glfs *glfs; > + int fds[2]; > + struct glfs_fd *fd; > + int qemu_aio_count; > + int event_reader_pos; > + GlusterAIOCB *event_acb; > +} BDRVGlusterState; > + > +#define GLUSTER_FD_READ 0 > +#define GLUSTER_FD_WRITE 1 > + > +#define GLUSTER_TRANSPORT_DEFAULT "gluster://" > +#define GLUSTER_TRANSPORT_DEFAULT_SZ strlen(GLUSTER_TRANSPORT_DEFAULT) > +#define GLUSTER_TRANSPORT_TCP "gluster+tcp://" > +#define GLUSTER_TRANSPORT_TCP_SZ strlen(GLUSTER_TRANSPORT_TCP) > +#define GLUSTER_TRANSPORT_UNIX "gluster+unix://" > +#define GLUSTER_TRANSPORT_UNIX_SZ strlen(GLUSTER_TRANSPORT_UNIX) > +#define GLUSTER_TRANSPORT_RDMA "gluster+rdma://" > +#define GLUSTER_TRANSPORT_RDMA_SZ strlen(GLUSTER_TRANSPORT_RDMA) > + > +typedef struct GlusterURI { > + char *server; > + int port; > + char *volname; > + char *image; > + char *transport; > + bool is_unix; > +} GlusterURI; > + > +static void qemu_gluster_uri_free(GlusterURI *uri) > +{ > + g_free(uri->server); > + g_free(uri->volname); > + g_free(uri->image); > + g_free(uri->transport); > + g_free(uri); > +} > + > +static int parse_socket(GlusterURI *uri, char *socket) > +{ > + char *token, *saveptr; > + > + if (!socket) { > + return 0; > + } > + token = strtok_r(socket, "=", &saveptr); > + if (!token || strcmp(token, "socket")) { > + return -EINVAL; > + } > + token = strtok_r(NULL, "=", &saveptr); > + if (!token) { > + return -EINVAL; > + } > + uri->server = g_strdup(token); > + uri->is_unix = true; > + return 0; > +} > + > +static int parse_gluster_spec(GlusterURI *uri, char *spec) > +{ > + char *token, *saveptr; > + int ret; > + QemuOpts *opts; > + char *p, *q; > + > + /* transport */ > + p = spec; > + if (!strncmp(p, GLUSTER_TRANSPORT_DEFAULT, GLUSTER_TRANSPORT_DEFAULT_SZ)) { > + uri->transport = g_strdup("tcp"); > + p += GLUSTER_TRANSPORT_DEFAULT_SZ; > + } else if (!strncmp(p, GLUSTER_TRANSPORT_TCP, GLUSTER_TRANSPORT_TCP_SZ)) { > + uri->transport = g_strdup("tcp"); > + p += GLUSTER_TRANSPORT_TCP_SZ; > + } else if (!strncmp(p, GLUSTER_TRANSPORT_UNIX, GLUSTER_TRANSPORT_UNIX_SZ)) { > + uri->transport = g_strdup("unix"); > + p += GLUSTER_TRANSPORT_UNIX_SZ; > + } else if (!strncmp(p, GLUSTER_TRANSPORT_RDMA, GLUSTER_TRANSPORT_RDMA_SZ)) { Would look a bit nicer with strstart() form cutils.c instead of strncmp(). > + uri->transport = g_strdup("rdma"); > + p += GLUSTER_TRANSPORT_RDMA_SZ; > + } else { > + return -EINVAL; > + } > + q = p; > + > + /* server */ > + if (!strcmp(uri->transport, "unix")) { > + if (!uri->is_unix) { > + return -EINVAL; > + } > + } else { > + if (uri->is_unix) { > + return -EINVAL; > + } > + p = strchr(p, '/'); > + if (!p) { > + return -EINVAL; > + } > + *p++ = '\0'; > + opts = qemu_opts_create(qemu_find_opts("inet"), NULL, 0, NULL); > + ret = inet_parse(opts, q); > + if (!ret) { > + uri->server = g_strdup(qemu_opt_get(opts, "host")); > + uri->port = strtoul(qemu_opt_get(opts, "port"), NULL, 0); > + if (uri->port < 0) { > + ret = -EINVAL; > + } > + } > + qemu_opts_del(opts); > + if (ret < 0) { > + return -EINVAL; > + } > + } > + > + /* volname */ > + token = strtok_r(p, "/", &saveptr); > + if (!token) { > + return -EINVAL; > + } > + uri->volname = g_strdup(token); > + > + /* image */ > + token = strtok_r(NULL, "?", &saveptr); > + if (!token) { > + return -EINVAL; > + } > + uri->image = g_strdup(token); > + return 0; > +} > + > +/* > + * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...] > + * > + * 'gluster' is the protocol. > + * > + * 'transport' specifies the transport type used to connect to gluster > + * management daemon (glusterd). Valid transport types are > + * tcp, unix and rdma. If the transport type isn't specified, then tcp > + * type is assumed. > + * > + * 'server' specifies the server where the volume file specification for > + * the given volume resides. This can be either hostname or ipv4 address > + * or ipv6 address. ipv6 address needs to be with in square brackets [ ]. > + * If transport type is 'unix', then server field is ignored, but the > + * 'socket' field needs to be populated with the path to unix domain > + * socket. > + * > + * 'port' is the port number on which glusterd is listening. This is optional > + * and if not specified, QEMU will send 0 which will make gluster to use the > + * default port. port is ignored for unix type of transport. > + * > + * 'volname' is the name of the gluster volume which contains the VM image. > + * > + * 'image' is the path to the actual VM image that resides on gluster volume. > + * > + * Examples: > + * > + * file=gluster://1.2.3.4/testvol/a.img > + * file=gluster+tcp://1.2.3.4/testvol/a.img > + * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img > + * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img > + * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img > + * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img > + * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket > + * file=gluster+rdma://1.2.3.4:24007/testvol/a.img > + */ > +static int qemu_gluster_parseuri(GlusterURI *uri, const char *filename) > +{ > + char *token, *saveptr; > + char *p, *q, *gluster_spec = NULL; > + int ret = -EINVAL; > + > + p = q = g_strdup(filename); Neither p nor q are changed, so one variable would be enough. > + > + /* Extract server, volname and image */ > + token = strtok_r(p, "?", &saveptr); > + if (!token) { > + goto out; > + } > + gluster_spec = g_strdup(token); > + > + /* socket */ > + token = strtok_r(NULL, "?", &saveptr); > + ret = parse_socket(uri, token); > + if (ret < 0) { > + goto out; > + } The is_unix thing feels a bit backwards. You set it whenever an option is present, and fail if a protocol other than gluster+unix:// is used. Wouldn't it make more sense to set it depending on the protocol and then check in the option parser if is_unix == true when there is a 'socket' option? Otherwise adding non-unix options later is going to become hard. The rest looks good now. Kevin
On Tue, Sep 18, 2012 at 04:01:58PM +0200, Kevin Wolf wrote: > > + > > +#define GLUSTER_TRANSPORT_DEFAULT "gluster://" > > +#define GLUSTER_TRANSPORT_DEFAULT_SZ strlen(GLUSTER_TRANSPORT_DEFAULT) > > +#define GLUSTER_TRANSPORT_TCP "gluster+tcp://" > > +#define GLUSTER_TRANSPORT_TCP_SZ strlen(GLUSTER_TRANSPORT_TCP) > > +#define GLUSTER_TRANSPORT_UNIX "gluster+unix://" > > +#define GLUSTER_TRANSPORT_UNIX_SZ strlen(GLUSTER_TRANSPORT_UNIX) > > +#define GLUSTER_TRANSPORT_RDMA "gluster+rdma://" > > +#define GLUSTER_TRANSPORT_RDMA_SZ strlen(GLUSTER_TRANSPORT_RDMA) > > + > > + > > +static int parse_gluster_spec(GlusterURI *uri, char *spec) > > +{ > > + char *token, *saveptr; > > + int ret; > > + QemuOpts *opts; > > + char *p, *q; > > + > > + /* transport */ > > + p = spec; > > + if (!strncmp(p, GLUSTER_TRANSPORT_DEFAULT, GLUSTER_TRANSPORT_DEFAULT_SZ)) { > > + uri->transport = g_strdup("tcp"); > > + p += GLUSTER_TRANSPORT_DEFAULT_SZ; > > + } else if (!strncmp(p, GLUSTER_TRANSPORT_TCP, GLUSTER_TRANSPORT_TCP_SZ)) { > > + uri->transport = g_strdup("tcp"); > > + p += GLUSTER_TRANSPORT_TCP_SZ; > > + } else if (!strncmp(p, GLUSTER_TRANSPORT_UNIX, GLUSTER_TRANSPORT_UNIX_SZ)) { > > + uri->transport = g_strdup("unix"); > > + p += GLUSTER_TRANSPORT_UNIX_SZ; > > + } else if (!strncmp(p, GLUSTER_TRANSPORT_RDMA, GLUSTER_TRANSPORT_RDMA_SZ)) { > > Would look a bit nicer with strstart() form cutils.c instead of strncmp(). strstart() works with const char pointers, but I have char pointers here which I need to modify. > > +static int qemu_gluster_parseuri(GlusterURI *uri, const char *filename) > > +{ > > + char *token, *saveptr; > > + char *p, *q, *gluster_spec = NULL; > > + int ret = -EINVAL; > > + > > + p = q = g_strdup(filename); > > Neither p nor q are changed, so one variable would be enough. Right, will fix. > > > + > > + /* Extract server, volname and image */ > > + token = strtok_r(p, "?", &saveptr); > > + if (!token) { > > + goto out; > > + } > > + gluster_spec = g_strdup(token); > > + > > + /* socket */ > > + token = strtok_r(NULL, "?", &saveptr); > > + ret = parse_socket(uri, token); > > + if (ret < 0) { > > + goto out; > > + } > > The is_unix thing feels a bit backwards. You set it whenever an option > is present, and fail if a protocol other than gluster+unix:// is used. > > Wouldn't it make more sense to set it depending on the protocol and then > check in the option parser if is_unix == true when there is a 'socket' > option? Otherwise adding non-unix options later is going to become hard. I see your point, will change this. > > The rest looks good now. Thanks for the review. Regards, Bharata.
Il 20/09/2012 08:41, Bharata B Rao ha scritto: >> > Would look a bit nicer with strstart() form cutils.c instead of strncmp(). > strstart() works with const char pointers, but I have char pointers here > which I need to modify. You can pass a char* to a function that accepts const char*. In your case, the last argument to strstart would be NULL. Paolo
Il 20/09/2012 09:53, Paolo Bonzini ha scritto: >>>> Would look a bit nicer with strstart() form cutils.c instead of strncmp(). >> > strstart() works with const char pointers, but I have char pointers here >> > which I need to modify. > You can pass a char* to a function that accepts const char*. In your > case, the last argument to strstart would be NULL. As you pointed out on IRC, you meant the last argument. I don't think it would be a problem to cast that from char ** to const char **. Perhaps it would be cleaner to make qemu_gluster_parseuri and parse_gluster_spec accept a const char *. You can replace strtok_r + g_strdup with strspn/strcspn followed by g_strndup. BTW, here the second strtok_r needs to stop at "&". > +static int parse_socket(GlusterURI *uri, char *socket) > +{ > + char *token, *saveptr; > + > + if (!socket) { > + return 0; > + } > + token = strtok_r(socket, "=", &saveptr); > + if (!token || strcmp(token, "socket")) { > + return -EINVAL; > + } > + token = strtok_r(NULL, "=", &saveptr); > + if (!token) { > + return -EINVAL; > + } And the same for the second strtok_r here too: > + /* socket */ > + token = strtok_r(NULL, "?", &saveptr); > + ret = parse_socket(uri, token); > + if (ret < 0) { > + goto out; > + } > + > + /* Flag error for extra options */ > + token = strtok_r(NULL, "?", &saveptr); > + if (token) { > + ret = -EINVAL; > + goto out; > + } > + Paolo
On Thu, Sep 20, 2012 at 10:20:33AM +0200, Paolo Bonzini wrote: > Il 20/09/2012 09:53, Paolo Bonzini ha scritto: > >>>> Would look a bit nicer with strstart() form cutils.c instead of strncmp(). > >> > strstart() works with const char pointers, but I have char pointers here > >> > which I need to modify. > > You can pass a char* to a function that accepts const char*. In your > > case, the last argument to strstart would be NULL. > > As you pointed out on IRC, you meant the last argument. I don't think > it would be a problem to cast that from char ** to const char **. > > Perhaps it would be cleaner to make qemu_gluster_parseuri and > parse_gluster_spec accept a const char *. You can replace strtok_r + > g_strdup with strspn/strcspn followed by g_strndup. I feel the current approach of using the combination of strncmp, strtok_r and g_strdup should be good enough. But if you feel and insist that the right way to do this is to use the combination of strstart, strspn and g_strndup, I could give it a try. Regards, Bharata.
Il 20/09/2012 11:12, Bharata B Rao ha scritto: >> > Perhaps it would be cleaner to make qemu_gluster_parseuri and >> > parse_gluster_spec accept a const char *. You can replace strtok_r + >> > g_strdup with strspn/strcspn followed by g_strndup. > I feel the current approach of using the combination of strncmp, strtok_r > and g_strdup should be good enough. > > But if you feel and insist that the right way to do this is to use the > combination of strstart, strspn and g_strndup, I could give it a try. I won't insist on anything :) but please check the delimiters as mentioned in the other message. Paolo
Am 20.09.2012 11:12, schrieb Bharata B Rao: > On Thu, Sep 20, 2012 at 10:20:33AM +0200, Paolo Bonzini wrote: >> Il 20/09/2012 09:53, Paolo Bonzini ha scritto: >>>>>> Would look a bit nicer with strstart() form cutils.c instead of strncmp(). >>>>> strstart() works with const char pointers, but I have char pointers here >>>>> which I need to modify. >>> You can pass a char* to a function that accepts const char*. In your >>> case, the last argument to strstart would be NULL. >> >> As you pointed out on IRC, you meant the last argument. I don't think >> it would be a problem to cast that from char ** to const char **. >> >> Perhaps it would be cleaner to make qemu_gluster_parseuri and >> parse_gluster_spec accept a const char *. You can replace strtok_r + >> g_strdup with strspn/strcspn followed by g_strndup. > > I feel the current approach of using the combination of strncmp, strtok_r > and g_strdup should be good enough. Now that Paolo agreed that there wouldn't be a problem with casting, let's use strstart with a cast to (const char**) for the third parameter. Using strtok_r is okay with me. Kevin
Il 20/09/2012 11:34, Kevin Wolf ha scritto: > Am 20.09.2012 11:12, schrieb Bharata B Rao: >> On Thu, Sep 20, 2012 at 10:20:33AM +0200, Paolo Bonzini wrote: >>> Il 20/09/2012 09:53, Paolo Bonzini ha scritto: >>>>>>> Would look a bit nicer with strstart() form cutils.c instead of strncmp(). >>>>>> strstart() works with const char pointers, but I have char pointers here >>>>>> which I need to modify. >>>> You can pass a char* to a function that accepts const char*. In your >>>> case, the last argument to strstart would be NULL. >>> >>> As you pointed out on IRC, you meant the last argument. I don't think >>> it would be a problem to cast that from char ** to const char **. >>> >>> Perhaps it would be cleaner to make qemu_gluster_parseuri and >>> parse_gluster_spec accept a const char *. You can replace strtok_r + >>> g_strdup with strspn/strcspn followed by g_strndup. >> >> I feel the current approach of using the combination of strncmp, strtok_r >> and g_strdup should be good enough. > > Now that Paolo agreed that there wouldn't be a problem with casting, > let's use strstart with a cast to (const char**) for the third > parameter. Using strtok_r is okay with me. Just shooting around a possibility: why reinvent the wheel poorly if we can use a full-blown URI parsing library? The libxml2 one is very good and easy to use. It is also pretty much self-contained and has hardly seen a commit in 3 years, so we can even make it completely self-contained and keep it in tree. See attachment, which also includes some functions taken from libvirt to parse query parameters. (I know this is controversial and would rather add a build dependency, but at the same time it is tempting not to). Paolo ps: to test the attachment: gcc -I/usr/include/libxml2 -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include uri.c -lglib-2.0 && ./a.out
On Thu, Sep 20, 2012 at 05:08:56PM +0200, Paolo Bonzini wrote: > > Just shooting around a possibility: why reinvent the wheel poorly if we > can use a full-blown URI parsing library? The libxml2 one is very good > and easy to use. > > It is also pretty much self-contained and has hardly seen a commit in 3 > years, so we can even make it completely self-contained and keep it in > tree. See attachment, which also includes some functions taken from > libvirt to parse query parameters. > > (I know this is controversial and would rather add a build dependency, > but at the same time it is tempting not to). It would be nice to have such URI parsing capability in QEMU. This would make it very easy to parse gluster URI. Regards, Bharata.
Il 21/09/2012 05:50, Bharata B Rao ha scritto: >> > Just shooting around a possibility: why reinvent the wheel poorly if we >> > can use a full-blown URI parsing library? The libxml2 one is very good >> > and easy to use. >> > >> > It is also pretty much self-contained and has hardly seen a commit in 3 >> > years, so we can even make it completely self-contained and keep it in >> > tree. See attachment, which also includes some functions taken from >> > libvirt to parse query parameters. >> > >> > (I know this is controversial and would rather add a build dependency, >> > but at the same time it is tempting not to). > It would be nice to have such URI parsing capability in QEMU. This would > make it very easy to parse gluster URI. Feel free to take the two attached files in v8. Paolo
diff --git a/block/Makefile.objs b/block/Makefile.objs index b5754d3..a1ae67f 100644 --- a/block/Makefile.objs +++ b/block/Makefile.objs @@ -9,3 +9,4 @@ block-obj-$(CONFIG_POSIX) += raw-posix.o block-obj-$(CONFIG_LIBISCSI) += iscsi.o block-obj-$(CONFIG_CURL) += curl.o block-obj-$(CONFIG_RBD) += rbd.o +block-obj-$(CONFIG_GLUSTERFS) += gluster.o diff --git a/block/gluster.c b/block/gluster.c new file mode 100644 index 0000000..0de3286 --- /dev/null +++ b/block/gluster.c @@ -0,0 +1,694 @@ +/* + * GlusterFS backend for QEMU + * + * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com> + * + * Pipe handling mechanism in AIO implementation is derived from + * block/rbd.c. Hence, + * + * Copyright (C) 2010-2011 Christian Brunner <chb@muc.de>, + * Josh Durgin <josh.durgin@dreamhost.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ +#include <glusterfs/api/glfs.h> +#include "block_int.h" +#include "qemu_socket.h" + +typedef struct GlusterAIOCB { + BlockDriverAIOCB common; + int64_t size; + int ret; + bool *finished; + QEMUBH *bh; +} GlusterAIOCB; + +typedef struct BDRVGlusterState { + struct glfs *glfs; + int fds[2]; + struct glfs_fd *fd; + int qemu_aio_count; + int event_reader_pos; + GlusterAIOCB *event_acb; +} BDRVGlusterState; + +#define GLUSTER_FD_READ 0 +#define GLUSTER_FD_WRITE 1 + +#define GLUSTER_TRANSPORT_DEFAULT "gluster://" +#define GLUSTER_TRANSPORT_DEFAULT_SZ strlen(GLUSTER_TRANSPORT_DEFAULT) +#define GLUSTER_TRANSPORT_TCP "gluster+tcp://" +#define GLUSTER_TRANSPORT_TCP_SZ strlen(GLUSTER_TRANSPORT_TCP) +#define GLUSTER_TRANSPORT_UNIX "gluster+unix://" +#define GLUSTER_TRANSPORT_UNIX_SZ strlen(GLUSTER_TRANSPORT_UNIX) +#define GLUSTER_TRANSPORT_RDMA "gluster+rdma://" +#define GLUSTER_TRANSPORT_RDMA_SZ strlen(GLUSTER_TRANSPORT_RDMA) + +typedef struct GlusterURI { + char *server; + int port; + char *volname; + char *image; + char *transport; + bool is_unix; +} GlusterURI; + +static void qemu_gluster_uri_free(GlusterURI *uri) +{ + g_free(uri->server); + g_free(uri->volname); + g_free(uri->image); + g_free(uri->transport); + g_free(uri); +} + +static int parse_socket(GlusterURI *uri, char *socket) +{ + char *token, *saveptr; + + if (!socket) { + return 0; + } + token = strtok_r(socket, "=", &saveptr); + if (!token || strcmp(token, "socket")) { + return -EINVAL; + } + token = strtok_r(NULL, "=", &saveptr); + if (!token) { + return -EINVAL; + } + uri->server = g_strdup(token); + uri->is_unix = true; + return 0; +} + +static int parse_gluster_spec(GlusterURI *uri, char *spec) +{ + char *token, *saveptr; + int ret; + QemuOpts *opts; + char *p, *q; + + /* transport */ + p = spec; + if (!strncmp(p, GLUSTER_TRANSPORT_DEFAULT, GLUSTER_TRANSPORT_DEFAULT_SZ)) { + uri->transport = g_strdup("tcp"); + p += GLUSTER_TRANSPORT_DEFAULT_SZ; + } else if (!strncmp(p, GLUSTER_TRANSPORT_TCP, GLUSTER_TRANSPORT_TCP_SZ)) { + uri->transport = g_strdup("tcp"); + p += GLUSTER_TRANSPORT_TCP_SZ; + } else if (!strncmp(p, GLUSTER_TRANSPORT_UNIX, GLUSTER_TRANSPORT_UNIX_SZ)) { + uri->transport = g_strdup("unix"); + p += GLUSTER_TRANSPORT_UNIX_SZ; + } else if (!strncmp(p, GLUSTER_TRANSPORT_RDMA, GLUSTER_TRANSPORT_RDMA_SZ)) { + uri->transport = g_strdup("rdma"); + p += GLUSTER_TRANSPORT_RDMA_SZ; + } else { + return -EINVAL; + } + q = p; + + /* server */ + if (!strcmp(uri->transport, "unix")) { + if (!uri->is_unix) { + return -EINVAL; + } + } else { + if (uri->is_unix) { + return -EINVAL; + } + p = strchr(p, '/'); + if (!p) { + return -EINVAL; + } + *p++ = '\0'; + opts = qemu_opts_create(qemu_find_opts("inet"), NULL, 0, NULL); + ret = inet_parse(opts, q); + if (!ret) { + uri->server = g_strdup(qemu_opt_get(opts, "host")); + uri->port = strtoul(qemu_opt_get(opts, "port"), NULL, 0); + if (uri->port < 0) { + ret = -EINVAL; + } + } + qemu_opts_del(opts); + if (ret < 0) { + return -EINVAL; + } + } + + /* volname */ + token = strtok_r(p, "/", &saveptr); + if (!token) { + return -EINVAL; + } + uri->volname = g_strdup(token); + + /* image */ + token = strtok_r(NULL, "?", &saveptr); + if (!token) { + return -EINVAL; + } + uri->image = g_strdup(token); + return 0; +} + +/* + * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...] + * + * 'gluster' is the protocol. + * + * 'transport' specifies the transport type used to connect to gluster + * management daemon (glusterd). Valid transport types are + * tcp, unix and rdma. If the transport type isn't specified, then tcp + * type is assumed. + * + * 'server' specifies the server where the volume file specification for + * the given volume resides. This can be either hostname or ipv4 address + * or ipv6 address. ipv6 address needs to be with in square brackets [ ]. + * If transport type is 'unix', then server field is ignored, but the + * 'socket' field needs to be populated with the path to unix domain + * socket. + * + * 'port' is the port number on which glusterd is listening. This is optional + * and if not specified, QEMU will send 0 which will make gluster to use the + * default port. port is ignored for unix type of transport. + * + * 'volname' is the name of the gluster volume which contains the VM image. + * + * 'image' is the path to the actual VM image that resides on gluster volume. + * + * Examples: + * + * file=gluster://1.2.3.4/testvol/a.img + * file=gluster+tcp://1.2.3.4/testvol/a.img + * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img + * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img + * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img + * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img + * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket + * file=gluster+rdma://1.2.3.4:24007/testvol/a.img + */ +static int qemu_gluster_parseuri(GlusterURI *uri, const char *filename) +{ + char *token, *saveptr; + char *p, *q, *gluster_spec = NULL; + int ret = -EINVAL; + + p = q = g_strdup(filename); + + /* Extract server, volname and image */ + token = strtok_r(p, "?", &saveptr); + if (!token) { + goto out; + } + gluster_spec = g_strdup(token); + + /* socket */ + token = strtok_r(NULL, "?", &saveptr); + ret = parse_socket(uri, token); + if (ret < 0) { + goto out; + } + + /* Flag error for extra options */ + token = strtok_r(NULL, "?", &saveptr); + if (token) { + ret = -EINVAL; + goto out; + } + + ret = parse_gluster_spec(uri, gluster_spec); + if (ret < 0) { + goto out; + } + ret = 0; +out: + g_free(q); + g_free(gluster_spec); + return ret; +} + +static struct glfs *qemu_gluster_init(GlusterURI *uri, const char *filename) +{ + struct glfs *glfs = NULL; + int ret; + + ret = qemu_gluster_parseuri(uri, filename); + if (ret < 0) { + error_report("Usage: file=gluster[+transport]://[server[:port]]/" + "volname/image[?socket=...]"); + errno = -ret; + goto out; + } + + glfs = glfs_new(uri->volname); + if (!glfs) { + goto out; + } + + ret = glfs_set_volfile_server(glfs, uri->transport, uri->server, uri->port); + if (ret < 0) { + goto out; + } + + /* + * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when + * GlusterFS exports it in a header. + */ + ret = glfs_set_logging(glfs, "-", 4); + if (ret < 0) { + goto out; + } + + ret = glfs_init(glfs); + if (ret) { + error_report("Gluster connection failed for server=%s port=%d " + "volume=%s image=%s transport=%s\n", uri->server, uri->port, + uri->volname, uri->image, uri->transport); + goto out; + } + return glfs; + +out: + if (glfs) { + glfs_fini(glfs); + } + return NULL; +} + +static void qemu_gluster_complete_aio(GlusterAIOCB *acb, BDRVGlusterState *s) +{ + int ret; + bool *finished = acb->finished; + BlockDriverCompletionFunc *cb = acb->common.cb; + void *opaque = acb->common.opaque; + + if (!acb->ret || acb->ret == acb->size) { + ret = 0; /* Success */ + } else if (acb->ret < 0) { + ret = acb->ret; /* Read/Write failed */ + } else { + ret = -EIO; /* Partial read/write - fail it */ + } + + s->qemu_aio_count--; + qemu_aio_release(acb); + cb(opaque, ret); + if (finished) { + *finished = true; + } +} + +static void qemu_gluster_aio_event_reader(void *opaque) +{ + BDRVGlusterState *s = opaque; + ssize_t ret; + + do { + char *p = (char *)&s->event_acb; + + ret = read(s->fds[GLUSTER_FD_READ], p + s->event_reader_pos, + sizeof(s->event_acb) - s->event_reader_pos); + if (ret > 0) { + s->event_reader_pos += ret; + if (s->event_reader_pos == sizeof(s->event_acb)) { + s->event_reader_pos = 0; + qemu_gluster_complete_aio(s->event_acb, s); + } + } + } while (ret < 0 && errno == EINTR); +} + +static int qemu_gluster_aio_flush_cb(void *opaque) +{ + BDRVGlusterState *s = opaque; + + return (s->qemu_aio_count > 0); +} + +static int qemu_gluster_open(BlockDriverState *bs, const char *filename, + int bdrv_flags) +{ + BDRVGlusterState *s = bs->opaque; + int open_flags = 0; + int ret = 0; + GlusterURI *uri = g_malloc0(sizeof(GlusterURI)); + + s->glfs = qemu_gluster_init(uri, filename); + if (!s->glfs) { + ret = -errno; + goto out; + } + + open_flags |= O_BINARY; + open_flags &= ~O_ACCMODE; + if (bdrv_flags & BDRV_O_RDWR) { + open_flags |= O_RDWR; + } else { + open_flags |= O_RDONLY; + } + + if ((bdrv_flags & BDRV_O_NOCACHE)) { + open_flags |= O_DIRECT; + } + + s->fd = glfs_open(s->glfs, uri->image, open_flags); + if (!s->fd) { + ret = -errno; + goto out; + } + + ret = qemu_pipe(s->fds); + if (ret < 0) { + ret = -errno; + goto out; + } + fcntl(s->fds[GLUSTER_FD_READ], F_SETFL, O_NONBLOCK); + qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], + qemu_gluster_aio_event_reader, NULL, qemu_gluster_aio_flush_cb, s); + +out: + qemu_gluster_uri_free(uri); + if (!ret) { + return ret; + } + if (s->fd) { + glfs_close(s->fd); + } + if (s->glfs) { + glfs_fini(s->glfs); + } + return ret; +} + +static int qemu_gluster_create(const char *filename, + QEMUOptionParameter *options) +{ + struct glfs *glfs; + struct glfs_fd *fd; + int ret = 0; + int64_t total_size = 0; + GlusterURI *uri = g_malloc0(sizeof(GlusterURI)); + + glfs = qemu_gluster_init(uri, filename); + if (!glfs) { + ret = -errno; + goto out; + } + + while (options && options->name) { + if (!strcmp(options->name, BLOCK_OPT_SIZE)) { + total_size = options->value.n / BDRV_SECTOR_SIZE; + } + options++; + } + + fd = glfs_creat(glfs, uri->image, + O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR); + if (!fd) { + ret = -errno; + } else { + if (glfs_ftruncate(fd, total_size * BDRV_SECTOR_SIZE) != 0) { + ret = -errno; + } + if (glfs_close(fd) != 0) { + ret = -errno; + } + } +out: + qemu_gluster_uri_free(uri); + if (glfs) { + glfs_fini(glfs); + } + return ret; +} + +static void qemu_gluster_aio_cancel(BlockDriverAIOCB *blockacb) +{ + GlusterAIOCB *acb = (GlusterAIOCB *)blockacb; + bool finished = false; + + acb->finished = &finished; + while (!finished) { + qemu_aio_wait(); + } +} + +static AIOPool gluster_aio_pool = { + .aiocb_size = sizeof(GlusterAIOCB), + .cancel = qemu_gluster_aio_cancel, +}; + +static int qemu_gluster_send_pipe(BDRVGlusterState *s, GlusterAIOCB *acb) +{ + int ret = 0; + + while (1) { + int fd = s->fds[GLUSTER_FD_WRITE]; + + ret = write(fd, (void *)&acb, sizeof(acb)); + if (ret >= 0) { + break; + } + if (errno == EINTR) { + continue; + } + if (errno != EAGAIN) { + break; + } + } + return ret; +} + +static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg) +{ + GlusterAIOCB *acb = (GlusterAIOCB *)arg; + BlockDriverState *bs = acb->common.bs; + BDRVGlusterState *s = bs->opaque; + + acb->ret = ret; + if (qemu_gluster_send_pipe(s, acb) < 0) { + /* + * Gluster AIO callback thread failed to notify the waiting + * QEMU thread about IO completion. + * + * Complete this IO request and make the disk inaccessible for + * subsequent reads and writes. + */ + error_report("Gluster failed to notify QEMU about IO completion"); + + qemu_mutex_lock_iothread(); /* We are in gluster thread context */ + acb->common.cb(acb->common.opaque, -EIO); + qemu_aio_release(acb); + s->qemu_aio_count--; + close(s->fds[GLUSTER_FD_READ]); + close(s->fds[GLUSTER_FD_WRITE]); + qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], NULL, NULL, NULL, + NULL); + bs->drv = NULL; /* Make the disk inaccessible */ + qemu_mutex_unlock_iothread(); + } +} + +static BlockDriverAIOCB *qemu_gluster_aio_rw(BlockDriverState *bs, + int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque, int write) +{ + int ret; + GlusterAIOCB *acb; + BDRVGlusterState *s = bs->opaque; + size_t size; + off_t offset; + + offset = sector_num * BDRV_SECTOR_SIZE; + size = nb_sectors * BDRV_SECTOR_SIZE; + s->qemu_aio_count++; + + acb = qemu_aio_get(&gluster_aio_pool, bs, cb, opaque); + acb->size = size; + acb->ret = 0; + acb->finished = NULL; + + if (write) { + ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0, + &gluster_finish_aiocb, acb); + } else { + ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0, + &gluster_finish_aiocb, acb); + } + + if (ret < 0) { + goto out; + } + return &acb->common; + +out: + s->qemu_aio_count--; + qemu_aio_release(acb); + return NULL; +} + +static BlockDriverAIOCB *qemu_gluster_aio_readv(BlockDriverState *bs, + int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + return qemu_gluster_aio_rw(bs, sector_num, qiov, nb_sectors, cb, opaque, 0); +} + +static BlockDriverAIOCB *qemu_gluster_aio_writev(BlockDriverState *bs, + int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + return qemu_gluster_aio_rw(bs, sector_num, qiov, nb_sectors, cb, opaque, 1); +} + +static BlockDriverAIOCB *qemu_gluster_aio_flush(BlockDriverState *bs, + BlockDriverCompletionFunc *cb, void *opaque) +{ + int ret; + GlusterAIOCB *acb; + BDRVGlusterState *s = bs->opaque; + + acb = qemu_aio_get(&gluster_aio_pool, bs, cb, opaque); + acb->size = 0; + acb->ret = 0; + acb->finished = NULL; + s->qemu_aio_count++; + + ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb); + if (ret < 0) { + goto out; + } + return &acb->common; + +out: + s->qemu_aio_count--; + qemu_aio_release(acb); + return NULL; +} + +static int64_t qemu_gluster_getlength(BlockDriverState *bs) +{ + BDRVGlusterState *s = bs->opaque; + int64_t ret; + + ret = glfs_lseek(s->fd, 0, SEEK_END); + if (ret < 0) { + return -errno; + } else { + return ret; + } +} + +static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs) +{ + BDRVGlusterState *s = bs->opaque; + struct stat st; + int ret; + + ret = glfs_fstat(s->fd, &st); + if (ret < 0) { + return -errno; + } else { + return st.st_blocks * 512; + } +} + +static void qemu_gluster_close(BlockDriverState *bs) +{ + BDRVGlusterState *s = bs->opaque; + + close(s->fds[GLUSTER_FD_READ]); + close(s->fds[GLUSTER_FD_WRITE]); + qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], NULL, NULL, NULL, NULL); + + if (s->fd) { + glfs_close(s->fd); + s->fd = NULL; + } + glfs_fini(s->glfs); +} + +static QEMUOptionParameter qemu_gluster_create_options[] = { + { + .name = BLOCK_OPT_SIZE, + .type = OPT_SIZE, + .help = "Virtual disk size" + }, + { NULL } +}; + +static BlockDriver bdrv_gluster = { + .format_name = "gluster", + .protocol_name = "gluster", + .instance_size = sizeof(BDRVGlusterState), + .bdrv_file_open = qemu_gluster_open, + .bdrv_close = qemu_gluster_close, + .bdrv_create = qemu_gluster_create, + .bdrv_getlength = qemu_gluster_getlength, + .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size, + .bdrv_aio_readv = qemu_gluster_aio_readv, + .bdrv_aio_writev = qemu_gluster_aio_writev, + .bdrv_aio_flush = qemu_gluster_aio_flush, + .create_options = qemu_gluster_create_options, +}; + +static BlockDriver bdrv_gluster_tcp = { + .format_name = "gluster", + .protocol_name = "gluster+tcp", + .instance_size = sizeof(BDRVGlusterState), + .bdrv_file_open = qemu_gluster_open, + .bdrv_close = qemu_gluster_close, + .bdrv_create = qemu_gluster_create, + .bdrv_getlength = qemu_gluster_getlength, + .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size, + .bdrv_aio_readv = qemu_gluster_aio_readv, + .bdrv_aio_writev = qemu_gluster_aio_writev, + .bdrv_aio_flush = qemu_gluster_aio_flush, + .create_options = qemu_gluster_create_options, +}; + +static BlockDriver bdrv_gluster_unix = { + .format_name = "gluster", + .protocol_name = "gluster+unix", + .instance_size = sizeof(BDRVGlusterState), + .bdrv_file_open = qemu_gluster_open, + .bdrv_close = qemu_gluster_close, + .bdrv_create = qemu_gluster_create, + .bdrv_getlength = qemu_gluster_getlength, + .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size, + .bdrv_aio_readv = qemu_gluster_aio_readv, + .bdrv_aio_writev = qemu_gluster_aio_writev, + .bdrv_aio_flush = qemu_gluster_aio_flush, + .create_options = qemu_gluster_create_options, +}; + +static BlockDriver bdrv_gluster_rdma = { + .format_name = "gluster", + .protocol_name = "gluster+rdma", + .instance_size = sizeof(BDRVGlusterState), + .bdrv_file_open = qemu_gluster_open, + .bdrv_close = qemu_gluster_close, + .bdrv_create = qemu_gluster_create, + .bdrv_getlength = qemu_gluster_getlength, + .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size, + .bdrv_aio_readv = qemu_gluster_aio_readv, + .bdrv_aio_writev = qemu_gluster_aio_writev, + .bdrv_aio_flush = qemu_gluster_aio_flush, + .create_options = qemu_gluster_create_options, +}; + +static void bdrv_gluster_init(void) +{ + bdrv_register(&bdrv_gluster_rdma); + bdrv_register(&bdrv_gluster_unix); + bdrv_register(&bdrv_gluster_tcp); + bdrv_register(&bdrv_gluster); +} + +block_init(bdrv_gluster_init);