@@ -1,7 +1,9 @@
#include "qemu/osdep.h"
#include <blkio.h>
#include "block/block_int.h"
+#include "exec/memory.h"
#include "qapi/error.h"
+#include "qemu/error-report.h"
#include "qapi/qmp/qdict.h"
#include "qemu/module.h"
@@ -25,6 +27,9 @@ typedef struct {
/* Can we skip adding/deleting blkio_mem_regions? */
bool needs_mem_regions;
+
+ /* Are file descriptors necessary for blkio_mem_regions? */
+ bool needs_mem_region_fd;
} BDRVBlkioState;
static void blkio_aiocb_complete(BlkioAIOCB *acb, int ret)
@@ -157,6 +162,8 @@ static BlockAIOCB *blkio_aio_preadv(BlockDriverState *bs, int64_t offset,
BlockCompletionFunc *cb, void *opaque)
{
BDRVBlkioState *s = bs->opaque;
+ bool needs_mem_regions =
+ s->needs_mem_regions && !(flags & BDRV_REQ_REGISTERED_BUF);
struct iovec *iov = qiov->iov;
int iovcnt = qiov->niov;
BlkioAIOCB *acb;
@@ -166,7 +173,7 @@ static BlockAIOCB *blkio_aio_preadv(BlockDriverState *bs, int64_t offset,
acb = blkio_aiocb_get(bs, cb, opaque);
- if (s->needs_mem_regions) {
+ if (needs_mem_regions) {
if (blkio_aiocb_init_mem_region_locked(acb, bytes) < 0) {
qemu_aio_unref(&acb->common);
return NULL;
@@ -181,7 +188,7 @@ static BlockAIOCB *blkio_aio_preadv(BlockDriverState *bs, int64_t offset,
ret = blkioq_readv(s->blkioq, offset, iov, iovcnt, acb, 0);
if (ret < 0) {
- if (s->needs_mem_regions) {
+ if (needs_mem_regions) {
blkio_free_mem_region(s->blkio, &acb->mem_region);
qemu_iovec_destroy(&acb->qiov);
}
@@ -202,6 +209,8 @@ static BlockAIOCB *blkio_aio_pwritev(BlockDriverState *bs, int64_t offset,
{
uint32_t blkio_flags = (flags & BDRV_REQ_FUA) ? BLKIO_REQ_FUA : 0;
BDRVBlkioState *s = bs->opaque;
+ bool needs_mem_regions =
+ s->needs_mem_regions && !(flags & BDRV_REQ_REGISTERED_BUF);
struct iovec *iov = qiov->iov;
int iovcnt = qiov->niov;
BlkioAIOCB *acb;
@@ -211,7 +220,7 @@ static BlockAIOCB *blkio_aio_pwritev(BlockDriverState *bs, int64_t offset,
acb = blkio_aiocb_get(bs, cb, opaque);
- if (s->needs_mem_regions) {
+ if (needs_mem_regions) {
if (blkio_aiocb_init_mem_region_locked(acb, bytes) < 0) {
qemu_aio_unref(&acb->common);
return NULL;
@@ -225,7 +234,7 @@ static BlockAIOCB *blkio_aio_pwritev(BlockDriverState *bs, int64_t offset,
ret = blkioq_writev(s->blkioq, offset, iov, iovcnt, acb, blkio_flags);
if (ret < 0) {
- if (s->needs_mem_regions) {
+ if (needs_mem_regions) {
blkio_free_mem_region(s->blkio, &acb->mem_region);
}
qemu_aio_unref(&acb->common);
@@ -273,6 +282,80 @@ static void blkio_io_unplug(BlockDriverState *bs)
}
}
+static void blkio_register_buf(BlockDriverState *bs, void *host, size_t size)
+{
+ BDRVBlkioState *s = bs->opaque;
+ int ret;
+ struct blkio_mem_region region = (struct blkio_mem_region){
+ .addr = host,
+ .len = size,
+ .fd = -1,
+ };
+
+ if (((uintptr_t)host | size) % s->mem_region_alignment) {
+ error_report_once("%s: skipping unaligned buf %p with size %zu",
+ __func__, host, size);
+ return; /* skip unaligned */
+ }
+
+ /* Attempt to find the fd for a MemoryRegion */
+ if (s->needs_mem_region_fd) {
+ int fd = -1;
+ ram_addr_t offset;
+ MemoryRegion *mr;
+
+ /*
+ * bdrv_register_buf() is called with the BQL held so mr lives at least
+ * until this function returns.
+ */
+ mr = memory_region_from_host(host, &offset);
+ if (mr) {
+ fd = memory_region_get_fd(mr);
+ }
+ if (fd == -1) {
+ error_report_once("%s: skipping fd-less buf %p with size %zu",
+ __func__, host, size);
+ return; /* skip if there is no fd */
+ }
+
+ region.fd = fd;
+ region.fd_offset = offset;
+ }
+
+ WITH_QEMU_LOCK_GUARD(&s->lock) {
+ ret = blkio_add_mem_region(s->blkio, ®ion);
+ }
+
+ if (ret < 0) {
+ error_report_once("Failed to add blkio mem region %p with size %zu: %s",
+ host, size, blkio_get_error_msg());
+ }
+}
+
+static void blkio_unregister_buf(BlockDriverState *bs, void *host, size_t size)
+{
+ BDRVBlkioState *s = bs->opaque;
+ int ret;
+ struct blkio_mem_region region = (struct blkio_mem_region){
+ .addr = host,
+ .len = size,
+ .fd = -1,
+ };
+
+ if (((uintptr_t)host | size) % s->mem_region_alignment) {
+ return; /* skip unaligned */
+ }
+
+ WITH_QEMU_LOCK_GUARD(&s->lock) {
+ ret = blkio_del_mem_region(s->blkio, ®ion);
+ }
+
+ if (ret < 0) {
+ error_report_once("Failed to delete blkio mem region %p with size %zu: %s",
+ host, size, blkio_get_error_msg());
+ }
+}
+
static void blkio_parse_filename_io_uring(const char *filename, QDict *options,
Error **errp)
{
@@ -341,6 +424,17 @@ static int blkio_file_open(BlockDriverState *bs, QDict *options, int flags,
return ret;
}
+ ret = blkio_get_bool(s->blkio,
+ "needs-mem-region-fd",
+ &s->needs_mem_region_fd);
+ if (ret < 0) {
+ error_setg_errno(errp, -ret,
+ "failed to get needs-mem-region-fd: %s",
+ blkio_get_error_msg());
+ blkio_destroy(&s->blkio);
+ return ret;
+ }
+
ret = blkio_get_uint64(s->blkio,
"mem-region-alignment",
&s->mem_region_alignment);
@@ -360,7 +454,7 @@ static int blkio_file_open(BlockDriverState *bs, QDict *options, int flags,
return ret;
}
- bs->supported_write_flags = BDRV_REQ_FUA;
+ bs->supported_write_flags = BDRV_REQ_FUA | BDRV_REQ_REGISTERED_BUF;
qemu_mutex_init(&s->lock);
s->blkioq = blkio_get_queue(s->blkio, 0);
@@ -514,6 +608,8 @@ static BlockDriver bdrv_io_uring = {
.bdrv_aio_flush = blkio_aio_flush,
.bdrv_io_unplug = blkio_io_unplug,
.bdrv_refresh_limits = blkio_refresh_limits,
+ .bdrv_register_buf = blkio_register_buf,
+ .bdrv_unregister_buf = blkio_unregister_buf,
/*
* TODO
Avoid bounce buffers when QEMUIOVector elements are within previously registered bdrv_register_buf() buffers. The idea is that emulated storage controllers will register guest RAM using bdrv_register_buf() and set the BDRV_REQ_REGISTERED_BUF on I/O requests. Therefore no blkio_add_mem_region() calls are necessary in the performance-critical I/O code path. This optimization doesn't apply if the I/O buffer is internally allocated by QEMU (e.g. qcow2 metadata). There we still take the slow path because BDRV_REQ_REGISTERED_BUF is not set. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- block/blkio.c | 106 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 5 deletions(-)