hw/block/nvme: Add doorbell buffer config support

This patch adds Doorbell Buffer Config support (NVMe 1.3) to QEMU NVMe,
based on Mihai Rusu / Lin Ming's Google vendor extension patch [1]. The
basic idea of this optimization is to use a shared buffer between guest
OS and QEMU to reduce # of MMIO operations (doorbell writes). This patch
ports the original code to work under current QEMU and make it also
work with SPDK.

Unlike Linux kernel NVMe driver which builds the shadow buffer first and
then creates SQ/CQ, SPDK first creates SQ/CQ and then issues this command
to create shadow buffer. Thus, in this implementation, we also try to
associate shadow buffer entry with each SQ/CQ during queue initialization.

[1] http://lists.nongnu.org/archive/html/qemu-devel/2015-11/msg04127.html

Peroformance results using a **ramdisk** backed virtual NVMe device in guest
Linux 4.14 is as below:

Note: "QEMU" represent stock QEMU and "+dbbuf" is QEMU with this patch.
For psync, QD represents # of threads being used.


IOPS (Linux kernel NVMe driver)
       psync                 libaio
QD QEMU  +dbbuf  QEMU +dbbuf
1      47k      50k      45k    47k
4      86k      107k     59k   143k
16    95k      198k     58k   185k
64    97k      259k     59k   216k


 IOPS (SPDK)
QD  QEMU  +dbbuf
1      62k     71k
4      61k     191k
16    60k     319k
64    62k     364k

We can see that this patch can greatly increase the IOPS (and lower the
latency, not shown) (2.7x for psync, 3.7x for libaio and 5.9x for SPDK).

==Setup==:

(1) VM script:
x86_64-softmmu/qemu-system-x86_64 \
-name "nvme-FEMU-test" \
-enable-kvm \
-cpu host \
-smp 4 \
-m 8G \
-drive file=$IMGDIR/u14s.qcow2,if=ide,aio=native,cache=none,format=qcow2,id=hd0 \
-drive file=/mnt/tmpfs/test1.raw,if=none,aio=threads,format=raw,id=id0 \
-device nvme,drive=id0,serial=serial0,id=nvme0 \
-net user,hostfwd=tcp::8080-:22 \
-net nic,model=virtio \
-nographic \

(2) FIO configuration:

[global]
ioengine=libaio
filename=/dev/nvme0n1
thread=1
group_reporting=1
direct=1
verify=0
time_based=1
ramp_time=0
runtime=30
;size=1G
iodepth=16
rw=randread
bs=4k

[test]
numjobs=1


Signed-off-by: Huaicheng Li <huaicheng@cs.uchicago.edu>
---
 hw/block/nvme.c      | 97 +++++++++++++++++++++++++++++++++++++++++++++++++---
 hw/block/nvme.h      |  7 ++++
 include/block/nvme.h |  2 ++
 3 files changed, 102 insertions(+), 4 deletions(-)

Ping... Keith, can you review this?

Thanks,

Paolo

On 05/03/2018 20:49, Huaicheng Li wrote:
> This patch adds Doorbell Buffer Config support (NVMe 1.3) to QEMU NVMe,
> based on Mihai Rusu / Lin Ming's Google vendor extension patch [1]. The
> basic idea of this optimization is to use a shared buffer between guest
> OS and QEMU to reduce # of MMIO operations (doorbell writes). This patch
> ports the original code to work under current QEMU and make it also
> work with SPDK.
> 
> Unlike Linux kernel NVMe driver which builds the shadow buffer first and
> then creates SQ/CQ, SPDK first creates SQ/CQ and then issues this command
> to create shadow buffer. Thus, in this implementation, we also try to
> associate shadow buffer entry with each SQ/CQ during queue initialization.
> 
> [1] http://lists.nongnu.org/archive/html/qemu-devel/2015-11/msg04127.html
> 
> Peroformance results using a **ramdisk** backed virtual NVMe device in
> guest
> Linux 4.14 is as below:
> 
> Note: "QEMU" represent stock QEMU and "+dbbuf" is QEMU with this patch.
> For psync, QD represents # of threads being used.
> 
> 
> IOPS (Linux kernel NVMe driver)
>       psync                 libaio
> QD QEMU  +dbbuf  QEMU +dbbuf
> 1      47k      50k      45k    47k
> 4      86k      107k     59k   143k
> 16    95k      198k     58k   185k
> 64    97k      259k     59k   216k
> 
> 
> IOPS (SPDK)
> QD  QEMU  +dbbuf
> 1      62k     71k
> 4      61k     191k
> 16    60k     319k
> 64    62k     364k
> 
> We can see that this patch can greatly increase the IOPS (and lower the
> latency, not shown) (2.7x for psync, 3.7x for libaio and 5.9x for SPDK).
> 
> ==Setup==:
> 
> (1) VM script:
> x86_64-softmmu/qemu-system-x86_64 \
> -name "nvme-FEMU-test" \
> -enable-kvm \
> -cpu host \
> -smp 4 \
> -m 8G \
> -drive
> file=$IMGDIR/u14s.qcow2,if=ide,aio=native,cache=none,format=qcow2,id=hd0 \
> -drive file=/mnt/tmpfs/test1.raw,if=none,aio=threads,format=raw,id=id0 \
> -device nvme,drive=id0,serial=serial0,id=nvme0 \
> -net user,hostfwd=tcp::8080-:22 \
> -net nic,model=virtio \
> -nographic \
> 
> (2) FIO configuration:
> 
> [global]
> ioengine=libaio
> filename=/dev/nvme0n1
> thread=1
> group_reporting=1
> direct=1
> verify=0
> time_based=1
> ramp_time=0
> runtime=30
> ;size=1G
> iodepth=16
> rw=randread
> bs=4k
> 
> [test]
> numjobs=1
> 
> 
> Signed-off-by: Huaicheng Li <huaicheng@cs.uchicago.edu>
> ---
> hw/block/nvme.c      | 97
> +++++++++++++++++++++++++++++++++++++++++++++++++---
> hw/block/nvme.h      |  7 ++++
> include/block/nvme.h |  2 ++
> 3 files changed, 102 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/block/nvme.c b/hw/block/nvme.c
> index 85d2406400..3882037e36 100644
> --- a/hw/block/nvme.c
> +++ b/hw/block/nvme.c
> @@ -9,7 +9,7 @@
>  */
> 
> /**
> - * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
> + * Reference Specs: http://www.nvmexpress.org, 1.3, 1.2, 1.1, 1.0e
>  *
>  *  http://www.nvmexpress.org/resources/
>  */
> @@ -33,6 +33,7 @@
> #include "qapi/error.h"
> #include "qapi/visitor.h"
> #include "sysemu/block-backend.h"
> +#include "exec/memory.h"
> 
> #include "qemu/log.h"
> #include "trace.h"
> @@ -244,6 +245,14 @@ static uint16_t nvme_dma_read_prp(NvmeCtrl *n,
> uint8_t *ptr, uint32_t len,
>     return status;
> }
> 
> +static void nvme_update_cq_head(NvmeCQueue *cq)
> +{
> +    if (cq->db_addr) {
> +        pci_dma_read(&cq->ctrl->parent_obj, cq->db_addr, &cq->head,
> +                sizeof(cq->head));
> +    }
> +}
> +
> static void nvme_post_cqes(void *opaque)
> {
>     NvmeCQueue *cq = opaque;
> @@ -254,6 +263,8 @@ static void nvme_post_cqes(void *opaque)
>         NvmeSQueue *sq;
>         hwaddr addr;
> 
> +        nvme_update_cq_head(cq);
> +
>         if (nvme_cq_full(cq)) {
>             break;
>         }
> @@ -461,6 +472,7 @@ static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
> static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
>     uint16_t sqid, uint16_t cqid, uint16_t size)
> {
> +    uint32_t stride = 4 << NVME_CAP_DSTRD(n->bar.cap);
>     int i;
>     NvmeCQueue *cq;
> 
> @@ -480,6 +492,11 @@ static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl
> *n, uint64_t dma_addr,
>     }
>     sq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_process_sq, sq);
> 
> +    if (sqid && n->dbbuf_dbs && n->dbbuf_eis) {
> +        sq->db_addr = n->dbbuf_dbs + 2 * sqid * stride;
> +        sq->ei_addr = n->dbbuf_eis + 2 * sqid * stride;
> +    }
> +
>     assert(n->cq[cqid]);
>     cq = n->cq[cqid];
>     QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
> @@ -559,6 +576,8 @@ static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
> static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
>     uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled)
> {
> +    uint32_t stride = 4 << NVME_CAP_DSTRD(n->bar.cap);
> +
>     cq->ctrl = n;
>     cq->cqid = cqid;
>     cq->size = size;
> @@ -569,11 +588,51 @@ static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl
> *n, uint64_t dma_addr,
>     cq->head = cq->tail = 0;
>     QTAILQ_INIT(&cq->req_list);
>     QTAILQ_INIT(&cq->sq_list);
> +    if (cqid && n->dbbuf_dbs && n->dbbuf_eis) {
> +        cq->db_addr = n->dbbuf_dbs + (2 * cqid + 1) * stride;
> +        cq->ei_addr = n->dbbuf_eis + (2 * cqid + 1) * stride;
> +    }
>     msix_vector_use(&n->parent_obj, cq->vector);
>     n->cq[cqid] = cq;
>     cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
> }
> 
> +static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeCmd *cmd)
> +{
> +    uint32_t stride = 4 << NVME_CAP_DSTRD(n->bar.cap);
> +    uint64_t dbs_addr = le64_to_cpu(cmd->prp1);
> +    uint64_t eis_addr = le64_to_cpu(cmd->prp2);
> +    int i;
> +
> +    /* Address should not be NULL and should be page aligned */
> +    if (dbs_addr == 0 || dbs_addr & (n->page_size - 1) ||
> +            eis_addr == 0 || eis_addr & (n->page_size - 1)) {
> +        return NVME_INVALID_FIELD | NVME_DNR;
> +    }
> +
> +    /* Save shadow buffer base addr for use during queue creation */
> +    n->dbbuf_dbs = dbs_addr;
> +    n->dbbuf_eis = eis_addr;
> +
> +    for (i = 1; i < n->num_queues; i++) {
> +        NvmeSQueue *sq = n->sq[i];
> +        NvmeCQueue *cq = n->cq[i];
> +
> +        if (sq) {
> +            /* Submission queue tail pointer location, 2 * QID * stride */
> +            sq->db_addr = dbs_addr + 2 * i * stride;
> +            sq->ei_addr = eis_addr + 2 * i * stride;
> +        }
> +
> +        if (cq) {
> +            /* Completion queue head pointer location, (2 * QID + 1) *
> stride */
> +            cq->db_addr = dbs_addr + (2 * i + 1) * stride;
> +            cq->ei_addr = eis_addr + (2 * i + 1) * stride;
> +        }
> +    }
> +    return NVME_SUCCESS;
> +}
> +
> static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
> {
>     NvmeCQueue *cq;
> @@ -753,12 +812,30 @@ static uint16_t nvme_admin_cmd(NvmeCtrl *n,
> NvmeCmd *cmd, NvmeRequest *req)
>         return nvme_set_feature(n, cmd, req);
>     case NVME_ADM_CMD_GET_FEATURES:
>         return nvme_get_feature(n, cmd, req);
> +    case NVME_ADM_CMD_DBBUF_CONFIG:
> +        return nvme_dbbuf_config(n, cmd);
>     default:
>         trace_nvme_err_invalid_admin_opc(cmd->opcode);
>         return NVME_INVALID_OPCODE | NVME_DNR;
>     }
> }
> 
> +static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
> +{
> +    if (sq->ei_addr) {
> +        pci_dma_write(&sq->ctrl->parent_obj, sq->ei_addr, &sq->tail,
> +                sizeof(sq->tail));
> +    }
> +}
> +
> +static void nvme_update_sq_tail(NvmeSQueue *sq)
> +{
> +    if (sq->db_addr) {
> +        pci_dma_read(&sq->ctrl->parent_obj, sq->db_addr, &sq->tail,
> +                sizeof(sq->tail));
> +    }
> +}
> +
> static void nvme_process_sq(void *opaque)
> {
>     NvmeSQueue *sq = opaque;
> @@ -770,6 +847,8 @@ static void nvme_process_sq(void *opaque)
>     NvmeCmd cmd;
>     NvmeRequest *req;
> 
> +    nvme_update_sq_tail(sq);
> +
>     while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
>         addr = sq->dma_addr + sq->head * n->sqe_size;
>         nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd));
> @@ -787,6 +866,9 @@ static void nvme_process_sq(void *opaque)
>             req->status = status;
>             nvme_enqueue_req_completion(cq, req);
>         }
> +
> +        nvme_update_sq_eventidx(sq);
> +        nvme_update_sq_tail(sq);
>     }
> }
> 
> @@ -1105,7 +1187,9 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr
> addr, int val)
>         }
> 
>         start_sqs = nvme_cq_full(cq) ? 1 : 0;
> -        cq->head = new_head;
> +        if (!cq->db_addr) {
> +            cq->head = new_head;
> +        }
>         if (start_sqs) {
>             NvmeSQueue *sq;
>             QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
> @@ -1142,7 +1226,9 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr
> addr, int val)
>             return;
>         }
> 
> -        sq->tail = new_tail;
> +        if (!sq->db_addr) {
> +            sq->tail = new_tail;
> +        }
>         timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
>     }
> }
> @@ -1256,7 +1342,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error
> **errp)
>     id->ieee[0] = 0x00;
>     id->ieee[1] = 0x02;
>     id->ieee[2] = 0xb3;
> -    id->oacs = cpu_to_le16(0);
> +    id->oacs = cpu_to_le16(NVME_OACS_DBBUF);
>     id->frmw = 7 << 1;
>     id->lpa = 1 << 0;
>     id->sqes = (0x6 << 4) | 0x6;
> @@ -1320,6 +1406,9 @@ static void nvme_realize(PCIDevice *pci_dev, Error
> **errp)
>             cpu_to_le64(n->ns_size >>
>                 id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
>     }
> +
> +    n->dbbuf_dbs = 0;
> +    n->dbbuf_eis = 0;
> }
> 
> static void nvme_exit(PCIDevice *pci_dev)
> diff --git a/hw/block/nvme.h b/hw/block/nvme.h
> index 8f3981121d..b532dbe160 100644
> --- a/hw/block/nvme.h
> +++ b/hw/block/nvme.h
> @@ -33,6 +33,8 @@ typedef struct NvmeSQueue {
>     QTAILQ_HEAD(sq_req_list, NvmeRequest) req_list;
>     QTAILQ_HEAD(out_req_list, NvmeRequest) out_req_list;
>     QTAILQ_ENTRY(NvmeSQueue) entry;
> +    uint64_t    db_addr;
> +    uint64_t    ei_addr;
> } NvmeSQueue;
> 
> typedef struct NvmeCQueue {
> @@ -48,6 +50,8 @@ typedef struct NvmeCQueue {
>     QEMUTimer   *timer;
>     QTAILQ_HEAD(sq_list, NvmeSQueue) sq_list;
>     QTAILQ_HEAD(cq_req_list, NvmeRequest) req_list;
> +    uint64_t    db_addr;
> +    uint64_t    ei_addr;
> } NvmeCQueue;
> 
> typedef struct NvmeNamespace {
> @@ -88,6 +92,9 @@ typedef struct NvmeCtrl {
>     NvmeSQueue      admin_sq;
>     NvmeCQueue      admin_cq;
>     NvmeIdCtrl      id_ctrl;
> +
> +    uint64_t        dbbuf_dbs;
> +    uint64_t        dbbuf_eis;
> } NvmeCtrl;
> 
> #endif /* HW_NVME_H */
> diff --git a/include/block/nvme.h b/include/block/nvme.h
> index 849a6f3fa3..4890aaf491 100644
> --- a/include/block/nvme.h
> +++ b/include/block/nvme.h
> @@ -235,6 +235,7 @@ enum NvmeAdminCommands {
>     NVME_ADM_CMD_ASYNC_EV_REQ   = 0x0c,
>     NVME_ADM_CMD_ACTIVATE_FW    = 0x10,
>     NVME_ADM_CMD_DOWNLOAD_FW    = 0x11,
> +    NVME_ADM_CMD_DBBUF_CONFIG   = 0x7c,
>     NVME_ADM_CMD_FORMAT_NVM     = 0x80,
>     NVME_ADM_CMD_SECURITY_SEND  = 0x81,
>     NVME_ADM_CMD_SECURITY_RECV  = 0x82,
> @@ -572,6 +573,7 @@ enum NvmeIdCtrlOacs {
>     NVME_OACS_SECURITY  = 1 << 0,
>     NVME_OACS_FORMAT    = 1 << 1,
>     NVME_OACS_FW        = 1 << 2,
> +    NVME_OACS_DBBUF     = 1 << 8,
> };
> 
> enum NvmeIdCtrlOncs {