Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/925070/?format=api
{ "id": 925070, "url": "http://patchwork.ozlabs.org/api/patches/925070/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20180604120601.18123-6-bjorn.topel@gmail.com/", "project": { "id": 46, "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api", "name": "Intel Wired Ethernet development", "link_name": "intel-wired-lan", "list_id": "intel-wired-lan.osuosl.org", "list_email": "intel-wired-lan@osuosl.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20180604120601.18123-6-bjorn.topel@gmail.com>", "list_archive_url": null, "date": "2018-06-04T12:05:55", "name": "[bpf-next,05/11] xsk: add zero-copy support for Rx", "commit_ref": null, "pull_url": null, "state": "awaiting-upstream", "archived": false, "hash": "cffe31c17a786a6a2bd8c3ea3581a05c200f0324", "submitter": { "id": 70569, "url": "http://patchwork.ozlabs.org/api/people/70569/?format=api", "name": "Björn Töpel", "email": "bjorn.topel@gmail.com" }, "delegate": null, "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20180604120601.18123-6-bjorn.topel@gmail.com/mbox/", "series": [ { "id": 48416, "url": "http://patchwork.ozlabs.org/api/series/48416/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/list/?series=48416", "date": "2018-06-04T12:05:50", "name": "AF_XDP: introducing zero-copy support", "version": 1, "mbox": "http://patchwork.ozlabs.org/series/48416/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/925070/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/925070/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<intel-wired-lan-bounces@osuosl.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Authentication-Results": [ "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=osuosl.org\n\t(client-ip=140.211.166.136; helo=silver.osuosl.org;\n\tenvelope-from=intel-wired-lan-bounces@osuosl.org;\n\treceiver=<UNKNOWN>)", "ozlabs.org;\n\tdmarc=fail (p=none dis=none) header.from=gmail.com" ], "Received": [ "from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136])\n\t(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 40zyrz5K5Xz9s08\n\tfor <incoming@patchwork.ozlabs.org>;\n\tTue, 5 Jun 2018 01:04:35 +1000 (AEST)", "from localhost (localhost [127.0.0.1])\n\tby silver.osuosl.org (Postfix) with ESMTP id 27C422DF13;\n\tMon, 4 Jun 2018 15:04:34 +0000 (UTC)", "from silver.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id qPpi1Y2VqMeQ; Mon, 4 Jun 2018 15:04:29 +0000 (UTC)", "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby silver.osuosl.org (Postfix) with ESMTP id 938F62DD8E;\n\tMon, 4 Jun 2018 15:04:28 +0000 (UTC)", "from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136])\n\tby ash.osuosl.org (Postfix) with ESMTP id 9881F1BFFD0\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:06:41 +0000 (UTC)", "from localhost (localhost [127.0.0.1])\n\tby silver.osuosl.org (Postfix) with ESMTP id 93A8C2CC8B\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:06:41 +0000 (UTC)", "from silver.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id UgAtH4yVGteD for <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:06:40 +0000 (UTC)", "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n\tby silver.osuosl.org (Postfix) with ESMTPS id A9B4C26E12\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:06:40 +0000 (UTC)", "from fmsmga004.fm.intel.com ([10.253.24.48])\n\tby orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t04 Jun 2018 05:06:40 -0700", "from btopel-mobl1.isw.intel.com (HELO\n\tbtopel-mobl1.hil-pdxphhh.sea.wayport.net) ([10.103.211.148])\n\tby fmsmga004.fm.intel.com with ESMTP; 04 Jun 2018 05:06:34 -0700" ], "X-Virus-Scanned": [ "amavisd-new at osuosl.org", "amavisd-new at osuosl.org" ], "X-Greylist": "domain auto-whitelisted by SQLgrey-1.7.6", "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.49,476,1520924400\"; d=\"scan'208\";a=\"60197192\"", "From": "=?utf-8?b?QmrDtnJuIFTDtnBlbA==?= <bjorn.topel@gmail.com>", "To": "bjorn.topel@gmail.com, magnus.karlsson@intel.com,\n\tmagnus.karlsson@gmail.com, alexander.h.duyck@intel.com,\n\talexander.duyck@gmail.com, ast@fb.com, brouer@redhat.com,\n\tdaniel@iogearbox.net, netdev@vger.kernel.org, mykyta.iziumtsev@linaro.org", "Date": "Mon, 4 Jun 2018 14:05:55 +0200", "Message-Id": "<20180604120601.18123-6-bjorn.topel@gmail.com>", "X-Mailer": "git-send-email 2.14.1", "In-Reply-To": "<20180604120601.18123-1-bjorn.topel@gmail.com>", "References": "<20180604120601.18123-1-bjorn.topel@gmail.com>", "MIME-Version": "1.0", "X-Mailman-Approved-At": "Mon, 04 Jun 2018 15:04:25 +0000", "Subject": "[Intel-wired-lan] [PATCH bpf-next 05/11] xsk: add zero-copy support\n\tfor Rx", "X-BeenThere": "intel-wired-lan@osuosl.org", "X-Mailman-Version": "2.1.24", "Precedence": "list", "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.osuosl.org>", "List-Unsubscribe": "<https://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=unsubscribe>", "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>", "List-Post": "<mailto:intel-wired-lan@osuosl.org>", "List-Help": "<mailto:intel-wired-lan-request@osuosl.org?subject=help>", "List-Subscribe": "<https://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=subscribe>", "Cc": "francois.ozog@linaro.org, willemdebruijn.kernel@gmail.com, mst@redhat.com,\n\tilias.apalodimas@linaro.org, michael.lundkvist@ericsson.com,\n\tbrian.brooks@linaro.org, intel-wired-lan@lists.osuosl.org, \n\tqi.z.zhang@intel.com, michael.chan@broadcom.com, =?utf-8?b?QmrDtnJu?=\n\t=?utf-8?q?_T=C3=B6pel?= <bjorn.topel@intel.com>, andy@greyhouse.net", "Content-Type": "text/plain; charset=\"utf-8\"", "Content-Transfer-Encoding": "base64", "Errors-To": "intel-wired-lan-bounces@osuosl.org", "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@osuosl.org>" }, "content": "From: Björn Töpel <bjorn.topel@intel.com>\n\nExtend the xsk_rcv to support the new MEM_TYPE_ZERO_COPY memory, and\nwireup ndo_bpf call in bind.\n\nSigned-off-by: Björn Töpel <bjorn.topel@intel.com>\n---\n include/net/xdp_sock.h | 6 +++\n include/uapi/linux/if_xdp.h | 4 +-\n net/xdp/xdp_umem.c | 77 ++++++++++++++++++++++++++++++++++++\n net/xdp/xdp_umem.h | 3 ++\n net/xdp/xsk.c | 96 +++++++++++++++++++++++++++++++++++----------\n 5 files changed, 165 insertions(+), 21 deletions(-)", "diff": "diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h\nindex caf343a7e224..d93d3aac3fc9 100644\n--- a/include/net/xdp_sock.h\n+++ b/include/net/xdp_sock.h\n@@ -22,6 +22,7 @@ struct xdp_umem_props {\n \n struct xdp_umem_page {\n \tvoid *addr;\n+\tdma_addr_t dma;\n };\n \n struct xdp_umem {\n@@ -38,6 +39,9 @@ struct xdp_umem {\n \tstruct work_struct work;\n \tstruct page **pgs;\n \tu32 npgs;\n+\tstruct net_device *dev;\n+\tu16 queue_id;\n+\tbool zc;\n };\n \n struct xdp_sock {\n@@ -60,6 +64,8 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);\n int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);\n void xsk_flush(struct xdp_sock *xs);\n bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs);\n+u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr);\n+void xsk_umem_discard_addr(struct xdp_umem *umem);\n #else\n static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)\n {\ndiff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h\nindex e411d6f9ac65..1fa0e977ea8d 100644\n--- a/include/uapi/linux/if_xdp.h\n+++ b/include/uapi/linux/if_xdp.h\n@@ -13,7 +13,9 @@\n #include <linux/types.h>\n \n /* Options for the sxdp_flags field */\n-#define XDP_SHARED_UMEM 1\n+#define XDP_SHARED_UMEM\t(1 << 0)\n+#define XDP_COPY\t(1 << 1) /* Force copy-mode */\n+#define XDP_ZEROCOPY\t(1 << 2) /* Force zero-copy mode */\n \n struct sockaddr_xdp {\n \t__u16 sxdp_family;\ndiff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c\nindex aca826011f6c..f729d79b8d91 100644\n--- a/net/xdp/xdp_umem.c\n+++ b/net/xdp/xdp_umem.c\n@@ -17,6 +17,81 @@\n \n #define XDP_UMEM_MIN_CHUNK_SIZE 2048\n \n+int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,\n+\t\t\tu32 queue_id, u16 flags)\n+{\n+\tbool force_zc, force_copy;\n+\tstruct netdev_bpf bpf;\n+\tint err;\n+\n+\tforce_zc = flags & XDP_ZEROCOPY;\n+\tforce_copy = flags & XDP_COPY;\n+\n+\tif (force_zc && force_copy)\n+\t\treturn -EINVAL;\n+\n+\tif (force_copy)\n+\t\treturn 0;\n+\n+\tdev_hold(dev);\n+\n+\tif (dev->netdev_ops->ndo_bpf) {\n+\t\tbpf.command = XDP_QUERY_XSK_UMEM;\n+\n+\t\trtnl_lock();\n+\t\terr = dev->netdev_ops->ndo_bpf(dev, &bpf);\n+\t\trtnl_unlock();\n+\n+\t\tif (err) {\n+\t\t\tdev_put(dev);\n+\t\t\treturn force_zc ? -ENOTSUPP : 0;\n+\t\t}\n+\n+\t\tbpf.command = XDP_SETUP_XSK_UMEM;\n+\t\tbpf.xsk.umem = umem;\n+\t\tbpf.xsk.queue_id = queue_id;\n+\n+\t\trtnl_lock();\n+\t\terr = dev->netdev_ops->ndo_bpf(dev, &bpf);\n+\t\trtnl_unlock();\n+\n+\t\tif (err) {\n+\t\t\tdev_put(dev);\n+\t\t\treturn force_zc ? err : 0; /* fail or fallback */\n+\t\t}\n+\n+\t\tumem->dev = dev;\n+\t\tumem->queue_id = queue_id;\n+\t\tumem->zc = true;\n+\t\treturn 0;\n+\t}\n+\n+\tdev_put(dev);\n+\treturn force_zc ? -ENOTSUPP : 0; /* fail or fallback */\n+}\n+\n+void xdp_umem_clear_dev(struct xdp_umem *umem)\n+{\n+\tstruct netdev_bpf bpf;\n+\tint err;\n+\n+\tif (umem->dev) {\n+\t\tbpf.command = XDP_SETUP_XSK_UMEM;\n+\t\tbpf.xsk.umem = NULL;\n+\t\tbpf.xsk.queue_id = umem->queue_id;\n+\n+\t\trtnl_lock();\n+\t\terr = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf);\n+\t\trtnl_unlock();\n+\n+\t\tif (err)\n+\t\t\tWARN(1, \"failed to disable umem!\\n\");\n+\n+\t\tdev_put(umem->dev);\n+\t\tumem->dev = NULL;\n+\t}\n+}\n+\n static void xdp_umem_unpin_pages(struct xdp_umem *umem)\n {\n \tunsigned int i;\n@@ -43,6 +118,8 @@ static void xdp_umem_release(struct xdp_umem *umem)\n \tstruct task_struct *task;\n \tstruct mm_struct *mm;\n \n+\txdp_umem_clear_dev(umem);\n+\n \tif (umem->fq) {\n \t\txskq_destroy(umem->fq);\n \t\tumem->fq = NULL;\ndiff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h\nindex 40e8fa4a92af..674508a32a4d 100644\n--- a/net/xdp/xdp_umem.h\n+++ b/net/xdp/xdp_umem.h\n@@ -13,6 +13,9 @@ static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)\n \treturn umem->pages[addr >> PAGE_SHIFT].addr + (addr & (PAGE_SIZE - 1));\n }\n \n+int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,\n+\t\t\tu32 queue_id, u16 flags);\n+void xdp_umem_clear_dev(struct xdp_umem *umem);\n bool xdp_umem_validate_queues(struct xdp_umem *umem);\n void xdp_get_umem(struct xdp_umem *umem);\n void xdp_put_umem(struct xdp_umem *umem);\ndiff --git a/net/xdp/xsk.c b/net/xdp/xsk.c\nindex 4688c750df1d..ab64bd8260ea 100644\n--- a/net/xdp/xsk.c\n+++ b/net/xdp/xsk.c\n@@ -36,19 +36,28 @@ static struct xdp_sock *xdp_sk(struct sock *sk)\n \n bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)\n {\n-\treturn !!xs->rx;\n+\treturn READ_ONCE(xs->rx) && READ_ONCE(xs->umem) &&\n+\t\tREAD_ONCE(xs->umem->fq);\n }\n \n-static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)\n+u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)\n+{\n+\treturn xskq_peek_addr(umem->fq, addr);\n+}\n+EXPORT_SYMBOL(xsk_umem_peek_addr);\n+\n+void xsk_umem_discard_addr(struct xdp_umem *umem)\n+{\n+\txskq_discard_addr(umem->fq);\n+}\n+EXPORT_SYMBOL(xsk_umem_discard_addr);\n+\n+static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)\n {\n-\tu32 len = xdp->data_end - xdp->data;\n \tvoid *buffer;\n \tu64 addr;\n \tint err;\n \n-\tif (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)\n-\t\treturn -EINVAL;\n-\n \tif (!xskq_peek_addr(xs->umem->fq, &addr) ||\n \t len > xs->umem->chunk_size_nohr) {\n \t\txs->rx_dropped++;\n@@ -60,25 +69,41 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)\n \tbuffer = xdp_umem_get_data(xs->umem, addr);\n \tmemcpy(buffer, xdp->data, len);\n \terr = xskq_produce_batch_desc(xs->rx, addr, len);\n-\tif (!err)\n+\tif (!err) {\n \t\txskq_discard_addr(xs->umem->fq);\n-\telse\n-\t\txs->rx_dropped++;\n+\t\txdp_return_buff(xdp);\n+\t\treturn 0;\n+\t}\n \n+\txs->rx_dropped++;\n \treturn err;\n }\n \n-int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)\n+static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)\n {\n-\tint err;\n+\tint err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);\n \n-\terr = __xsk_rcv(xs, xdp);\n-\tif (likely(!err))\n+\tif (err) {\n \t\txdp_return_buff(xdp);\n+\t\txs->rx_dropped++;\n+\t}\n \n \treturn err;\n }\n \n+int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)\n+{\n+\tu32 len;\n+\n+\tif (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)\n+\t\treturn -EINVAL;\n+\n+\tlen = xdp->data_end - xdp->data;\n+\n+\treturn (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?\n+\t\t__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);\n+}\n+\n void xsk_flush(struct xdp_sock *xs)\n {\n \txskq_produce_flush_desc(xs->rx);\n@@ -87,12 +112,29 @@ void xsk_flush(struct xdp_sock *xs)\n \n int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)\n {\n+\tu32 len = xdp->data_end - xdp->data;\n+\tvoid *buffer;\n+\tu64 addr;\n \tint err;\n \n-\terr = __xsk_rcv(xs, xdp);\n-\tif (!err)\n+\tif (!xskq_peek_addr(xs->umem->fq, &addr) ||\n+\t len > xs->umem->chunk_size_nohr) {\n+\t\txs->rx_dropped++;\n+\t\treturn -ENOSPC;\n+\t}\n+\n+\taddr += xs->umem->headroom;\n+\n+\tbuffer = xdp_umem_get_data(xs->umem, addr);\n+\tmemcpy(buffer, xdp->data, len);\n+\terr = xskq_produce_batch_desc(xs->rx, addr, len);\n+\tif (!err) {\n+\t\txskq_discard_addr(xs->umem->fq);\n \t\txsk_flush(xs);\n+\t\treturn 0;\n+\t}\n \n+\txs->rx_dropped++;\n \treturn err;\n }\n \n@@ -291,6 +333,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)\n \tstruct sock *sk = sock->sk;\n \tstruct xdp_sock *xs = xdp_sk(sk);\n \tstruct net_device *dev;\n+\tu32 flags, qid;\n \tint err = 0;\n \n \tif (addr_len < sizeof(struct sockaddr_xdp))\n@@ -315,16 +358,26 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)\n \t\tgoto out_unlock;\n \t}\n \n-\tif ((xs->rx && sxdp->sxdp_queue_id >= dev->real_num_rx_queues) ||\n-\t (xs->tx && sxdp->sxdp_queue_id >= dev->real_num_tx_queues)) {\n+\tqid = sxdp->sxdp_queue_id;\n+\n+\tif ((xs->rx && qid >= dev->real_num_rx_queues) ||\n+\t (xs->tx && qid >= dev->real_num_tx_queues)) {\n \t\terr = -EINVAL;\n \t\tgoto out_unlock;\n \t}\n \n-\tif (sxdp->sxdp_flags & XDP_SHARED_UMEM) {\n+\tflags = sxdp->sxdp_flags;\n+\n+\tif (flags & XDP_SHARED_UMEM) {\n \t\tstruct xdp_sock *umem_xs;\n \t\tstruct socket *sock;\n \n+\t\tif ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) {\n+\t\t\t/* Cannot specify flags for shared sockets. */\n+\t\t\terr = -EINVAL;\n+\t\t\tgoto out_unlock;\n+\t\t}\n+\n \t\tif (xs->umem) {\n \t\t\t/* We have already our own. */\n \t\t\terr = -EINVAL;\n@@ -343,8 +396,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)\n \t\t\terr = -EBADF;\n \t\t\tsockfd_put(sock);\n \t\t\tgoto out_unlock;\n-\t\t} else if (umem_xs->dev != dev ||\n-\t\t\t umem_xs->queue_id != sxdp->sxdp_queue_id) {\n+\t\t} else if (umem_xs->dev != dev || umem_xs->queue_id != qid) {\n \t\t\terr = -EINVAL;\n \t\t\tsockfd_put(sock);\n \t\t\tgoto out_unlock;\n@@ -360,6 +412,10 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)\n \t\t/* This xsk has its own umem. */\n \t\txskq_set_umem(xs->umem->fq, &xs->umem->props);\n \t\txskq_set_umem(xs->umem->cq, &xs->umem->props);\n+\n+\t\terr = xdp_umem_assign_dev(xs->umem, dev, qid, flags);\n+\t\tif (err)\n+\t\t\tgoto out_unlock;\n \t}\n \n \txs->dev = dev;\n", "prefixes": [ "bpf-next", "05/11" ] }