From patchwork Tue May 15 19:06:09 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?b?QmrDtnJuIFTDtnBlbA==?= X-Patchwork-Id: 913876 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=gmail.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 40lnBD1Qxpz9s01 for ; Wed, 16 May 2018 05:07:16 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752535AbeEOTHN (ORCPT ); Tue, 15 May 2018 15:07:13 -0400 Received: from mga12.intel.com ([192.55.52.136]:46728 "EHLO mga12.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752474AbeEOTHK (ORCPT ); Tue, 15 May 2018 15:07:10 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga003.jf.intel.com ([10.7.209.27]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 15 May 2018 12:07:08 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,404,1520924400"; d="scan'208";a="51095757" Received: from twgeistx-mobl2.ger.corp.intel.com (HELO btopel-mobl1.ger.corp.intel.com) ([10.249.39.165]) by orsmga003.jf.intel.com with ESMTP; 15 May 2018 12:07:02 -0700 From: =?utf-8?b?QmrDtnJuIFTDtnBlbA==?= To: bjorn.topel@gmail.com, magnus.karlsson@gmail.com, magnus.karlsson@intel.com, alexander.h.duyck@intel.com, alexander.duyck@gmail.com, john.fastabend@gmail.com, ast@fb.com, brouer@redhat.com, willemdebruijn.kernel@gmail.com, daniel@iogearbox.net, mst@redhat.com, netdev@vger.kernel.org Cc: =?utf-8?b?QmrDtnJuIFTDtnBlbA==?= , michael.lundkvist@ericsson.com, jesse.brandeburg@intel.com, anjali.singhai@intel.com, qi.z.zhang@intel.com, intel-wired-lan@lists.osuosl.org Subject: [RFC PATCH bpf-next 06/12] xsk: add zero-copy support for Rx Date: Tue, 15 May 2018 21:06:09 +0200 Message-Id: <20180515190615.23099-7-bjorn.topel@gmail.com> X-Mailer: git-send-email 2.14.1 In-Reply-To: <20180515190615.23099-1-bjorn.topel@gmail.com> References: <20180515190615.23099-1-bjorn.topel@gmail.com> MIME-Version: 1.0 Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org From: Björn Töpel Extend the xsk_rcv to support the new MEM_TYPE_ZERO_COPY memory, and wireup ndo_bpf call in bind. Signed-off-by: Björn Töpel --- include/net/xdp_sock.h | 7 +++++ net/xdp/xdp_umem.c | 60 +++++++++++++++++++++++++++++++++++++++++++ net/xdp/xdp_umem.h | 3 +++ net/xdp/xsk.c | 69 ++++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 125 insertions(+), 14 deletions(-) diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 09068c4f068e..644684eb2caf 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -31,6 +31,7 @@ struct xdp_umem_props { struct xdp_umem_frame { void *addr; + dma_addr_t dma; }; struct xdp_umem { @@ -47,6 +48,8 @@ struct xdp_umem { size_t size; atomic_t users; struct work_struct work; + struct net_device *dev; + u16 queue_id; }; struct xdp_sock { @@ -69,6 +72,10 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); void xsk_flush(struct xdp_sock *xs); bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs); + +u32 *xsk_umem_peek_id(struct xdp_umem *umem); +void xsk_umem_discard_id(struct xdp_umem *umem); + #else static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) { diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index b426cbe3151a..f70cdaa2ef4d 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -26,6 +26,64 @@ #define XDP_UMEM_MIN_FRAME_SIZE 2048 +int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, + u16 queue_id) +{ + struct netdev_bpf bpf; + int err; + + if (umem->dev) { + if (dev != umem->dev || queue_id != umem->queue_id) + return -EBUSY; + return 0; + } + + dev_hold(dev); + if (dev->netdev_ops->ndo_bpf) { + bpf.command = XDP_SETUP_XSK_UMEM; + bpf.xsk.umem = umem; + bpf.xsk.queue_id = queue_id; + + rtnl_lock(); + err = dev->netdev_ops->ndo_bpf(dev, &bpf); + rtnl_unlock(); + + if (err) { + dev_put(dev); + return 0; + } + + umem->dev = dev; + umem->queue_id = queue_id; + return 0; + } + + dev_put(dev); + return 0; +} + +void xdp_umem_clear_dev(struct xdp_umem *umem) +{ + struct netdev_bpf bpf; + int err; + + if (umem->dev) { + bpf.command = XDP_SETUP_XSK_UMEM; + bpf.xsk.umem = NULL; + bpf.xsk.queue_id = umem->queue_id; + + rtnl_lock(); + err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); + rtnl_unlock(); + + if (err) + WARN(1, "failed to disable umem!\n"); + + dev_put(umem->dev); + umem->dev = NULL; + } +} + int xdp_umem_create(struct xdp_umem **umem) { *umem = kzalloc(sizeof(**umem), GFP_KERNEL); @@ -66,6 +124,8 @@ static void xdp_umem_release(struct xdp_umem *umem) struct task_struct *task; struct mm_struct *mm; + xdp_umem_clear_dev(umem); + if (umem->fq) { xskq_destroy(umem->fq); umem->fq = NULL; diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h index 0a969384af93..3bb96d156b40 100644 --- a/net/xdp/xdp_umem.h +++ b/net/xdp/xdp_umem.h @@ -34,4 +34,7 @@ void xdp_get_umem(struct xdp_umem *umem); void xdp_put_umem(struct xdp_umem *umem); int xdp_umem_create(struct xdp_umem **umem); +int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, + u16 queue_id); + #endif /* XDP_UMEM_H_ */ diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index e59ca8e2618d..a0cf9c042ed2 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -43,6 +43,18 @@ static struct xdp_sock *xdp_sk(struct sock *sk) return (struct xdp_sock *)sk; } +u32 *xsk_umem_peek_id(struct xdp_umem *umem) +{ + return xskq_peek_id(umem->fq); +} +EXPORT_SYMBOL(xsk_umem_peek_id); + +void xsk_umem_discard_id(struct xdp_umem *umem) +{ + xskq_discard_id(umem->fq); +} +EXPORT_SYMBOL(xsk_umem_discard_id); + bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) { return !!xs->rx; @@ -50,40 +62,54 @@ bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) { - u32 *id, len = xdp->data_end - xdp->data; + u32 *id, len; void *buffer; int err = 0; - if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) - return -EINVAL; - id = xskq_peek_id(xs->umem->fq); if (!id) return -ENOSPC; buffer = xdp_umem_get_data_with_headroom(xs->umem, *id); + len = xdp->data_end - xdp->data; memcpy(buffer, xdp->data, len); err = xskq_produce_batch_desc(xs->rx, *id, len, xs->umem->frame_headroom); - if (!err) + if (!err) { xskq_discard_id(xs->umem->fq); + xdp_return_buff(xdp); + return 0; + } + xs->rx_dropped++; return err; } -int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) +static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp) { + u16 off = xdp->data - xdp->data_hard_start; + u32 len = xdp->data_end - xdp->data; int err; - err = __xsk_rcv(xs, xdp); - if (likely(!err)) + err = xskq_produce_batch_desc(xs->rx, (u32)xdp->handle, len, + xs->umem->frame_headroom + off); + if (err) { xdp_return_buff(xdp); - else xs->rx_dropped++; + } return err; } +int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) +{ + if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) + return -EINVAL; + + return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ? + __xsk_rcv_zc(xs, xdp) : __xsk_rcv(xs, xdp); +} + void xsk_flush(struct xdp_sock *xs) { xskq_produce_flush_desc(xs->rx); @@ -92,14 +118,26 @@ void xsk_flush(struct xdp_sock *xs) int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) { - int err; + u32 *id, len; + void *buffer; + int err = 0; - err = __xsk_rcv(xs, xdp); - if (!err) + id = xskq_peek_id(xs->umem->fq); + if (!id) + return -ENOSPC; + + buffer = xdp_umem_get_data_with_headroom(xs->umem, *id); + len = xdp->data_end - xdp->data; + memcpy(buffer, xdp->data, len); + err = xskq_produce_batch_desc(xs->rx, *id, len, + xs->umem->frame_headroom); + if (!err) { + xskq_discard_id(xs->umem->fq); xsk_flush(xs); - else - xs->rx_dropped++; + return 0; + } + xs->rx_dropped++; return err; } @@ -362,6 +400,9 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) xs->dev = dev; xs->queue_id = sxdp->sxdp_queue_id; + err = xdp_umem_assign_dev(xs->umem, dev, xs->queue_id); + if (err) + goto out_unlock; xskq_set_umem(xs->rx, &xs->umem->props); xskq_set_umem(xs->tx, &xs->umem->props);