Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/977674/?format=api
{ "id": 977674, "url": "http://patchwork.ozlabs.org/api/patches/977674/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20181002080034.11754-6-bjorn.topel@gmail.com/", "project": { "id": 46, "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api", "name": "Intel Wired Ethernet development", "link_name": "intel-wired-lan", "list_id": "intel-wired-lan.osuosl.org", "list_email": "intel-wired-lan@osuosl.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20181002080034.11754-6-bjorn.topel@gmail.com>", "list_archive_url": null, "date": "2018-10-02T08:00:34", "name": "[v2,5/5] ixgbe: add AF_XDP zero-copy Tx support", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "d4fbac9461b2e3b5d2a1fea0721584f4753901a1", "submitter": { "id": 70569, "url": "http://patchwork.ozlabs.org/api/people/70569/?format=api", "name": "Björn Töpel", "email": "bjorn.topel@gmail.com" }, "delegate": { "id": 68, "url": "http://patchwork.ozlabs.org/api/users/68/?format=api", "username": "jtkirshe", "first_name": "Jeff", "last_name": "Kirsher", "email": "jeffrey.t.kirsher@intel.com" }, "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20181002080034.11754-6-bjorn.topel@gmail.com/mbox/", "series": [ { "id": 68583, "url": "http://patchwork.ozlabs.org/api/series/68583/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/list/?series=68583", "date": "2018-10-02T08:00:30", "name": "Introducing ixgbe AF_XDP ZC support", "version": 2, "mbox": "http://patchwork.ozlabs.org/series/68583/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/977674/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/977674/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<intel-wired-lan-bounces@osuosl.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Authentication-Results": [ "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=osuosl.org\n\t(client-ip=140.211.166.136; helo=silver.osuosl.org;\n\tenvelope-from=intel-wired-lan-bounces@osuosl.org;\n\treceiver=<UNKNOWN>)", "ozlabs.org;\n\tdmarc=fail (p=none dis=none) header.from=gmail.com" ], "Received": [ "from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136])\n\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256\n\tbits)) (No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 42PWnX3Z6nz9sj1\n\tfor <incoming@patchwork.ozlabs.org>;\n\tTue, 2 Oct 2018 18:01:36 +1000 (AEST)", "from localhost (localhost [127.0.0.1])\n\tby silver.osuosl.org (Postfix) with ESMTP id 1E4CD23673;\n\tTue, 2 Oct 2018 08:01:35 +0000 (UTC)", "from silver.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id 9K0ZSM+MJGXt; Tue, 2 Oct 2018 08:01:33 +0000 (UTC)", "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby silver.osuosl.org (Postfix) with ESMTP id A69AF2372E;\n\tTue, 2 Oct 2018 08:01:33 +0000 (UTC)", "from hemlock.osuosl.org (smtp2.osuosl.org [140.211.166.133])\n\tby ash.osuosl.org (Postfix) with ESMTP id 51D361BF479\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tTue, 2 Oct 2018 08:01:33 +0000 (UTC)", "from localhost (localhost [127.0.0.1])\n\tby hemlock.osuosl.org (Postfix) with ESMTP id 4EA0787274\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tTue, 2 Oct 2018 08:01:33 +0000 (UTC)", "from hemlock.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id 4yvIEGHiKSn3 for <intel-wired-lan@lists.osuosl.org>;\n\tTue, 2 Oct 2018 08:01:32 +0000 (UTC)", "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n\tby hemlock.osuosl.org (Postfix) with ESMTPS id 876D4869DC\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tTue, 2 Oct 2018 08:01:31 +0000 (UTC)", "from orsmga003.jf.intel.com ([10.7.209.27])\n\tby orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t02 Oct 2018 01:01:31 -0700", "from pvenkata-mobl.amr.corp.intel.com (HELO\n\tbtopel-mobl1.ger.corp.intel.com) ([10.252.35.165])\n\tby orsmga003.jf.intel.com with ESMTP; 02 Oct 2018 01:01:27 -0700" ], "X-Virus-Scanned": [ "amavisd-new at osuosl.org", "amavisd-new at osuosl.org" ], "X-Greylist": "from auto-whitelisted by SQLgrey-1.7.6", "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.54,331,1534834800\"; d=\"scan'208\";a=\"88439318\"", "From": "=?utf-8?b?QmrDtnJuIFTDtnBlbA==?= <bjorn.topel@gmail.com>", "To": "jeffrey.t.kirsher@intel.com,\n\tintel-wired-lan@lists.osuosl.org", "Date": "Tue, 2 Oct 2018 10:00:34 +0200", "Message-Id": "<20181002080034.11754-6-bjorn.topel@gmail.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20181002080034.11754-1-bjorn.topel@gmail.com>", "References": "<20181002080034.11754-1-bjorn.topel@gmail.com>", "MIME-Version": "1.0", "Subject": "[Intel-wired-lan] [PATCH v2 5/5] ixgbe: add AF_XDP zero-copy Tx\n\tsupport", "X-BeenThere": "intel-wired-lan@osuosl.org", "X-Mailman-Version": "2.1.24", "Precedence": "list", "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.osuosl.org>", "List-Unsubscribe": "<https://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=unsubscribe>", "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>", "List-Post": "<mailto:intel-wired-lan@osuosl.org>", "List-Help": "<mailto:intel-wired-lan-request@osuosl.org?subject=help>", "List-Subscribe": "<https://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=subscribe>", "Cc": "jakub.kicinski@netronome.com, daniel@iogearbox.net,\n\tnetdev@vger.kernel.org, ast@kernel.org, tuc@vmware.com,\n\tu9012063@gmail.com, brouer@redhat.com, =?utf-8?b?QmrDtnJuIFTDtnBlbA==?=\n\t<bjorn.topel@intel.com>, magnus.karlsson@gmail.com,\n\tmagnus.karlsson@intel.com", "Content-Type": "text/plain; charset=\"utf-8\"", "Content-Transfer-Encoding": "base64", "Errors-To": "intel-wired-lan-bounces@osuosl.org", "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@osuosl.org>" }, "content": "From: Björn Töpel <bjorn.topel@intel.com>\n\nThis patch adds zero-copy Tx support for AF_XDP sockets. It implements\nthe ndo_xsk_async_xmit netdev ndo and performs all the Tx logic from a\nNAPI context. This means pulling egress packets from the Tx ring,\nplacing the frames on the NIC HW descriptor ring and completing sent\nframes back to the application via the completion ring.\n\nThe regular XDP Tx ring is used for AF_XDP as well. This rationale for\nthis is as follows: XDP_REDIRECT guarantees mutual exclusion between\ndifferent NAPI contexts based on CPU id. In other words, a netdev can\nXDP_REDIRECT to another netdev with a different NAPI context, since\nthe operation is bound to a specific core and each core has its own\nhardware ring.\n\nAs the AF_XDP Tx action is running in the same NAPI context and using\nthe same ring, it will also be protected from XDP_REDIRECT actions\nwith the exact same mechanism.\n\nAs with AF_XDP Rx, all AF_XDP Tx specific functions are added to\nixgbe_xsk.c.\n\nSigned-off-by: Björn Töpel <bjorn.topel@intel.com>\n---\n drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 17 +-\n .../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 4 +\n drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 175 ++++++++++++++++++\n 3 files changed, 195 insertions(+), 1 deletion(-)", "diff": "diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c\nindex b211032f8682..ec31b32d6674 100644\n--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c\n+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c\n@@ -3161,7 +3161,11 @@ int ixgbe_poll(struct napi_struct *napi, int budget)\n #endif\n \n \tixgbe_for_each_ring(ring, q_vector->tx) {\n-\t\tif (!ixgbe_clean_tx_irq(q_vector, ring, budget))\n+\t\tbool wd = ring->xsk_umem ?\n+\t\t\t ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) :\n+\t\t\t ixgbe_clean_tx_irq(q_vector, ring, budget);\n+\n+\t\tif (!wd)\n \t\t\tclean_complete = false;\n \t}\n \n@@ -3472,6 +3476,10 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,\n \tu32 txdctl = IXGBE_TXDCTL_ENABLE;\n \tu8 reg_idx = ring->reg_idx;\n \n+\tring->xsk_umem = NULL;\n+\tif (ring_is_xdp(ring))\n+\t\tring->xsk_umem = ixgbe_xsk_umem(adapter, ring);\n+\n \t/* disable queue to avoid issues while updating state */\n \tIXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);\n \tIXGBE_WRITE_FLUSH(hw);\n@@ -5944,6 +5952,11 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)\n \tu16 i = tx_ring->next_to_clean;\n \tstruct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];\n \n+\tif (tx_ring->xsk_umem) {\n+\t\tixgbe_xsk_clean_tx_ring(tx_ring);\n+\t\tgoto out;\n+\t}\n+\n \twhile (i != tx_ring->next_to_use) {\n \t\tunion ixgbe_adv_tx_desc *eop_desc, *tx_desc;\n \n@@ -5995,6 +6008,7 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)\n \tif (!ring_is_xdp(tx_ring))\n \t\tnetdev_tx_reset_queue(txring_txq(tx_ring));\n \n+out:\n \t/* reset next_to_use and next_to_clean */\n \ttx_ring->next_to_use = 0;\n \ttx_ring->next_to_clean = 0;\n@@ -10350,6 +10364,7 @@ static const struct net_device_ops ixgbe_netdev_ops = {\n \t.ndo_features_check\t= ixgbe_features_check,\n \t.ndo_bpf\t\t= ixgbe_xdp,\n \t.ndo_xdp_xmit\t\t= ixgbe_xdp_xmit,\n+\t.ndo_xsk_async_xmit\t= ixgbe_xsk_async_xmit,\n };\n \n static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter,\ndiff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h\nindex 56afb685c648..53d4089f5644 100644\n--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h\n+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h\n@@ -42,5 +42,9 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,\n \t\t\t struct ixgbe_ring *rx_ring,\n \t\t\t const int budget);\n void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring);\n+bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector,\n+\t\t\t struct ixgbe_ring *tx_ring, int napi_budget);\n+int ixgbe_xsk_async_xmit(struct net_device *dev, u32 queue_id);\n+void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring);\n \n #endif /* #define _IXGBE_TXRX_COMMON_H_ */\ndiff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c\nindex 61259036ff4b..cf1c6f2d97e5 100644\n--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c\n+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c\n@@ -626,3 +626,178 @@ void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring)\n \t\t}\n \t}\n }\n+\n+static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)\n+{\n+\tunion ixgbe_adv_tx_desc *tx_desc = NULL;\n+\tstruct ixgbe_tx_buffer *tx_bi;\n+\tbool work_done = true;\n+\tu32 len, cmd_type;\n+\tdma_addr_t dma;\n+\n+\twhile (budget-- > 0) {\n+\t\tif (unlikely(!ixgbe_desc_unused(xdp_ring))) {\n+\t\t\twork_done = false;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))\n+\t\t\tbreak;\n+\n+\t\tdma_sync_single_for_device(xdp_ring->dev, dma, len,\n+\t\t\t\t\t DMA_BIDIRECTIONAL);\n+\n+\t\ttx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];\n+\t\ttx_bi->bytecount = len;\n+\t\ttx_bi->xdpf = NULL;\n+\n+\t\ttx_desc = IXGBE_TX_DESC(xdp_ring, xdp_ring->next_to_use);\n+\t\ttx_desc->read.buffer_addr = cpu_to_le64(dma);\n+\n+\t\t/* put descriptor type bits */\n+\t\tcmd_type = IXGBE_ADVTXD_DTYP_DATA |\n+\t\t\t IXGBE_ADVTXD_DCMD_DEXT |\n+\t\t\t IXGBE_ADVTXD_DCMD_IFCS;\n+\t\tcmd_type |= len | IXGBE_TXD_CMD;\n+\t\ttx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);\n+\t\ttx_desc->read.olinfo_status =\n+\t\t\tcpu_to_le32(len << IXGBE_ADVTXD_PAYLEN_SHIFT);\n+\n+\t\txdp_ring->next_to_use++;\n+\t\tif (xdp_ring->next_to_use == xdp_ring->count)\n+\t\t\txdp_ring->next_to_use = 0;\n+\t}\n+\n+\tif (tx_desc) {\n+\t\tixgbe_xdp_ring_update_tail(xdp_ring);\n+\t\txsk_umem_consume_tx_done(xdp_ring->xsk_umem);\n+\t}\n+\n+\treturn !!budget && work_done;\n+}\n+\n+static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring,\n+\t\t\t\t struct ixgbe_tx_buffer *tx_bi)\n+{\n+\txdp_return_frame(tx_bi->xdpf);\n+\tdma_unmap_single(tx_ring->dev,\n+\t\t\t dma_unmap_addr(tx_bi, dma),\n+\t\t\t dma_unmap_len(tx_bi, len), DMA_TO_DEVICE);\n+\tdma_unmap_len_set(tx_bi, len, 0);\n+}\n+\n+bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector,\n+\t\t\t struct ixgbe_ring *tx_ring, int napi_budget)\n+{\n+\tunsigned int total_packets = 0, total_bytes = 0;\n+\tu32 i = tx_ring->next_to_clean, xsk_frames = 0;\n+\tunsigned int budget = q_vector->tx.work_limit;\n+\tstruct xdp_umem *umem = tx_ring->xsk_umem;\n+\tunion ixgbe_adv_tx_desc *tx_desc;\n+\tstruct ixgbe_tx_buffer *tx_bi;\n+\tbool xmit_done;\n+\n+\ttx_bi = &tx_ring->tx_buffer_info[i];\n+\ttx_desc = IXGBE_TX_DESC(tx_ring, i);\n+\ti -= tx_ring->count;\n+\n+\tdo {\n+\t\tif (!(tx_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))\n+\t\t\tbreak;\n+\n+\t\ttotal_bytes += tx_bi->bytecount;\n+\t\ttotal_packets += tx_bi->gso_segs;\n+\n+\t\tif (tx_bi->xdpf)\n+\t\t\tixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi);\n+\t\telse\n+\t\t\txsk_frames++;\n+\n+\t\ttx_bi->xdpf = NULL;\n+\t\ttotal_bytes += tx_bi->bytecount;\n+\n+\t\ttx_bi++;\n+\t\ttx_desc++;\n+\t\ti++;\n+\t\tif (unlikely(!i)) {\n+\t\t\ti -= tx_ring->count;\n+\t\t\ttx_bi = tx_ring->tx_buffer_info;\n+\t\t\ttx_desc = IXGBE_TX_DESC(tx_ring, 0);\n+\t\t}\n+\n+\t\t/* issue prefetch for next Tx descriptor */\n+\t\tprefetch(tx_desc);\n+\n+\t\t/* update budget accounting */\n+\t\tbudget--;\n+\t} while (likely(budget));\n+\n+\ti += tx_ring->count;\n+\ttx_ring->next_to_clean = i;\n+\n+\tu64_stats_update_begin(&tx_ring->syncp);\n+\ttx_ring->stats.bytes += total_bytes;\n+\ttx_ring->stats.packets += total_packets;\n+\tu64_stats_update_end(&tx_ring->syncp);\n+\tq_vector->tx.total_bytes += total_bytes;\n+\tq_vector->tx.total_packets += total_packets;\n+\n+\tif (xsk_frames)\n+\t\txsk_umem_complete_tx(umem, xsk_frames);\n+\n+\txmit_done = ixgbe_xmit_zc(tx_ring, q_vector->tx.work_limit);\n+\treturn budget > 0 && xmit_done;\n+}\n+\n+int ixgbe_xsk_async_xmit(struct net_device *dev, u32 qid)\n+{\n+\tstruct ixgbe_adapter *adapter = netdev_priv(dev);\n+\tstruct ixgbe_ring *ring;\n+\n+\tif (test_bit(__IXGBE_DOWN, &adapter->state))\n+\t\treturn -ENETDOWN;\n+\n+\tif (!READ_ONCE(adapter->xdp_prog))\n+\t\treturn -ENXIO;\n+\n+\tif (qid >= adapter->num_xdp_queues)\n+\t\treturn -ENXIO;\n+\n+\tif (!adapter->xsk_umems || !adapter->xsk_umems[qid])\n+\t\treturn -ENXIO;\n+\n+\tring = adapter->xdp_ring[qid];\n+\tif (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) {\n+\t\tu64 eics = BIT_ULL(ring->q_vector->v_idx);\n+\n+\t\tixgbe_irq_rearm_queues(adapter, eics);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring)\n+{\n+\tu16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use;\n+\tstruct xdp_umem *umem = tx_ring->xsk_umem;\n+\tstruct ixgbe_tx_buffer *tx_bi;\n+\tu32 xsk_frames = 0;\n+\n+\twhile (ntc != ntu) {\n+\t\ttx_bi = &tx_ring->tx_buffer_info[ntc];\n+\n+\t\tif (tx_bi->xdpf)\n+\t\t\tixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi);\n+\t\telse\n+\t\t\txsk_frames++;\n+\n+\t\ttx_bi->xdpf = NULL;\n+\n+\t\tntc++;\n+\t\tif (ntc == tx_ring->count)\n+\t\t\tntc = 0;\n+\t}\n+\n+\tif (xsk_frames)\n+\t\txsk_umem_complete_tx(umem, xsk_frames);\n+}\n", "prefixes": [ "v2", "5/5" ] }