Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/974017/?format=api
{ "id": 974017, "url": "http://patchwork.ozlabs.org/api/patches/974017/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20180924163557.1187-6-bjorn.topel@gmail.com/", "project": { "id": 46, "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api", "name": "Intel Wired Ethernet development", "link_name": "intel-wired-lan", "list_id": "intel-wired-lan.osuosl.org", "list_email": "intel-wired-lan@osuosl.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20180924163557.1187-6-bjorn.topel@gmail.com>", "list_archive_url": null, "date": "2018-09-24T16:35:57", "name": "[5/5] ixgbe: add AF_XDP zero-copy Tx support", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "7f8aaf4e23915ffa2e07c653a7c85ab9603e96f8", "submitter": { "id": 70569, "url": "http://patchwork.ozlabs.org/api/people/70569/?format=api", "name": "Björn Töpel", "email": "bjorn.topel@gmail.com" }, "delegate": { "id": 68, "url": "http://patchwork.ozlabs.org/api/users/68/?format=api", "username": "jtkirshe", "first_name": "Jeff", "last_name": "Kirsher", "email": "jeffrey.t.kirsher@intel.com" }, "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20180924163557.1187-6-bjorn.topel@gmail.com/mbox/", "series": [ { "id": 67233, "url": "http://patchwork.ozlabs.org/api/series/67233/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/list/?series=67233", "date": "2018-09-24T16:35:52", "name": "Introducing ixgbe AF_XDP ZC support", "version": 1, "mbox": "http://patchwork.ozlabs.org/series/67233/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/974017/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/974017/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<intel-wired-lan-bounces@osuosl.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Authentication-Results": [ "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=osuosl.org\n\t(client-ip=140.211.166.138; helo=whitealder.osuosl.org;\n\tenvelope-from=intel-wired-lan-bounces@osuosl.org;\n\treceiver=<UNKNOWN>)", "ozlabs.org;\n\tdmarc=fail (p=none dis=none) header.from=gmail.com" ], "Received": [ "from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138])\n\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256\n\tbits)) (No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 42JqfF4j80z9s8T\n\tfor <incoming@patchwork.ozlabs.org>;\n\tTue, 25 Sep 2018 02:39:01 +1000 (AEST)", "from localhost (localhost [127.0.0.1])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id 578CD80784;\n\tMon, 24 Sep 2018 16:38:59 +0000 (UTC)", "from whitealder.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id nAQ20F-8Fu0U; Mon, 24 Sep 2018 16:38:56 +0000 (UTC)", "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id 90CEC85AB4;\n\tMon, 24 Sep 2018 16:38:56 +0000 (UTC)", "from hemlock.osuosl.org (smtp2.osuosl.org [140.211.166.133])\n\tby ash.osuosl.org (Postfix) with ESMTP id 29E211BFB70\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 24 Sep 2018 16:38:55 +0000 (UTC)", "from localhost (localhost [127.0.0.1])\n\tby hemlock.osuosl.org (Postfix) with ESMTP id 275D1863A5\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 24 Sep 2018 16:38:55 +0000 (UTC)", "from hemlock.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id i0r8+TdVoQf1 for <intel-wired-lan@lists.osuosl.org>;\n\tMon, 24 Sep 2018 16:38:54 +0000 (UTC)", "from mga06.intel.com (mga06.intel.com [134.134.136.31])\n\tby hemlock.osuosl.org (Postfix) with ESMTPS id 6A805863C4\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 24 Sep 2018 16:38:54 +0000 (UTC)", "from fmsmga003.fm.intel.com ([10.253.24.29])\n\tby orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t24 Sep 2018 09:38:53 -0700", "from dandretz-mobl2.ger.corp.intel.com (HELO\n\tbtopel-mobl1.isw.intel.com) ([10.252.37.237])\n\tby FMSMGA003.fm.intel.com with ESMTP; 24 Sep 2018 09:36:53 -0700" ], "X-Virus-Scanned": [ "amavisd-new at osuosl.org", "amavisd-new at osuosl.org" ], "X-Greylist": "from auto-whitelisted by SQLgrey-1.7.6", "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.54,298,1534834800\"; d=\"scan'208\";a=\"82877951\"", "From": "=?utf-8?b?QmrDtnJuIFTDtnBlbA==?= <bjorn.topel@gmail.com>", "To": "jeffrey.t.kirsher@intel.com,\n\tintel-wired-lan@lists.osuosl.org", "Date": "Mon, 24 Sep 2018 18:35:57 +0200", "Message-Id": "<20180924163557.1187-6-bjorn.topel@gmail.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20180924163557.1187-1-bjorn.topel@gmail.com>", "References": "<20180924163557.1187-1-bjorn.topel@gmail.com>", "MIME-Version": "1.0", "Subject": "[Intel-wired-lan] [PATCH 5/5] ixgbe: add AF_XDP zero-copy Tx support", "X-BeenThere": "intel-wired-lan@osuosl.org", "X-Mailman-Version": "2.1.24", "Precedence": "list", "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.osuosl.org>", "List-Unsubscribe": "<https://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=unsubscribe>", "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>", "List-Post": "<mailto:intel-wired-lan@osuosl.org>", "List-Help": "<mailto:intel-wired-lan-request@osuosl.org?subject=help>", "List-Subscribe": "<https://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=subscribe>", "Cc": "daniel@iogearbox.net, netdev@vger.kernel.org, ast@kernel.org,\n\ttuc@vmware.com, u9012063@gmail.com, brouer@redhat.com, =?utf-8?q?Bj?=\n\t=?utf-8?b?w7ZybiBUw7ZwZWw=?= <bjorn.topel@intel.com>,\n\tmagnus.karlsson@gmail.com, magnus.karlsson@intel.com", "Content-Type": "text/plain; charset=\"utf-8\"", "Content-Transfer-Encoding": "base64", "Errors-To": "intel-wired-lan-bounces@osuosl.org", "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@osuosl.org>" }, "content": "From: Björn Töpel <bjorn.topel@intel.com>\n\nThis patch adds zero-copy Tx support for AF_XDP sockets. It implements\nthe ndo_xsk_async_xmit netdev ndo and performs all the Tx logic from a\nNAPI context. This means pulling egress packets from the Tx ring,\nplacing the frames on the NIC HW descriptor ring and completing sent\nframes back to the application via the completion ring.\n\nThe regular XDP Tx ring is used for AF_XDP as well. This rationale for\nthis is as follows: XDP_REDIRECT guarantees mutual exclusion between\ndifferent NAPI contexts based on CPU id. In other words, a netdev can\nXDP_REDIRECT to another netdev with a different NAPI context, since\nthe operation is bound to a specific core and each core has its own\nhardware ring.\n\nAs the AF_XDP Tx action is running in the same NAPI context and using\nthe same ring, it will also be protected from XDP_REDIRECT actions\nwith the exact same mechanism.\n\nAs with AF_XDP Rx, all AF_XDP Tx specific functions are added to\nixgbe_xsk.c.\n\nSigned-off-by: Björn Töpel <bjorn.topel@intel.com>\n---\n drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 16 +-\n .../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 4 +\n drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 175 ++++++++++++++++++\n 3 files changed, 194 insertions(+), 1 deletion(-)", "diff": "diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c\nindex 4e9b28894a5b..64b19346396d 100644\n--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c\n+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c\n@@ -3161,7 +3161,11 @@ int ixgbe_poll(struct napi_struct *napi, int budget)\n #endif\n \n \tixgbe_for_each_ring(ring, q_vector->tx) {\n-\t\tif (!ixgbe_clean_tx_irq(q_vector, ring, budget))\n+\t\tbool wd = ring->xsk_umem ?\n+\t\t\t ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) :\n+\t\t\t ixgbe_clean_tx_irq(q_vector, ring, budget);\n+\n+\t\tif (!wd)\n \t\t\tclean_complete = false;\n \t}\n \n@@ -3472,6 +3476,9 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,\n \tu32 txdctl = IXGBE_TXDCTL_ENABLE;\n \tu8 reg_idx = ring->reg_idx;\n \n+\tif (ring_is_xdp(ring))\n+\t\tring->xsk_umem = ixgbe_xsk_umem(adapter, ring);\n+\n \t/* disable queue to avoid issues while updating state */\n \tIXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);\n \tIXGBE_WRITE_FLUSH(hw);\n@@ -5938,6 +5945,11 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)\n \tu16 i = tx_ring->next_to_clean;\n \tstruct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];\n \n+\tif (tx_ring->xsk_umem) {\n+\t\tixgbe_xsk_clean_tx_ring(tx_ring);\n+\t\tgoto out;\n+\t}\n+\n \twhile (i != tx_ring->next_to_use) {\n \t\tunion ixgbe_adv_tx_desc *eop_desc, *tx_desc;\n \n@@ -5989,6 +6001,7 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)\n \tif (!ring_is_xdp(tx_ring))\n \t\tnetdev_tx_reset_queue(txring_txq(tx_ring));\n \n+out:\n \t/* reset next_to_use and next_to_clean */\n \ttx_ring->next_to_use = 0;\n \ttx_ring->next_to_clean = 0;\n@@ -10369,6 +10382,7 @@ static const struct net_device_ops ixgbe_netdev_ops = {\n \t.ndo_features_check\t= ixgbe_features_check,\n \t.ndo_bpf\t\t= ixgbe_xdp,\n \t.ndo_xdp_xmit\t\t= ixgbe_xdp_xmit,\n+\t.ndo_xsk_async_xmit\t= ixgbe_xsk_async_xmit,\n };\n \n static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter,\ndiff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h\nindex 56afb685c648..53d4089f5644 100644\n--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h\n+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h\n@@ -42,5 +42,9 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,\n \t\t\t struct ixgbe_ring *rx_ring,\n \t\t\t const int budget);\n void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring);\n+bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector,\n+\t\t\t struct ixgbe_ring *tx_ring, int napi_budget);\n+int ixgbe_xsk_async_xmit(struct net_device *dev, u32 queue_id);\n+void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring);\n \n #endif /* #define _IXGBE_TXRX_COMMON_H_ */\ndiff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c\nindex 253ce3cfbcf1..e998ed880460 100644\n--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c\n+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c\n@@ -637,3 +637,178 @@ void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring)\n \t\t}\n \t}\n }\n+\n+static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)\n+{\n+\tunion ixgbe_adv_tx_desc *tx_desc = NULL;\n+\tstruct ixgbe_tx_buffer *tx_bi;\n+\tbool work_done = true;\n+\tu32 len, cmd_type;\n+\tdma_addr_t dma;\n+\n+\twhile (budget-- > 0) {\n+\t\tif (unlikely(!ixgbe_desc_unused(xdp_ring))) {\n+\t\t\twork_done = false;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))\n+\t\t\tbreak;\n+\n+\t\tdma_sync_single_for_device(xdp_ring->dev, dma, len,\n+\t\t\t\t\t DMA_BIDIRECTIONAL);\n+\n+\t\ttx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];\n+\t\ttx_bi->bytecount = len;\n+\t\ttx_bi->xdpf = NULL;\n+\n+\t\ttx_desc = IXGBE_TX_DESC(xdp_ring, xdp_ring->next_to_use);\n+\t\ttx_desc->read.buffer_addr = cpu_to_le64(dma);\n+\n+\t\t/* put descriptor type bits */\n+\t\tcmd_type = IXGBE_ADVTXD_DTYP_DATA |\n+\t\t\t IXGBE_ADVTXD_DCMD_DEXT |\n+\t\t\t IXGBE_ADVTXD_DCMD_IFCS;\n+\t\tcmd_type |= len | IXGBE_TXD_CMD;\n+\t\ttx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);\n+\t\ttx_desc->read.olinfo_status =\n+\t\t\tcpu_to_le32(len << IXGBE_ADVTXD_PAYLEN_SHIFT);\n+\n+\t\txdp_ring->next_to_use++;\n+\t\tif (xdp_ring->next_to_use == xdp_ring->count)\n+\t\t\txdp_ring->next_to_use = 0;\n+\t}\n+\n+\tif (tx_desc) {\n+\t\tixgbe_xdp_ring_update_tail(xdp_ring);\n+\t\txsk_umem_consume_tx_done(xdp_ring->xsk_umem);\n+\t}\n+\n+\treturn !!budget && work_done;\n+}\n+\n+static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring,\n+\t\t\t\t struct ixgbe_tx_buffer *tx_bi)\n+{\n+\txdp_return_frame(tx_bi->xdpf);\n+\tdma_unmap_single(tx_ring->dev,\n+\t\t\t dma_unmap_addr(tx_bi, dma),\n+\t\t\t dma_unmap_len(tx_bi, len), DMA_TO_DEVICE);\n+\tdma_unmap_len_set(tx_bi, len, 0);\n+}\n+\n+bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector,\n+\t\t\t struct ixgbe_ring *tx_ring, int napi_budget)\n+{\n+\tunsigned int total_packets = 0, total_bytes = 0;\n+\tu32 i = tx_ring->next_to_clean, xsk_frames = 0;\n+\tunsigned int budget = q_vector->tx.work_limit;\n+\tstruct xdp_umem *umem = tx_ring->xsk_umem;\n+\tunion ixgbe_adv_tx_desc *tx_desc;\n+\tstruct ixgbe_tx_buffer *tx_bi;\n+\tbool xmit_done;\n+\n+\ttx_bi = &tx_ring->tx_buffer_info[i];\n+\ttx_desc = IXGBE_TX_DESC(tx_ring, i);\n+\ti -= tx_ring->count;\n+\n+\tdo {\n+\t\tif (!(tx_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))\n+\t\t\tbreak;\n+\n+\t\ttotal_bytes += tx_bi->bytecount;\n+\t\ttotal_packets += tx_bi->gso_segs;\n+\n+\t\tif (tx_bi->xdpf)\n+\t\t\tixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi);\n+\t\telse\n+\t\t\txsk_frames++;\n+\n+\t\ttx_bi->xdpf = NULL;\n+\t\ttotal_bytes += tx_bi->bytecount;\n+\n+\t\ttx_bi++;\n+\t\ttx_desc++;\n+\t\ti++;\n+\t\tif (unlikely(!i)) {\n+\t\t\ti -= tx_ring->count;\n+\t\t\ttx_bi = tx_ring->tx_buffer_info;\n+\t\t\ttx_desc = IXGBE_TX_DESC(tx_ring, 0);\n+\t\t}\n+\n+\t\t/* issue prefetch for next Tx descriptor */\n+\t\tprefetch(tx_desc);\n+\n+\t\t/* update budget accounting */\n+\t\tbudget--;\n+\t} while (likely(budget));\n+\n+\ti += tx_ring->count;\n+\ttx_ring->next_to_clean = i;\n+\n+\tu64_stats_update_begin(&tx_ring->syncp);\n+\ttx_ring->stats.bytes += total_bytes;\n+\ttx_ring->stats.packets += total_packets;\n+\tu64_stats_update_end(&tx_ring->syncp);\n+\tq_vector->tx.total_bytes += total_bytes;\n+\tq_vector->tx.total_packets += total_packets;\n+\n+\tif (xsk_frames)\n+\t\txsk_umem_complete_tx(umem, xsk_frames);\n+\n+\txmit_done = ixgbe_xmit_zc(tx_ring, q_vector->tx.work_limit);\n+\treturn budget > 0 && xmit_done;\n+}\n+\n+int ixgbe_xsk_async_xmit(struct net_device *dev, u32 queue_id)\n+{\n+\tstruct ixgbe_adapter *adapter = netdev_priv(dev);\n+\tstruct ixgbe_ring *ring;\n+\n+\tif (test_bit(__IXGBE_DOWN, &adapter->state))\n+\t\treturn -ENETDOWN;\n+\n+\tif (!READ_ONCE(adapter->xdp_prog))\n+\t\treturn -ENXIO;\n+\n+\tif (queue_id >= adapter->num_rx_queues)\n+\t\treturn -ENXIO;\n+\n+\tif (!adapter->xsk_umems[queue_id])\n+\t\treturn -ENXIO;\n+\n+\tring = adapter->xdp_ring[queue_id];\n+\tif (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) {\n+\t\tu64 eics = BIT_ULL(ring->q_vector->v_idx);\n+\n+\t\tixgbe_irq_rearm_queues(adapter, eics);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring)\n+{\n+\tu16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use;\n+\tstruct xdp_umem *umem = tx_ring->xsk_umem;\n+\tstruct ixgbe_tx_buffer *tx_bi;\n+\tu32 xsk_frames = 0;\n+\n+\twhile (ntc != ntu) {\n+\t\ttx_bi = &tx_ring->tx_buffer_info[ntc];\n+\n+\t\tif (tx_bi->xdpf)\n+\t\t\tixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi);\n+\t\telse\n+\t\t\txsk_frames++;\n+\n+\t\ttx_bi->xdpf = NULL;\n+\n+\t\tntc++;\n+\t\tif (ntc == tx_ring->count)\n+\t\t\tntc = 0;\n+\t}\n+\n+\tif (xsk_frames)\n+\t\txsk_umem_complete_tx(umem, xsk_frames);\n+}\n", "prefixes": [ "5/5" ] }