Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/925072/?format=api
{ "id": 925072, "url": "http://patchwork.ozlabs.org/api/patches/925072/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20180604120601.18123-11-bjorn.topel@gmail.com/", "project": { "id": 46, "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api", "name": "Intel Wired Ethernet development", "link_name": "intel-wired-lan", "list_id": "intel-wired-lan.osuosl.org", "list_email": "intel-wired-lan@osuosl.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20180604120601.18123-11-bjorn.topel@gmail.com>", "list_archive_url": null, "date": "2018-06-04T12:06:00", "name": "[bpf-next,10/11] i40e: implement AF_XDP zero-copy support for Tx", "commit_ref": null, "pull_url": null, "state": "changes-requested", "archived": false, "hash": "76ca6c2c52b4de5620ea01cb9153af816e616e4c", "submitter": { "id": 70569, "url": "http://patchwork.ozlabs.org/api/people/70569/?format=api", "name": "Björn Töpel", "email": "bjorn.topel@gmail.com" }, "delegate": { "id": 68, "url": "http://patchwork.ozlabs.org/api/users/68/?format=api", "username": "jtkirshe", "first_name": "Jeff", "last_name": "Kirsher", "email": "jeffrey.t.kirsher@intel.com" }, "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20180604120601.18123-11-bjorn.topel@gmail.com/mbox/", "series": [ { "id": 48416, "url": "http://patchwork.ozlabs.org/api/series/48416/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/list/?series=48416", "date": "2018-06-04T12:05:50", "name": "AF_XDP: introducing zero-copy support", "version": 1, "mbox": "http://patchwork.ozlabs.org/series/48416/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/925072/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/925072/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<intel-wired-lan-bounces@osuosl.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Authentication-Results": [ "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=osuosl.org\n\t(client-ip=140.211.166.138; helo=whitealder.osuosl.org;\n\tenvelope-from=intel-wired-lan-bounces@osuosl.org;\n\treceiver=<UNKNOWN>)", "ozlabs.org;\n\tdmarc=fail (p=none dis=none) header.from=gmail.com" ], "Received": [ "from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138])\n\t(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 40zys02KW6z9s0w\n\tfor <incoming@patchwork.ozlabs.org>;\n\tTue, 5 Jun 2018 01:04:36 +1000 (AEST)", "from localhost (localhost [127.0.0.1])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id B21988798E;\n\tMon, 4 Jun 2018 15:04:34 +0000 (UTC)", "from whitealder.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id nIOOC04UNC4j; Mon, 4 Jun 2018 15:04:29 +0000 (UTC)", "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id D11C087B49;\n\tMon, 4 Jun 2018 15:04:29 +0000 (UTC)", "from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136])\n\tby ash.osuosl.org (Postfix) with ESMTP id 3767C1BFFD0\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:07:09 +0000 (UTC)", "from localhost (localhost [127.0.0.1])\n\tby silver.osuosl.org (Postfix) with ESMTP id 2FA6E26E12\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:07:09 +0000 (UTC)", "from silver.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id HEhjuXX-10IR for <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:07:08 +0000 (UTC)", "from mga07.intel.com (mga07.intel.com [134.134.136.100])\n\tby silver.osuosl.org (Postfix) with ESMTPS id F3BDC2CD1B\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tMon, 4 Jun 2018 12:07:07 +0000 (UTC)", "from fmsmga004.fm.intel.com ([10.253.24.48])\n\tby orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t04 Jun 2018 05:07:07 -0700", "from btopel-mobl1.isw.intel.com (HELO\n\tbtopel-mobl1.hil-pdxphhh.sea.wayport.net) ([10.103.211.148])\n\tby fmsmga004.fm.intel.com with ESMTP; 04 Jun 2018 05:07:02 -0700" ], "X-Virus-Scanned": [ "amavisd-new at osuosl.org", "amavisd-new at osuosl.org" ], "X-Greylist": "domain auto-whitelisted by SQLgrey-1.7.6", "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.49,476,1520924400\"; d=\"scan'208\";a=\"60197251\"", "From": "=?utf-8?b?QmrDtnJuIFTDtnBlbA==?= <bjorn.topel@gmail.com>", "To": "bjorn.topel@gmail.com, magnus.karlsson@intel.com,\n\tmagnus.karlsson@gmail.com, alexander.h.duyck@intel.com,\n\talexander.duyck@gmail.com, ast@fb.com, brouer@redhat.com,\n\tdaniel@iogearbox.net, netdev@vger.kernel.org, mykyta.iziumtsev@linaro.org", "Date": "Mon, 4 Jun 2018 14:06:00 +0200", "Message-Id": "<20180604120601.18123-11-bjorn.topel@gmail.com>", "X-Mailer": "git-send-email 2.14.1", "In-Reply-To": "<20180604120601.18123-1-bjorn.topel@gmail.com>", "References": "<20180604120601.18123-1-bjorn.topel@gmail.com>", "X-Mailman-Approved-At": "Mon, 04 Jun 2018 15:04:26 +0000", "Subject": "[Intel-wired-lan] [PATCH bpf-next 10/11] i40e: implement AF_XDP\n\tzero-copy support for Tx", "X-BeenThere": "intel-wired-lan@osuosl.org", "X-Mailman-Version": "2.1.24", "Precedence": "list", "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.osuosl.org>", "List-Unsubscribe": "<https://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=unsubscribe>", "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>", "List-Post": "<mailto:intel-wired-lan@osuosl.org>", "List-Help": "<mailto:intel-wired-lan-request@osuosl.org?subject=help>", "List-Subscribe": "<https://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=subscribe>", "Cc": "francois.ozog@linaro.org, willemdebruijn.kernel@gmail.com, mst@redhat.com,\n\tilias.apalodimas@linaro.org, michael.lundkvist@ericsson.com,\n\tbrian.brooks@linaro.org, intel-wired-lan@lists.osuosl.org,\n\tqi.z.zhang@intel.com, michael.chan@broadcom.com, andy@greyhouse.net", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "intel-wired-lan-bounces@osuosl.org", "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@osuosl.org>" }, "content": "From: Magnus Karlsson <magnus.karlsson@intel.com>\n\nHere, ndo_xsk_async_xmit is implemented. As a shortcut, the existing\nXDP Tx rings are used for zero-copy. This will result in other devices\ndoing XDP_REDIRECT to an AF_XDP enabled queue will have its packets\ndropped.\n\nSigned-off-by: Magnus Karlsson <magnus.karlsson@intel.com>\n---\n drivers/net/ethernet/intel/i40e/i40e_main.c | 7 +-\n drivers/net/ethernet/intel/i40e/i40e_txrx.c | 93 +++++++++++-------\n drivers/net/ethernet/intel/i40e/i40e_txrx.h | 23 +++++\n drivers/net/ethernet/intel/i40e/i40e_xsk.c | 140 ++++++++++++++++++++++++++++\n drivers/net/ethernet/intel/i40e/i40e_xsk.h | 2 +\n include/net/xdp_sock.h | 14 +++\n 6 files changed, 242 insertions(+), 37 deletions(-)", "diff": "diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c\nindex 8c602424d339..98c18c41809d 100644\n--- a/drivers/net/ethernet/intel/i40e/i40e_main.c\n+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c\n@@ -3073,8 +3073,12 @@ static int i40e_configure_tx_ring(struct i40e_ring *ring)\n \ti40e_status err = 0;\n \tu32 qtx_ctl = 0;\n \n-\tif (ring_is_xdp(ring))\n+\tring->clean_tx_irq = i40e_clean_tx_irq;\n+\tif (ring_is_xdp(ring)) {\n \t\tring->xsk_umem = i40e_xsk_umem(ring);\n+\t\tif (ring->xsk_umem)\n+\t\t\tring->clean_tx_irq = i40e_clean_tx_irq_zc;\n+\t}\n \n \t/* some ATR related tx ring init */\n \tif (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {\n@@ -12162,6 +12166,7 @@ static const struct net_device_ops i40e_netdev_ops = {\n \t.ndo_bpf\t\t= i40e_xdp,\n \t.ndo_xdp_xmit\t\t= i40e_xdp_xmit,\n \t.ndo_xdp_flush\t\t= i40e_xdp_flush,\n+\t.ndo_xsk_async_xmit\t= i40e_xsk_async_xmit,\n };\n \n /**\ndiff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c\nindex 6b1142fbc697..923bb84a93ab 100644\n--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c\n+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c\n@@ -10,16 +10,6 @@\n #include \"i40e_trace.h\"\n #include \"i40e_prototype.h\"\n \n-static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,\n-\t\t\t\tu32 td_tag)\n-{\n-\treturn cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |\n-\t\t\t ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |\n-\t\t\t ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |\n-\t\t\t ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |\n-\t\t\t ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));\n-}\n-\n #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)\n /**\n * i40e_fdir - Generate a Flow Director descriptor based on fdata\n@@ -649,9 +639,13 @@ void i40e_clean_tx_ring(struct i40e_ring *tx_ring)\n \tif (!tx_ring->tx_bi)\n \t\treturn;\n \n-\t/* Free all the Tx ring sk_buffs */\n-\tfor (i = 0; i < tx_ring->count; i++)\n-\t\ti40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);\n+\t/* Cleanup only needed for non XSK TX ZC rings */\n+\tif (!tx_ring->xsk_umem) {\n+\t\t/* Free all the Tx ring sk_buffs */\n+\t\tfor (i = 0; i < tx_ring->count; i++)\n+\t\t\ti40e_unmap_and_free_tx_resource(tx_ring,\n+\t\t\t\t\t\t\t&tx_ring->tx_bi[i]);\n+\t}\n \n \tbi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;\n \tmemset(tx_ring->tx_bi, 0, bi_size);\n@@ -768,8 +762,40 @@ void i40e_detect_recover_hung(struct i40e_vsi *vsi)\n \t}\n }\n \n+void i40e_update_tx_stats(struct i40e_ring *tx_ring,\n+\t\t\t unsigned int total_packets,\n+\t\t\t unsigned int total_bytes)\n+{\n+\tu64_stats_update_begin(&tx_ring->syncp);\n+\ttx_ring->stats.bytes += total_bytes;\n+\ttx_ring->stats.packets += total_packets;\n+\tu64_stats_update_end(&tx_ring->syncp);\n+\ttx_ring->q_vector->tx.total_bytes += total_bytes;\n+\ttx_ring->q_vector->tx.total_packets += total_packets;\n+}\n+\n #define WB_STRIDE 4\n \n+void i40e_arm_wb(struct i40e_ring *tx_ring,\n+\t\t struct i40e_vsi *vsi,\n+\t\t int budget)\n+{\n+\tif (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {\n+\t\t/* check to see if there are < 4 descriptors\n+\t\t * waiting to be written back, then kick the hardware to force\n+\t\t * them to be written back in case we stay in NAPI.\n+\t\t * In this mode on X722 we do not enable Interrupt.\n+\t\t */\n+\t\tunsigned int j = i40e_get_tx_pending(tx_ring, false);\n+\n+\t\tif (budget &&\n+\t\t ((j / WB_STRIDE) == 0) && (j > 0) &&\n+\t\t !test_bit(__I40E_VSI_DOWN, vsi->state) &&\n+\t\t (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))\n+\t\t\ttx_ring->arm_wb = true;\n+\t}\n+}\n+\n /**\n * i40e_clean_tx_irq - Reclaim resources after transmit completes\n * @vsi: the VSI we care about\n@@ -778,8 +804,8 @@ void i40e_detect_recover_hung(struct i40e_vsi *vsi)\n *\n * Returns true if there's any budget left (e.g. the clean is finished)\n **/\n-static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,\n-\t\t\t struct i40e_ring *tx_ring, int napi_budget)\n+bool i40e_clean_tx_irq(struct i40e_vsi *vsi,\n+\t\t struct i40e_ring *tx_ring, int napi_budget)\n {\n \tu16 i = tx_ring->next_to_clean;\n \tstruct i40e_tx_buffer *tx_buf;\n@@ -874,27 +900,9 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,\n \n \ti += tx_ring->count;\n \ttx_ring->next_to_clean = i;\n-\tu64_stats_update_begin(&tx_ring->syncp);\n-\ttx_ring->stats.bytes += total_bytes;\n-\ttx_ring->stats.packets += total_packets;\n-\tu64_stats_update_end(&tx_ring->syncp);\n-\ttx_ring->q_vector->tx.total_bytes += total_bytes;\n-\ttx_ring->q_vector->tx.total_packets += total_packets;\n-\n-\tif (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {\n-\t\t/* check to see if there are < 4 descriptors\n-\t\t * waiting to be written back, then kick the hardware to force\n-\t\t * them to be written back in case we stay in NAPI.\n-\t\t * In this mode on X722 we do not enable Interrupt.\n-\t\t */\n-\t\tunsigned int j = i40e_get_tx_pending(tx_ring, false);\n \n-\t\tif (budget &&\n-\t\t ((j / WB_STRIDE) == 0) && (j > 0) &&\n-\t\t !test_bit(__I40E_VSI_DOWN, vsi->state) &&\n-\t\t (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))\n-\t\t\ttx_ring->arm_wb = true;\n-\t}\n+\ti40e_update_tx_stats(tx_ring, total_packets, total_bytes);\n+\ti40e_arm_wb(tx_ring, vsi, budget);\n \n \tif (ring_is_xdp(tx_ring))\n \t\treturn !!budget;\n@@ -2467,10 +2475,11 @@ int i40e_napi_poll(struct napi_struct *napi, int budget)\n \t * budget and be more aggressive about cleaning up the Tx descriptors.\n \t */\n \ti40e_for_each_ring(ring, q_vector->tx) {\n-\t\tif (!i40e_clean_tx_irq(vsi, ring, budget)) {\n+\t\tif (!ring->clean_tx_irq(vsi, ring, budget)) {\n \t\t\tclean_complete = false;\n \t\t\tcontinue;\n \t\t}\n+\n \t\tarm_wb |= ring->arm_wb;\n \t\tring->arm_wb = false;\n \t}\n@@ -3595,6 +3604,12 @@ int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,\n \tif (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs)\n \t\treturn -ENXIO;\n \n+\t/* NB! For now, AF_XDP zero-copy hijacks the XDP ring, and\n+\t * will drop incoming packets redirected by other devices!\n+\t */\n+\tif (vsi->xdp_rings[queue_index]->xsk_umem)\n+\t\treturn -ENXIO;\n+\n \tif (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))\n \t\treturn -EINVAL;\n \n@@ -3633,5 +3648,11 @@ void i40e_xdp_flush(struct net_device *dev)\n \tif (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs)\n \t\treturn;\n \n+\t/* NB! For now, AF_XDP zero-copy hijacks the XDP ring, and\n+\t * will drop incoming packets redirected by other devices!\n+\t */\n+\tif (vsi->xdp_rings[queue_index]->xsk_umem)\n+\t\treturn;\n+\n \ti40e_xdp_ring_update_tail(vsi->xdp_rings[queue_index]);\n }\ndiff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h\nindex cddb185cd2f8..b9c42c352a8d 100644\n--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h\n+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h\n@@ -426,6 +426,8 @@ struct i40e_ring {\n \n \tint (*clean_rx_irq)(struct i40e_ring *ring, int budget);\n \tbool (*alloc_rx_buffers)(struct i40e_ring *ring, u16 n);\n+\tbool (*clean_tx_irq)(struct i40e_vsi *vsi, struct i40e_ring *ring,\n+\t\t\t int budget);\n \tstruct xdp_umem *xsk_umem;\n \n \tstruct zero_copy_allocator zca; /* ZC allocator anchor */\n@@ -506,6 +508,9 @@ int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,\n \t\t u32 flags);\n void i40e_xdp_flush(struct net_device *dev);\n int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget);\n+bool i40e_clean_tx_irq(struct i40e_vsi *vsi,\n+\t\t struct i40e_ring *tx_ring, int napi_budget);\n+int i40e_xsk_async_xmit(struct net_device *dev, u32 queue_id);\n \n /**\n * i40e_get_head - Retrieve head from head writeback\n@@ -687,6 +692,16 @@ static inline void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring)\n \twritel_relaxed(xdp_ring->next_to_use, xdp_ring->tail);\n }\n \n+static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,\n+\t\t\t\tu32 td_tag)\n+{\n+\treturn cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |\n+\t\t\t ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |\n+\t\t\t ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |\n+\t\t\t ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |\n+\t\t\t ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));\n+}\n+\n void i40e_fd_handle_status(struct i40e_ring *rx_ring,\n \t\t\t union i40e_rx_desc *rx_desc, u8 prog_id);\n int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp,\n@@ -696,4 +711,12 @@ void i40e_process_skb_fields(struct i40e_ring *rx_ring,\n \t\t\t u8 rx_ptype);\n void i40e_receive_skb(struct i40e_ring *rx_ring,\n \t\t struct sk_buff *skb, u16 vlan_tag);\n+\n+void i40e_update_tx_stats(struct i40e_ring *tx_ring,\n+\t\t\t unsigned int total_packets,\n+\t\t\t unsigned int total_bytes);\n+void i40e_arm_wb(struct i40e_ring *tx_ring,\n+\t\t struct i40e_vsi *vsi,\n+\t\t int budget);\n+\n #endif /* _I40E_TXRX_H_ */\ndiff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c\nindex 9d16924415b9..021fec5b5799 100644\n--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c\n+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c\n@@ -535,3 +535,143 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)\n \treturn failure ? budget : (int)total_rx_packets;\n }\n \n+/* Returns true if the work is finished */\n+static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)\n+{\n+\tunsigned int total_packets = 0, total_bytes = 0;\n+\tstruct i40e_tx_buffer *tx_bi;\n+\tstruct i40e_tx_desc *tx_desc;\n+\tbool work_done = true;\n+\tdma_addr_t dma;\n+\tu32 len;\n+\n+\twhile (budget-- > 0) {\n+\t\tif (!unlikely(I40E_DESC_UNUSED(xdp_ring))) {\n+\t\t\txdp_ring->tx_stats.tx_busy++;\n+\t\t\twork_done = false;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))\n+\t\t\tbreak;\n+\n+\t\tdma_sync_single_for_device(xdp_ring->dev, dma, len,\n+\t\t\t\t\t DMA_BIDIRECTIONAL);\n+\n+\t\ttx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use];\n+\t\ttx_bi->bytecount = len;\n+\t\ttx_bi->gso_segs = 1;\n+\n+\t\ttx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use);\n+\t\ttx_desc->buffer_addr = cpu_to_le64(dma);\n+\t\ttx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC\n+\t\t\t\t\t\t\t| I40E_TX_DESC_CMD_EOP,\n+\t\t\t\t\t\t\t 0, len, 0);\n+\n+\t\ttotal_packets++;\n+\t\ttotal_bytes += len;\n+\n+\t\txdp_ring->next_to_use++;\n+\t\tif (xdp_ring->next_to_use == xdp_ring->count)\n+\t\t\txdp_ring->next_to_use = 0;\n+\t}\n+\n+\tif (total_packets > 0) {\n+\t\t/* Request an interrupt for the last frame and bump tail ptr. */\n+\t\ttx_desc->cmd_type_offset_bsz |= (I40E_TX_DESC_CMD_RS <<\n+\t\t\t\t\t\t I40E_TXD_QW1_CMD_SHIFT);\n+\t\ti40e_xdp_ring_update_tail(xdp_ring);\n+\n+\t\txsk_umem_consume_tx_done(xdp_ring->xsk_umem);\n+\t\ti40e_update_tx_stats(xdp_ring, total_packets, total_bytes);\n+\t}\n+\n+\treturn !!budget && work_done;\n+}\n+\n+bool i40e_clean_tx_irq_zc(struct i40e_vsi *vsi,\n+\t\t\t struct i40e_ring *tx_ring, int napi_budget)\n+{\n+\tstruct xdp_umem *umem = tx_ring->xsk_umem;\n+\tu32 head_idx = i40e_get_head(tx_ring);\n+\tunsigned int budget = vsi->work_limit;\n+\tbool work_done = true, xmit_done;\n+\tu32 completed_frames;\n+\tu32 frames_ready;\n+\n+\tif (head_idx < tx_ring->next_to_clean)\n+\t\thead_idx += tx_ring->count;\n+\tframes_ready = head_idx - tx_ring->next_to_clean;\n+\n+\tif (frames_ready == 0) {\n+\t\tgoto out_xmit;\n+\t} else if (frames_ready > budget) {\n+\t\tcompleted_frames = budget;\n+\t\twork_done = false;\n+\t} else {\n+\t\tcompleted_frames = frames_ready;\n+\t}\n+\n+\ttx_ring->next_to_clean += completed_frames;\n+\tif (unlikely(tx_ring->next_to_clean >= tx_ring->count))\n+\t\ttx_ring->next_to_clean -= tx_ring->count;\n+\n+\txsk_umem_complete_tx(umem, completed_frames);\n+\n+\ti40e_arm_wb(tx_ring, vsi, budget);\n+\n+out_xmit:\n+\txmit_done = i40e_xmit_zc(tx_ring, budget);\n+\n+\treturn work_done && xmit_done;\n+}\n+\n+/**\n+ * i40e_napi_is_scheduled - If napi is running, set the NAPIF_STATE_MISSED\n+ * @n: napi context\n+ *\n+ * Returns true if NAPI is scheduled.\n+ **/\n+static bool i40e_napi_is_scheduled(struct napi_struct *n)\n+{\n+\tunsigned long val, new;\n+\n+\tdo {\n+\t\tval = READ_ONCE(n->state);\n+\t\tif (val & NAPIF_STATE_DISABLE)\n+\t\t\treturn true;\n+\n+\t\tif (!(val & NAPIF_STATE_SCHED))\n+\t\t\treturn false;\n+\n+\t\tnew = val | NAPIF_STATE_MISSED;\n+\t} while (cmpxchg(&n->state, val, new) != val);\n+\n+\treturn true;\n+}\n+\n+int i40e_xsk_async_xmit(struct net_device *dev, u32 queue_id)\n+{\n+\tstruct i40e_netdev_priv *np = netdev_priv(dev);\n+\tstruct i40e_vsi *vsi = np->vsi;\n+\tstruct i40e_ring *ring;\n+\n+\tif (test_bit(__I40E_VSI_DOWN, vsi->state))\n+\t\treturn -ENETDOWN;\n+\n+\tif (!i40e_enabled_xdp_vsi(vsi))\n+\t\treturn -ENXIO;\n+\n+\tif (queue_id >= vsi->num_queue_pairs)\n+\t\treturn -ENXIO;\n+\n+\tif (!vsi->xdp_rings[queue_id]->xsk_umem)\n+\t\treturn -ENXIO;\n+\n+\tring = vsi->xdp_rings[queue_id];\n+\n+\tif (!i40e_napi_is_scheduled(&ring->q_vector->napi))\n+\t\ti40e_force_wb(vsi, ring->q_vector);\n+\n+\treturn 0;\n+}\ndiff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.h b/drivers/net/ethernet/intel/i40e/i40e_xsk.h\nindex 757ac5ca8511..bd006f1a4397 100644\n--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.h\n+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.h\n@@ -13,5 +13,7 @@ int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,\n void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);\n bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 cleaned_count);\n int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget);\n+bool i40e_clean_tx_irq_zc(struct i40e_vsi *vsi,\n+\t\t\t struct i40e_ring *tx_ring, int napi_budget);\n \n #endif /* _I40E_XSK_H_ */\ndiff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h\nindex ec8fd3314097..63aa05abf11d 100644\n--- a/include/net/xdp_sock.h\n+++ b/include/net/xdp_sock.h\n@@ -103,6 +103,20 @@ static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)\n static inline void xsk_umem_discard_addr(struct xdp_umem *umem)\n {\n }\n+\n+static inline void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)\n+{\n+}\n+\n+static inline bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma,\n+\t\t\t\t u32 *len)\n+{\n+\treturn false;\n+}\n+\n+static inline void xsk_umem_consume_tx_done(struct xdp_umem *umem)\n+{\n+}\n #endif /* CONFIG_XDP_SOCKETS */\n \n static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)\n", "prefixes": [ "bpf-next", "10/11" ] }