Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/1187708/?format=api
{ "id": 1187708, "url": "http://patchwork.ozlabs.org/api/patches/1187708/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20191031143005.5002-3-anthony.l.nguyen@intel.com/", "project": { "id": 46, "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api", "name": "Intel Wired Ethernet development", "link_name": "intel-wired-lan", "list_id": "intel-wired-lan.osuosl.org", "list_email": "intel-wired-lan@osuosl.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20191031143005.5002-3-anthony.l.nguyen@intel.com>", "list_archive_url": null, "date": "2019-10-31T14:30:05", "name": "[S30,v4,5/9] ice: Add support for AF_XDP", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "30760b56bde7df1d174eb744701219285a2be730", "submitter": { "id": 68875, "url": "http://patchwork.ozlabs.org/api/people/68875/?format=api", "name": "Tony Nguyen", "email": "anthony.l.nguyen@intel.com" }, "delegate": { "id": 68, "url": "http://patchwork.ozlabs.org/api/users/68/?format=api", "username": "jtkirshe", "first_name": "Jeff", "last_name": "Kirsher", "email": "jeffrey.t.kirsher@intel.com" }, "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20191031143005.5002-3-anthony.l.nguyen@intel.com/mbox/", "series": [ { "id": 139992, "url": "http://patchwork.ozlabs.org/api/series/139992/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/list/?series=139992", "date": "2019-10-31T14:30:04", "name": null, "version": 4, "mbox": "http://patchwork.ozlabs.org/series/139992/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/1187708/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/1187708/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<intel-wired-lan-bounces@osuosl.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Authentication-Results": [ "ozlabs.org; spf=pass (sender SPF authorized)\n\tsmtp.mailfrom=osuosl.org (client-ip=140.211.166.133;\n\thelo=hemlock.osuosl.org;\n\tenvelope-from=intel-wired-lan-bounces@osuosl.org;\n\treceiver=<UNKNOWN>)", "ozlabs.org;\n\tdmarc=fail (p=none dis=none) header.from=intel.com" ], "Received": [ "from hemlock.osuosl.org (smtp2.osuosl.org [140.211.166.133])\n\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256\n\tbits)) (No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 47415D4KKWz9sPj\n\tfor <incoming@patchwork.ozlabs.org>;\n\tFri, 1 Nov 2019 10:00:48 +1100 (AEDT)", "from localhost (localhost [127.0.0.1])\n\tby hemlock.osuosl.org (Postfix) with ESMTP id EF04F8836A;\n\tThu, 31 Oct 2019 23:00:46 +0000 (UTC)", "from hemlock.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id ZNUqDWyYhHTN; Thu, 31 Oct 2019 23:00:39 +0000 (UTC)", "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby hemlock.osuosl.org (Postfix) with ESMTP id 297F58836B;\n\tThu, 31 Oct 2019 23:00:39 +0000 (UTC)", "from hemlock.osuosl.org (smtp2.osuosl.org [140.211.166.133])\n\tby ash.osuosl.org (Postfix) with ESMTP id E0A821BF27A\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tThu, 31 Oct 2019 23:00:36 +0000 (UTC)", "from localhost (localhost [127.0.0.1])\n\tby hemlock.osuosl.org (Postfix) with ESMTP id BBED888170\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tThu, 31 Oct 2019 23:00:36 +0000 (UTC)", "from hemlock.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id hpx99W9ja9UR for <intel-wired-lan@lists.osuosl.org>;\n\tThu, 31 Oct 2019 23:00:33 +0000 (UTC)", "from mga07.intel.com (mga07.intel.com [134.134.136.100])\n\tby hemlock.osuosl.org (Postfix) with ESMTPS id 40B3288360\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tThu, 31 Oct 2019 23:00:33 +0000 (UTC)", "from fmsmga003.fm.intel.com ([10.253.24.29])\n\tby orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t31 Oct 2019 16:00:27 -0700", "from unknown (HELO localhost.jf.intel.com) ([10.166.244.174])\n\tby FMSMGA003.fm.intel.com with ESMTP; 31 Oct 2019 16:00:27 -0700" ], "X-Virus-Scanned": [ "amavisd-new at osuosl.org", "amavisd-new at osuosl.org" ], "X-Greylist": "domain auto-whitelisted by SQLgrey-1.7.6", "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.68,253,1569308400\"; d=\"scan'208\";a=\"206251864\"", "From": "Tony Nguyen <anthony.l.nguyen@intel.com>", "To": "intel-wired-lan@lists.osuosl.org", "Date": "Thu, 31 Oct 2019 07:30:05 -0700", "Message-Id": "<20191031143005.5002-3-anthony.l.nguyen@intel.com>", "X-Mailer": "git-send-email 2.20.1", "In-Reply-To": "<20191031143005.5002-1-anthony.l.nguyen@intel.com>", "References": "<20191031143005.5002-1-anthony.l.nguyen@intel.com>", "MIME-Version": "1.0", "Subject": "[Intel-wired-lan] [PATCH S30 v4 5/9] ice: Add support for AF_XDP", "X-BeenThere": "intel-wired-lan@osuosl.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.osuosl.org>", "List-Unsubscribe": "<https://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=unsubscribe>", "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>", "List-Post": "<mailto:intel-wired-lan@osuosl.org>", "List-Help": "<mailto:intel-wired-lan-request@osuosl.org?subject=help>", "List-Subscribe": "<https://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "intel-wired-lan-bounces@osuosl.org", "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@osuosl.org>" }, "content": "From: Krzysztof Kazimierczak <krzysztof.kazimierczak@intel.com>\n\nAdd zero copy AF_XDP support. This patch adds zero copy support for\nTx and Rx; code for zero copy is added to ice_xsk.h and ice_xsk.c.\n\nFor Tx, implement ndo_xsk_wakeup. As with other drivers, reuse\nexisting XDP Tx queues for this task, since XDP_REDIRECT guarantees\nmutual exclusion between different NAPI contexts based on CPU ID. In\nturn, a netdev can XDP_REDIRECT to another netdev with a different\nNAPI context, since the operation is bound to a specific core and each\ncore has its own hardware ring.\n\nFor Rx, allocate frames as MEM_TYPE_ZERO_COPY on queues that AF_XDP is\nenabled.\n\nSigned-off-by: Krzysztof Kazimierczak <krzysztof.kazimierczak@intel.com>\nCo-developed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>\nSigned-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>\nSigned-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>\n---\nv4:\n- Change ice_build_ctob() call to build_ctob()\nv3:\n- Remove unused prototype ice_xsk_umem_query()\n---\n drivers/net/ethernet/intel/ice/Makefile | 1 +\n drivers/net/ethernet/intel/ice/ice.h | 26 +\n drivers/net/ethernet/intel/ice/ice_base.c | 53 +-\n drivers/net/ethernet/intel/ice/ice_ethtool.c | 7 +\n drivers/net/ethernet/intel/ice/ice_lib.c | 57 +-\n drivers/net/ethernet/intel/ice/ice_lib.h | 4 +\n drivers/net/ethernet/intel/ice/ice_main.c | 16 +\n drivers/net/ethernet/intel/ice/ice_txrx.c | 46 +-\n drivers/net/ethernet/intel/ice/ice_txrx.h | 20 +-\n drivers/net/ethernet/intel/ice/ice_xsk.c | 1181 ++++++++++++++++++\n drivers/net/ethernet/intel/ice/ice_xsk.h | 72 ++\n 11 files changed, 1456 insertions(+), 27 deletions(-)\n create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.c\n create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.h", "diff": "diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile\nindex 94fe430fdfdd..774aad3ce237 100644\n--- a/drivers/net/ethernet/intel/ice/Makefile\n+++ b/drivers/net/ethernet/intel/ice/Makefile\n@@ -22,3 +22,4 @@ ice-y := ice_main.o\t\\\n \t ice_ethtool.o\n ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o\n ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_lib.o\n+ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o\ndiff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h\nindex 0a90de35ca96..61e2030283aa 100644\n--- a/drivers/net/ethernet/intel/ice/ice.h\n+++ b/drivers/net/ethernet/intel/ice/ice.h\n@@ -36,6 +36,7 @@\n #include <linux/avf/virtchnl.h>\n #include <linux/mfd/core.h>\n #include <net/ipv6.h>\n+#include <net/xdp_sock.h>\n #include \"ice_devids.h\"\n #include \"ice_type.h\"\n #include \"ice_txrx.h\"\n@@ -46,6 +47,7 @@\n #include \"ice_idc_int.h\"\n #include \"ice_virtchnl_pf.h\"\n #include \"ice_sriov.h\"\n+#include \"ice_xsk.h\"\n \n extern const char ice_drv_ver[];\n #define ICE_BAR0\t\t0\n@@ -291,6 +293,9 @@ struct ice_vsi {\n \tstruct ice_ring **xdp_rings;\t /* XDP ring array */\n \tu16 num_xdp_txq;\t\t /* Used XDP queues */\n \tu8 xdp_mapping_mode;\t\t /* ICE_MAP_MODE_[CONTIG|SCATTER] */\n+\tstruct xdp_umem **xsk_umems;\n+\tu16 num_xsk_umems_used;\n+\tu16 num_xsk_umems;\n } ____cacheline_internodealigned_in_smp;\n \n /* struct that defines an interrupt vector */\n@@ -453,6 +458,27 @@ static inline void ice_set_ring_xdp(struct ice_ring *ring)\n \tring->flags |= ICE_TX_FLAGS_RING_XDP;\n }\n \n+/**\n+ * ice_xsk_umem - get XDP UMEM bound to a ring\n+ * @ring - ring to use\n+ *\n+ * Returns a pointer to xdp_umem structure if there is an UMEM present,\n+ * NULL otherwise.\n+ */\n+static inline struct xdp_umem *ice_xsk_umem(struct ice_ring *ring)\n+{\n+\tstruct xdp_umem **umems = ring->vsi->xsk_umems;\n+\tint qid = ring->q_index;\n+\n+\tif (ice_ring_is_xdp(ring))\n+\t\tqid -= ring->vsi->num_xdp_txq;\n+\n+\tif (!umems || !umems[qid] || !ice_is_xdp_ena_vsi(ring->vsi))\n+\t\treturn NULL;\n+\n+\treturn umems[qid];\n+}\n+\n /**\n * ice_get_main_vsi - Get the PF VSI\n * @pf: PF instance\ndiff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c\nindex e335256d483d..4db042e88476 100644\n--- a/drivers/net/ethernet/intel/ice/ice_base.c\n+++ b/drivers/net/ethernet/intel/ice/ice_base.c\n@@ -276,14 +276,17 @@ ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q)\n */\n int ice_setup_rx_ctx(struct ice_ring *ring)\n {\n+\tint chain_len = ICE_MAX_CHAINED_RX_BUFS;\n \tstruct ice_vsi *vsi = ring->vsi;\n-\tstruct ice_hw *hw = &vsi->back->hw;\n \tu32 rxdid = ICE_RXDID_FLEX_NIC;\n \tstruct ice_rlan_ctx rlan_ctx;\n+\tstruct ice_hw *hw;\n \tu32 regval;\n \tu16 pf_q;\n \tint err;\n \n+\thw = &vsi->back->hw;\n+\n \t/* what is Rx queue number in global space of 2K Rx queues */\n \tpf_q = vsi->rxq_map[ring->q_index];\n \n@@ -297,10 +300,38 @@ int ice_setup_rx_ctx(struct ice_ring *ring)\n \t\t\txdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,\n \t\t\t\t\t ring->q_index);\n \n-\t\terr = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,\n-\t\t\t\t\t\t MEM_TYPE_PAGE_SHARED, NULL);\n-\t\tif (err)\n-\t\t\treturn err;\n+\t\tring->xsk_umem = ice_xsk_umem(ring);\n+\t\tif (ring->xsk_umem) {\n+\t\t\txdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);\n+\n+\t\t\tring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -\n+\t\t\t\t\t XDP_PACKET_HEADROOM;\n+\t\t\t/* For AF_XDP ZC, we disallow packets to span on\n+\t\t\t * multiple buffers, thus letting us skip that\n+\t\t\t * handling in the fast-path.\n+\t\t\t */\n+\t\t\tchain_len = 1;\n+\t\t\tring->zca.free = ice_zca_free;\n+\t\t\terr = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,\n+\t\t\t\t\t\t\t MEM_TYPE_ZERO_COPY,\n+\t\t\t\t\t\t\t &ring->zca);\n+\t\t\tif (err)\n+\t\t\t\treturn err;\n+\n+\t\t\tdev_info(&vsi->back->pdev->dev, \"Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\\n\",\n+\t\t\t\t ring->q_index);\n+\t\t} else {\n+\t\t\tif (!xdp_rxq_info_is_reg(&ring->xdp_rxq))\n+\t\t\t\txdp_rxq_info_reg(&ring->xdp_rxq,\n+\t\t\t\t\t\t ring->netdev,\n+\t\t\t\t\t\t ring->q_index);\n+\n+\t\t\terr = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,\n+\t\t\t\t\t\t\t MEM_TYPE_PAGE_SHARED,\n+\t\t\t\t\t\t\t NULL);\n+\t\t\tif (err)\n+\t\t\t\treturn err;\n+\t\t}\n \t}\n \t/* Receive Queue Base Address.\n \t * Indicates the starting address of the descriptor queue defined in\n@@ -340,7 +371,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring)\n \t * than 5 x DBUF\n \t */\n \trlan_ctx.rxmax = min_t(u16, vsi->max_frame,\n-\t\t\t ICE_MAX_CHAINED_RX_BUFS * vsi->rx_buf_len);\n+\t\t\t chain_len * ring->rx_buf_len);\n \n \t/* Rx queue threshold in units of 64 */\n \trlan_ctx.lrxqthresh = 1;\n@@ -378,7 +409,15 @@ int ice_setup_rx_ctx(struct ice_ring *ring)\n \t/* init queue specific tail register */\n \tring->tail = hw->hw_addr + QRX_TAIL(pf_q);\n \twritel(0, ring->tail);\n-\tice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));\n+\n+\terr = ring->xsk_umem ?\n+\t ice_alloc_rx_bufs_slow_zc(ring, ICE_DESC_UNUSED(ring)) :\n+\t ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));\n+\tif (err)\n+\t\tdev_info(&vsi->back->pdev->dev,\n+\t\t\t \"Failed allocate some buffers on %sRx ring %d (pf_q %d)\\n\",\n+\t\t\t ring->xsk_umem ? \"UMEM enabled \" : \"\",\n+\t\t\t ring->q_index, pf_q);\n \n \treturn 0;\n }\ndiff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c\nindex 6cee99b5865b..42b032620f66 100644\n--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c\n+++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c\n@@ -2612,6 +2612,13 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring)\n \t\treturn 0;\n \t}\n \n+\t/* If there is a AF_XDP UMEM attached to any of Rx rings,\n+\t * disallow changing the number of descriptors -- regardless\n+\t * if the netdev is running or not.\n+\t */\n+\tif (ice_xsk_any_rx_ring_ena(vsi))\n+\t\treturn -EBUSY;\n+\n \twhile (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) {\n \t\ttimeout--;\n \t\tif (!timeout)\ndiff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c\nindex d3baad3d6b18..4d2da81e0b9d 100644\n--- a/drivers/net/ethernet/intel/ice/ice_lib.c\n+++ b/drivers/net/ethernet/intel/ice/ice_lib.c\n@@ -1318,7 +1318,17 @@ int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi)\n */\n int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi)\n {\n-\treturn ice_vsi_cfg_txqs(vsi, vsi->xdp_rings);\n+\tint ret;\n+\tint i;\n+\n+\tret = ice_vsi_cfg_txqs(vsi, vsi->xdp_rings);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tfor (i = 0; i < vsi->num_xdp_txq; i++)\n+\t\tvsi->xdp_rings[i]->xsk_umem = ice_xsk_umem(vsi->xdp_rings[i]);\n+\n+\treturn ret;\n }\n \n /**\n@@ -2557,6 +2567,51 @@ char *ice_nvm_version_str(struct ice_hw *hw)\n \treturn buf;\n }\n \n+/**\n+ * ice_update_ring_stats - Update ring statistics\n+ * @ring: ring to update\n+ * @cont: used to increment per-vector counters\n+ * @pkts: number of processed packets\n+ * @bytes: number of processed bytes\n+ *\n+ * This function assumes that caller has acquired a u64_stats_sync lock.\n+ */\n+static void\n+ice_update_ring_stats(struct ice_ring *ring, struct ice_ring_container *cont,\n+\t\t u64 pkts, u64 bytes)\n+{\n+\tring->stats.bytes += bytes;\n+\tring->stats.pkts += pkts;\n+\tcont->total_bytes += bytes;\n+\tcont->total_pkts += pkts;\n+}\n+\n+/**\n+ * ice_update_tx_ring_stats - Update Tx ring specific counters\n+ * @tx_ring: ring to update\n+ * @pkts: number of processed packets\n+ * @bytes: number of processed bytes\n+ */\n+void ice_update_tx_ring_stats(struct ice_ring *tx_ring, u64 pkts, u64 bytes)\n+{\n+\tu64_stats_update_begin(&tx_ring->syncp);\n+\tice_update_ring_stats(tx_ring, &tx_ring->q_vector->tx, pkts, bytes);\n+\tu64_stats_update_end(&tx_ring->syncp);\n+}\n+\n+/**\n+ * ice_update_rx_ring_stats - Update Rx ring specific counters\n+ * @rx_ring: ring to update\n+ * @pkts: number of processed packets\n+ * @bytes: number of processed bytes\n+ */\n+void ice_update_rx_ring_stats(struct ice_ring *rx_ring, u64 pkts, u64 bytes)\n+{\n+\tu64_stats_update_begin(&rx_ring->syncp);\n+\tice_update_ring_stats(rx_ring, &rx_ring->q_vector->rx, pkts, bytes);\n+\tu64_stats_update_end(&rx_ring->syncp);\n+}\n+\n /**\n * ice_vsi_cfg_mac_fltr - Add or remove a MAC address filter for a VSI\n * @vsi: the VSI being configured MAC filter\ndiff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h\nindex 53758ec71d3a..12cb89e28631 100644\n--- a/drivers/net/ethernet/intel/ice/ice_lib.h\n+++ b/drivers/net/ethernet/intel/ice/ice_lib.h\n@@ -85,6 +85,10 @@ void ice_vsi_free_tx_rings(struct ice_vsi *vsi);\n \n int ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena);\n \n+void ice_update_tx_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes);\n+\n+void ice_update_rx_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes);\n+\n void ice_vsi_cfg_frame_size(struct ice_vsi *vsi);\n \n u32 ice_intrl_usec_to_reg(u8 intrl, u8 gran);\ndiff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c\nindex fff7e5567910..5b270fe0e349 100644\n--- a/drivers/net/ethernet/intel/ice/ice_main.c\n+++ b/drivers/net/ethernet/intel/ice/ice_main.c\n@@ -1734,6 +1734,7 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)\n \t\tif (ice_setup_tx_ring(xdp_ring))\n \t\t\tgoto free_xdp_rings;\n \t\tice_set_ring_xdp(xdp_ring);\n+\t\txdp_ring->xsk_umem = ice_xsk_umem(xdp_ring);\n \t}\n \n \treturn 0;\n@@ -1976,6 +1977,17 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,\n \tif (if_running)\n \t\tret = ice_up(vsi);\n \n+\tif (!ret && prog && vsi->xsk_umems) {\n+\t\tint i;\n+\n+\t\tice_for_each_rxq(vsi, i) {\n+\t\t\tstruct ice_ring *rx_ring = vsi->rx_rings[i];\n+\n+\t\t\tif (rx_ring->xsk_umem)\n+\t\t\t\tnapi_schedule(&rx_ring->q_vector->napi);\n+\t\t}\n+\t}\n+\n \treturn (ret || xdp_ring_err) ? -ENOMEM : 0;\n }\n \n@@ -2001,6 +2013,9 @@ static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)\n \tcase XDP_QUERY_PROG:\n \t\txdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;\n \t\treturn 0;\n+\tcase XDP_SETUP_XSK_UMEM:\n+\t\treturn ice_xsk_umem_setup(vsi, xdp->xsk.umem,\n+\t\t\t\t\t xdp->xsk.queue_id);\n \tdefault:\n \t\treturn -EINVAL;\n \t}\n@@ -5334,4 +5349,5 @@ static const struct net_device_ops ice_netdev_ops = {\n \t.ndo_tx_timeout = ice_tx_timeout,\n \t.ndo_bpf = ice_xdp,\n \t.ndo_xdp_xmit = ice_xdp_xmit,\n+\t.ndo_xsk_wakeup = ice_xsk_wakeup,\n };\ndiff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c\nindex 279e5ec7d15f..86a23036f420 100644\n--- a/drivers/net/ethernet/intel/ice/ice_txrx.c\n+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c\n@@ -11,6 +11,7 @@\n #include \"ice_lib.h\"\n #include \"ice.h\"\n #include \"ice_dcb_lib.h\"\n+#include \"ice_xsk.h\"\n \n #define ICE_RX_HDR_SIZE\t\t256\n \n@@ -58,6 +59,11 @@ void ice_clean_tx_ring(struct ice_ring *tx_ring)\n {\n \tu16 i;\n \n+\tif (ice_ring_is_xdp(tx_ring) && tx_ring->xsk_umem) {\n+\t\tice_xsk_clean_xdp_ring(tx_ring);\n+\t\tgoto tx_skip_free;\n+\t}\n+\n \t/* ring already cleared, nothing to do */\n \tif (!tx_ring->tx_buf)\n \t\treturn;\n@@ -66,6 +72,7 @@ void ice_clean_tx_ring(struct ice_ring *tx_ring)\n \tfor (i = 0; i < tx_ring->count; i++)\n \t\tice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]);\n \n+tx_skip_free:\n \tmemset(tx_ring->tx_buf, 0, sizeof(*tx_ring->tx_buf) * tx_ring->count);\n \n \t/* Zero out the descriptor ring */\n@@ -198,12 +205,8 @@ static bool ice_clean_tx_irq(struct ice_ring *tx_ring, int napi_budget)\n \n \ti += tx_ring->count;\n \ttx_ring->next_to_clean = i;\n-\tu64_stats_update_begin(&tx_ring->syncp);\n-\ttx_ring->stats.bytes += total_bytes;\n-\ttx_ring->stats.pkts += total_pkts;\n-\tu64_stats_update_end(&tx_ring->syncp);\n-\ttx_ring->q_vector->tx.total_bytes += total_bytes;\n-\ttx_ring->q_vector->tx.total_pkts += total_pkts;\n+\n+\tice_update_tx_ring_stats(tx_ring, total_pkts, total_bytes);\n \n \tif (ice_ring_is_xdp(tx_ring))\n \t\treturn !!budget;\n@@ -286,6 +289,11 @@ void ice_clean_rx_ring(struct ice_ring *rx_ring)\n \tif (!rx_ring->rx_buf)\n \t\treturn;\n \n+\tif (rx_ring->xsk_umem) {\n+\t\tice_xsk_clean_rx_ring(rx_ring);\n+\t\tgoto rx_skip_free;\n+\t}\n+\n \t/* Free all the Rx ring sk_buffs */\n \tfor (i = 0; i < rx_ring->count; i++) {\n \t\tstruct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];\n@@ -313,6 +321,7 @@ void ice_clean_rx_ring(struct ice_ring *rx_ring)\n \t\trx_buf->page_offset = 0;\n \t}\n \n+rx_skip_free:\n \tmemset(rx_ring->rx_buf, 0, sizeof(*rx_ring->rx_buf) * rx_ring->count);\n \n \t/* Zero out the descriptor ring */\n@@ -1073,13 +1082,7 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget)\n \tif (xdp_prog)\n \t\tice_finalize_xdp_rx(rx_ring, xdp_xmit);\n \n-\t/* update queue and vector specific stats */\n-\tu64_stats_update_begin(&rx_ring->syncp);\n-\trx_ring->stats.pkts += total_rx_pkts;\n-\trx_ring->stats.bytes += total_rx_bytes;\n-\tu64_stats_update_end(&rx_ring->syncp);\n-\trx_ring->q_vector->rx.total_pkts += total_rx_pkts;\n-\trx_ring->q_vector->rx.total_bytes += total_rx_bytes;\n+\tice_update_rx_ring_stats(rx_ring, total_rx_pkts, total_rx_bytes);\n \n \t/* guarantee a trip back through this routine if there was a failure */\n \treturn failure ? budget : (int)total_rx_pkts;\n@@ -1457,9 +1460,14 @@ int ice_napi_poll(struct napi_struct *napi, int budget)\n \t/* Since the actual Tx work is minimal, we can give the Tx a larger\n \t * budget and be more aggressive about cleaning up the Tx descriptors.\n \t */\n-\tice_for_each_ring(ring, q_vector->tx)\n-\t\tif (!ice_clean_tx_irq(ring, budget))\n+\tice_for_each_ring(ring, q_vector->tx) {\n+\t\tbool wd = ring->xsk_umem ?\n+\t\t\t ice_clean_tx_irq_zc(ring, budget) :\n+\t\t\t ice_clean_tx_irq(ring, budget);\n+\n+\t\tif (!wd)\n \t\t\tclean_complete = false;\n+\t}\n \n \t/* Handle case where we are called by netpoll with a budget of 0 */\n \tif (unlikely(budget <= 0))\n@@ -1479,7 +1487,13 @@ int ice_napi_poll(struct napi_struct *napi, int budget)\n \tice_for_each_ring(ring, q_vector->rx) {\n \t\tint cleaned;\n \n-\t\tcleaned = ice_clean_rx_irq(ring, budget_per_ring);\n+\t\t/* A dedicated path for zero-copy allows making a single\n+\t\t * comparison in the irq context instead of many inside the\n+\t\t * ice_clean_rx_irq function and makes the codebase cleaner.\n+\t\t */\n+\t\tcleaned = ring->xsk_umem ?\n+\t\t\t ice_clean_rx_irq_zc(ring, budget_per_ring) :\n+\t\t\t ice_clean_rx_irq(ring, budget_per_ring);\n \t\twork_done += cleaned;\n \t\t/* if we clean as many as budgeted, we must not be done */\n \t\tif (cleaned >= budget_per_ring)\ndiff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h\nindex a07101b13226..d5d243b8e69f 100644\n--- a/drivers/net/ethernet/intel/ice/ice_txrx.h\n+++ b/drivers/net/ethernet/intel/ice/ice_txrx.h\n@@ -4,6 +4,8 @@\n #ifndef _ICE_TXRX_H_\n #define _ICE_TXRX_H_\n \n+#include \"ice_type.h\"\n+\n #define ICE_DFLT_IRQ_WORK\t256\n #define ICE_RXBUF_2048\t\t2048\n #define ICE_MAX_CHAINED_RX_BUFS\t5\n@@ -88,9 +90,17 @@ struct ice_tx_offload_params {\n struct ice_rx_buf {\n \tstruct sk_buff *skb;\n \tdma_addr_t dma;\n-\tstruct page *page;\n-\tunsigned int page_offset;\n-\tu16 pagecnt_bias;\n+\tunion {\n+\t\tstruct {\n+\t\t\tstruct page *page;\n+\t\t\tunsigned int page_offset;\n+\t\t\tu16 pagecnt_bias;\n+\t\t};\n+\t\tstruct {\n+\t\t\tvoid *addr;\n+\t\t\tu64 handle;\n+\t\t};\n+\t};\n };\n \n struct ice_q_stats {\n@@ -211,6 +221,8 @@ struct ice_ring {\n \n \tstruct rcu_head rcu;\t\t/* to avoid race on free */\n \tstruct bpf_prog *xdp_prog;\n+\tstruct xdp_umem *xsk_umem;\n+\tstruct zero_copy_allocator zca;\n \t/* CL3 - 3rd cacheline starts here */\n \tstruct xdp_rxq_info xdp_rxq;\n \t/* CLX - the below items are only accessed infrequently and should be\n@@ -250,6 +262,8 @@ struct ice_ring_container {\n #define ice_for_each_ring(pos, head) \\\n \tfor (pos = (head).ring; pos; pos = pos->next)\n \n+union ice_32b_rx_flex_desc;\n+\n bool ice_alloc_rx_bufs(struct ice_ring *rxr, u16 cleaned_count);\n netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev);\n void ice_clean_tx_ring(struct ice_ring *tx_ring);\ndiff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c\nnew file mode 100644\nindex 000000000000..fcffad0069d6\n--- /dev/null\n+++ b/drivers/net/ethernet/intel/ice/ice_xsk.c\n@@ -0,0 +1,1181 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/* Copyright (c) 2019, Intel Corporation. */\n+\n+#include <linux/bpf_trace.h>\n+#include <net/xdp_sock.h>\n+#include <net/xdp.h>\n+#include \"ice.h\"\n+#include \"ice_base.h\"\n+#include \"ice_type.h\"\n+#include \"ice_xsk.h\"\n+#include \"ice_txrx.h\"\n+#include \"ice_txrx_lib.h\"\n+#include \"ice_lib.h\"\n+\n+/**\n+ * ice_qp_reset_stats - Resets all stats for rings of given index\n+ * @vsi: VSI that contains rings of interest\n+ * @q_idx: ring index in array\n+ */\n+static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)\n+{\n+\tmemset(&vsi->rx_rings[q_idx]->rx_stats, 0,\n+\t sizeof(vsi->rx_rings[q_idx]->rx_stats));\n+\tmemset(&vsi->tx_rings[q_idx]->stats, 0,\n+\t sizeof(vsi->tx_rings[q_idx]->stats));\n+\tif (ice_is_xdp_ena_vsi(vsi))\n+\t\tmemset(&vsi->xdp_rings[q_idx]->stats, 0,\n+\t\t sizeof(vsi->xdp_rings[q_idx]->stats));\n+}\n+\n+/**\n+ * ice_qp_clean_rings - Cleans all the rings of a given index\n+ * @vsi: VSI that contains rings of interest\n+ * @q_idx: ring index in array\n+ */\n+static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)\n+{\n+\tice_clean_tx_ring(vsi->tx_rings[q_idx]);\n+\tif (ice_is_xdp_ena_vsi(vsi))\n+\t\tice_clean_tx_ring(vsi->xdp_rings[q_idx]);\n+\tice_clean_rx_ring(vsi->rx_rings[q_idx]);\n+}\n+\n+/**\n+ * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector\n+ * @vsi: VSI that has netdev\n+ * @q_vector: q_vector that has NAPI context\n+ * @enable: true for enable, false for disable\n+ */\n+static void\n+ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector,\n+\t\t bool enable)\n+{\n+\tif (!vsi->netdev || !q_vector)\n+\t\treturn;\n+\n+\tif (enable)\n+\t\tnapi_enable(&q_vector->napi);\n+\telse\n+\t\tnapi_disable(&q_vector->napi);\n+}\n+\n+/**\n+ * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring\n+ * @vsi: the VSI that contains queue vector being un-configured\n+ * @rx_ring: Rx ring that will have its IRQ disabled\n+ * @q_vector: queue vector\n+ */\n+static void\n+ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_ring *rx_ring,\n+\t\t struct ice_q_vector *q_vector)\n+{\n+\tstruct ice_pf *pf = vsi->back;\n+\tstruct ice_hw *hw = &pf->hw;\n+\tint base = vsi->base_vector;\n+\tu16 reg;\n+\tu32 val;\n+\n+\t/* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle\n+\t * here only QINT_RQCTL\n+\t */\n+\treg = rx_ring->reg_idx;\n+\tval = rd32(hw, QINT_RQCTL(reg));\n+\tval &= ~QINT_RQCTL_CAUSE_ENA_M;\n+\twr32(hw, QINT_RQCTL(reg), val);\n+\n+\tif (q_vector) {\n+\t\tu16 v_idx = q_vector->v_idx;\n+\n+\t\twr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);\n+\t\tice_flush(hw);\n+\t\tsynchronize_irq(pf->msix_entries[v_idx + base].vector);\n+\t}\n+}\n+\n+/**\n+ * ice_qvec_cfg_msix - Enable IRQ for given queue vector\n+ * @vsi: the VSI that contains queue vector\n+ * @q_vector: queue vector\n+ */\n+static void\n+ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector)\n+{\n+\tu16 reg_idx = q_vector->reg_idx;\n+\tstruct ice_pf *pf = vsi->back;\n+\tstruct ice_hw *hw = &pf->hw;\n+\tstruct ice_ring *ring;\n+\n+\tice_cfg_itr(hw, q_vector);\n+\n+\twr32(hw, GLINT_RATE(reg_idx),\n+\t ice_intrl_usec_to_reg(q_vector->intrl, hw->intrl_gran));\n+\n+\tice_for_each_ring(ring, q_vector->tx)\n+\t\tice_cfg_txq_interrupt(vsi, ring->reg_idx, reg_idx,\n+\t\t\t\t q_vector->tx.itr_idx);\n+\n+\tice_for_each_ring(ring, q_vector->rx)\n+\t\tice_cfg_rxq_interrupt(vsi, ring->reg_idx, reg_idx,\n+\t\t\t\t q_vector->rx.itr_idx);\n+\n+\tice_flush(hw);\n+}\n+\n+/**\n+ * ice_qvec_ena_irq - Enable IRQ for given queue vector\n+ * @vsi: the VSI that contains queue vector\n+ * @q_vector: queue vector\n+ */\n+static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector)\n+{\n+\tstruct ice_pf *pf = vsi->back;\n+\tstruct ice_hw *hw = &pf->hw;\n+\n+\tice_irq_dynamic_ena(hw, vsi, q_vector);\n+\n+\tice_flush(hw);\n+}\n+\n+/**\n+ * ice_qp_dis - Disables a queue pair\n+ * @vsi: VSI of interest\n+ * @q_idx: ring index in array\n+ *\n+ * Returns 0 on success, negative on failure.\n+ */\n+static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx)\n+{\n+\tstruct ice_txq_meta txq_meta = { };\n+\tstruct ice_ring *tx_ring, *rx_ring;\n+\tstruct ice_q_vector *q_vector;\n+\tint timeout = 50;\n+\tint err;\n+\n+\tif (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)\n+\t\treturn -EINVAL;\n+\n+\ttx_ring = vsi->tx_rings[q_idx];\n+\trx_ring = vsi->rx_rings[q_idx];\n+\tq_vector = rx_ring->q_vector;\n+\n+\twhile (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) {\n+\t\ttimeout--;\n+\t\tif (!timeout)\n+\t\t\treturn -EBUSY;\n+\t\tusleep_range(1000, 2000);\n+\t}\n+\tnetif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx));\n+\n+\tice_qvec_dis_irq(vsi, rx_ring, q_vector);\n+\n+\tice_fill_txq_meta(vsi, tx_ring, &txq_meta);\n+\terr = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta);\n+\tif (err)\n+\t\treturn err;\n+\tif (ice_is_xdp_ena_vsi(vsi)) {\n+\t\tstruct ice_ring *xdp_ring = vsi->xdp_rings[q_idx];\n+\n+\t\tmemset(&txq_meta, 0, sizeof(txq_meta));\n+\t\tice_fill_txq_meta(vsi, xdp_ring, &txq_meta);\n+\t\terr = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring,\n+\t\t\t\t\t &txq_meta);\n+\t\tif (err)\n+\t\t\treturn err;\n+\t}\n+\terr = ice_vsi_ctrl_rx_ring(vsi, false, q_idx);\n+\tif (err)\n+\t\treturn err;\n+\n+\tice_qvec_toggle_napi(vsi, q_vector, false);\n+\tice_qp_clean_rings(vsi, q_idx);\n+\tice_qp_reset_stats(vsi, q_idx);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ice_qp_ena - Enables a queue pair\n+ * @vsi: VSI of interest\n+ * @q_idx: ring index in array\n+ *\n+ * Returns 0 on success, negative on failure.\n+ */\n+static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx)\n+{\n+\tstruct ice_aqc_add_tx_qgrp *qg_buf;\n+\tstruct ice_ring *tx_ring, *rx_ring;\n+\tstruct ice_q_vector *q_vector;\n+\tint err;\n+\n+\tif (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)\n+\t\treturn -EINVAL;\n+\n+\tqg_buf = kzalloc(sizeof(*qg_buf), GFP_KERNEL);\n+\tif (!qg_buf)\n+\t\treturn -ENOMEM;\n+\n+\tqg_buf->num_txqs = 1;\n+\n+\ttx_ring = vsi->tx_rings[q_idx];\n+\trx_ring = vsi->rx_rings[q_idx];\n+\tq_vector = rx_ring->q_vector;\n+\n+\terr = ice_vsi_cfg_txq(vsi, tx_ring, qg_buf);\n+\tif (err)\n+\t\tgoto free_buf;\n+\n+\tif (ice_is_xdp_ena_vsi(vsi)) {\n+\t\tstruct ice_ring *xdp_ring = vsi->xdp_rings[q_idx];\n+\n+\t\tmemset(qg_buf, 0, sizeof(*qg_buf));\n+\t\tqg_buf->num_txqs = 1;\n+\t\terr = ice_vsi_cfg_txq(vsi, xdp_ring, qg_buf);\n+\t\tif (err)\n+\t\t\tgoto free_buf;\n+\t\tice_set_ring_xdp(xdp_ring);\n+\t\txdp_ring->xsk_umem = ice_xsk_umem(xdp_ring);\n+\t}\n+\n+\terr = ice_setup_rx_ctx(rx_ring);\n+\tif (err)\n+\t\tgoto free_buf;\n+\n+\tice_qvec_cfg_msix(vsi, q_vector);\n+\n+\terr = ice_vsi_ctrl_rx_ring(vsi, true, q_idx);\n+\tif (err)\n+\t\tgoto free_buf;\n+\n+\tclear_bit(__ICE_CFG_BUSY, vsi->state);\n+\tice_qvec_toggle_napi(vsi, q_vector, true);\n+\tice_qvec_ena_irq(vsi, q_vector);\n+\n+\tnetif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx));\n+free_buf:\n+\tkfree(qg_buf);\n+\treturn err;\n+}\n+\n+/**\n+ * ice_xsk_alloc_umems - allocate a UMEM region for an XDP socket\n+ * @vsi: VSI to allocate the UMEM on\n+ *\n+ * Returns 0 on success, negative on error\n+ */\n+static int ice_xsk_alloc_umems(struct ice_vsi *vsi)\n+{\n+\tif (vsi->xsk_umems)\n+\t\treturn 0;\n+\n+\tvsi->xsk_umems = kcalloc(vsi->num_xsk_umems, sizeof(*vsi->xsk_umems),\n+\t\t\t\t GFP_KERNEL);\n+\n+\tif (!vsi->xsk_umems) {\n+\t\tvsi->num_xsk_umems = 0;\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ice_xsk_add_umem - add a UMEM region for XDP sockets\n+ * @vsi: VSI to which the UMEM will be added\n+ * @umem: pointer to a requested UMEM region\n+ * @qid: queue ID\n+ *\n+ * Returns 0 on success, negative on error\n+ */\n+static int ice_xsk_add_umem(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)\n+{\n+\tint err;\n+\n+\terr = ice_xsk_alloc_umems(vsi);\n+\tif (err)\n+\t\treturn err;\n+\n+\tvsi->xsk_umems[qid] = umem;\n+\tvsi->num_xsk_umems_used++;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ice_xsk_remove_umem - Remove an UMEM for a certain ring/qid\n+ * @vsi: VSI from which the VSI will be removed\n+ * @qid: Ring/qid associated with the UMEM\n+ */\n+static void ice_xsk_remove_umem(struct ice_vsi *vsi, u16 qid)\n+{\n+\tvsi->xsk_umems[qid] = NULL;\n+\tvsi->num_xsk_umems_used--;\n+\n+\tif (vsi->num_xsk_umems_used == 0) {\n+\t\tkfree(vsi->xsk_umems);\n+\t\tvsi->xsk_umems = NULL;\n+\t\tvsi->num_xsk_umems = 0;\n+\t}\n+}\n+\n+/**\n+ * ice_xsk_umem_dma_map - DMA map UMEM region for XDP sockets\n+ * @vsi: VSI to map the UMEM region\n+ * @umem: UMEM to map\n+ *\n+ * Returns 0 on success, negative on error\n+ */\n+static int ice_xsk_umem_dma_map(struct ice_vsi *vsi, struct xdp_umem *umem)\n+{\n+\tstruct ice_pf *pf = vsi->back;\n+\tstruct device *dev;\n+\tunsigned int i;\n+\n+\tdev = &pf->pdev->dev;\n+\tfor (i = 0; i < umem->npgs; i++) {\n+\t\tdma_addr_t dma = dma_map_page_attrs(dev, umem->pgs[i], 0,\n+\t\t\t\t\t\t PAGE_SIZE,\n+\t\t\t\t\t\t DMA_BIDIRECTIONAL,\n+\t\t\t\t\t\t ICE_RX_DMA_ATTR);\n+\t\tif (dma_mapping_error(dev, dma)) {\n+\t\t\tdev_dbg(dev,\n+\t\t\t\t\"XSK UMEM DMA mapping error on page num %d\", i);\n+\t\t\tgoto out_unmap;\n+\t\t}\n+\n+\t\tumem->pages[i].dma = dma;\n+\t}\n+\n+\treturn 0;\n+\n+out_unmap:\n+\tfor (; i > 0; i--) {\n+\t\tdma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,\n+\t\t\t\t DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);\n+\t\tumem->pages[i].dma = 0;\n+\t}\n+\n+\treturn -EFAULT;\n+}\n+\n+/**\n+ * ice_xsk_umem_dma_unmap - DMA unmap UMEM region for XDP sockets\n+ * @vsi: VSI from which the UMEM will be unmapped\n+ * @umem: UMEM to unmap\n+ */\n+static void ice_xsk_umem_dma_unmap(struct ice_vsi *vsi, struct xdp_umem *umem)\n+{\n+\tstruct ice_pf *pf = vsi->back;\n+\tstruct device *dev;\n+\tunsigned int i;\n+\n+\tdev = &pf->pdev->dev;\n+\tfor (i = 0; i < umem->npgs; i++) {\n+\t\tdma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,\n+\t\t\t\t DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);\n+\n+\t\tumem->pages[i].dma = 0;\n+\t}\n+}\n+\n+/**\n+ * ice_xsk_umem_disable - disable a UMEM region\n+ * @vsi: Current VSI\n+ * @qid: queue ID\n+ *\n+ * Returns 0 on success, negative on failure\n+ */\n+static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid)\n+{\n+\tif (!vsi->xsk_umems || qid >= vsi->num_xsk_umems ||\n+\t !vsi->xsk_umems[qid])\n+\t\treturn -EINVAL;\n+\n+\tice_xsk_umem_dma_unmap(vsi, vsi->xsk_umems[qid]);\n+\tice_xsk_remove_umem(vsi, qid);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ice_xsk_umem_enable - enable a UMEM region\n+ * @vsi: Current VSI\n+ * @umem: pointer to a requested UMEM region\n+ * @qid: queue ID\n+ *\n+ * Returns 0 on success, negative on failure\n+ */\n+static int\n+ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)\n+{\n+\tstruct xdp_umem_fq_reuse *reuseq;\n+\tint err;\n+\n+\tif (vsi->type != ICE_VSI_PF)\n+\t\treturn -EINVAL;\n+\n+\tvsi->num_xsk_umems = min_t(u16, vsi->num_rxq, vsi->num_txq);\n+\tif (qid >= vsi->num_xsk_umems)\n+\t\treturn -EINVAL;\n+\n+\tif (vsi->xsk_umems && vsi->xsk_umems[qid])\n+\t\treturn -EBUSY;\n+\n+\treuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);\n+\tif (!reuseq)\n+\t\treturn -ENOMEM;\n+\n+\txsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));\n+\n+\terr = ice_xsk_umem_dma_map(vsi, umem);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = ice_xsk_add_umem(vsi, umem, qid);\n+\tif (err)\n+\t\treturn err;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ice_xsk_umem_setup - enable/disable a UMEM region depending on its state\n+ * @vsi: Current VSI\n+ * @umem: UMEM to enable/associate to a ring, NULL to disable\n+ * @qid: queue ID\n+ *\n+ * Returns 0 on success, negative on failure\n+ */\n+int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)\n+{\n+\tbool if_running, umem_present = !!umem;\n+\tint ret = 0, umem_failure = 0;\n+\n+\tif_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi);\n+\n+\tif (if_running) {\n+\t\tret = ice_qp_dis(vsi, qid);\n+\t\tif (ret) {\n+\t\t\tnetdev_err(vsi->netdev, \"ice_qp_dis error = %d\", ret);\n+\t\t\tgoto xsk_umem_if_up;\n+\t\t}\n+\t}\n+\n+\tumem_failure = umem_present ? ice_xsk_umem_enable(vsi, umem, qid) :\n+\t\t\t\t ice_xsk_umem_disable(vsi, qid);\n+\n+xsk_umem_if_up:\n+\tif (if_running) {\n+\t\tret = ice_qp_ena(vsi, qid);\n+\t\tif (!ret && umem_present)\n+\t\t\tnapi_schedule(&vsi->xdp_rings[qid]->q_vector->napi);\n+\t\telse if (ret)\n+\t\t\tnetdev_err(vsi->netdev, \"ice_qp_ena error = %d\", ret);\n+\t}\n+\n+\tif (umem_failure) {\n+\t\tnetdev_err(vsi->netdev, \"Could not %sable UMEM, error = %d\",\n+\t\t\t umem_present ? \"en\" : \"dis\", umem_failure);\n+\t\treturn umem_failure;\n+\t}\n+\n+\treturn ret;\n+}\n+\n+/**\n+ * ice_zca_free - Callback for MEM_TYPE_ZERO_COPY allocations\n+ * @zca: zero-cpoy allocator\n+ * @handle: Buffer handle\n+ */\n+void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle)\n+{\n+\tstruct ice_rx_buf *rx_buf;\n+\tstruct ice_ring *rx_ring;\n+\tstruct xdp_umem *umem;\n+\tu64 hr, mask;\n+\tu16 nta;\n+\n+\trx_ring = container_of(zca, struct ice_ring, zca);\n+\tumem = rx_ring->xsk_umem;\n+\thr = umem->headroom + XDP_PACKET_HEADROOM;\n+\n+\tmask = umem->chunk_mask;\n+\n+\tnta = rx_ring->next_to_alloc;\n+\trx_buf = &rx_ring->rx_buf[nta];\n+\n+\tnta++;\n+\trx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;\n+\n+\thandle &= mask;\n+\n+\trx_buf->dma = xdp_umem_get_dma(umem, handle);\n+\trx_buf->dma += hr;\n+\n+\trx_buf->addr = xdp_umem_get_data(umem, handle);\n+\trx_buf->addr += hr;\n+\n+\trx_buf->handle = (u64)handle + umem->headroom;\n+}\n+\n+/**\n+ * ice_alloc_buf_fast_zc - Retrieve buffer address from XDP umem\n+ * @rx_ring: ring with an xdp_umem bound to it\n+ * @rx_buf: buffer to which xsk page address will be assigned\n+ *\n+ * This function allocates an Rx buffer in the hot path.\n+ * The buffer can come from fill queue or recycle queue.\n+ *\n+ * Returns true if an assignment was successful, false if not.\n+ */\n+static __always_inline bool\n+ice_alloc_buf_fast_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)\n+{\n+\tstruct xdp_umem *umem = rx_ring->xsk_umem;\n+\tvoid *addr = rx_buf->addr;\n+\tu64 handle, hr;\n+\n+\tif (addr) {\n+\t\trx_ring->rx_stats.page_reuse_count++;\n+\t\treturn true;\n+\t}\n+\n+\tif (!xsk_umem_peek_addr(umem, &handle)) {\n+\t\trx_ring->rx_stats.alloc_page_failed++;\n+\t\treturn false;\n+\t}\n+\n+\thr = umem->headroom + XDP_PACKET_HEADROOM;\n+\n+\trx_buf->dma = xdp_umem_get_dma(umem, handle);\n+\trx_buf->dma += hr;\n+\n+\trx_buf->addr = xdp_umem_get_data(umem, handle);\n+\trx_buf->addr += hr;\n+\n+\trx_buf->handle = handle + umem->headroom;\n+\n+\txsk_umem_discard_addr(umem);\n+\treturn true;\n+}\n+\n+/**\n+ * ice_alloc_buf_slow_zc - Retrieve buffer address from XDP umem\n+ * @rx_ring: ring with an xdp_umem bound to it\n+ * @rx_buf: buffer to which xsk page address will be assigned\n+ *\n+ * This function allocates an Rx buffer in the slow path.\n+ * The buffer can come from fill queue or recycle queue.\n+ *\n+ * Returns true if an assignment was successful, false if not.\n+ */\n+static __always_inline bool\n+ice_alloc_buf_slow_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)\n+{\n+\tstruct xdp_umem *umem = rx_ring->xsk_umem;\n+\tu64 handle, headroom;\n+\n+\tif (!xsk_umem_peek_addr_rq(umem, &handle)) {\n+\t\trx_ring->rx_stats.alloc_page_failed++;\n+\t\treturn false;\n+\t}\n+\n+\thandle &= umem->chunk_mask;\n+\theadroom = umem->headroom + XDP_PACKET_HEADROOM;\n+\n+\trx_buf->dma = xdp_umem_get_dma(umem, handle);\n+\trx_buf->dma += headroom;\n+\n+\trx_buf->addr = xdp_umem_get_data(umem, handle);\n+\trx_buf->addr += headroom;\n+\n+\trx_buf->handle = handle + umem->headroom;\n+\n+\txsk_umem_discard_addr_rq(umem);\n+\treturn true;\n+}\n+\n+/**\n+ * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers\n+ * @rx_ring: Rx ring\n+ * @count: The number of buffers to allocate\n+ * @alloc: the function pointer to call for allocation\n+ *\n+ * This function allocates a number of Rx buffers from the fill ring\n+ * or the internal recycle mechanism and places them on the Rx ring.\n+ *\n+ * Returns false if all allocations were successful, true if any fail.\n+ */\n+static bool\n+ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,\n+\t\t bool alloc(struct ice_ring *, struct ice_rx_buf *))\n+{\n+\tunion ice_32b_rx_flex_desc *rx_desc;\n+\tu16 ntu = rx_ring->next_to_use;\n+\tstruct ice_rx_buf *rx_buf;\n+\tbool ret = false;\n+\n+\tif (!count)\n+\t\treturn false;\n+\n+\trx_desc = ICE_RX_DESC(rx_ring, ntu);\n+\trx_buf = &rx_ring->rx_buf[ntu];\n+\n+\tdo {\n+\t\tif (!alloc(rx_ring, rx_buf)) {\n+\t\t\tret = true;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tdma_sync_single_range_for_device(rx_ring->dev, rx_buf->dma, 0,\n+\t\t\t\t\t\t rx_ring->rx_buf_len,\n+\t\t\t\t\t\t DMA_BIDIRECTIONAL);\n+\n+\t\trx_desc->read.pkt_addr = cpu_to_le64(rx_buf->dma);\n+\t\trx_desc->wb.status_error0 = 0;\n+\n+\t\trx_desc++;\n+\t\trx_buf++;\n+\t\tntu++;\n+\n+\t\tif (unlikely(ntu == rx_ring->count)) {\n+\t\t\trx_desc = ICE_RX_DESC(rx_ring, 0);\n+\t\t\trx_buf = rx_ring->rx_buf;\n+\t\t\tntu = 0;\n+\t\t}\n+\t} while (--count);\n+\n+\tif (rx_ring->next_to_use != ntu)\n+\t\tice_release_rx_desc(rx_ring, ntu);\n+\n+\treturn ret;\n+}\n+\n+/**\n+ * ice_alloc_rx_bufs_fast_zc - allocate zero copy bufs in the hot path\n+ * @rx_ring: Rx ring\n+ * @count: number of bufs to allocate\n+ *\n+ * Returns false on success, true on failure.\n+ */\n+static bool ice_alloc_rx_bufs_fast_zc(struct ice_ring *rx_ring, u16 count)\n+{\n+\treturn ice_alloc_rx_bufs_zc(rx_ring, count,\n+\t\t\t\t ice_alloc_buf_fast_zc);\n+}\n+\n+/**\n+ * ice_alloc_rx_bufs_slow_zc - allocate zero copy bufs in the slow path\n+ * @rx_ring: Rx ring\n+ * @count: number of bufs to allocate\n+ *\n+ * Returns false on success, true on failure.\n+ */\n+bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count)\n+{\n+\treturn ice_alloc_rx_bufs_zc(rx_ring, count,\n+\t\t\t\t ice_alloc_buf_slow_zc);\n+}\n+\n+/**\n+ * ice_bump_ntc - Bump the next_to_clean counter of an Rx ring\n+ * @rx_ring: Rx ring\n+ */\n+static void ice_bump_ntc(struct ice_ring *rx_ring)\n+{\n+\tint ntc = rx_ring->next_to_clean + 1;\n+\n+\tntc = (ntc < rx_ring->count) ? ntc : 0;\n+\trx_ring->next_to_clean = ntc;\n+\tprefetch(ICE_RX_DESC(rx_ring, ntc));\n+}\n+\n+/**\n+ * ice_get_rx_buf_zc - Fetch the current Rx buffer\n+ * @rx_ring: Rx ring\n+ * @size: size of a buffer\n+ *\n+ * This function returns the current, received Rx buffer and does\n+ * DMA synchronization.\n+ *\n+ * Returns a pointer to the received Rx buffer.\n+ */\n+static struct ice_rx_buf *ice_get_rx_buf_zc(struct ice_ring *rx_ring, int size)\n+{\n+\tstruct ice_rx_buf *rx_buf;\n+\n+\trx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];\n+\n+\tdma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 0,\n+\t\t\t\t size, DMA_BIDIRECTIONAL);\n+\n+\treturn rx_buf;\n+}\n+\n+/**\n+ * ice_reuse_rx_buf_zc - reuse an Rx buffer\n+ * @rx_ring: Rx ring\n+ * @old_buf: The buffer to recycle\n+ *\n+ * This function recycles a finished Rx buffer, and places it on the recycle\n+ * queue (next_to_alloc).\n+ */\n+static void\n+ice_reuse_rx_buf_zc(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf)\n+{\n+\tunsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask;\n+\tu64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;\n+\tu16 nta = rx_ring->next_to_alloc;\n+\tstruct ice_rx_buf *new_buf;\n+\n+\tnew_buf = &rx_ring->rx_buf[nta++];\n+\trx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;\n+\n+\tnew_buf->dma = old_buf->dma & mask;\n+\tnew_buf->dma += hr;\n+\n+\tnew_buf->addr = (void *)((unsigned long)old_buf->addr & mask);\n+\tnew_buf->addr += hr;\n+\n+\tnew_buf->handle = old_buf->handle & mask;\n+\tnew_buf->handle += rx_ring->xsk_umem->headroom;\n+\n+\told_buf->addr = NULL;\n+}\n+\n+/**\n+ * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer\n+ * @rx_ring: Rx ring\n+ * @rx_buf: zero-copy Rx buffer\n+ * @xdp: XDP buffer\n+ *\n+ * This function allocates a new skb from a zero-copy Rx buffer.\n+ *\n+ * Returns the skb on success, NULL on failure.\n+ */\n+static struct sk_buff *\n+ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,\n+\t\t struct xdp_buff *xdp)\n+{\n+\tunsigned int metasize = xdp->data - xdp->data_meta;\n+\tunsigned int datasize = xdp->data_end - xdp->data;\n+\tunsigned int datasize_hard = xdp->data_end -\n+\t\t\t\t xdp->data_hard_start;\n+\tstruct sk_buff *skb;\n+\n+\tskb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard,\n+\t\t\t GFP_ATOMIC | __GFP_NOWARN);\n+\tif (unlikely(!skb))\n+\t\treturn NULL;\n+\n+\tskb_reserve(skb, xdp->data - xdp->data_hard_start);\n+\tmemcpy(__skb_put(skb, datasize), xdp->data, datasize);\n+\tif (metasize)\n+\t\tskb_metadata_set(skb, metasize);\n+\n+\tice_reuse_rx_buf_zc(rx_ring, rx_buf);\n+\n+\treturn skb;\n+}\n+\n+/**\n+ * ice_run_xdp_zc - Executes an XDP program in zero-copy path\n+ * @rx_ring: Rx ring\n+ * @xdp: xdp_buff used as input to the XDP program\n+ *\n+ * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}\n+ */\n+static int\n+ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)\n+{\n+\tint err, result = ICE_XDP_PASS;\n+\tstruct bpf_prog *xdp_prog;\n+\tstruct ice_ring *xdp_ring;\n+\tu32 act;\n+\n+\trcu_read_lock();\n+\txdp_prog = READ_ONCE(rx_ring->xdp_prog);\n+\tif (!xdp_prog) {\n+\t\trcu_read_unlock();\n+\t\treturn ICE_XDP_PASS;\n+\t}\n+\n+\tact = bpf_prog_run_xdp(xdp_prog, xdp);\n+\txdp->handle += xdp->data - xdp->data_hard_start;\n+\tswitch (act) {\n+\tcase XDP_PASS:\n+\t\tbreak;\n+\tcase XDP_TX:\n+\t\txdp_ring = rx_ring->vsi->xdp_rings[rx_ring->q_index];\n+\t\tresult = ice_xmit_xdp_buff(xdp, xdp_ring);\n+\t\tbreak;\n+\tcase XDP_REDIRECT:\n+\t\terr = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);\n+\t\tresult = !err ? ICE_XDP_REDIR : ICE_XDP_CONSUMED;\n+\t\tbreak;\n+\tdefault:\n+\t\tbpf_warn_invalid_xdp_action(act);\n+\t\t/* fallthrough -- not supported action */\n+\tcase XDP_ABORTED:\n+\t\ttrace_xdp_exception(rx_ring->netdev, xdp_prog, act);\n+\t\t/* fallthrough -- handle aborts by dropping frame */\n+\tcase XDP_DROP:\n+\t\tresult = ICE_XDP_CONSUMED;\n+\t\tbreak;\n+\t}\n+\n+\trcu_read_unlock();\n+\treturn result;\n+}\n+\n+/**\n+ * ice_clean_rx_irq_zc - consumes packets from the hardware ring\n+ * @rx_ring: AF_XDP Rx ring\n+ * @budget: NAPI budget\n+ *\n+ * Returns number of processed packets on success, remaining budget on failure.\n+ */\n+int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)\n+{\n+\tunsigned int total_rx_bytes = 0, total_rx_packets = 0;\n+\tu16 cleaned_count = ICE_DESC_UNUSED(rx_ring);\n+\tunsigned int xdp_xmit = 0;\n+\tstruct xdp_buff xdp;\n+\tbool failure = 0;\n+\n+\txdp.rxq = &rx_ring->xdp_rxq;\n+\n+\twhile (likely(total_rx_packets < (unsigned int)budget)) {\n+\t\tunion ice_32b_rx_flex_desc *rx_desc;\n+\t\tunsigned int size, xdp_res = 0;\n+\t\tstruct ice_rx_buf *rx_buf;\n+\t\tstruct sk_buff *skb;\n+\t\tu16 stat_err_bits;\n+\t\tu16 vlan_tag = 0;\n+\t\tu8 rx_ptype;\n+\n+\t\tif (cleaned_count >= ICE_RX_BUF_WRITE) {\n+\t\t\tfailure |= ice_alloc_rx_bufs_fast_zc(rx_ring,\n+\t\t\t\t\t\t\t cleaned_count);\n+\t\t\tcleaned_count = 0;\n+\t\t}\n+\n+\t\trx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean);\n+\n+\t\tstat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);\n+\t\tif (!ice_test_staterr(rx_desc, stat_err_bits))\n+\t\t\tbreak;\n+\n+\t\t/* This memory barrier is needed to keep us from reading\n+\t\t * any other fields out of the rx_desc until we have\n+\t\t * verified the descriptor has been written back.\n+\t\t */\n+\t\tdma_rmb();\n+\n+\t\tsize = le16_to_cpu(rx_desc->wb.pkt_len) &\n+\t\t\t\t ICE_RX_FLX_DESC_PKT_LEN_M;\n+\t\tif (!size)\n+\t\t\tbreak;\n+\n+\t\trx_buf = ice_get_rx_buf_zc(rx_ring, size);\n+\t\tif (!rx_buf->addr)\n+\t\t\tbreak;\n+\n+\t\txdp.data = rx_buf->addr;\n+\t\txdp.data_meta = xdp.data;\n+\t\txdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;\n+\t\txdp.data_end = xdp.data + size;\n+\t\txdp.handle = rx_buf->handle;\n+\n+\t\txdp_res = ice_run_xdp_zc(rx_ring, &xdp);\n+\t\tif (xdp_res) {\n+\t\t\tif (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) {\n+\t\t\t\txdp_xmit |= xdp_res;\n+\t\t\t\trx_buf->addr = NULL;\n+\t\t\t} else {\n+\t\t\t\tice_reuse_rx_buf_zc(rx_ring, rx_buf);\n+\t\t\t}\n+\n+\t\t\ttotal_rx_bytes += size;\n+\t\t\ttotal_rx_packets++;\n+\t\t\tcleaned_count++;\n+\n+\t\t\tice_bump_ntc(rx_ring);\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\t/* XDP_PASS path */\n+\t\tskb = ice_construct_skb_zc(rx_ring, rx_buf, &xdp);\n+\t\tif (!skb) {\n+\t\t\trx_ring->rx_stats.alloc_buf_failed++;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tcleaned_count++;\n+\t\tice_bump_ntc(rx_ring);\n+\n+\t\tif (eth_skb_pad(skb)) {\n+\t\t\tskb = NULL;\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\ttotal_rx_bytes += skb->len;\n+\t\ttotal_rx_packets++;\n+\n+\t\tstat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S);\n+\t\tif (ice_test_staterr(rx_desc, stat_err_bits))\n+\t\t\tvlan_tag = le16_to_cpu(rx_desc->wb.l2tag1);\n+\n+\t\trx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) &\n+\t\t\t\t ICE_RX_FLEX_DESC_PTYPE_M;\n+\n+\t\tice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);\n+\t\tice_receive_skb(rx_ring, skb, vlan_tag);\n+\t}\n+\n+\tice_finalize_xdp_rx(rx_ring, xdp_xmit);\n+\tice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes);\n+\n+\treturn failure ? budget : (int)total_rx_packets;\n+}\n+\n+/**\n+ * ice_xmit_zc - Completes AF_XDP entries, and cleans XDP entries\n+ * @xdp_ring: XDP Tx ring\n+ * @budget: max number of frames to xmit\n+ *\n+ * Returns true if cleanup/transmission is done.\n+ */\n+static bool ice_xmit_zc(struct ice_ring *xdp_ring, int budget)\n+{\n+\tstruct ice_tx_desc *tx_desc = NULL;\n+\tbool work_done = true;\n+\tstruct xdp_desc desc;\n+\tdma_addr_t dma;\n+\n+\twhile (likely(budget-- > 0)) {\n+\t\tstruct ice_tx_buf *tx_buf;\n+\n+\t\tif (unlikely(!ICE_DESC_UNUSED(xdp_ring))) {\n+\t\t\txdp_ring->tx_stats.tx_busy++;\n+\t\t\twork_done = false;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\ttx_buf = &xdp_ring->tx_buf[xdp_ring->next_to_use];\n+\n+\t\tif (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))\n+\t\t\tbreak;\n+\n+\t\tdma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);\n+\n+\t\tdma_sync_single_for_device(xdp_ring->dev, dma, desc.len,\n+\t\t\t\t\t DMA_BIDIRECTIONAL);\n+\n+\t\ttx_buf->bytecount = desc.len;\n+\n+\t\ttx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use);\n+\t\ttx_desc->buf_addr = cpu_to_le64(dma);\n+\t\ttx_desc->cmd_type_offset_bsz = build_ctob(ICE_TXD_LAST_DESC_CMD,\n+\t\t\t\t\t\t\t 0, desc.len, 0);\n+\n+\t\txdp_ring->next_to_use++;\n+\t\tif (xdp_ring->next_to_use == xdp_ring->count)\n+\t\t\txdp_ring->next_to_use = 0;\n+\t}\n+\n+\tif (tx_desc) {\n+\t\tice_xdp_ring_update_tail(xdp_ring);\n+\t\txsk_umem_consume_tx_done(xdp_ring->xsk_umem);\n+\t}\n+\n+\treturn budget > 0 && work_done;\n+}\n+\n+/**\n+ * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer\n+ * @xdp_ring: XDP Tx ring\n+ * @tx_buf: Tx buffer to clean\n+ */\n+static void\n+ice_clean_xdp_tx_buf(struct ice_ring *xdp_ring, struct ice_tx_buf *tx_buf)\n+{\n+\txdp_return_frame((struct xdp_frame *)tx_buf->raw_buf);\n+\tdma_unmap_single(xdp_ring->dev, dma_unmap_addr(tx_buf, dma),\n+\t\t\t dma_unmap_len(tx_buf, len), DMA_TO_DEVICE);\n+\tdma_unmap_len_set(tx_buf, len, 0);\n+}\n+\n+/**\n+ * ice_clean_tx_irq_zc - Completes AF_XDP entries, and cleans XDP entries\n+ * @xdp_ring: XDP Tx ring\n+ * @budget: NAPI budget\n+ *\n+ * Returns true if cleanup/tranmission is done.\n+ */\n+bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget)\n+{\n+\tint total_packets = 0, total_bytes = 0;\n+\ts16 ntc = xdp_ring->next_to_clean;\n+\tstruct ice_tx_desc *tx_desc;\n+\tstruct ice_tx_buf *tx_buf;\n+\tbool xmit_done = true;\n+\tu32 xsk_frames = 0;\n+\n+\ttx_desc = ICE_TX_DESC(xdp_ring, ntc);\n+\ttx_buf = &xdp_ring->tx_buf[ntc];\n+\tntc -= xdp_ring->count;\n+\n+\tdo {\n+\t\tif (!(tx_desc->cmd_type_offset_bsz &\n+\t\t cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))\n+\t\t\tbreak;\n+\n+\t\ttotal_bytes += tx_buf->bytecount;\n+\t\ttotal_packets++;\n+\n+\t\tif (tx_buf->raw_buf) {\n+\t\t\tice_clean_xdp_tx_buf(xdp_ring, tx_buf);\n+\t\t\ttx_buf->raw_buf = NULL;\n+\t\t} else {\n+\t\t\txsk_frames++;\n+\t\t}\n+\n+\t\ttx_desc->cmd_type_offset_bsz = 0;\n+\t\ttx_buf++;\n+\t\ttx_desc++;\n+\t\tntc++;\n+\n+\t\tif (unlikely(!ntc)) {\n+\t\t\tntc -= xdp_ring->count;\n+\t\t\ttx_buf = xdp_ring->tx_buf;\n+\t\t\ttx_desc = ICE_TX_DESC(xdp_ring, 0);\n+\t\t}\n+\n+\t\tprefetch(tx_desc);\n+\n+\t} while (likely(--budget));\n+\n+\tntc += xdp_ring->count;\n+\txdp_ring->next_to_clean = ntc;\n+\n+\tif (xsk_frames)\n+\t\txsk_umem_complete_tx(xdp_ring->xsk_umem, xsk_frames);\n+\n+\tice_update_tx_ring_stats(xdp_ring, total_packets, total_bytes);\n+\txmit_done = ice_xmit_zc(xdp_ring, ICE_DFLT_IRQ_WORK);\n+\n+\treturn budget > 0 && xmit_done;\n+}\n+\n+/**\n+ * ice_xsk_wakeup - Implements ndo_xsk_wakeup\n+ * @netdev: net_device\n+ * @queue_id: queue to wake up\n+ * @flags: ignored in our case, since we have Rx and Tx in the same NAPI\n+ *\n+ * Returns negative on error, zero otherwise.\n+ */\n+int\n+ice_xsk_wakeup(struct net_device *netdev, u32 queue_id,\n+\t u32 __always_unused flags)\n+{\n+\tstruct ice_netdev_priv *np = netdev_priv(netdev);\n+\tstruct ice_q_vector *q_vector;\n+\tstruct ice_vsi *vsi = np->vsi;\n+\tstruct ice_ring *ring;\n+\n+\tif (test_bit(__ICE_DOWN, vsi->state))\n+\t\treturn -ENETDOWN;\n+\n+\tif (!ice_is_xdp_ena_vsi(vsi))\n+\t\treturn -ENXIO;\n+\n+\tif (queue_id >= vsi->num_txq)\n+\t\treturn -ENXIO;\n+\n+\tif (!vsi->xdp_rings[queue_id]->xsk_umem)\n+\t\treturn -ENXIO;\n+\n+\tring = vsi->xdp_rings[queue_id];\n+\n+\t/* The idea here is that if NAPI is running, mark a miss, so\n+\t * it will run again. If not, trigger an interrupt and\n+\t * schedule the NAPI from interrupt context. If NAPI would be\n+\t * scheduled here, the interrupt affinity would not be\n+\t * honored.\n+\t */\n+\tq_vector = ring->q_vector;\n+\tif (!napi_if_scheduled_mark_missed(&q_vector->napi))\n+\t\tice_trigger_sw_intr(&vsi->back->hw, q_vector);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP UMEM attached\n+ * @vsi: VSI to be checked\n+ *\n+ * Returns true if any of the Rx rings has an AF_XDP UMEM attached\n+ */\n+bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)\n+{\n+\tint i;\n+\n+\tif (!vsi->xsk_umems)\n+\t\treturn false;\n+\n+\tfor (i = 0; i < vsi->num_xsk_umems; i++) {\n+\t\tif (vsi->xsk_umems[i])\n+\t\t\treturn true;\n+\t}\n+\n+\treturn false;\n+}\n+\n+/**\n+ * ice_xsk_clean_rx_ring - clean UMEM queues connected to a given Rx ring\n+ * @rx_ring: ring to be cleaned\n+ */\n+void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring)\n+{\n+\tu16 i;\n+\n+\tfor (i = 0; i < rx_ring->count; i++) {\n+\t\tstruct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];\n+\n+\t\tif (!rx_buf->addr)\n+\t\t\tcontinue;\n+\n+\t\txsk_umem_fq_reuse(rx_ring->xsk_umem, rx_buf->handle);\n+\t\trx_buf->addr = NULL;\n+\t}\n+}\n+\n+/**\n+ * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its UMEM queues\n+ * @xdp_ring: XDP_Tx ring\n+ */\n+void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring)\n+{\n+\tu16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use;\n+\tu32 xsk_frames = 0;\n+\n+\twhile (ntc != ntu) {\n+\t\tstruct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];\n+\n+\t\tif (tx_buf->raw_buf)\n+\t\t\tice_clean_xdp_tx_buf(xdp_ring, tx_buf);\n+\t\telse\n+\t\t\txsk_frames++;\n+\n+\t\ttx_buf->raw_buf = NULL;\n+\n+\t\tntc++;\n+\t\tif (ntc >= xdp_ring->count)\n+\t\t\tntc = 0;\n+\t}\n+\n+\tif (xsk_frames)\n+\t\txsk_umem_complete_tx(xdp_ring->xsk_umem, xsk_frames);\n+}\ndiff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h\nnew file mode 100644\nindex 000000000000..3479e1de98fe\n--- /dev/null\n+++ b/drivers/net/ethernet/intel/ice/ice_xsk.h\n@@ -0,0 +1,72 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/* Copyright (c) 2019, Intel Corporation. */\n+\n+#ifndef _ICE_XSK_H_\n+#define _ICE_XSK_H_\n+#include \"ice_txrx.h\"\n+#include \"ice.h\"\n+\n+struct ice_vsi;\n+\n+#ifdef CONFIG_XDP_SOCKETS\n+int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid);\n+void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle);\n+int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget);\n+bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget);\n+int ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags);\n+bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count);\n+bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi);\n+void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring);\n+void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring);\n+#else\n+static inline int\n+ice_xsk_umem_setup(struct ice_vsi __always_unused *vsi,\n+\t\t struct xdp_umem __always_unused *umem,\n+\t\t u16 __always_unused qid)\n+{\n+\treturn -ENOTSUPP;\n+}\n+\n+static inline void\n+ice_zca_free(struct zero_copy_allocator __always_unused *zca,\n+\t unsigned long __always_unused handle)\n+{\n+}\n+\n+static inline int\n+ice_clean_rx_irq_zc(struct ice_ring __always_unused *rx_ring,\n+\t\t int __always_unused budget)\n+{\n+\treturn 0;\n+}\n+\n+static inline bool\n+ice_clean_tx_irq_zc(struct ice_ring __always_unused *xdp_ring,\n+\t\t int __always_unused budget)\n+{\n+\treturn false;\n+}\n+\n+static inline bool\n+ice_alloc_rx_bufs_slow_zc(struct ice_ring __always_unused *rx_ring,\n+\t\t\t u16 __always_unused count)\n+{\n+\treturn false;\n+}\n+\n+static inline bool ice_xsk_any_rx_ring_ena(struct ice_vsi __always_unused *vsi)\n+{\n+\treturn false;\n+}\n+\n+static inline int\n+ice_xsk_wakeup(struct net_device __always_unused *netdev,\n+\t u32 __always_unused queue_id, u32 __always_unused flags)\n+{\n+\treturn -ENOTSUPP;\n+}\n+\n+#define ice_xsk_clean_rx_ring(rx_ring) do {} while (0)\n+#define ice_xsk_clean_xdp_ring(xdp_ring) do {} while (0)\n+#endif /* CONFIG_XDP_SOCKETS */\n+#endif /* !_ICE_XSK_H_ */\n", "prefixes": [ "S30", "v4", "5/9" ] }