Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 1124849,
    "url": "http://patchwork.ozlabs.org/api/patches/1124849/?format=api",
    "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20190629185405.1601-4-shiraz.saleem@intel.com/",
    "project": {
        "id": 46,
        "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api",
        "name": "Intel Wired Ethernet development",
        "link_name": "intel-wired-lan",
        "list_id": "intel-wired-lan.osuosl.org",
        "list_email": "intel-wired-lan@osuosl.org",
        "web_url": "",
        "scm_url": "",
        "webscm_url": "",
        "list_archive_url": "",
        "list_archive_url_format": "",
        "commit_url_format": ""
    },
    "msgid": "<20190629185405.1601-4-shiraz.saleem@intel.com>",
    "list_archive_url": null,
    "date": "2019-06-29T18:53:51",
    "name": "[rdma-next,03/17] RDMA/irdma: Implement HW Admin Queue OPs",
    "commit_ref": null,
    "pull_url": null,
    "state": "rejected",
    "archived": false,
    "hash": "9ca24d8ddb3b952ab7833725deefb20f46b219ce",
    "submitter": {
        "id": 69500,
        "url": "http://patchwork.ozlabs.org/api/people/69500/?format=api",
        "name": "Saleem, Shiraz",
        "email": "shiraz.saleem@intel.com"
    },
    "delegate": {
        "id": 68,
        "url": "http://patchwork.ozlabs.org/api/users/68/?format=api",
        "username": "jtkirshe",
        "first_name": "Jeff",
        "last_name": "Kirsher",
        "email": "jeffrey.t.kirsher@intel.com"
    },
    "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20190629185405.1601-4-shiraz.saleem@intel.com/mbox/",
    "series": [
        {
            "id": 116886,
            "url": "http://patchwork.ozlabs.org/api/series/116886/?format=api",
            "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/list/?series=116886",
            "date": "2019-06-29T18:53:48",
            "name": "Add unified Intel Ethernet RDMA driver (irdma)",
            "version": 1,
            "mbox": "http://patchwork.ozlabs.org/series/116886/mbox/"
        }
    ],
    "comments": "http://patchwork.ozlabs.org/api/patches/1124849/comments/",
    "check": "pending",
    "checks": "http://patchwork.ozlabs.org/api/patches/1124849/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<intel-wired-lan-bounces@osuosl.org>",
        "X-Original-To": [
            "incoming@patchwork.ozlabs.org",
            "intel-wired-lan@lists.osuosl.org"
        ],
        "Delivered-To": [
            "patchwork-incoming@bilbo.ozlabs.org",
            "intel-wired-lan@lists.osuosl.org"
        ],
        "Authentication-Results": [
            "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=osuosl.org\n\t(client-ip=140.211.166.138; helo=whitealder.osuosl.org;\n\tenvelope-from=intel-wired-lan-bounces@osuosl.org;\n\treceiver=<UNKNOWN>)",
            "ozlabs.org;\n\tdmarc=fail (p=none dis=none) header.from=intel.com"
        ],
        "Received": [
            "from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138])\n\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256\n\tbits)) (No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 45bqJp0yQZz9s3l\n\tfor <incoming@patchwork.ozlabs.org>;\n\tSun, 30 Jun 2019 09:16:42 +1000 (AEST)",
            "from localhost (localhost [127.0.0.1])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id 8848D858AE;\n\tSat, 29 Jun 2019 23:16:40 +0000 (UTC)",
            "from whitealder.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id c6x-GeFr9j2V; Sat, 29 Jun 2019 23:15:53 +0000 (UTC)",
            "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id A2A5586A1E;\n\tSat, 29 Jun 2019 23:15:36 +0000 (UTC)",
            "from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138])\n\tby ash.osuosl.org (Postfix) with ESMTP id 319B51BF3AD\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tSat, 29 Jun 2019 18:55:11 +0000 (UTC)",
            "from localhost (localhost [127.0.0.1])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id 2331E821AE\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tSat, 29 Jun 2019 18:55:11 +0000 (UTC)",
            "from whitealder.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id Wq9DcnG8XtOJ for <intel-wired-lan@lists.osuosl.org>;\n\tSat, 29 Jun 2019 18:54:19 +0000 (UTC)",
            "from mga14.intel.com (mga14.intel.com [192.55.52.115])\n\tby whitealder.osuosl.org (Postfix) with ESMTPS id A7D9386B68\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tSat, 29 Jun 2019 18:54:17 +0000 (UTC)",
            "from fmsmga004.fm.intel.com ([10.253.24.48])\n\tby fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t29 Jun 2019 11:54:17 -0700",
            "from ssaleem-mobl.amr.corp.intel.com ([10.254.177.95])\n\tby fmsmga004.fm.intel.com with ESMTP; 29 Jun 2019 11:54:16 -0700"
        ],
        "X-Virus-Scanned": [
            "amavisd-new at osuosl.org",
            "amavisd-new at osuosl.org"
        ],
        "X-Greylist": "domain auto-whitelisted by SQLgrey-1.7.6",
        "X-Amp-Result": "SKIPPED(no attachment in message)",
        "X-Amp-File-Uploaded": "False",
        "X-ExtLoop1": "1",
        "X-IronPort-AV": "E=Sophos;i=\"5.63,432,1557212400\"; d=\"scan'208\";a=\"185972859\"",
        "From": "Shiraz Saleem <shiraz.saleem@intel.com>",
        "To": "intel-wired-lan@lists.osuosl.org",
        "Date": "Sat, 29 Jun 2019 13:53:51 -0500",
        "Message-Id": "<20190629185405.1601-4-shiraz.saleem@intel.com>",
        "X-Mailer": "git-send-email 2.21.0",
        "In-Reply-To": "<20190629185405.1601-1-shiraz.saleem@intel.com>",
        "References": "<20190629185405.1601-1-shiraz.saleem@intel.com>",
        "MIME-Version": "1.0",
        "X-Mailman-Approved-At": "Sat, 29 Jun 2019 23:15:33 +0000",
        "Subject": "[Intel-wired-lan] [PATCH rdma-next 03/17] RDMA/irdma: Implement HW\n\tAdmin Queue OPs",
        "X-BeenThere": "intel-wired-lan@osuosl.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.osuosl.org>",
        "List-Unsubscribe": "<https://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=unsubscribe>",
        "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>",
        "List-Post": "<mailto:intel-wired-lan@osuosl.org>",
        "List-Help": "<mailto:intel-wired-lan-request@osuosl.org?subject=help>",
        "List-Subscribe": "<https://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@osuosl.org?subject=subscribe>",
        "Cc": "Mustafa Ismail <mustafa.ismail@intel.com>,\n\tShiraz Saleem <shiraz.saleem@intel.com>",
        "Content-Type": "text/plain; charset=\"us-ascii\"",
        "Content-Transfer-Encoding": "7bit",
        "Errors-To": "intel-wired-lan-bounces@osuosl.org",
        "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@osuosl.org>"
    },
    "content": "From: Mustafa Ismail <mustafa.ismail@intel.com>\n\nThe driver posts privileged commands to the HW\nAdmin Queue (Control QP or CQP) to request administrative\nactions from the HW. Implement create/destroy of CQP\nand the supporting functions, data structures and headers\nto handle the different CQP commands\n\nSigned-off-by: Mustafa Ismail <mustafa.ismail@intel.com>\nSigned-off-by: Shiraz Saleem <shiraz.saleem@intel.com>\n---\n drivers/infiniband/hw/irdma/ctrl.c  | 5958 +++++++++++++++++++++++++++++++++++\n drivers/infiniband/hw/irdma/defs.h  | 2126 +++++++++++++\n drivers/infiniband/hw/irdma/irdma.h |  191 ++\n drivers/infiniband/hw/irdma/type.h  | 1701 ++++++++++\n 4 files changed, 9976 insertions(+)\n create mode 100644 drivers/infiniband/hw/irdma/ctrl.c\n create mode 100644 drivers/infiniband/hw/irdma/defs.h\n create mode 100644 drivers/infiniband/hw/irdma/irdma.h\n create mode 100644 drivers/infiniband/hw/irdma/type.h",
    "diff": "diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c\nnew file mode 100644\nindex 0000000..e70c72a\n--- /dev/null\n+++ b/drivers/infiniband/hw/irdma/ctrl.c\n@@ -0,0 +1,5958 @@\n+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB\n+/* Copyright (c) 2019, Intel Corporation. */\n+\n+#include \"osdep.h\"\n+#include \"status.h\"\n+#include \"hmc.h\"\n+#include \"defs.h\"\n+#include \"type.h\"\n+#include \"protos.h\"\n+#include \"ws.h\"\n+\n+/**\n+ * irdma_get_qp_from_list - get next qp from a list\n+ * @head: Listhead of qp's\n+ * @qp: current qp\n+ */\n+struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,\n+\t\t\t\t\t   struct irdma_sc_qp *qp)\n+{\n+\tstruct list_head *lastentry;\n+\tstruct list_head *entry = NULL;\n+\n+\tif (list_empty(head))\n+\t\treturn NULL;\n+\n+\tif (!qp) {\n+\t\tentry = head->next;\n+\t} else {\n+\t\tlastentry = &qp->list;\n+\t\tentry = lastentry->next;\n+\t\tif (entry == head)\n+\t\t\treturn NULL;\n+\t}\n+\n+\treturn container_of(entry, struct irdma_sc_qp, list);\n+}\n+\n+/**\n+ * irdma_suspend_qps - suspend all qp's on VSI\n+ * @vsi: the VSI struct pointer\n+ */\n+void irdma_suspend_qps(struct irdma_sc_vsi *vsi)\n+{\n+\tstruct irdma_sc_qp *qp = NULL;\n+\tunsigned long flags;\n+\tint i;\n+\n+\tfor (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {\n+\t\tspin_lock_irqsave(&vsi->qos[i].lock, flags);\n+\t\tqp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);\n+\t\twhile (qp) {\n+\t\t\t/* issue cqp suspend command */\n+\t\t\tif (!irdma_qp_suspend_resume(qp, true))\n+\t\t\t\tatomic_inc(&vsi->qp_suspend_reqs);\n+\t\t\tqp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);\n+\t\t}\n+\t\tspin_unlock_irqrestore(&vsi->qos[i].lock, flags);\n+\t}\n+}\n+\n+/**\n+ * irdma_change_l2params - given the new l2 parameters, change all qp\n+ * @vsi: RDMA VSI pointer\n+ * @l2params: New parameters from l2\n+ */\n+void irdma_change_l2params(struct irdma_sc_vsi *vsi,\n+\t\t\t   struct irdma_l2params *l2params)\n+{\n+\tstruct irdma_sc_qp *qp = NULL;\n+\tu8 i;\n+\tunsigned long flags;\n+\n+\tif (l2params->mtu_changed && vsi->mtu != l2params->mtu) {\n+\t\tvsi->mtu = l2params->mtu;\n+\t\tirdma_reinitialize_ieq(vsi);\n+\t}\n+\n+\tif (!l2params->tc_changed)\n+\t\treturn;\n+\n+\tvsi->tc_change_pending = false;\n+\tfor (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {\n+\t\tvsi->qos[i].traffic_class = l2params->up2tc[i];\n+\t\tspin_lock_irqsave(&vsi->qos[i].lock, flags);\n+\t\tqp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);\n+\t\twhile (qp) {\n+\t\t\tif (!irdma_ws_add(vsi, i)) {\n+\t\t\t\tqp->qs_handle = vsi->qos[qp->user_pri].qs_handle;\n+\t\t\t\tirdma_qp_suspend_resume(qp, false);\n+\t\t\t} else {\n+\t\t\t\tirdma_qp_suspend_resume(qp, false);\n+\t\t\t\tspin_unlock_irqrestore(&vsi->qos[i].lock, flags);\n+\t\t\t\tirdma_modify_qp_to_err(qp);\n+\t\t\t\tspin_lock_irqsave(&vsi->qos[i].lock, flags);\n+\t\t\t}\n+\t\t\tqp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);\n+\t\t}\n+\t\tspin_unlock_irqrestore(&vsi->qos[i].lock, flags);\n+\t}\n+}\n+\n+/**\n+ * irdma_qp_rem_qos - remove qp from qos lists during destroy qp\n+ * @qp: qp to be removed from qos\n+ */\n+void irdma_qp_rem_qos(struct irdma_sc_qp *qp)\n+{\n+\tstruct irdma_sc_vsi *vsi = qp->vsi;\n+\tunsigned long flags;\n+\n+\tif (!qp->on_qoslist)\n+\t\treturn;\n+\n+\tspin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);\n+\tqp->on_qoslist = false;\n+\tlist_del(&qp->list);\n+\tspin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);\n+\tdev_dbg(rfdev_to_dev(qp->dev),\n+\t\t\"DCB: DCB: Remove qp[%d] UP[%d] qset[%d]\\n\", qp->qp_uk.qp_id,\n+\t\tqp->user_pri, qp->qs_handle);\n+}\n+\n+/**\n+ * irdma_qp_add_qos - called during setctx for qp to be added to qos\n+ * @qp: qp to be added to qos\n+ */\n+void irdma_qp_add_qos(struct irdma_sc_qp *qp)\n+{\n+\tstruct irdma_sc_vsi *vsi = qp->vsi;\n+\tunsigned long flags;\n+\n+\tif (qp->on_qoslist)\n+\t\treturn;\n+\n+\tspin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);\n+\tlist_add(&qp->list, &vsi->qos[qp->user_pri].qplist);\n+\tqp->on_qoslist = true;\n+\tqp->qs_handle = vsi->qos[qp->user_pri].qs_handle;\n+\tspin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);\n+\tdev_dbg(rfdev_to_dev(qp->dev),\n+\t\t\"DCB: DCB: Add qp[%d] UP[%d] qset[%d]\\n\", qp->qp_uk.qp_id,\n+\t\tqp->user_pri, qp->qs_handle);\n+}\n+\n+/**\n+ * irdma_sc_pd_init - initialize sc pd struct\n+ * @dev: sc device struct\n+ * @pd: sc pd ptr\n+ * @pd_id: pd_id for allocated pd\n+ * @abi_ver: ABI version from user context, -1 if not valid\n+ */\n+static void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd,\n+\t\t\t     u32 pd_id, int abi_ver)\n+{\n+\tpd->pd_id = pd_id;\n+\tpd->abi_ver = abi_ver;\n+\tpd->dev = dev;\n+}\n+\n+/**\n+ * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry\n+ * @cqp: struct for cqp hw\n+ * @info: arp entry information\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,\n+\t\t\t     struct irdma_add_arp_cache_entry_info *info,\n+\t\t\t     u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 temp, hdr;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\tset_64bit_val(wqe, 8, info->reach_max);\n+\n+\ttemp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |\n+\t       LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |\n+\t       LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);\n+\tset_64bit_val(wqe, 16, temp);\n+\n+\thdr = info->arp_index |\n+\t      LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64((info->permanent ? 1 : 0), IRDMA_CQPSQ_MAT_PERMANENT) |\n+\t      LS_64(1, IRDMA_CQPSQ_MAT_ENTRYVALID) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"ARP_CACHE_ENTRY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_del_arp_cache_entry - dele arp cache entry\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @arp_index: arp index to delete arp entry\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,\n+\t\t\t     u16 arp_index, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\thdr = arp_index | LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"ARP_CACHE_DEL_ENTRY WQE\",\n+\t\t\twqe, IRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_query_arp_cache_entry - cqp wqe to query arp and arp index\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @arp_index: arp index to delete arp entry\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_query_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,\n+\t\t\t       u16 arp_index, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\thdr = arp_index | LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMA_CQPSQ_MAT_QUERY) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QUERY_ARP_CACHE_ENTRY WQE\",\n+\t\t\twqe, IRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries\n+ * @cqp: struct for cqp hw\n+ * @info: info for apbvt entry to add or delete\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,\n+\t\t\t    struct irdma_apbvt_info *info, u64 scratch,\n+\t\t\t    bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, info->port);\n+\n+\thdr = LS_64(IRDMA_CQP_OP_MANAGE_APBVT, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->add, IRDMA_CQPSQ_MAPT_ADDPORT) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"MANAGE_APBVT WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_manage_qhash_table_entry - manage quad hash entries\n+ * @cqp: struct for cqp hw\n+ * @info: info for quad hash to manage\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ *\n+ * This is called before connection establishment is started.\n+ * For passive connections, when listener is created, it will\n+ * call with entry type of  IRDMA_QHASH_TYPE_TCP_SYN with local\n+ * ip address and tcp port. When SYN is received (passive\n+ * connections) or sent (active connections), this routine is\n+ * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED\n+ * and quad is passed in info.\n+ *\n+ * When iwarp connection is done and its state moves to RTS, the\n+ * quad hash entry in the hardware will point to iwarp's qp\n+ * number and requires no calls from the driver.\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,\n+\t\t\t\t  struct irdma_qhash_table_info *info,\n+\t\t\t\t  u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 qw1 = 0;\n+\tu64 qw2 = 0;\n+\tu64 temp;\n+\tstruct irdma_sc_vsi *vsi = info->vsi;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\ttemp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |\n+\t       LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |\n+\t       LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);\n+\tset_64bit_val(wqe, 0, temp);\n+\n+\tqw1 = LS_64(info->qp_num, IRDMA_CQPSQ_QHASH_QPN) |\n+\t      LS_64(info->dest_port, IRDMA_CQPSQ_QHASH_DEST_PORT);\n+\tif (info->ipv4_valid) {\n+\t\tset_64bit_val(wqe, 48,\n+\t\t\t      LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));\n+\t} else {\n+\t\tset_64bit_val(wqe, 56,\n+\t\t\t      LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |\n+\t\t\t      LS_64(info->dest_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));\n+\n+\t\tset_64bit_val(wqe, 48,\n+\t\t\t      LS_64(info->dest_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |\n+\t\t\t      LS_64(info->dest_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));\n+\t}\n+\tqw2 = LS_64(vsi->qos[info->user_pri].qs_handle,\n+\t\t    IRDMA_CQPSQ_QHASH_QS_HANDLE);\n+\tif (info->vlan_valid)\n+\t\tqw2 |= LS_64(info->vlan_id, IRDMA_CQPSQ_QHASH_VLANID);\n+\tset_64bit_val(wqe, 16, qw2);\n+\tif (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {\n+\t\tqw1 |= LS_64(info->src_port, IRDMA_CQPSQ_QHASH_SRC_PORT);\n+\t\tif (!info->ipv4_valid) {\n+\t\t\tset_64bit_val(wqe, 40,\n+\t\t\t\t      LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |\n+\t\t\t\t      LS_64(info->src_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));\n+\t\t\tset_64bit_val(wqe, 32,\n+\t\t\t\t      LS_64(info->src_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |\n+\t\t\t\t      LS_64(info->src_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));\n+\t\t} else {\n+\t\t\tset_64bit_val(wqe, 32,\n+\t\t\t\t      LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));\n+\t\t}\n+\t}\n+\n+\tset_64bit_val(wqe, 8, qw1);\n+\ttemp = LS_64(cqp->polarity, IRDMA_CQPSQ_QHASH_WQEVALID) |\n+\t       LS_64(IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY,\n+\t\t     IRDMA_CQPSQ_QHASH_OPCODE) |\n+\t       LS_64(info->manage, IRDMA_CQPSQ_QHASH_MANAGE) |\n+\t       LS_64(info->ipv4_valid, IRDMA_CQPSQ_QHASH_IPV4VALID) |\n+\t       LS_64(info->vlan_valid, IRDMA_CQPSQ_QHASH_VLANVALID) |\n+\t       LS_64(info->entry_type, IRDMA_CQPSQ_QHASH_ENTRYTYPE);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, temp);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"MANAGE_QHASH WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cqp_nop - send a nop wqe\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_cqp_nop(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t       u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\thdr = LS_64(IRDMA_CQP_OP_NOP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"NOP WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_init - initialize qp\n+ * @qp: sc qp\n+ * @info: initialization qp info\n+ */\n+static enum irdma_status_code irdma_sc_qp_init(struct irdma_sc_qp *qp,\n+\t\t\t\t\t       struct irdma_qp_init_info *info)\n+{\n+\tenum irdma_status_code ret_code;\n+\tu32 pble_obj_cnt;\n+\tu16 wqe_size;\n+\n+\tif (info->qp_uk_init_info.max_sq_frag_cnt >\n+\t    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags ||\n+\t    info->qp_uk_init_info.max_rq_frag_cnt >\n+\t    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags)\n+\t\treturn IRDMA_ERR_INVALID_FRAG_COUNT;\n+\n+\tqp->dev = info->pd->dev;\n+\tqp->vsi = info->vsi;\n+\tqp->ieq_qp = info->vsi->exception_lan_q;\n+\tqp->sq_pa = info->sq_pa;\n+\tqp->rq_pa = info->rq_pa;\n+\tqp->hw_host_ctx_pa = info->host_ctx_pa;\n+\tqp->q2_pa = info->q2_pa;\n+\tqp->shadow_area_pa = info->shadow_area_pa;\n+\tqp->q2_buf = info->q2;\n+\tqp->pd = info->pd;\n+\tqp->hw_host_ctx = info->host_ctx;\n+\tinfo->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;\n+\tret_code = irdma_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\tqp->virtual_map = info->virtual_map;\n+\tpble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\n+\tif ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||\n+\t    (info->virtual_map && info->rq_pa >= pble_obj_cnt))\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\tqp->llp_stream_handle = (void *)(-1);\n+\tqp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;\n+\tqp->qp_uk.force_fence = true;\n+\tqp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, false);\n+\tdev_dbg(rfdev_to_dev(qp->dev),\n+\t\t\"WQE: hw_sq_size[%04d] sq_ring.size[%04d]\\n\", qp->hw_sq_size,\n+\t\tqp->qp_uk.sq_ring.size);\n+\tif (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4)\n+\t\twqe_size = IRDMA_WQE_SIZE_128;\n+\telse\n+\t\tret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,\n+\t\t\t\t\t\t       &wqe_size);\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\tqp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *\n+\t\t\t\t(wqe_size / IRDMA_QP_WQE_MIN_SIZE), false);\n+\tdev_dbg(rfdev_to_dev(qp->dev),\n+\t\t\"WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\\n\",\n+\t\tqp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);\n+\tqp->sq_tph_val = info->sq_tph_val;\n+\tqp->rq_tph_val = info->rq_tph_val;\n+\tqp->sq_tph_en = info->sq_tph_en;\n+\tqp->rq_tph_en = info->rq_tph_en;\n+\tqp->rcv_tph_en = info->rcv_tph_en;\n+\tqp->xmit_tph_en = info->xmit_tph_en;\n+\tqp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;\n+\tqp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_create - create qp\n+ * @qp: sc qp\n+ * @info: qp create info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info,\n+\t\t   u64 scratch, bool post_sq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\tcqp = qp->dev->cqp;\n+\tif (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||\n+\t    qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1))\n+\t\treturn IRDMA_ERR_INVALID_QP_ID;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, qp->hw_host_ctx_pa);\n+\tset_64bit_val(wqe, 40, qp->shadow_area_pa);\n+\n+\thdr = qp->qp_uk.qp_id |\n+\t      LS_64(IRDMA_CQP_OP_CREATE_QP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64((info->ord_valid ? 1 : 0), IRDMA_CQPSQ_QP_ORDVALID) |\n+\t      LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |\n+\t      LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |\n+\t      LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |\n+\t      LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |\n+\t      LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |\n+\t      LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |\n+\t      LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |\n+\t      LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QP_CREATE WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_modify - modify qp cqp wqe\n+ * @qp: sc qp\n+ * @info: modify qp info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info,\n+\t\t   u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\tu8 term_actions = 0;\n+\tu8 term_len = 0;\n+\n+\tcqp = qp->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tif (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {\n+\t\tif (info->dont_send_fin)\n+\t\t\tterm_actions += IRDMAQP_TERM_SEND_TERM_ONLY;\n+\t\tif (info->dont_send_term)\n+\t\t\tterm_actions += IRDMAQP_TERM_SEND_FIN_ONLY;\n+\t\tif (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||\n+\t\t    term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)\n+\t\t\tterm_len = info->termlen;\n+\t}\n+\n+\tset_64bit_val(wqe, 8,\n+\t\t      LS_64(info->new_mss, IRDMA_CQPSQ_QP_NEWMSS) |\n+\t\t      LS_64(term_len, IRDMA_CQPSQ_QP_TERMLEN));\n+\tset_64bit_val(wqe, 16, qp->hw_host_ctx_pa);\n+\tset_64bit_val(wqe, 40, qp->shadow_area_pa);\n+\n+\thdr = qp->qp_uk.qp_id |\n+\t      LS_64(IRDMA_CQP_OP_MODIFY_QP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->ord_valid, IRDMA_CQPSQ_QP_ORDVALID) |\n+\t      LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |\n+\t      LS_64(info->cached_var_valid, IRDMA_CQPSQ_QP_CACHEDVARVALID) |\n+\t      LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |\n+\t      LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |\n+\t      LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |\n+\t      LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |\n+\t      LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |\n+\t      LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |\n+\t      LS_64(info->mss_change, IRDMA_CQPSQ_QP_MSSCHANGE) |\n+\t      LS_64(info->remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |\n+\t      LS_64(term_actions, IRDMA_CQPSQ_QP_TERMACT) |\n+\t      LS_64(info->reset_tcp_conn, IRDMA_CQPSQ_QP_RESETCON) |\n+\t      LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |\n+\t      LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QP_MODIFY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_destroy - cqp destroy qp\n+ * @qp: sc qp\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @remove_hash_idx: flag if to remove hash idx\n+ * @ignore_mw_bnd: memory window bind flag\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, bool remove_hash_idx,\n+\t\t    bool ignore_mw_bnd, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\n+\tirdma_qp_rem_qos(qp);\n+\tcqp = qp->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, qp->hw_host_ctx_pa);\n+\tset_64bit_val(wqe, 40, qp->shadow_area_pa);\n+\n+\thdr = qp->qp_uk.qp_id |\n+\t      LS_64(IRDMA_CQP_OP_DESTROY_QP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |\n+\t      LS_64(ignore_mw_bnd, IRDMA_CQPSQ_QP_IGNOREMWBOUND) |\n+\t      LS_64(remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QP_DESTROY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_setctx_roce - set qp's context\n+ * @qp: sc qp\n+ * @qp_ctx: context ptr\n+ * @info: ctx info\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,\n+\t\t\tstruct irdma_qp_host_ctx_info *info)\n+{\n+\tstruct irdma_roce_offload_info *roce_info;\n+\tstruct irdma_udp_offload_info *udp;\n+\tu8 push_mode_en;\n+\tu16 push_idx;\n+\tu64 mac;\n+\n+\troce_info = info->roce_info;\n+\tudp = info->udp_info;\n+\n+\tmac = LS_64_1(roce_info->mac_addr[5], 16) |\n+\t      LS_64_1(roce_info->mac_addr[4], 24) |\n+\t      LS_64_1(roce_info->mac_addr[3], 32) |\n+\t      LS_64_1(roce_info->mac_addr[2], 40) |\n+\t      LS_64_1(roce_info->mac_addr[1], 48) |\n+\t      LS_64_1(roce_info->mac_addr[0], 56);\n+\n+\tqp->user_pri = info->user_pri;\n+\tif (info->add_to_qoslist)\n+\t\tirdma_qp_add_qos(qp);\n+\tif (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {\n+\t\tpush_mode_en = 0;\n+\t\tpush_idx = 0;\n+\t} else {\n+\t\tpush_mode_en = 1;\n+\t\tpush_idx = qp->push_idx;\n+\t}\n+\tset_64bit_val(qp_ctx, 0,\n+\t\t      LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |\n+\t\t      LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |\n+\t\t      LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |\n+\t\t      LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |\n+\t\t      LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |\n+\t\t      LS_64(push_idx, IRDMAQPC_PPIDX) |\n+\t\t      LS_64(push_mode_en, IRDMAQPC_PMENA) |\n+\t\t      LS_64(roce_info->pd_id >> 16, IRDMAQPC_PDIDXHI) |\n+\t\t      LS_64(roce_info->dctcp_en, IRDMAQPC_DC_TCP_EN) |\n+\t\t      LS_64(roce_info->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |\n+\t\t      LS_64(roce_info->is_qp1, IRDMAQPC_ISQP1) |\n+\t\t      LS_64(roce_info->roce_tver, IRDMAQPC_ROCE_TVER) |\n+\t\t      LS_64(roce_info->ecn_en, IRDMAQPC_ECN_EN) |\n+\t\t      LS_64(udp->ipv4, IRDMAQPC_IPV4) |\n+\t\t      LS_64(udp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG));\n+\tset_64bit_val(qp_ctx, 8, qp->sq_pa);\n+\tset_64bit_val(qp_ctx, 16, qp->rq_pa);\n+\tif ((roce_info->dcqcn_en || roce_info->ecn_en || roce_info->dctcp_en) &&\n+\t    !(udp->tos & 0x03))\n+\t\tudp->tos |= ECN_CODE_PT_VAL;\n+\tset_64bit_val(qp_ctx, 24,\n+\t\t      LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |\n+\t\t      LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE) |\n+\t\t      LS_64(udp->ttl, IRDMAQPC_TTL) | LS_64(udp->tos, IRDMAQPC_TOS) |\n+\t\t      LS_64(udp->src_port, IRDMAQPC_SRCPORTNUM) |\n+\t\t      LS_64(udp->dst_port, IRDMAQPC_DESTPORTNUM));\n+\tset_64bit_val(qp_ctx, 32,\n+\t\t      LS_64(udp->dest_ip_addr2, IRDMAQPC_DESTIPADDR2) |\n+\t\t      LS_64(udp->dest_ip_addr3, IRDMAQPC_DESTIPADDR3));\n+\tset_64bit_val(qp_ctx, 40,\n+\t\t      LS_64(udp->dest_ip_addr0, IRDMAQPC_DESTIPADDR0) |\n+\t\t      LS_64(udp->dest_ip_addr1, IRDMAQPC_DESTIPADDR1));\n+\tset_64bit_val(qp_ctx, 48,\n+\t\t      LS_64(udp->snd_mss, IRDMAQPC_SNDMSS) |\n+\t\t      LS_64(udp->vlan_tag, IRDMAQPC_VLANTAG) |\n+\t\t      LS_64(udp->arp_idx, IRDMAQPC_ARPIDX));\n+\tset_64bit_val(qp_ctx, 56,\n+\t\t      LS_64(roce_info->p_key, IRDMAQPC_PKEY) |\n+\t\t      LS_64(roce_info->pd_id, IRDMAQPC_PDIDX) |\n+\t\t      LS_64(roce_info->ack_credits, IRDMAQPC_ACKCREDITS) |\n+\t\t      LS_64(udp->flow_label, IRDMAQPC_FLOWLABEL));\n+\tset_64bit_val(qp_ctx, 64,\n+\t\t      LS_64(roce_info->qkey, IRDMAQPC_QKEY) |\n+\t\t      LS_64(roce_info->dest_qp, IRDMAQPC_DESTQP));\n+\tset_64bit_val(qp_ctx, 80,\n+\t\t      LS_64(udp->psn_nxt, IRDMAQPC_PSNNXT) |\n+\t\t      LS_64(udp->lsn, IRDMAQPC_LSN));\n+\tset_64bit_val(qp_ctx, 88, LS_64(udp->epsn, IRDMAQPC_EPSN));\n+\tset_64bit_val(qp_ctx, 96,\n+\t\t      LS_64(udp->psn_max, IRDMAQPC_PSNMAX) |\n+\t\t      LS_64(udp->psn_una, IRDMAQPC_PSNUNA));\n+\tset_64bit_val(qp_ctx, 112,\n+\t\t      LS_64(udp->cwnd, IRDMAQPC_CWNDROCE));\n+\tset_64bit_val(qp_ctx, 128,\n+\t\t      LS_64(roce_info->err_rq_idx, IRDMAQPC_ERR_RQ_IDX) |\n+\t\t      LS_64(udp->rnr_nak_thresh, IRDMAQPC_RNRNAK_THRESH) |\n+\t\t      LS_64(udp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH));\n+\tset_64bit_val(qp_ctx, 136,\n+\t\t      LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |\n+\t\t      LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));\n+\tset_64bit_val(qp_ctx, 144,\n+\t\t      LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));\n+\tset_64bit_val(qp_ctx, 152, mac);\n+\tset_64bit_val(qp_ctx, 160,\n+\t\t      LS_64(roce_info->ord_size, IRDMAQPC_ORDSIZE) |\n+\t\t      LS_64(roce_info->ird_size, IRDMAQPC_IRDSIZE) |\n+\t\t      LS_64(roce_info->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |\n+\t\t      LS_64(roce_info->rd_en, IRDMAQPC_RDOK) |\n+\t\t      LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |\n+\t\t      LS_64(roce_info->bind_en, IRDMAQPC_BINDEN) |\n+\t\t      LS_64(roce_info->fast_reg_en, IRDMAQPC_FASTREGEN) |\n+\t\t      LS_64(roce_info->dcqcn_en, IRDMAQPC_DCQCNENABLE) |\n+\t\t      LS_64(roce_info->rcv_no_icrc, IRDMAQPC_RCVNOICRC) |\n+\t\t      LS_64(roce_info->fw_cc_enable, IRDMAQPC_FW_CC_ENABLE) |\n+\t\t      LS_64(roce_info->udprivcq_en, IRDMAQPC_UDPRIVCQENABLE) |\n+\t\t      LS_64(roce_info->priv_mode_en, IRDMAQPC_PRIVEN) |\n+\t\t      LS_64(roce_info->timely_en, IRDMAQPC_TIMELYENABLE));\n+\tset_64bit_val(qp_ctx, 168,\n+\t\t      LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));\n+\tset_64bit_val(qp_ctx, 176,\n+\t\t      LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |\n+\t\t      LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |\n+\t\t      LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE));\n+\tset_64bit_val(qp_ctx, 184,\n+\t\t      LS_64(udp->local_ipaddr3, IRDMAQPC_LOCAL_IPADDR3) |\n+\t\t      LS_64(udp->local_ipaddr2, IRDMAQPC_LOCAL_IPADDR2));\n+\tset_64bit_val(qp_ctx, 192,\n+\t\t      LS_64(udp->local_ipaddr1, IRDMAQPC_LOCAL_IPADDR1) |\n+\t\t      LS_64(udp->local_ipaddr0, IRDMAQPC_LOCAL_IPADDR0));\n+\tset_64bit_val(qp_ctx, 200,\n+\t\t      LS_64(roce_info->t_high, IRDMAQPC_THIGH) |\n+\t\t      LS_64(roce_info->t_low, IRDMAQPC_TLOW));\n+\tset_64bit_val(qp_ctx, 208,\n+\t\t      LS_64(info->rem_endpoint_idx, IRDMAQPC_REMENDPOINTIDX));\n+\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, \"QP_HOST CTX WQE\", qp_ctx,\n+\t\t\tIRDMA_QP_CTX_SIZE);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_alloc_local_mac_entry - allocate a mac entry\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,\n+\t\t\t       bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\twqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\thdr = LS_64(IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY,\n+\t\t    IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"ALLOCATE_LOCAL_MAC WQE\",\n+\t\t\twqe, IRDMA_CQP_WQE_SIZE * 8);\n+\n+\tif (post_sq)\n+\t\tcqp->dev->cqp_ops->cqp_post_sq(cqp);\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_add_local_mac_entry - add mac enry\n+ * @cqp: struct for cqp hw\n+ * @info:mac addr info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,\n+\t\t\t     struct irdma_local_mac_entry_info *info,\n+\t\t\t     u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 temp, header;\n+\n+\twqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\ttemp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |\n+\t       LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |\n+\t       LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);\n+\n+\tset_64bit_val(wqe, 32, temp);\n+\n+\theader = LS_64(info->entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |\n+\t\t LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |\n+\t\t LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, header);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"ADD_LOCAL_MAC WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\n+\tif (post_sq)\n+\t\tcqp->dev->cqp_ops->cqp_post_sq(cqp);\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @entry_idx: index of mac entry\n+ * @ignore_ref_count: to force mac adde delete\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,\n+\t\t\t     u16 entry_idx, u8 ignore_ref_count, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 header;\n+\n+\twqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\theader = LS_64(entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |\n+\t\t LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |\n+\t\t LS_64(1, IRDMA_CQPSQ_MLM_FREEENTRY) |\n+\t\t LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |\n+\t\t LS_64(ignore_ref_count, IRDMA_CQPSQ_MLM_IGNORE_REF_CNT);\n+\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, header);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"DEL_LOCAL_MAC_IPADDR WQE\",\n+\t\t\twqe, IRDMA_CQP_WQE_SIZE * 8);\n+\n+\tif (post_sq)\n+\t\tcqp->dev->cqp_ops->cqp_post_sq(cqp);\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_setctx - set qp's context\n+ * @qp: sc qp\n+ * @qp_ctx: context ptr\n+ * @info: ctx info\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,\n+\t\t   struct irdma_qp_host_ctx_info *info)\n+{\n+\tstruct irdma_iwarp_offload_info *iw;\n+\tstruct irdma_tcp_offload_info *tcp;\n+\tstruct irdma_sc_dev *dev;\n+\tu8 push_mode_en;\n+\tu16 push_idx;\n+\tu64 qw0, qw3, qw7 = 0;\n+\tu64 mac = 0;\n+\n+\tiw = info->iwarp_info;\n+\ttcp = info->tcp_info;\n+\tdev = qp->dev;\n+\n+\tif (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {\n+\t\tmac = LS_64_1(iw->mac_addr[5], 16) |\n+\t\t      LS_64_1(iw->mac_addr[4], 24) |\n+\t\t      LS_64_1(iw->mac_addr[3], 32) |\n+\t\t      LS_64_1(iw->mac_addr[2], 40) |\n+\t\t      LS_64_1(iw->mac_addr[1], 48) |\n+\t\t      LS_64_1(iw->mac_addr[0], 56);\n+\t}\n+\n+\tqp->user_pri = info->user_pri;\n+\tif (info->add_to_qoslist)\n+\t\tirdma_qp_add_qos(qp);\n+\n+\tif (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {\n+\t\tpush_mode_en = 0;\n+\t\tpush_idx = 0;\n+\t} else {\n+\t\tpush_mode_en = 1;\n+\t\tpush_idx = qp->push_idx;\n+\t}\n+\tqw0 = LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |\n+\t      LS_64(iw->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |\n+\t      LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |\n+\t      LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |\n+\t      LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |\n+\t      LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |\n+\t      LS_64(push_idx, IRDMAQPC_PPIDX) |\n+\t      LS_64(push_mode_en, IRDMAQPC_PMENA) |\n+\t      LS_64(iw->ib_rd_en, IRDMAQPC_IBRDENABLE) |\n+\t      LS_64(iw->pd_id >> 16, IRDMAQPC_PDIDXHI);\n+\n+\tset_64bit_val(qp_ctx, 8, qp->sq_pa);\n+\tset_64bit_val(qp_ctx, 16, qp->rq_pa);\n+\n+\tqw3 = LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |\n+\t      LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE);\n+\tif (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)\n+\t\tqw3 |= LS_64(qp->src_mac_addr_idx, IRDMAQPC_GEN1_SRCMACADDRIDX);\n+\tset_64bit_val(qp_ctx, 128,\n+\t\t      LS_64(iw->err_rq_idx, IRDMAQPC_ERR_RQ_IDX));\n+\tset_64bit_val(qp_ctx, 136,\n+\t\t      LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |\n+\t\t      LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));\n+\tset_64bit_val(qp_ctx, 168,\n+\t\t      LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));\n+\tset_64bit_val(qp_ctx, 176,\n+\t\t      LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |\n+\t\t      LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |\n+\t\t      LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE) |\n+\t\t      LS_64(qp->ieq_qp, IRDMAQPC_EXCEPTION_LAN_QUEUE));\n+\tif (info->iwarp_info_valid) {\n+\t\tqw0 |= LS_64(iw->ddp_ver, IRDMAQPC_DDP_VER) |\n+\t\t       LS_64(iw->rdmap_ver, IRDMAQPC_RDMAP_VER) |\n+\t\t       LS_64(iw->dctcp_en, IRDMAQPC_DC_TCP_EN) |\n+\t\t       LS_64(iw->ecn_en, IRDMAQPC_ECN_EN);\n+\t\tqw7 |= LS_64(iw->pd_id, IRDMAQPC_PDIDX);\n+\t\tset_64bit_val(qp_ctx, 144,\n+\t\t\t      LS_64(qp->q2_pa >> 8, IRDMAQPC_Q2ADDR) |\n+\t\t\t      LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));\n+\t\tset_64bit_val(qp_ctx, 152,\n+\t\t\t      mac | LS_64(iw->last_byte_sent, IRDMAQPC_LASTBYTESENT));\n+\t\tset_64bit_val(qp_ctx, 160,\n+\t\t\t      LS_64(iw->ord_size, IRDMAQPC_ORDSIZE) |\n+\t\t\t      LS_64(iw->ird_size, IRDMAQPC_IRDSIZE) |\n+\t\t\t      LS_64(iw->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |\n+\t\t\t      LS_64(iw->rd_en, IRDMAQPC_RDOK) |\n+\t\t\t      LS_64(iw->snd_mark_en, IRDMAQPC_SNDMARKERS) |\n+\t\t\t      LS_64(iw->bind_en, IRDMAQPC_BINDEN) |\n+\t\t\t      LS_64(iw->fast_reg_en, IRDMAQPC_FASTREGEN) |\n+\t\t\t      LS_64(iw->priv_mode_en, IRDMAQPC_PRIVEN) |\n+\t\t\t      LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |\n+\t\t\t      LS_64(1, IRDMAQPC_IWARPMODE) |\n+\t\t\t      LS_64(iw->rcv_mark_en, IRDMAQPC_RCVMARKERS) |\n+\t\t\t      LS_64(iw->align_hdrs, IRDMAQPC_ALIGNHDRS) |\n+\t\t\t      LS_64(iw->rcv_no_mpa_crc, IRDMAQPC_RCVNOMPACRC) |\n+\t\t\t      LS_64(iw->rcv_mark_offset, IRDMAQPC_RCVMARKOFFSET) |\n+\t\t\t      LS_64(iw->snd_mark_offset, IRDMAQPC_SNDMARKOFFSET) |\n+\t\t\t      LS_64(iw->timely_en, IRDMAQPC_TIMELYENABLE));\n+\t}\n+\tif (info->tcp_info_valid) {\n+\t\tqw0 |= LS_64(tcp->ipv4, IRDMAQPC_IPV4) |\n+\t\t       LS_64(tcp->no_nagle, IRDMAQPC_NONAGLE) |\n+\t\t       LS_64(tcp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG) |\n+\t\t       LS_64(tcp->time_stamp, IRDMAQPC_TIMESTAMP) |\n+\t\t       LS_64(tcp->cwnd_inc_limit, IRDMAQPC_LIMIT) |\n+\t\t       LS_64(tcp->drop_ooo_seg, IRDMAQPC_DROPOOOSEG) |\n+\t\t       LS_64(tcp->dup_ack_thresh, IRDMAQPC_DUPACK_THRESH);\n+\n+\t\tif ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03))\n+\t\t\ttcp->tos |= ECN_CODE_PT_VAL;\n+\n+\t\tqw3 |= LS_64(tcp->ttl, IRDMAQPC_TTL) |\n+\t\t       LS_64(tcp->avoid_stretch_ack, IRDMAQPC_AVOIDSTRETCHACK) |\n+\t\t       LS_64(tcp->tos, IRDMAQPC_TOS) |\n+\t\t       LS_64(tcp->src_port, IRDMAQPC_SRCPORTNUM) |\n+\t\t       LS_64(tcp->dst_port, IRDMAQPC_DESTPORTNUM);\n+\t\tif (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {\n+\t\t\tqw3 |= LS_64(tcp->src_mac_addr_idx,\n+\t\t\t\t     IRDMAQPC_GEN1_SRCMACADDRIDX);\n+\n+\t\t\tqp->src_mac_addr_idx = tcp->src_mac_addr_idx;\n+\t\t}\n+\t\tset_64bit_val(qp_ctx, 32,\n+\t\t\t      LS_64(tcp->dest_ip_addr2, IRDMAQPC_DESTIPADDR2) |\n+\t\t\t      LS_64(tcp->dest_ip_addr3, IRDMAQPC_DESTIPADDR3));\n+\t\tset_64bit_val(qp_ctx, 40,\n+\t\t\t      LS_64(tcp->dest_ip_addr0, IRDMAQPC_DESTIPADDR0) |\n+\t\t\t      LS_64(tcp->dest_ip_addr1, IRDMAQPC_DESTIPADDR1));\n+\t\tset_64bit_val(qp_ctx, 48,\n+\t\t\t      LS_64(tcp->snd_mss, IRDMAQPC_SNDMSS) |\n+\t\t\t      LS_64(tcp->syn_rst_handling, IRDMAQPC_SYN_RST_HANDLING) |\n+\t\t\t      LS_64(tcp->vlan_tag, IRDMAQPC_VLANTAG) |\n+\t\t\t      LS_64(tcp->arp_idx, IRDMAQPC_ARPIDX));\n+\t\tqw7 |= LS_64(tcp->flow_label, IRDMAQPC_FLOWLABEL) |\n+\t\t       LS_64(tcp->wscale, IRDMAQPC_WSCALE) |\n+\t\t       LS_64(tcp->ignore_tcp_opt, IRDMAQPC_IGNORE_TCP_OPT) |\n+\t\t       LS_64(tcp->ignore_tcp_uns_opt,\n+\t\t\t     IRDMAQPC_IGNORE_TCP_UNS_OPT) |\n+\t\t       LS_64(tcp->tcp_state, IRDMAQPC_TCPSTATE) |\n+\t\t       LS_64(tcp->rcv_wscale, IRDMAQPC_RCVSCALE) |\n+\t\t       LS_64(tcp->snd_wscale, IRDMAQPC_SNDSCALE);\n+\t\tset_64bit_val(qp_ctx, 72,\n+\t\t\t      LS_64(tcp->time_stamp_recent, IRDMAQPC_TIMESTAMP_RECENT) |\n+\t\t\t      LS_64(tcp->time_stamp_age, IRDMAQPC_TIMESTAMP_AGE));\n+\t\tset_64bit_val(qp_ctx, 80,\n+\t\t\t      LS_64(tcp->snd_nxt, IRDMAQPC_SNDNXT) |\n+\t\t\t      LS_64(tcp->snd_wnd, IRDMAQPC_SNDWND));\n+\t\tset_64bit_val(qp_ctx, 88,\n+\t\t\t      LS_64(tcp->rcv_nxt, IRDMAQPC_RCVNXT) |\n+\t\t\t      LS_64(tcp->rcv_wnd, IRDMAQPC_RCVWND));\n+\t\tset_64bit_val(qp_ctx, 96,\n+\t\t\t      LS_64(tcp->snd_max, IRDMAQPC_SNDMAX) |\n+\t\t\t      LS_64(tcp->snd_una, IRDMAQPC_SNDUNA));\n+\t\tset_64bit_val(qp_ctx, 104,\n+\t\t\t      LS_64(tcp->srtt, IRDMAQPC_SRTT) |\n+\t\t\t      LS_64(tcp->rtt_var, IRDMAQPC_RTTVAR));\n+\t\tset_64bit_val(qp_ctx, 112,\n+\t\t\t      LS_64(tcp->ss_thresh, IRDMAQPC_SSTHRESH) |\n+\t\t\t      LS_64(tcp->cwnd, IRDMAQPC_CWND));\n+\t\tset_64bit_val(qp_ctx, 120,\n+\t\t\t      LS_64(tcp->snd_wl1, IRDMAQPC_SNDWL1) |\n+\t\t\t      LS_64(tcp->snd_wl2, IRDMAQPC_SNDWL2));\n+\t\tset_64bit_val(qp_ctx, 128,\n+\t\t\t      LS_64(tcp->max_snd_window, IRDMAQPC_MAXSNDWND) |\n+\t\t\t      LS_64(tcp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH));\n+\t\tset_64bit_val(qp_ctx, 184,\n+\t\t\t      LS_64(tcp->local_ipaddr3, IRDMAQPC_LOCAL_IPADDR3) |\n+\t\t\t      LS_64(tcp->local_ipaddr2, IRDMAQPC_LOCAL_IPADDR2));\n+\t\tset_64bit_val(qp_ctx, 192,\n+\t\t\t      LS_64(tcp->local_ipaddr1, IRDMAQPC_LOCAL_IPADDR1) |\n+\t\t\t      LS_64(tcp->local_ipaddr0, IRDMAQPC_LOCAL_IPADDR0));\n+\t\tset_64bit_val(qp_ctx, 200,\n+\t\t\t      LS_64(iw->t_high, IRDMAQPC_THIGH) |\n+\t\t\t      LS_64(iw->t_low, IRDMAQPC_TLOW));\n+\t\tset_64bit_val(qp_ctx, 208,\n+\t\t\t      LS_64(info->rem_endpoint_idx, IRDMAQPC_REMENDPOINTIDX));\n+\t}\n+\n+\tset_64bit_val(qp_ctx, 0, qw0);\n+\tset_64bit_val(qp_ctx, 24, qw3);\n+\tset_64bit_val(qp_ctx, 56, qw7);\n+\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, \"QP_HOST CTX\", qp_ctx,\n+\t\t\tIRDMA_QP_CTX_SIZE);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_alloc_stag - mr stag alloc\n+ * @dev: sc device struct\n+ * @info: stag info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_alloc_stag(struct irdma_sc_dev *dev,\n+\t\t    struct irdma_allocate_stag_info *info, u64 scratch,\n+\t\t    bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\tenum irdma_page_size page_size;\n+\n+\tif (info->page_size == 0x40000000)\n+\t\tpage_size = IRDMA_PAGE_SIZE_1G;\n+\telse if (info->page_size == 0x200000)\n+\t\tpage_size = IRDMA_PAGE_SIZE_2M;\n+\telse\n+\t\tpage_size = IRDMA_PAGE_SIZE_4K;\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 8,\n+\t\t      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |\n+\t\t      LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN));\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));\n+\tset_64bit_val(wqe, 40,\n+\t\t      LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STAG_HMCFNIDX));\n+\n+\tif (info->chunk_size)\n+\t\tset_64bit_val(wqe, 48,\n+\t\t\t      LS_64(info->first_pm_pbl_idx, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));\n+\n+\thdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMA_CQPSQ_STAG_MR) |\n+\t      LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |\n+\t      LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |\n+\t      LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |\n+\t      LS_64(info->remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |\n+\t      LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |\n+\t      LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"ALLOC_STAG WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_mr_reg_non_shared - non-shared mr registration\n+ * @dev: sc device struct\n+ * @info: mr info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,\n+\t\t\t   struct irdma_reg_ns_stag_info *info, u64 scratch,\n+\t\t\t   bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 temp;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\tu32 pble_obj_cnt;\n+\tbool remote_access;\n+\tu8 addr_type;\n+\tenum irdma_page_size page_size;\n+\n+\tif (info->page_size == 0x40000000)\n+\t\tpage_size = IRDMA_PAGE_SIZE_1G;\n+\telse if (info->page_size == 0x200000)\n+\t\tpage_size = IRDMA_PAGE_SIZE_2M;\n+\telse\n+\t\tpage_size = IRDMA_PAGE_SIZE_4K;\n+\n+\tif (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |\n+\t\t\t\t   IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))\n+\t\tremote_access = true;\n+\telse\n+\t\tremote_access = false;\n+\n+\tpble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\tif (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\ttemp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?\n+\t\t(uintptr_t)info->va : info->fbo;\n+\n+\tset_64bit_val(wqe, 0, temp);\n+\tset_64bit_val(wqe, 8,\n+\t\t      LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN) |\n+\t\t      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(info->stag_key, IRDMA_CQPSQ_STAG_KEY) |\n+\t\t      LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));\n+\tif (!info->chunk_size) {\n+\t\tset_64bit_val(wqe, 32, info->reg_addr_pa);\n+\t\tset_64bit_val(wqe, 48, 0);\n+\t} else {\n+\t\tset_64bit_val(wqe, 32, 0);\n+\t\tset_64bit_val(wqe, 48,\n+\t\t\t      LS_64(info->first_pm_pbl_index, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));\n+\t}\n+\tset_64bit_val(wqe, 40, info->hmc_fcn_index);\n+\tset_64bit_val(wqe, 56, 0);\n+\n+\taddr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;\n+\thdr = LS_64(IRDMA_CQP_OP_REG_MR, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMA_CQPSQ_STAG_MR) |\n+\t      LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |\n+\t      LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |\n+\t      LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |\n+\t      LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |\n+\t      LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |\n+\t      LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |\n+\t      LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"MR_REG_NS WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_mr_reg_shared - registered shared memory region\n+ * @dev: sc device struct\n+ * @info: info for shared memory registration\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_mr_reg_shared(struct irdma_sc_dev *dev,\n+\t\t       struct irdma_register_shared_stag *info, u64 scratch,\n+\t\t       bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 temp, va64, fbo, hdr;\n+\tu32 va32;\n+\tbool remote_access;\n+\tu8 addr_type;\n+\n+\tif (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |\n+\t\t\t\t   IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))\n+\t\tremote_access = true;\n+\telse\n+\t\tremote_access = false;\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\tva64 = (uintptr_t)(info->va);\n+\tva32 = (u32)(va64 & 0x00000000FFFFFFFF);\n+\tfbo = (u64)(va32 & (4096 - 1));\n+\n+\tset_64bit_val(wqe, 0,\n+\t\t      (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?\n+\t\t       (uintptr_t)info->va : fbo));\n+\tset_64bit_val(wqe, 8,\n+\t\t      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));\n+\ttemp = LS_64(info->new_stag_key, IRDMA_CQPSQ_STAG_KEY) |\n+\t       LS_64(info->new_stag_idx, IRDMA_CQPSQ_STAG_IDX) |\n+\t       LS_64(info->parent_stag_idx, IRDMA_CQPSQ_STAG_PARENTSTAGIDX);\n+\tset_64bit_val(wqe, 16, temp);\n+\n+\taddr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;\n+\thdr = LS_64(IRDMA_CQP_OP_REG_SMR, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMA_CQPSQ_STAG_MR) |\n+\t      LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |\n+\t      LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |\n+\t      LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"MR_REG_SHARED WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_dealloc_stag - deallocate stag\n+ * @dev: sc device struct\n+ * @info: dealloc stag info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,\n+\t\t      struct irdma_dealloc_stag_info *info, u64 scratch,\n+\t\t      bool post_sq)\n+{\n+\tu64 hdr;\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 8,\n+\t\t      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));\n+\n+\thdr = LS_64(IRDMA_CQP_OP_DEALLOC_STAG, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->mr, IRDMA_CQPSQ_STAG_MR) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"DEALLOC_STAG WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_query_stag - query hardware for stag\n+ * @dev: sc device struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @stag_index: stag index for query\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_query_stag(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t  u64 scratch, u32 stag_index,\n+\t\t\t\t\t\t  bool post_sq)\n+{\n+\tu64 hdr;\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(stag_index, IRDMA_CQPSQ_QUERYSTAG_IDX));\n+\n+\thdr = LS_64(IRDMA_CQP_OP_QUERY_STAG, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"QUERY_STAG WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_mw_alloc - mw allocate\n+ * @dev: sc device struct\n+ * @info: memory window allocation information\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_mw_alloc(struct irdma_sc_dev *dev, struct irdma_mw_alloc_info *info,\n+\t\t  u64 scratch, bool post_sq)\n+{\n+\tu64 hdr;\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 8,\n+\t\t      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(info->mw_stag_index, IRDMA_CQPSQ_STAG_IDX));\n+\n+\thdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->mw_wide, IRDMA_CQPSQ_STAG_MWTYPE) |\n+\t      LS_64(info->mw1_bind_dont_vldt_key,\n+\t\t    IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"MW_ALLOC WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp\n+ * @qp: sc qp struct\n+ * @info: fast mr info\n+ * @post_sq: flag for cqp db to ring\n+ */\n+enum irdma_status_code\n+irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,\n+\t\t\t  struct irdma_fast_reg_stag_info *info, bool post_sq)\n+{\n+\tu64 temp, hdr;\n+\t__le64 *wqe;\n+\tu32 wqe_idx;\n+\tenum irdma_page_size page_size;\n+\tstruct irdma_post_sq_info sq_info = {};\n+\n+\tif (info->page_size == 0x40000000)\n+\t\tpage_size = IRDMA_PAGE_SIZE_1G;\n+\telse if (info->page_size == 0x200000)\n+\t\tpage_size = IRDMA_PAGE_SIZE_2M;\n+\telse\n+\t\tpage_size = IRDMA_PAGE_SIZE_4K;\n+\n+\tsq_info.wr_id = info->wr_id;\n+\tsq_info.push_wqe = info->push_wqe;\n+\n+\twqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,\n+\t\t\t\t\t IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_QP_TOOMANY_WRS_POSTED;\n+\n+\tirdma_clr_wqes(&qp->qp_uk, wqe_idx);\n+\n+\tdev_dbg(rfdev_to_dev(qp->dev),\n+\t\t\"MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\\n\", info->wr_id,\n+\t\twqe_idx, &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);\n+\n+\ttemp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?\n+\t\t(uintptr_t)info->va : info->fbo;\n+\tset_64bit_val(wqe, 0, temp);\n+\n+\ttemp = RS_64(info->first_pm_pbl_index >> 16, IRDMAQPSQ_FIRSTPMPBLIDXHI);\n+\tset_64bit_val(wqe, 8,\n+\t\t      LS_64(temp, IRDMAQPSQ_FIRSTPMPBLIDXHI) |\n+\t\t      LS_64(info->reg_addr_pa >> IRDMAQPSQ_PBLADDR_S, IRDMAQPSQ_PBLADDR));\n+\tset_64bit_val(wqe, 16,\n+\t\t      info->total_len |\n+\t\t      LS_64(info->first_pm_pbl_index, IRDMAQPSQ_FIRSTPMPBLIDXLO));\n+\n+\thdr = LS_64(info->stag_key, IRDMAQPSQ_STAGKEY) |\n+\t      LS_64(info->stag_idx, IRDMAQPSQ_STAGINDEX) |\n+\t      LS_64(IRDMAQP_OP_FAST_REGISTER, IRDMAQPSQ_OPCODE) |\n+\t      LS_64(info->chunk_size, IRDMAQPSQ_LPBLSIZE) |\n+\t      LS_64(page_size, IRDMAQPSQ_HPAGESIZE) |\n+\t      LS_64(info->access_rights, IRDMAQPSQ_STAGRIGHTS) |\n+\t      LS_64(info->addr_type, IRDMAQPSQ_VABASEDTO) |\n+\t      LS_64((sq_info.push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |\n+\t      LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |\n+\t      LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |\n+\t      LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |\n+\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, \"FAST_REG WQE\", wqe,\n+\t\t\tIRDMA_QP_WQE_MIN_SIZE);\n+\tif (sq_info.push_wqe) {\n+\t\tirdma_qp_push_wqe(&qp->qp_uk, wqe, IRDMA_QP_WQE_MIN_QUANTA,\n+\t\t\t\t  wqe_idx, post_sq);\n+\t} else {\n+\t\tif (post_sq)\n+\t\t\tirdma_qp_post_wr(&qp->qp_uk);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_gen_rts_ae - request AE generated after RTS\n+ * @qp: sc qp struct\n+ */\n+static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tstruct irdma_qp_uk *qp_uk;\n+\n+\tqp_uk = &qp->qp_uk;\n+\n+\twqe = qp_uk->sq_base[1].elem;\n+\n+\thdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMAQPSQ_LOCALFENCE) |\n+\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, \"NOP W/LOCAL FENCE WQE\", wqe,\n+\t\t\tIRDMA_QP_WQE_MIN_SIZE);\n+\n+\twqe = qp_uk->sq_base[2].elem;\n+\thdr = LS_64(IRDMAQP_OP_GEN_RTS_AE, IRDMAQPSQ_OPCODE) |\n+\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, \"CONN EST WQE\", wqe,\n+\t\t\tIRDMA_QP_WQE_MIN_SIZE);\n+}\n+\n+/**\n+ * irdma_sc_send_lsmm - send last streaming mode message\n+ * @qp: sc qp struct\n+ * @lsmm_buf: buffer with lsmm message\n+ * @size: size of lsmm buffer\n+ * @stag: stag of lsmm buffer\n+ */\n+static void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,\n+\t\t\t       irdma_stag stag)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tstruct irdma_qp_uk *qp_uk;\n+\n+\tqp_uk = &qp->qp_uk;\n+\twqe = qp_uk->sq_base->elem;\n+\n+\tset_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);\n+\tif (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {\n+\t\tset_64bit_val(wqe, 8,\n+\t\t\t      LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN) |\n+\t\t\t      LS_64(stag, IRDMAQPSQ_GEN1_FRAG_STAG));\n+\t} else {\n+\t\tset_64bit_val(wqe, 8,\n+\t\t\t      LS_64(size, IRDMAQPSQ_FRAG_LEN) |\n+\t\t\t      LS_64(stag, IRDMAQPSQ_FRAG_STAG) |\n+\t\t\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));\n+\t}\n+\tset_64bit_val(wqe, 16, 0);\n+\n+\thdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMAQPSQ_STREAMMODE) |\n+\t      LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |\n+\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, \"SEND_LSMM WQE\", wqe,\n+\t\t\tIRDMA_QP_WQE_MIN_SIZE);\n+\n+\tif (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)\n+\t\tirdma_sc_gen_rts_ae(qp);\n+}\n+\n+/**\n+ * irdma_sc_send_lsmm_nostag - for privilege qp\n+ * @qp: sc qp struct\n+ * @lsmm_buf: buffer with lsmm message\n+ * @size: size of lsmm buffer\n+ */\n+static void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf,\n+\t\t\t\t      u32 size)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tstruct irdma_qp_uk *qp_uk;\n+\n+\tqp_uk = &qp->qp_uk;\n+\twqe = qp_uk->sq_base->elem;\n+\n+\tset_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);\n+\n+\tif (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1)\n+\t\tset_64bit_val(wqe, 8,\n+\t\t\t      LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN));\n+\telse\n+\t\tset_64bit_val(wqe, 8,\n+\t\t\t      LS_64(size, IRDMAQPSQ_FRAG_LEN) |\n+\t\t\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));\n+\tset_64bit_val(wqe, 16, 0);\n+\n+\thdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMAQPSQ_STREAMMODE) |\n+\t      LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |\n+\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, \"SEND_LSMM_NOSTAG WQE\", wqe,\n+\t\t\tIRDMA_QP_WQE_MIN_SIZE);\n+}\n+\n+/**\n+ * irdma_sc_send_rtt - send last read0 or write0\n+ * @qp: sc qp struct\n+ * @read: Do read0 or write0\n+ */\n+static void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tstruct irdma_qp_uk *qp_uk;\n+\n+\tqp_uk = &qp->qp_uk;\n+\twqe = qp_uk->sq_base->elem;\n+\n+\tset_64bit_val(wqe, 0, 0);\n+\tset_64bit_val(wqe, 16, 0);\n+\tif (read) {\n+\t\tif (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {\n+\t\t\tset_64bit_val(wqe, 8,\n+\t\t\t\t      LS_64(0xabcd, IRDMAQPSQ_GEN1_FRAG_STAG));\n+\t\t} else {\n+\t\t\tset_64bit_val(wqe, 8,\n+\t\t\t\t      (u64)0xabcd | LS_64(qp->qp_uk.swqe_polarity,\n+\t\t\t\t      IRDMAQPSQ_VALID));\n+\t\t}\n+\t\thdr = LS_64(0x1234, IRDMAQPSQ_REMSTAG) |\n+\t\t      LS_64(IRDMAQP_OP_RDMA_READ, IRDMAQPSQ_OPCODE) |\n+\t\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\n+\t} else {\n+\t\tif (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {\n+\t\t\tset_64bit_val(wqe, 8, 0);\n+\t\t} else {\n+\t\t\tset_64bit_val(wqe, 8,\n+\t\t\t\t      LS_64(qp->qp_uk.swqe_polarity,\n+\t\t\t\t\t    IRDMAQPSQ_VALID));\n+\t\t}\n+\t\thdr = LS_64(IRDMAQP_OP_RDMA_WRITE, IRDMAQPSQ_OPCODE) |\n+\t\t      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);\n+\t}\n+\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, \"RTR WQE\", wqe,\n+\t\t\tIRDMA_QP_WQE_MIN_SIZE);\n+\n+\tif (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)\n+\t\tirdma_sc_gen_rts_ae(qp);\n+}\n+\n+/**\n+ * irdma_iwarp_opcode - determine if incoming is rdma layer\n+ * @info: aeq info for the packet\n+ * @pkt: packet for error\n+ */\n+static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)\n+{\n+\t__be16 *mpa;\n+\tu32 opcode = 0xffffffff;\n+\n+\tif (info->q2_data_written) {\n+\t\tmpa = (__be16 *)pkt;\n+\t\topcode = ntohs(mpa[1]) & 0xf;\n+\t}\n+\n+\treturn opcode;\n+}\n+\n+/**\n+ * irdma_locate_mpa - return pointer to mpa in the pkt\n+ * @pkt: packet with data\n+ */\n+static u8 *irdma_locate_mpa(u8 *pkt)\n+{\n+\t/* skip over ethernet header */\n+\tpkt += IRDMA_MAC_HLEN;\n+\n+\t/* Skip over IP and TCP headers */\n+\tpkt += 4 * (pkt[0] & 0x0f);\n+\tpkt += 4 * ((pkt[12] >> 4) & 0x0f);\n+\n+\treturn pkt;\n+}\n+\n+/**\n+ * irdma_bld_termhdr_ctrl - setup terminate hdr control fields\n+ * @qp: sc qp ptr for pkt\n+ * @hdr: term hdr\n+ * @opcode: flush opcode for termhdr\n+ * @layer_etype: error layer + error type\n+ * @err: error cod ein the header\n+ */\n+static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,\n+\t\t\t\t   struct irdma_terminate_hdr *hdr,\n+\t\t\t\t   enum irdma_flush_opcode opcode,\n+\t\t\t\t   u8 layer_etype, u8 err)\n+{\n+\tqp->flush_code = opcode;\n+\thdr->layer_etype = layer_etype;\n+\thdr->error_code = err;\n+}\n+\n+/**\n+ * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr\n+ * @pkt: ptr to mpa in offending pkt\n+ * @hdr: term hdr\n+ * @copy_len: offending pkt length to be copied to term hdr\n+ * @is_tagged: DDP tagged or untagged\n+ */\n+static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr,\n+\t\t\t\t       int *copy_len, u8 *is_tagged)\n+{\n+\tu16 ddp_seg_len;\n+\n+\tddp_seg_len = ntohs(*(__be16 *)pkt);\n+\tif (ddp_seg_len) {\n+\t\t*copy_len = 2;\n+\t\thdr->hdrct = DDP_LEN_FLAG;\n+\t\tif (pkt[2] & 0x80) {\n+\t\t\t*is_tagged = 1;\n+\t\t\tif (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {\n+\t\t\t\t*copy_len += TERM_DDP_LEN_TAGGED;\n+\t\t\t\thdr->hdrct |= DDP_HDR_FLAG;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tif (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {\n+\t\t\t\t*copy_len += TERM_DDP_LEN_UNTAGGED;\n+\t\t\t\thdr->hdrct |= DDP_HDR_FLAG;\n+\t\t\t}\n+\t\t\tif (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &&\n+\t\t\t    ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {\n+\t\t\t\t*copy_len += TERM_RDMA_LEN;\n+\t\t\t\thdr->hdrct |= RDMA_HDR_FLAG;\n+\t\t\t}\n+\t\t}\n+\t}\n+}\n+\n+/**\n+ * irdma_bld_terminate_hdr - build terminate message header\n+ * @qp: qp associated with received terminate AE\n+ * @info: the struct contiaing AE information\n+ */\n+static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,\n+\t\t\t\t   struct irdma_aeqe_info *info)\n+{\n+\tu8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;\n+\tint copy_len = 0;\n+\tu8 is_tagged = 0;\n+\tu32 opcode;\n+\tstruct irdma_terminate_hdr *termhdr;\n+\n+\ttermhdr = (struct irdma_terminate_hdr *)qp->q2_buf;\n+\tmemset(termhdr, 0, Q2_BAD_FRAME_OFFSET);\n+\n+\tif (info->q2_data_written) {\n+\t\tpkt = irdma_locate_mpa(pkt);\n+\t\tirdma_bld_termhdr_ddp_rdma(pkt, termhdr, &copy_len, &is_tagged);\n+\t}\n+\n+\topcode = irdma_iwarp_opcode(info, pkt);\n+\tqp->eventtype = TERM_EVENT_QP_FATAL;\n+\n+\tswitch (info->ae_id) {\n+\tcase IRDMA_AE_AMP_UNALLOCATED_STAG:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tif (opcode == IRDMA_OP_TYPE_RDMA_WRITE)\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,\n+\t\t\t\t\t       (LAYER_DDP << 4) | DDP_TAGGED_BUF,\n+\t\t\t\t\t       DDP_TAGGED_INV_STAG);\n+\t\telse\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t\t       RDMAP_INV_STAG);\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_BOUNDS_VIOLATION:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tif (info->q2_data_written)\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,\n+\t\t\t\t\t       (LAYER_DDP << 4) | DDP_TAGGED_BUF,\n+\t\t\t\t\t       DDP_TAGGED_BOUNDS);\n+\t\telse\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t\t       RDMAP_INV_BOUNDS);\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_BAD_PD:\n+\t\tswitch (opcode) {\n+\t\tcase IRDMA_OP_TYPE_RDMA_WRITE:\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,\n+\t\t\t\t\t       (LAYER_DDP << 4) | DDP_TAGGED_BUF,\n+\t\t\t\t\t       DDP_TAGGED_UNASSOC_STAG);\n+\t\t\tbreak;\n+\t\tcase IRDMA_OP_TYPE_SEND_INV:\n+\t\tcase IRDMA_OP_TYPE_SEND_SOL_INV:\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t\t       RDMAP_CANT_INV_STAG);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t\t       RDMAP_UNASSOC_STAG);\n+\t\t}\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_INVALID_STAG:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t       RDMAP_INV_STAG);\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_BAD_QP:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t       DDP_UNTAGGED_INV_QN);\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_BAD_STAG_KEY:\n+\tcase IRDMA_AE_AMP_BAD_STAG_INDEX:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tswitch (opcode) {\n+\t\tcase IRDMA_OP_TYPE_SEND_INV:\n+\t\tcase IRDMA_OP_TYPE_SEND_SOL_INV:\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,\n+\t\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,\n+\t\t\t\t\t       RDMAP_CANT_INV_STAG);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,\n+\t\t\t\t\t       RDMAP_INV_STAG);\n+\t\t}\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_RIGHTS_VIOLATION:\n+\tcase IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:\n+\tcase IRDMA_AE_PRIV_OPERATION_DENIED:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t       RDMAP_ACCESS);\n+\t\tbreak;\n+\tcase IRDMA_AE_AMP_TO_WRAP:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,\n+\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,\n+\t\t\t\t       RDMAP_TO_WRAP);\n+\t\tbreak;\n+\tcase IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t       (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);\n+\t\tbreak;\n+\tcase IRDMA_AE_LLP_SEGMENT_TOO_SMALL:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_CATASTROPHIC,\n+\t\t\t\t       DDP_CATASTROPHIC_LOCAL);\n+\t\tbreak;\n+\tcase IRDMA_AE_LCE_QP_CATASTROPHIC:\n+\tcase IRDMA_AE_DDP_NO_L_BIT:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_CATASTROPHIC,\n+\t\t\t\t       DDP_CATASTROPHIC_LOCAL);\n+\t\tbreak;\n+\tcase IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t       DDP_UNTAGGED_INV_MSN_RANGE);\n+\t\tbreak;\n+\tcase IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:\n+\t\tqp->eventtype = TERM_EVENT_QP_ACCESS_ERR;\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t       DDP_UNTAGGED_INV_TOO_LONG);\n+\t\tbreak;\n+\tcase IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:\n+\t\tif (is_tagged)\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t\t       (LAYER_DDP << 4) | DDP_TAGGED_BUF,\n+\t\t\t\t\t       DDP_TAGGED_INV_DDP_VER);\n+\t\telse\n+\t\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t\t       DDP_UNTAGGED_INV_DDP_VER);\n+\t\tbreak;\n+\tcase IRDMA_AE_DDP_UBE_INVALID_MO:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t       DDP_UNTAGGED_INV_MO);\n+\t\tbreak;\n+\tcase IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t       DDP_UNTAGGED_INV_MSN_NO_BUF);\n+\t\tbreak;\n+\tcase IRDMA_AE_DDP_UBE_INVALID_QN:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,\n+\t\t\t\t       DDP_UNTAGGED_INV_QN);\n+\t\tbreak;\n+\tcase IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,\n+\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,\n+\t\t\t\t       RDMAP_INV_RDMAP_VER);\n+\t\tbreak;\n+\tdefault:\n+\t\tirdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,\n+\t\t\t\t       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,\n+\t\t\t\t       RDMAP_UNSPECIFIED);\n+\t\tbreak;\n+\t}\n+\n+\tif (copy_len)\n+\t\tmemcpy(termhdr + 1, pkt, copy_len);\n+\n+\treturn sizeof(struct irdma_terminate_hdr) + copy_len;\n+}\n+\n+/**\n+ * irdma_terminate_send_fin() - Send fin for terminate message\n+ * @qp: qp associated with received terminate AE\n+ */\n+void irdma_terminate_send_fin(struct irdma_sc_qp *qp)\n+{\n+\tirdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,\n+\t\t\t     IRDMAQP_TERM_SEND_FIN_ONLY, 0);\n+}\n+\n+/**\n+ * irdma_terminate_connection() - Bad AE and send terminate to remote QP\n+ * @qp: qp associated with received terminate AE\n+ * @info: the struct contiaing AE information\n+ */\n+void irdma_terminate_connection(struct irdma_sc_qp *qp,\n+\t\t\t\tstruct irdma_aeqe_info *info)\n+{\n+\tu8 termlen = 0;\n+\n+\tif (qp->term_flags & IRDMA_TERM_SENT)\n+\t\treturn;\n+\n+\ttermlen = irdma_bld_terminate_hdr(qp, info);\n+\tirdma_terminate_start_timer(qp);\n+\tqp->term_flags |= IRDMA_TERM_SENT;\n+\tirdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,\n+\t\t\t     IRDMAQP_TERM_SEND_TERM_ONLY, termlen);\n+}\n+\n+/**\n+ * irdma_terminate_received - handle terminate received AE\n+ * @qp: qp associated with received terminate AE\n+ * @info: the struct contiaing AE information\n+ */\n+void irdma_terminate_received(struct irdma_sc_qp *qp,\n+\t\t\t      struct irdma_aeqe_info *info)\n+{\n+\tu8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;\n+\t__be32 *mpa;\n+\tu8 ddp_ctl;\n+\tu8 rdma_ctl;\n+\tu16 aeq_id = 0;\n+\tstruct irdma_terminate_hdr *termhdr;\n+\n+\tmpa = (__be32 *)irdma_locate_mpa(pkt);\n+\tif (info->q2_data_written) {\n+\t\t/* did not validate the frame - do it now */\n+\t\tddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;\n+\t\trdma_ctl = ntohl(mpa[0]) & 0xff;\n+\t\tif ((ddp_ctl & 0xc0) != 0x40)\n+\t\t\taeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;\n+\t\telse if ((ddp_ctl & 0x03) != 1)\n+\t\t\taeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;\n+\t\telse if (ntohl(mpa[2]) != 2)\n+\t\t\taeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;\n+\t\telse if (ntohl(mpa[3]) != 1)\n+\t\t\taeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;\n+\t\telse if (ntohl(mpa[4]) != 0)\n+\t\t\taeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;\n+\t\telse if ((rdma_ctl & 0xc0) != 0x40)\n+\t\t\taeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;\n+\n+\t\tinfo->ae_id = aeq_id;\n+\t\tif (info->ae_id) {\n+\t\t\t/* Bad terminate recvd - send back a terminate */\n+\t\t\tirdma_terminate_connection(qp, info);\n+\t\t\treturn;\n+\t\t}\n+\t}\n+\n+\tqp->term_flags |= IRDMA_TERM_RCVD;\n+\tqp->eventtype = TERM_EVENT_QP_FATAL;\n+\ttermhdr = (struct irdma_terminate_hdr *)&mpa[5];\n+\tif (termhdr->layer_etype == RDMAP_REMOTE_PROT ||\n+\t    termhdr->layer_etype == RDMAP_REMOTE_OP) {\n+\t\tirdma_terminate_done(qp, 0);\n+\t} else {\n+\t\tirdma_terminate_start_timer(qp);\n+\t\tirdma_terminate_send_fin(qp);\n+\t}\n+}\n+\n+static enum irdma_status_code irdma_null_ws_add(struct irdma_sc_vsi *vsi,\n+\t\t\t\t\t\tu8 user_pri)\n+{\n+\treturn 0;\n+}\n+\n+static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)\n+{\n+\t/* do nothing */\n+}\n+\n+static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi)\n+{\n+\t/* do nothing */\n+}\n+\n+/**\n+ * irdma_sc_vsi_init - Init the vsi structure\n+ * @vsi: pointer to vsi structure to initialize\n+ * @info: the info used to initialize the vsi struct\n+ */\n+void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,\n+\t\t       struct irdma_vsi_init_info *info)\n+{\n+\tint i;\n+\tu32 reg_data = 0;\n+\tu32 reg_offset = 0;\n+\tstruct irdma_l2params *l2p;\n+\n+\tvsi->dev = info->dev;\n+\tvsi->back_vsi = info->back_vsi;\n+\tvsi->mtu = info->params->mtu;\n+\tvsi->exception_lan_q = info->exception_lan_q;\n+\tvsi->vsi_idx = info->pf_data_vsi_num;\n+\tvsi->vm_vf_type = info->vm_vf_type;\n+\tvsi->vm_id = info->vm_id;\n+\tif (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)\n+\t\tvsi->fcn_id = info->dev->hmc_fn_id;\n+\n+\tl2p = info->params;\n+\tvsi->qos_rel_bw = l2p->vsi_rel_bw;\n+\tvsi->qos_prio_type = l2p->vsi_prio_type;\n+\tfor (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {\n+\t\tif (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)\n+\t\t\tvsi->qos[i].qs_handle = l2p->qs_handle_list[i];\n+\t\tvsi->qos[i].traffic_class = info->params->up2tc[i];\n+\t\tvsi->qos[i].rel_bw =\n+\t\t\tl2p->tc_info[vsi->qos[i].traffic_class].rel_bw;\n+\t\tvsi->qos[i].prio_type =\n+\t\t\tl2p->tc_info[vsi->qos[i].traffic_class].prio_type;\n+\t\tspin_lock_init(&vsi->qos[i].lock);\n+\t\tINIT_LIST_HEAD(&vsi->qos[i].qplist);\n+\t}\n+\tif (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {\n+\t\tvsi->dev->ws_add = irdma_null_ws_add;\n+\t\tvsi->dev->ws_remove = irdma_null_ws_remove;\n+\t\tvsi->dev->ws_reset = irdma_null_ws_reset;\n+\t} else {\n+\t\tvsi->dev->ws_add = irdma_ws_add;\n+\t\tvsi->dev->ws_remove = irdma_ws_remove;\n+\t\tvsi->dev->ws_reset = irdma_ws_reset;\n+\t}\n+\tif (info->dev->is_pf) {\n+\t\treg_offset = info->dev->hw_regs[IRDMA_VSIQF_PE_CTL1] +\n+\t\t\t     4 * (vsi->vsi_idx);\n+\t\tif (vsi->dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {\n+\t\t\twr32(info->dev->hw, reg_offset, 0x1);\n+\t\t} else {\n+\t\t\treg_data = rd32(info->dev->hw, reg_offset);\n+\t\t\treg_data |= 0x2;\n+\t\t\twr32(info->dev->hw, reg_offset, reg_data);\n+\t\t}\n+\t}\n+}\n+\n+/**\n+ * irdma_get_fcn_id - Return the function id\n+ * @vsi: pointer to the vsi\n+ */\n+static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)\n+{\n+\tstruct irdma_stats_inst_info stats_info = {};\n+\tstruct irdma_sc_dev *dev = vsi->dev;\n+\tu8 fcn_id = IRDMA_INVALID_FCN_ID;\n+\tu8 start_idx, max_stats, i;\n+\n+\tif (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {\n+\t\tif (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,\n+\t\t\t\t\t      &stats_info))\n+\t\t\treturn stats_info.stats_idx;\n+\t}\n+\n+\tstart_idx = 1;\n+\tmax_stats = 16;\n+\tfor (i = start_idx; i < max_stats; i++)\n+\t\tif (!dev->fcn_id_array[i]) {\n+\t\t\tfcn_id = i;\n+\t\t\tdev->fcn_id_array[i] = true;\n+\t\t\tbreak;\n+\t\t}\n+\n+\treturn fcn_id;\n+}\n+\n+/**\n+ * irdma_vsi_stats_init - Initialize the vsi statistics\n+ * @vsi: pointer to the vsi structure\n+ * @info: The info structure used for initialization\n+ */\n+enum irdma_status_code irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,\n+\t\t\t\t\t    struct irdma_vsi_stats_info *info)\n+{\n+\tu8 fcn_id = info->fcn_id;\n+\tstruct irdma_dma_mem *stats_buff_mem;\n+\n+\tvsi->pestat = info->pestat;\n+\tvsi->pestat->hw = vsi->dev->hw;\n+\tvsi->pestat->vsi = vsi;\n+\tstats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;\n+\tstats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);\n+\tstats_buff_mem->va = dma_alloc_coherent(hw_to_dev(vsi->pestat->hw),\n+\t\t\t\t\t\tstats_buff_mem->size,\n+\t\t\t\t\t\t&stats_buff_mem->pa,\n+\t\t\t\t\t\tGFP_KERNEL);\n+\tif (!stats_buff_mem->va)\n+\t\treturn IRDMA_ERR_NO_MEMORY;\n+\n+\tvsi->pestat->gather_info.gather_stats = stats_buff_mem->va;\n+\tvsi->pestat->gather_info.last_gather_stats =\n+\t\t(void *)((uintptr_t)stats_buff_mem->va +\n+\t\t\t IRDMA_GATHER_STATS_BUF_SIZE);\n+\n+\tif (vsi->dev->is_pf)\n+\t\tirdma_hw_stats_start_timer(vsi);\n+\n+\tif (info->alloc_fcn_id)\n+\t\tfcn_id = irdma_get_fcn_id(vsi);\n+\tif (fcn_id == IRDMA_INVALID_FCN_ID)\n+\t\tgoto stats_error;\n+\n+\tvsi->stats_fcn_id_alloc = info->alloc_fcn_id;\n+\tvsi->fcn_id = fcn_id;\n+\tif (info->alloc_fcn_id) {\n+\t\tvsi->pestat->gather_info.use_stats_inst = true;\n+\t\tvsi->pestat->gather_info.stats_inst_index = fcn_id;\n+\t}\n+\n+\treturn 0;\n+\n+stats_error:\n+\tdma_free_coherent(hw_to_dev(vsi->pestat->hw), stats_buff_mem->size,\n+\t\t\t  stats_buff_mem->va, stats_buff_mem->pa);\n+\tstats_buff_mem->va = NULL;\n+\n+\treturn IRDMA_ERR_CQP_COMPL_ERROR;\n+}\n+\n+/**\n+ * irdma_vsi_stats_free - Free the vsi stats\n+ * @vsi: pointer to the vsi structure\n+ */\n+void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)\n+{\n+\tstruct irdma_stats_inst_info stats_info = {};\n+\tu8 fcn_id = vsi->fcn_id;\n+\tstruct irdma_sc_dev *dev = vsi->dev;\n+\n+\tif (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {\n+\t\tif (vsi->stats_fcn_id_alloc) {\n+\t\t\tstats_info.stats_idx = vsi->fcn_id;\n+\t\t\tirdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,\n+\t\t\t\t\t\t &stats_info);\n+\t\t}\n+\t} else {\n+\t\tif (vsi->stats_fcn_id_alloc &&\n+\t\t    fcn_id < vsi->dev->hw_attrs.max_stat_inst)\n+\t\t\tvsi->dev->fcn_id_array[fcn_id] = false;\n+\t}\n+\n+\tif (!vsi->pestat)\n+\t\treturn;\n+\tif (vsi->dev->is_pf)\n+\t\tirdma_hw_stats_stop_timer(vsi);\n+\tdma_free_coherent(hw_to_dev(vsi->pestat->hw),\n+\t\t\t  vsi->pestat->gather_info.stats_buff_mem.size,\n+\t\t\t  vsi->pestat->gather_info.stats_buff_mem.va,\n+\t\t\t  vsi->pestat->gather_info.stats_buff_mem.pa);\n+\tvsi->pestat->gather_info.stats_buff_mem.va = NULL;\n+}\n+\n+/**\n+ * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size\n+ * @wqsize: size of the wq (sq, rq) to encoded_size\n+ * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's\n+ */\n+u8 irdma_get_encoded_wqe_size(u32 wqsize, bool cqpsq)\n+{\n+\tu8 encoded_size = 0;\n+\n+\t/* cqp sq's hw coded value starts from 1 for size of 4\n+\t * while it starts from 0 for qp' wq's.\n+\t */\n+\tif (cqpsq)\n+\t\tencoded_size = 1;\n+\twqsize >>= 2;\n+\twhile (wqsize >>= 1)\n+\t\tencoded_size++;\n+\n+\treturn encoded_size;\n+}\n+\n+/**\n+ * irdma_sc_gather_stats - collect the statistics\n+ * @cqp: struct for cqp hw\n+ * @info: gather stats info structure\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,\n+\t\t      struct irdma_stats_gather_info *info, u64 scratch)\n+{\n+\t__le64 *wqe;\n+\tu64 temp;\n+\n+\tif (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)\n+\t\treturn IRDMA_ERR_BUF_TOO_SHORT;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 40,\n+\t\t      LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));\n+\tset_64bit_val(wqe, 32, info->stats_buff_mem.pa);\n+\n+\ttemp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |\n+\t       LS_64(info->use_stats_inst, IRDMA_CQPSQ_STATS_USE_INST) |\n+\t       LS_64(info->stats_inst_index, IRDMA_CQPSQ_STATS_INST_INDEX) |\n+\t       LS_64(info->use_hmc_fcn_index,\n+\t\t     IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |\n+\t       LS_64(IRDMA_CQP_OP_GATHER_STATS, IRDMA_CQPSQ_STATS_OP);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, temp);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_STATS, \"GATHER_STATS WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPDB],\n+\t     IRDMA_RING_CURRENT_HEAD(cqp->sq_ring));\n+\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\"STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\\n\",\n+\t\tcqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_manage_stats_inst - allocate or free stats instance\n+ * @cqp: struct for cqp hw\n+ * @info: stats info structure\n+ * @alloc: alloc vs. delete flag\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,\n+\t\t\t   struct irdma_stats_inst_info *info, bool alloc,\n+\t\t\t   u64 scratch)\n+{\n+\t__le64 *wqe;\n+\tu64 temp;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 40,\n+\t\t      LS_64(info->hmc_fn_id, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));\n+\ttemp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |\n+\t       LS_64(alloc, IRDMA_CQPSQ_STATS_ALLOC_INST) |\n+\t       LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |\n+\t       LS_64(info->stats_idx, IRDMA_CQPSQ_STATS_INST_INDEX) |\n+\t       LS_64(IRDMA_CQP_OP_MANAGE_STATS, IRDMA_CQPSQ_STATS_OP);\n+\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, temp);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"MANAGE_STATS WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\n+\tirdma_sc_cqp_post_sq(cqp);\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_set_up_mapping - set the up map table\n+ * @cqp: struct for cqp hw\n+ * @info: User priority map info\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t  struct irdma_up_info *info,\n+\t\t\t\t\t\t  u64 scratch)\n+{\n+\t__le64 *wqe;\n+\tu64 temp;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\ttemp = info->map[0] | LS_64_1(info->map[1], 8) |\n+\t       LS_64_1(info->map[2], 16) | LS_64_1(info->map[3], 24) |\n+\t       LS_64_1(info->map[4], 32) | LS_64_1(info->map[5], 40) |\n+\t       LS_64_1(info->map[6], 48) | LS_64_1(info->map[7], 56);\n+\n+\tset_64bit_val(wqe, 0, temp);\n+\tset_64bit_val(wqe, 40,\n+\t\t      LS_64(info->cnp_up_override, IRDMA_CQPSQ_UP_CNPOVERRIDE) |\n+\t\t      LS_64(info->hmc_fcn_idx, IRDMA_CQPSQ_UP_HMCFCNIDX));\n+\n+\ttemp = LS_64(cqp->polarity, IRDMA_CQPSQ_UP_WQEVALID) |\n+\t       LS_64(info->use_vlan, IRDMA_CQPSQ_UP_USEVLAN) |\n+\t       LS_64(info->use_cnp_up_override, IRDMA_CQPSQ_UP_USEOVERRIDE) |\n+\t       LS_64(IRDMA_CQP_OP_UP_MAP, IRDMA_CQPSQ_UP_OP);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, temp);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"UPMAP WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_manage_ws_node - create/modify/destroy WS node\n+ * @cqp: struct for cqp hw\n+ * @info: node info structure\n+ * @node_op: 0 for add 1 for modify, 2 for delete\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,\n+\t\t\tstruct irdma_ws_node_info *info,\n+\t\t\tenum irdma_ws_node_op node_op, u64 scratch)\n+{\n+\t__le64 *wqe;\n+\tu64 temp = 0;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 32,\n+\t\t      LS_64(info->vsi, IRDMA_CQPSQ_WS_VSI) |\n+\t\t      LS_64(info->weight, IRDMA_CQPSQ_WS_WEIGHT));\n+\n+\ttemp = LS_64(cqp->polarity, IRDMA_CQPSQ_WS_WQEVALID) |\n+\t       LS_64(node_op, IRDMA_CQPSQ_WS_NODEOP) |\n+\t       LS_64(info->enable, IRDMA_CQPSQ_WS_ENABLENODE) |\n+\t       LS_64(info->type_leaf, IRDMA_CQPSQ_WS_NODETYPE) |\n+\t       LS_64(info->prio_type, IRDMA_CQPSQ_WS_PRIOTYPE) |\n+\t       LS_64(info->tc, IRDMA_CQPSQ_WS_TC) |\n+\t       LS_64(IRDMA_CQP_OP_WORK_SCHED_NODE, IRDMA_CQPSQ_WS_OP) |\n+\t       LS_64(info->parent_id, IRDMA_CQPSQ_WS_PARENTID) |\n+\t       LS_64(info->id, IRDMA_CQPSQ_WS_NODEID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, temp);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"MANAGE_WS WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_qp_flush_wqes - flush qp's wqe\n+ * @qp: sc qp\n+ * @info: dlush information\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, struct irdma_qp_flush_info *info,\n+\t\t       u64 scratch, bool post_sq)\n+{\n+\tu64 temp = 0;\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\tbool flush_sq = false, flush_rq = false;\n+\n+\tif (info->rq && !qp->flush_rq)\n+\t\tflush_rq = true;\n+\tif (info->sq && !qp->flush_sq)\n+\t\tflush_sq = true;\n+\tqp->flush_sq |= flush_sq;\n+\tqp->flush_rq |= flush_rq;\n+\n+\tif (!flush_sq && !flush_rq) {\n+\t\tdev_dbg(rfdev_to_dev(qp->dev),\n+\t\t\t\"CQP: Additional flush request ignored\\n\");\n+\t\treturn 0;\n+\t}\n+\n+\tcqp = qp->pd->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tif (info->userflushcode) {\n+\t\tif (flush_rq)\n+\t\t\ttemp |= LS_64(info->rq_minor_code, IRDMA_CQPSQ_FWQE_RQMNERR) |\n+\t\t\t\tLS_64(info->rq_major_code, IRDMA_CQPSQ_FWQE_RQMJERR);\n+\t\tif (flush_sq)\n+\t\t\ttemp |= LS_64(info->sq_minor_code, IRDMA_CQPSQ_FWQE_SQMNERR) |\n+\t\t\t\tLS_64(info->sq_major_code, IRDMA_CQPSQ_FWQE_SQMJERR);\n+\t}\n+\tset_64bit_val(wqe, 16, temp);\n+\n+\ttemp = (info->generate_ae) ?\n+\t\tinfo->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE) : 0;\n+\tset_64bit_val(wqe, 8, temp);\n+\n+\thdr = qp->qp_uk.qp_id |\n+\t      LS_64(IRDMA_CQP_OP_FLUSH_WQES, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->generate_ae, IRDMA_CQPSQ_FWQE_GENERATE_AE) |\n+\t      LS_64(info->userflushcode, IRDMA_CQPSQ_FWQE_USERFLCODE) |\n+\t      LS_64(flush_sq, IRDMA_CQPSQ_FWQE_FLUSHSQ) |\n+\t      LS_64(flush_rq, IRDMA_CQPSQ_FWQE_FLUSHRQ) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QP_FLUSH WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP\n+ * @qp: sc qp\n+ * @info: gen ae information\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_gen_ae(struct irdma_sc_qp *qp,\n+\t\t\t\t\t      struct irdma_gen_ae_info *info,\n+\t\t\t\t\t      u64 scratch, bool post_sq)\n+{\n+\tu64 temp;\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\n+\tcqp = qp->pd->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\ttemp = info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE);\n+\tset_64bit_val(wqe, 8, temp);\n+\n+\thdr = qp->qp_uk.qp_id | LS_64(IRDMA_CQP_OP_GEN_AE, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(1, IRDMA_CQPSQ_FWQE_GENERATE_AE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"GEN_AE WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/*** irdma_sc_qp_upload_context - upload qp's context\n+ * @dev: sc device struct\n+ * @info: upload context info ptr for return\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,\n+\t\t\t   struct irdma_upload_context_info *info, u64 scratch,\n+\t\t\t   bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, info->buf_pa);\n+\n+\thdr = LS_64(info->qp_id, IRDMA_CQPSQ_UCTX_QPID) |\n+\t      LS_64(IRDMA_CQP_OP_UPLOAD_CONTEXT, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->qp_type, IRDMA_CQPSQ_UCTX_QPTYPE) |\n+\t      LS_64(info->raw_format, IRDMA_CQPSQ_UCTX_RAWFORMAT) |\n+\t      LS_64(info->freeze_qp, IRDMA_CQPSQ_UCTX_FREEZEQP) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"QP_UPLOAD_CTX WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_manage_push_page - Handle push page\n+ * @cqp: struct for cqp hw\n+ * @info: push page info\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,\n+\t\t\t  struct irdma_cqp_manage_push_page_info *info,\n+\t\t\t  u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\tif (info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)\n+\t\treturn IRDMA_ERR_INVALID_PUSH_PAGE_INDEX;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, info->qs_handle);\n+\thdr = LS_64(info->push_idx, IRDMA_CQPSQ_MPP_PPIDX) |\n+\t      LS_64(info->push_page_type, IRDMA_CQPSQ_MPP_PPTYPE) |\n+\t      LS_64(IRDMA_CQP_OP_MANAGE_PUSH_PAGES, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |\n+\t      LS_64(info->free_page, IRDMA_CQPSQ_MPP_FREE_PAGE);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"MANAGE_PUSH_PAGES WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_suspend_qp - suspend qp for param change\n+ * @cqp: struct for cqp hw\n+ * @qp: sc qp struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t  struct irdma_sc_qp *qp,\n+\t\t\t\t\t\t  u64 scratch)\n+{\n+\tu64 hdr;\n+\t__le64 *wqe;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\thdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_SUSPENDQP_QPID) |\n+\t      LS_64(IRDMA_CQP_OP_SUSPEND_QP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"SUSPEND_QP WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_resume_qp - resume qp after suspend\n+ * @cqp: struct for cqp hw\n+ * @qp: sc qp struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code irdma_sc_resume_qp(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t struct irdma_sc_qp *qp,\n+\t\t\t\t\t\t u64 scratch)\n+{\n+\tu64 hdr;\n+\t__le64 *wqe;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(qp->qs_handle, IRDMA_CQPSQ_RESUMEQP_QSHANDLE));\n+\n+\thdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_RESUMEQP_QPID) |\n+\t      LS_64(IRDMA_CQP_OP_RESUME_QP, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"RESUME_QP WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cq_ack - acknowledge completion q\n+ * @cq: cq struct\n+ */\n+static void irdma_sc_cq_ack(struct irdma_sc_cq *cq)\n+{\n+\twritel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);\n+}\n+\n+/**\n+ * irdma_sc_cq_init - initialize completion q\n+ * @cq: cq struct\n+ * @info: cq initialization info\n+ */\n+static enum irdma_status_code irdma_sc_cq_init(struct irdma_sc_cq *cq,\n+\t\t\t\t\t       struct irdma_cq_init_info *info)\n+{\n+\tenum irdma_status_code ret_code;\n+\tu32 pble_obj_cnt;\n+\n+\tif (info->cq_uk_init_info.cq_size < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||\n+\t    info->cq_uk_init_info.cq_size > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tpble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\tif (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\tcq->cq_pa = info->cq_base_pa;\n+\tcq->dev = info->dev;\n+\tcq->ceq_id = info->ceq_id;\n+\tinfo->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;\n+\tinfo->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;\n+\tret_code = irdma_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\tcq->virtual_map = info->virtual_map;\n+\tcq->pbl_chunk_size = info->pbl_chunk_size;\n+\tcq->ceqe_mask = info->ceqe_mask;\n+\tcq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;\n+\tcq->shadow_area_pa = info->shadow_area_pa;\n+\tcq->shadow_read_threshold = info->shadow_read_threshold;\n+\tcq->ceq_id_valid = info->ceq_id_valid;\n+\tcq->tph_en = info->tph_en;\n+\tcq->tph_val = info->tph_val;\n+\tcq->first_pm_pbl_idx = info->first_pm_pbl_idx;\n+\tcq->vsi = info->vsi;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cq_create - create completion q\n+ * @cq: cq struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @check_overflow: flag for overflow check\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_cq_create(struct irdma_sc_cq *cq,\n+\t\t\t\t\t\t u64 scratch,\n+\t\t\t\t\t\t bool check_overflow,\n+\t\t\t\t\t\t bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\tstruct irdma_sc_ceq *ceq;\n+\tenum irdma_status_code ret_code = 0;\n+\n+\tcqp = cq->dev->cqp;\n+\tif (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1))\n+\t\treturn IRDMA_ERR_INVALID_CQ_ID;\n+\n+\tif (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))\n+\t\treturn IRDMA_ERR_INVALID_CEQ_ID;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tceq = cq->dev->ceq[cq->ceq_id];\n+\tif (ceq && ceq->reg_cq)\n+\t\tret_code = irdma_sc_add_cq_ctx(ceq, cq);\n+\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\tset_64bit_val(wqe, 0, cq->cq_uk.cq_size);\n+\tset_64bit_val(wqe, 8, RS_64_1(cq, 1));\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(cq->shadow_read_threshold,\n+\t\t\t    IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));\n+\tset_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));\n+\tset_64bit_val(wqe, 40, cq->shadow_area_pa);\n+\tset_64bit_val(wqe, 48,\n+\t\t      LS_64((cq->virtual_map ? cq->first_pm_pbl_idx : 0),\n+\t\t\t    IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX));\n+\tset_64bit_val(wqe, 56,\n+\t\t      LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |\n+\t\t      LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));\n+\n+\thdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |\n+\t      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),\n+\t\t\tIRDMA_CQPSQ_CQ_CEQID) |\n+\t      LS_64(IRDMA_CQP_OP_CREATE_CQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |\n+\t      LS_64(check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |\n+\t      LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |\n+\t      LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |\n+\t      LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |\n+\t      LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |\n+\t      LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CQ_CREATE WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cq_destroy - destroy completion q\n+ * @cq: cq struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_cq_destroy(struct irdma_sc_cq *cq,\n+\t\t\t\t\t\t  u64 scratch, bool post_sq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tstruct irdma_sc_ceq *ceq;\n+\n+\tcqp = cq->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tceq = cq->dev->ceq[cq->ceq_id];\n+\tif (ceq && ceq->reg_cq)\n+\t\tirdma_sc_remove_cq_ctx(ceq, cq);\n+\n+\tset_64bit_val(wqe, 0, cq->cq_uk.cq_size);\n+\tset_64bit_val(wqe, 8, RS_64_1(cq, 1));\n+\tset_64bit_val(wqe, 40, cq->shadow_area_pa);\n+\tset_64bit_val(wqe, 48,\n+\t\t      (cq->virtual_map ? cq->first_pm_pbl_idx : 0));\n+\n+\thdr = cq->cq_uk.cq_id |\n+\t      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),\n+\t\t\tIRDMA_CQPSQ_CQ_CEQID) |\n+\t      LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |\n+\t      LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |\n+\t      LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |\n+\t      LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |\n+\t      LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |\n+\t      LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CQ_DESTROY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cq_resize - set resized cq buffer info\n+ * @cq: resized cq\n+ * @info: resized cq buffer info\n+ */\n+static void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info)\n+{\n+\tcq->virtual_map = info->virtual_map;\n+\tcq->cq_pa = info->cq_pa;\n+\tcq->first_pm_pbl_idx = info->first_pm_pbl_idx;\n+\tcq->pbl_chunk_size = info->pbl_chunk_size;\n+\tcq->cq_uk.ops.iw_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size);\n+}\n+\n+/**\n+ * irdma_sc_cq_modify - modify a Completion Queue\n+ * @cq: cq struct\n+ * @info: modification info struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag to post to sq\n+ */\n+static enum irdma_status_code\n+irdma_sc_cq_modify(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info,\n+\t\t   u64 scratch, bool post_sq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tu32 pble_obj_cnt;\n+\n+\tif (info->ceq_valid &&\n+\t    info->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))\n+\t\treturn IRDMA_ERR_INVALID_CEQ_ID;\n+\n+\tpble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\tif (info->cq_resize && info->virtual_map &&\n+\t    info->first_pm_pbl_idx >= pble_obj_cnt)\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\tcqp = cq->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 0, info->cq_size);\n+\tset_64bit_val(wqe, 8, RS_64_1(cq, 1));\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(info->shadow_read_threshold,\n+\t\t\t    IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));\n+\tset_64bit_val(wqe, 32, info->cq_pa);\n+\tset_64bit_val(wqe, 40, cq->shadow_area_pa);\n+\tset_64bit_val(wqe, 48, info->first_pm_pbl_idx);\n+\tset_64bit_val(wqe, 56,\n+\t\t      LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |\n+\t\t      LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));\n+\n+\thdr = cq->cq_uk.cq_id |\n+\t      FLD_LS_64(cq->dev, (info->ceq_valid ? cq->ceq_id : 0),\n+\t\t\tIRDMA_CQPSQ_CQ_CEQID) |\n+\t      LS_64(IRDMA_CQP_OP_MODIFY_CQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(info->cq_resize, IRDMA_CQPSQ_CQ_CQRESIZE) |\n+\t      LS_64(info->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |\n+\t      LS_64(info->check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |\n+\t      LS_64(info->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |\n+\t      LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |\n+\t      LS_64(info->ceq_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |\n+\t      LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |\n+\t      LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CQ_MODIFY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_check_cqp_progress - check cqp processing progress\n+ * @timeout: timeout info struct\n+ * @dev: sc device struct\n+ */\n+static void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout,\n+\t\t\t\t     struct irdma_sc_dev *dev)\n+{\n+\tif (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {\n+\t\ttimeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];\n+\t\ttimeout->count = 0;\n+\t} else {\n+\t\tif (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] !=\n+\t\t    timeout->compl_cqp_cmds)\n+\t\t\ttimeout->count++;\n+\t}\n+}\n+\n+/**\n+ * irdma_get_cqp_reg_info - get head and tail for cqp using registers\n+ * @cqp: struct for cqp hw\n+ * @val: cqp tail register value\n+ * @tail: wqtail register value\n+ * @error: cqp processing err\n+ */\n+static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val,\n+\t\t\t\t\t  u32 *tail, u32 *error)\n+{\n+\t*val = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPTAIL]);\n+\t*tail = RS_32(*val, IRDMA_CQPTAIL_WQTAIL);\n+\t*error = RS_32(*val, IRDMA_CQPTAIL_CQP_OP_ERR);\n+}\n+\n+/**\n+ * irdma_cqp_poll_registers - poll cqp registers\n+ * @cqp: struct for cqp hw\n+ * @tail: wqtail register value\n+ * @count: how many times to try for completion\n+ */\n+static enum irdma_status_code irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t       u32 tail, u32 count)\n+{\n+\tu32 i = 0;\n+\tu32 newtail, error, val;\n+\n+\twhile (i++ < count) {\n+\t\tirdma_get_cqp_reg_info(cqp, &val, &newtail, &error);\n+\t\tif (error) {\n+\t\t\terror = rd32(cqp->dev->hw,\n+\t\t\t\t     cqp->dev->hw_regs[IRDMA_CQPERRCODES]);\n+\t\t\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\t\t\"CQP: CQPERRCODES error_code[x%08X]\\n\", error);\n+\t\t\treturn IRDMA_ERR_CQP_COMPL_ERROR;\n+\t\t}\n+\t\tif (newtail != tail) {\n+\t\t\t/* SUCCESS */\n+\t\t\tIRDMA_RING_MOVE_TAIL(cqp->sq_ring);\n+\t\t\tcqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tudelay(cqp->dev->hw_attrs.max_sleep_count);\n+\t}\n+\n+\treturn IRDMA_ERR_TIMEOUT;\n+}\n+\n+/**\n+ * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base\n+ * @buf: pointer to commit buffer\n+ * @buf_idx: buffer index\n+ * @obj_info: object info pointer\n+ * @rsrc_idx: indexs of memory resource\n+ */\n+static u64 irdma_sc_decode_fpm_commit(__le64 *buf, u32 buf_idx,\n+\t\t\t\t      struct irdma_hmc_obj_info *obj_info,\n+\t\t\t\t      u32 rsrc_idx)\n+{\n+\tu64 temp;\n+\n+\tget_64bit_val(buf, buf_idx, &temp);\n+\n+\tswitch (rsrc_idx) {\n+\tcase IRDMA_HMC_IW_QP:\n+\t\tobj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_QPCNT);\n+\t\tbreak;\n+\tcase IRDMA_HMC_IW_CQ:\n+\t\tobj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_CQCNT);\n+\t\tbreak;\n+\tcase IRDMA_HMC_IW_APBVT_ENTRY:\n+\t\tobj_info[rsrc_idx].cnt = 1;\n+\t\tbreak;\n+\tdefault:\n+\t\tobj_info[rsrc_idx].cnt = (u32)temp;\n+\t\tbreak;\n+\t}\n+\n+\tobj_info[rsrc_idx].base = (u64)RS_64_1(temp, IRDMA_COMMIT_FPM_BASE_S) * 512;\n+\n+\treturn temp;\n+}\n+\n+/**\n+ * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer\n+ * @dev: pointer to dev struct\n+ * @buf: ptr to fpm commit buffer\n+ * @info: ptr to irdma_hmc_obj_info struct\n+ * @sd: number of SDs for HMC objects\n+ *\n+ * parses fpm commit info and copy base value\n+ * of hmc objects in hmc_info\n+ */\n+static enum irdma_status_code\n+irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 *buf,\n+\t\t\t      struct irdma_hmc_obj_info *info, u32 *sd)\n+{\n+\tu64 size;\n+\tu32 i;\n+\tu64 max_base = 0;\n+\tu32 last_hmc_obj = 0;\n+\n+\tirdma_sc_decode_fpm_commit(buf, 0, info, IRDMA_HMC_IW_QP);\n+\tirdma_sc_decode_fpm_commit(buf, 8, info, IRDMA_HMC_IW_CQ);\n+\t/* skiping RSRVD */\n+\tirdma_sc_decode_fpm_commit(buf, 24, info, IRDMA_HMC_IW_HTE);\n+\tirdma_sc_decode_fpm_commit(buf, 32, info, IRDMA_HMC_IW_ARP);\n+\tirdma_sc_decode_fpm_commit(buf, 40, info,\n+\t\t\t\t   IRDMA_HMC_IW_APBVT_ENTRY);\n+\tirdma_sc_decode_fpm_commit(buf, 48, info, IRDMA_HMC_IW_MR);\n+\tirdma_sc_decode_fpm_commit(buf, 56, info, IRDMA_HMC_IW_XF);\n+\tirdma_sc_decode_fpm_commit(buf, 64, info, IRDMA_HMC_IW_XFFL);\n+\tirdma_sc_decode_fpm_commit(buf, 72, info, IRDMA_HMC_IW_Q1);\n+\tirdma_sc_decode_fpm_commit(buf, 80, info, IRDMA_HMC_IW_Q1FL);\n+\tirdma_sc_decode_fpm_commit(buf, 88, info,\n+\t\t\t\t   IRDMA_HMC_IW_TIMER);\n+\tirdma_sc_decode_fpm_commit(buf, 112, info,\n+\t\t\t\t   IRDMA_HMC_IW_PBLE);\n+\t/* skipping RSVD. */\n+\tif (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {\n+\t\tirdma_sc_decode_fpm_commit(buf, 96, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_FSIMC);\n+\t\tirdma_sc_decode_fpm_commit(buf, 104, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_FSIAV);\n+\t\tirdma_sc_decode_fpm_commit(buf, 128, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_RRF);\n+\t\tirdma_sc_decode_fpm_commit(buf, 136, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_RRFFL);\n+\t\tirdma_sc_decode_fpm_commit(buf, 144, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_HDR);\n+\t\tirdma_sc_decode_fpm_commit(buf, 152, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_MD);\n+\t\tirdma_sc_decode_fpm_commit(buf, 160, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_OOISC);\n+\t\tirdma_sc_decode_fpm_commit(buf, 168, info,\n+\t\t\t\t\t   IRDMA_HMC_IW_OOISCFFL);\n+\t}\n+\n+\t/* searching for the last object in HMC to find the size of the HMC area. */\n+\tfor (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {\n+\t\tif (info[i].base > max_base) {\n+\t\t\tmax_base = info[i].base;\n+\t\t\tlast_hmc_obj = i;\n+\t\t}\n+\t}\n+\n+\tsize = info[last_hmc_obj].cnt * info[last_hmc_obj].size +\n+\t       info[last_hmc_obj].base;\n+\n+\tif (size & 0x1FFFFF)\n+\t\t*sd = (u32)((size >> 21) + 1); /* add 1 for remainder */\n+\telse\n+\t\t*sd = (u32)(size >> 21);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size\n+ * @buf: ptr to fpm query buffer\n+ * @buf_idx: index into buf\n+ * @obj_info: ptr to irdma_hmc_obj_info struct\n+ * @rsrc_idx: resource index into info\n+ *\n+ * Decode a 64 bit value from fpm query buffer into max count and size\n+ */\n+static u64 irdma_sc_decode_fpm_query(__le64 *buf, u32 buf_idx,\n+\t\t\t\t     struct irdma_hmc_obj_info *obj_info,\n+\t\t\t\t     u32 rsrc_idx)\n+{\n+\tu64 temp;\n+\tu32 size;\n+\n+\tget_64bit_val(buf, buf_idx, &temp);\n+\tobj_info[rsrc_idx].max_cnt = (u32)temp;\n+\tsize = (u32)RS_64_1(temp, 32);\n+\tobj_info[rsrc_idx].size = LS_64_1(1, size);\n+\n+\treturn temp;\n+}\n+\n+/**\n+ * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer\n+ * @dev: ptr to shared code device\n+ * @buf: ptr to fpm query buffer\n+ * @hmc_info: ptr to irdma_hmc_obj_info struct\n+ * @hmc_fpm_misc: ptr to fpm data\n+ *\n+ * parses fpm query buffer and copy max_cnt and\n+ * size value of hmc objects in hmc_info\n+ */\n+static enum irdma_status_code\n+irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf,\n+\t\t\t     struct irdma_hmc_info *hmc_info,\n+\t\t\t     struct irdma_hmc_fpm_misc *hmc_fpm_misc)\n+{\n+\tstruct irdma_hmc_obj_info *obj_info;\n+\tu64 temp;\n+\tu32 size;\n+\tu16 max_pe_sds;\n+\n+\tobj_info = hmc_info->hmc_obj;\n+\n+\tget_64bit_val(buf, 0, &temp);\n+\thmc_info->first_sd_index = (u16)RS_64(temp, IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX);\n+\tmax_pe_sds = (u16)RS_64(temp, IRDMA_QUERY_FPM_MAX_PE_SDS);\n+\n+\t/* Reduce SD count for VFs by 1 to account\n+\t * for PBLE backing page rounding\n+\t */\n+\tif (hmc_info->hmc_fn_id >= dev->hw_attrs.first_hw_vf_fpm_id)\n+\t\tmax_pe_sds--;\n+\thmc_fpm_misc->max_sds = max_pe_sds;\n+\thmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;\n+\tget_64bit_val(buf, 8, &temp);\n+\tobj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)RS_64(temp, IRDMA_QUERY_FPM_MAX_QPS);\n+\tsize = (u32)RS_64_1(temp, 32);\n+\tobj_info[IRDMA_HMC_IW_QP].size = LS_64_1(1, size);\n+\n+\tget_64bit_val(buf, 16, &temp);\n+\tobj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, IRDMA_QUERY_FPM_MAX_CQS);\n+\tsize = (u32)RS_64_1(temp, 32);\n+\tobj_info[IRDMA_HMC_IW_CQ].size = LS_64_1(1, size);\n+\n+\tirdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE);\n+\tirdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP);\n+\n+\tobj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192;\n+\tobj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1;\n+\n+\tirdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR);\n+\tirdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF);\n+\n+\tget_64bit_val(buf, 64, &temp);\n+\tobj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;\n+\tobj_info[IRDMA_HMC_IW_XFFL].size = 4;\n+\thmc_fpm_misc->xf_block_size = RS_64(temp, IRDMA_QUERY_FPM_XFBLOCKSIZE);\n+\tif (!hmc_fpm_misc->xf_block_size)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tirdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1);\n+\tget_64bit_val(buf, 80, &temp);\n+\tobj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;\n+\tobj_info[IRDMA_HMC_IW_Q1FL].size = 4;\n+\n+\thmc_fpm_misc->q1_block_size = RS_64(temp, IRDMA_QUERY_FPM_Q1BLOCKSIZE);\n+\tif (!hmc_fpm_misc->q1_block_size)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tirdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER);\n+\n+\tget_64bit_val(buf, 112, &temp);\n+\tobj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp;\n+\tobj_info[IRDMA_HMC_IW_PBLE].size = 8;\n+\n+\tget_64bit_val(buf, 120, &temp);\n+\thmc_fpm_misc->max_ceqs = RS_64(temp, IRDMA_QUERY_FPM_MAX_CEQS);\n+\thmc_fpm_misc->ht_multiplier = RS_64(temp, IRDMA_QUERY_FPM_HTMULTIPLIER);\n+\thmc_fpm_misc->timer_bucket = RS_64(temp, IRDMA_QUERY_FPM_TIMERBUCKET);\n+\tif (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)\n+\t\treturn 0;\n+\tirdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);\n+\tirdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV);\n+\tirdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF);\n+\n+\tget_64bit_val(buf, 136, &temp);\n+\tobj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;\n+\tobj_info[IRDMA_HMC_IW_RRFFL].size = 4;\n+\thmc_fpm_misc->rrf_block_size = RS_64(temp, IRDMA_QUERY_FPM_RRFBLOCKSIZE);\n+\tif (!hmc_fpm_misc->rrf_block_size &&\n+\t    obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tirdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);\n+\tirdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);\n+\tirdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);\n+\n+\tget_64bit_val(buf, 168, &temp);\n+\tobj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;\n+\tobj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;\n+\thmc_fpm_misc->ooiscf_block_size = RS_64(temp, IRDMA_QUERY_FPM_OOISCFBLOCKSIZE);\n+\tif (!hmc_fpm_misc->ooiscf_block_size &&\n+\t    obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_find_reg_cq - find cq ctx index\n+ * @ceq: ceq sc structure\n+ * @cq: cq sc structure\n+ */\n+static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,\n+\t\t\t\tstruct irdma_sc_cq *cq)\n+{\n+\tu32 i;\n+\n+\tfor (i = 0; i < ceq->reg_cq_size; i++) {\n+\t\tif (cq == ceq->reg_cq[i])\n+\t\t\treturn i;\n+\t}\n+\n+\treturn IRDMA_INVALID_CQ_IDX;\n+}\n+\n+/**\n+ * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq\n+ * @ceq: ceq sc structure\n+ * @cq: cq sc structure\n+ */\n+enum irdma_status_code irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t   struct irdma_sc_cq *cq)\n+{\n+\tunsigned long flags;\n+\n+\tspin_lock_irqsave(&ceq->req_cq_lock, flags);\n+\n+\tif (ceq->reg_cq_size == ceq->elem_cnt) {\n+\t\tspin_unlock_irqrestore(&ceq->req_cq_lock, flags);\n+\t\treturn IRDMA_ERR_REG_CQ_FULL;\n+\t}\n+\n+\tceq->reg_cq[ceq->reg_cq_size++] = cq;\n+\n+\tspin_unlock_irqrestore(&ceq->req_cq_lock, flags);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq\n+ * @ceq: ceq sc structure\n+ * @cq: cq sc structure\n+ */\n+void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)\n+{\n+\tunsigned long flags;\n+\tu32 cq_ctx_idx;\n+\n+\tspin_lock_irqsave(&ceq->req_cq_lock, flags);\n+\tcq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);\n+\tif (cq_ctx_idx == IRDMA_INVALID_CQ_IDX)\n+\t\tgoto exit;\n+\n+\tceq->reg_cq_size--;\n+\tif (cq_ctx_idx != ceq->reg_cq_size)\n+\t\tceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];\n+\tceq->reg_cq[ceq->reg_cq_size] = NULL;\n+\n+exit:\n+\tspin_unlock_irqrestore(&ceq->req_cq_lock, flags);\n+}\n+\n+/**\n+ * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair\n+ * @cqp: IWARP control queue pair pointer\n+ * @info: IWARP control queue pair init info pointer\n+ *\n+ * Initializes the object and context buffers for a control Queue Pair.\n+ */\n+static enum irdma_status_code\n+irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, struct irdma_cqp_init_info *info)\n+{\n+\tu8 hw_sq_size;\n+\n+\tif (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 ||\n+\t    info->sq_size < IRDMA_CQP_SW_SQSIZE_4 ||\n+\t    ((info->sq_size & (info->sq_size - 1))))\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\thw_sq_size = irdma_get_encoded_wqe_size(info->sq_size, true);\n+\tcqp->size = sizeof(*cqp);\n+\tcqp->sq_size = info->sq_size;\n+\tcqp->hw_sq_size = hw_sq_size;\n+\tcqp->sq_base = info->sq;\n+\tcqp->host_ctx = info->host_ctx;\n+\tcqp->sq_pa = info->sq_pa;\n+\tcqp->host_ctx_pa = info->host_ctx_pa;\n+\tcqp->dev = info->dev;\n+\tcqp->struct_ver = info->struct_ver;\n+\tcqp->hw_maj_ver = info->hw_maj_ver;\n+\tcqp->hw_min_ver = info->hw_min_ver;\n+\tcqp->scratch_array = info->scratch_array;\n+\tcqp->polarity = 0;\n+\tcqp->en_datacenter_tcp = info->en_datacenter_tcp;\n+\tcqp->ena_vf_count = info->ena_vf_count;\n+\tcqp->hmc_profile = info->hmc_profile;\n+\tcqp->ceqs_per_vf = info->ceqs_per_vf;\n+\tcqp->disable_packed = info->disable_packed;\n+\tcqp->rocev2_rto_policy = info->rocev2_rto_policy;\n+\tcqp->protocol_used = info->protocol_used;\n+\tinfo->dev->cqp = cqp;\n+\n+\tIRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);\n+\tcqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0;\n+\tcqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0;\n+\t/* for the cqp commands backlog. */\n+\tINIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);\n+\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPTAIL], 0);\n+\tif (cqp->dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2)\n+\t\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPDB], 0);\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPSTATUS], 0);\n+\n+\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\"WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\\n\",\n+\t\tcqp->sq_size, cqp->hw_sq_size, cqp->sq_base,\n+\t\t(u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cqp_create - create cqp during bringup\n+ * @cqp: struct for cqp hw\n+ * @maj_err: If error, major err number\n+ * @min_err: If error, minor err number\n+ */\n+static enum irdma_status_code irdma_sc_cqp_create(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t  u16 *maj_err, u16 *min_err)\n+{\n+\tu64 temp;\n+\tu32 cnt = 0, p1, p2, val = 0, err_code;\n+\tenum irdma_status_code ret_code;\n+\n+\tcqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size,\n+\t\t\t\tIRDMA_SD_BUF_ALIGNMENT);\n+\tcqp->sdbuf.va = dma_alloc_coherent(hw_to_dev(cqp->dev->hw),\n+\t\t\t\t\t   cqp->sdbuf.size, &cqp->sdbuf.pa,\n+\t\t\t\t\t   GFP_KERNEL);\n+\tif (!cqp->sdbuf.va)\n+\t\treturn IRDMA_ERR_NO_MEMORY;\n+\n+\ttemp = LS_64(cqp->hw_sq_size, IRDMA_CQPHC_SQSIZE) |\n+\t       LS_64(cqp->struct_ver, IRDMA_CQPHC_SVER) |\n+\t       LS_64(cqp->rocev2_rto_policy, IRDMA_CQPHC_ROCEV2_RTO_POLICY) |\n+\t       LS_64(cqp->protocol_used, IRDMA_CQPHC_PROTOCOL_USED) |\n+\t       LS_64(cqp->disable_packed, IRDMA_CQPHC_DISABLE_PFPDUS) |\n+\t       LS_64(cqp->ceqs_per_vf, IRDMA_CQPHC_CEQPERVF);\n+\tset_64bit_val(cqp->host_ctx, 0, temp);\n+\tset_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);\n+\n+\ttemp = LS_64(cqp->ena_vf_count, IRDMA_CQPHC_ENABLED_VFS) |\n+\t       LS_64(cqp->hmc_profile, IRDMA_CQPHC_HMC_PROFILE);\n+\tset_64bit_val(cqp->host_ctx, 16, temp);\n+\tset_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);\n+\tset_64bit_val(cqp->host_ctx, 32, 0);\n+\ttemp = LS_64(cqp->hw_maj_ver, IRDMA_CQPHC_HW_MAJVER) |\n+\t       LS_64(cqp->hw_min_ver, IRDMA_CQPHC_HW_MINVER);\n+\tset_64bit_val(cqp->host_ctx, 32, temp);\n+\tset_64bit_val(cqp->host_ctx, 40, 0);\n+\tset_64bit_val(cqp->host_ctx, 48, 0);\n+\tset_64bit_val(cqp->host_ctx, 56, 0);\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CQP_HOST_CTX WQE\",\n+\t\t\tcqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8);\n+\tp1 = RS_32_1(cqp->host_ctx_pa, 32);\n+\tp2 = (u32)cqp->host_ctx_pa;\n+\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPHIGH], p1);\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPLOW], p2);\n+\n+\tdo {\n+\t\tif (cnt++ > cqp->dev->hw_attrs.max_done_count) {\n+\t\t\tret_code = IRDMA_ERR_TIMEOUT;\n+\t\t\tgoto err;\n+\t\t}\n+\t\tudelay(cqp->dev->hw_attrs.max_sleep_count);\n+\t\tval = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);\n+\t} while (!val);\n+\n+\tif (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {\n+\t\tret_code = IRDMA_ERR_DEVICE_NOT_SUPPORTED;\n+\t\tgoto err;\n+\t}\n+\n+\tcqp->process_cqp_sds = irdma_update_sds_noccq;\n+\treturn 0;\n+\n+err:\n+\tdma_free_coherent(hw_to_dev(cqp->dev->hw), cqp->sdbuf.size,\n+\t\t\t  cqp->sdbuf.va, cqp->sdbuf.pa);\n+\tcqp->sdbuf.va = NULL;\n+\terr_code = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPERRCODES]);\n+\t*min_err = RS_32(err_code, IRDMA_CQPERRCODES_CQP_MINOR_CODE);\n+\t*maj_err = RS_32(err_code, IRDMA_CQPERRCODES_CQP_MAJOR_CODE);\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_cqp_post_sq - post of cqp's sq\n+ * @cqp: struct for cqp hw\n+ */\n+void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp)\n+{\n+\twritel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db);\n+\n+\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\"WQE: CQP SQ head 0x%x tail 0x%x size 0x%x\\n\",\n+\t\tcqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);\n+}\n+\n+/**\n+ * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq\n+ * and pass back index\n+ * @cqp: CQP HW structure\n+ * @scratch: private data for CQP WQE\n+ * @wqe_idx: WQE index of CQP SQ\n+ */\n+static __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t  u64 scratch, u32 *wqe_idx)\n+{\n+\t__le64 *wqe = NULL;\n+\tenum irdma_status_code ret_code;\n+\n+\tif (IRDMA_RING_FULL_ERR(cqp->sq_ring)) {\n+\t\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\t\"WQE: CQP SQ is full, head 0x%x tail 0x%x size 0x%x\\n\",\n+\t\t\tcqp->sq_ring.head, cqp->sq_ring.tail,\n+\t\t\tcqp->sq_ring.size);\n+\t\treturn NULL;\n+\t}\n+\tIRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);\n+\tif (ret_code)\n+\t\treturn NULL;\n+\n+\tcqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++;\n+\tif (!*wqe_idx)\n+\t\tcqp->polarity = !cqp->polarity;\n+\twqe = cqp->sq_base[*wqe_idx].elem;\n+\tcqp->scratch_array[*wqe_idx] = scratch;\n+\tIRDMA_CQP_INIT_WQE(wqe);\n+\n+\treturn wqe;\n+}\n+\n+/**\n+ * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq\n+ * @cqp: struct for cqp hw\n+ * @scratch: private data for CQP WQE\n+ */\n+__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)\n+{\n+\tu32 wqe_idx;\n+\n+\treturn irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);\n+}\n+\n+/**\n+ * irdma_sc_cqp_destroy - destroy cqp during close\n+ * @cqp: struct for cqp hw\n+ */\n+static enum irdma_status_code irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)\n+{\n+\tu32 cnt = 0, val = 1;\n+\tenum irdma_status_code ret_code = 0;\n+\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPHIGH], 0);\n+\twr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPLOW], 0);\n+\tdo {\n+\t\tif (cnt++ > cqp->dev->hw_attrs.max_done_count) {\n+\t\t\tret_code = IRDMA_ERR_TIMEOUT;\n+\t\t\tbreak;\n+\t\t}\n+\t\tudelay(cqp->dev->hw_attrs.max_sleep_count);\n+\t\tval = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);\n+\t} while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));\n+\n+\tdma_free_coherent(hw_to_dev(cqp->dev->hw), cqp->sdbuf.size,\n+\t\t\t  cqp->sdbuf.va, cqp->sdbuf.pa);\n+\tcqp->sdbuf.va = NULL;\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_ccq_arm - enable intr for control cq\n+ * @ccq: ccq sc struct\n+ */\n+static void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)\n+{\n+\tu64 temp_val;\n+\tu16 sw_cq_sel;\n+\tu8 arm_next_se;\n+\tu8 arm_seq_num;\n+\n+\tget_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);\n+\tsw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);\n+\tarm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);\n+\tarm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);\n+\tarm_seq_num++;\n+\ttemp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |\n+\t\t   LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |\n+\t\t   LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |\n+\t\t   LS_64(1, IRDMA_CQ_DBSA_ARM_NEXT);\n+\tset_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);\n+\n+\tdma_wmb(); /* make sure shadow area is updated before arming */\n+\n+\twritel(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db);\n+}\n+\n+/**\n+ * irdma_sc_ccq_get_cqe_info - get ccq's cq entry\n+ * @ccq: ccq sc struct\n+ * @info: completion q entry to return\n+ */\n+static enum irdma_status_code\n+irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,\n+\t\t\t  struct irdma_ccq_cqe_info *info)\n+{\n+\tu64 qp_ctx, temp, temp1;\n+\t__le64 *cqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu32 wqe_idx;\n+\tu32 error;\n+\tu8 polarity;\n+\tenum irdma_status_code ret_code = 0;\n+\n+\tif (ccq->cq_uk.avoid_mem_cflct)\n+\t\tcqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);\n+\telse\n+\t\tcqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);\n+\n+\tget_64bit_val(cqe, 24, &temp);\n+\tpolarity = (u8)RS_64(temp, IRDMA_CQ_VALID);\n+\tif (polarity != ccq->cq_uk.polarity)\n+\t\treturn IRDMA_ERR_Q_EMPTY;\n+\n+\tget_64bit_val(cqe, 8, &qp_ctx);\n+\tcqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;\n+\tinfo->error = (bool)RS_64(temp, IRDMA_CQ_ERROR);\n+\tinfo->min_err_code = (u16)RS_64(temp, IRDMA_CQ_MINERR);\n+\tif (info->error) {\n+\t\tinfo->maj_err_code = (u16)RS_64(temp, IRDMA_CQ_MAJERR);\n+\t\tinfo->min_err_code = (u16)RS_64(temp, IRDMA_CQ_MINERR);\n+\t\terror = rd32(cqp->dev->hw,\n+\t\t\t     cqp->dev->hw_regs[IRDMA_CQPERRCODES]);\n+\t\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\t\"CQP: CQPERRCODES error_code[x%08X]\\n\", error);\n+\t}\n+\twqe_idx = (u32)RS_64(temp, IRDMA_CQ_WQEIDX);\n+\tinfo->scratch = cqp->scratch_array[wqe_idx];\n+\n+\tget_64bit_val(cqe, 16, &temp1);\n+\tinfo->op_ret_val = (u32)RS_64(temp1, IRDMA_CCQ_OPRETVAL);\n+\tget_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);\n+\tinfo->op_code = (u8)RS_64(temp1, IRDMA_CQPSQ_OPCODE);\n+\tinfo->cqp = cqp;\n+\n+\t/*  move the head for cq */\n+\tIRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);\n+\tif (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring))\n+\t\tccq->cq_uk.polarity ^= 1;\n+\n+\t/* update cq tail in cq shadow memory also */\n+\tIRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);\n+\tset_64bit_val(ccq->cq_uk.shadow_area, 0,\n+\t\t      IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring));\n+\n+\tdma_wmb(); /* make sure shadow area is updated before moving tail */\n+\n+\tIRDMA_RING_MOVE_TAIL(cqp->sq_ring);\n+\tccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ\n+ * @cqp: struct for cqp hw\n+ * @op_code: cqp opcode for completion\n+ * @compl_info: completion q entry to return\n+ */\n+static enum irdma_status_code\n+irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code,\n+\t\t\t      struct irdma_ccq_cqe_info *compl_info)\n+{\n+\tstruct irdma_ccq_cqe_info info = {};\n+\tstruct irdma_sc_cq *ccq;\n+\tenum irdma_status_code ret_code = 0;\n+\tu32 cnt = 0;\n+\n+\tccq = cqp->dev->ccq;\n+\twhile (1) {\n+\t\tif (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)\n+\t\t\treturn IRDMA_ERR_TIMEOUT;\n+\n+\t\tif (irdma_sc_ccq_get_cqe_info(ccq, &info)) {\n+\t\t\tudelay(cqp->dev->hw_attrs.max_sleep_count);\n+\t\t\tcontinue;\n+\t\t}\n+\t\tif (info.error) {\n+\t\t\tret_code = IRDMA_ERR_CQP_COMPL_ERROR;\n+\t\t\tbreak;\n+\t\t}\n+\t\t/* make sure op code matches*/\n+\t\tif (op_code == info.op_code)\n+\t\t\tbreak;\n+\t\tdev_dbg(rfdev_to_dev(cqp->dev),\n+\t\t\t\"WQE: opcode mismatch for my op code 0x%x, returned opcode %x\\n\",\n+\t\t\top_code, info.op_code);\n+\t}\n+\n+\tif (compl_info)\n+\t\tmemcpy(compl_info, &info, sizeof(*compl_info));\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_manage_hmc_pm_func_table - manage of function table\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @vf_index: vf index for cqp\n+ * @free_pm_fcn: function number\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, u64 scratch,\n+\t\t\t\t  u8 vf_index, bool free_pm_fcn, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\thdr = LS_64(vf_index, IRDMA_CQPSQ_MHMC_VFIDX) |\n+\t      LS_64(IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(free_pm_fcn, IRDMA_CQPSQ_MHMC_FREEPMFN) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE,\n+\t\t\t\"MANAGE_HMC_PM_FUNC_TABLE WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table\n+ * @cqp: struct for cqp hw\n+ */\n+static enum irdma_status_code\n+irdma_sc_manage_hmc_pm_func_table_done(struct irdma_sc_cqp *cqp)\n+{\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp,\n+\t\t\t\t\t     IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit\n+ * @cqp: struct for cqp hw\n+ */\n+static enum irdma_status_code\n+irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp)\n+{\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @hmc_fn_id: hmc function id\n+ * @commit_fpm_mem: Memory for fpm values\n+ * @post_sq: flag for cqp db to ring\n+ * @wait_type: poll ccq or cqp registers for cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, u8 hmc_fn_id,\n+\t\t\tstruct irdma_dma_mem *commit_fpm_mem, bool post_sq,\n+\t\t\tu8 wait_type)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tu32 tail, val, error;\n+\tenum irdma_status_code ret_code = 0;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, hmc_fn_id);\n+\tset_64bit_val(wqe, 32, commit_fpm_mem->pa);\n+\n+\thdr = LS_64(IRDMA_COMMIT_FPM_BUF_SIZE, IRDMA_CQPSQ_BUFSIZE) |\n+\t      LS_64(IRDMA_CQP_OP_COMMIT_FPM_VAL, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"COMMIT_FPM_VAL WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\n+\tirdma_get_cqp_reg_info(cqp, &val, &tail, &error);\n+\n+\tif (post_sq) {\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\t\tif (wait_type == IRDMA_CQP_WAIT_POLL_REGS)\n+\t\t\tret_code = irdma_cqp_poll_registers(cqp, tail,\n+\t\t\t\t\t\t\t    cqp->dev->hw_attrs.max_done_count);\n+\t\telse if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)\n+\t\t\tret_code = irdma_sc_commit_fpm_val_done(cqp);\n+\t}\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm\n+ * @cqp: struct for cqp hw\n+ */\n+static enum irdma_status_code\n+irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp)\n+{\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_query_fpm_val - cqp wqe query fpm values\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @hmc_fn_id: hmc function id\n+ * @query_fpm_mem: memory for return fpm values\n+ * @post_sq: flag for cqp db to ring\n+ * @wait_type: poll ccq or cqp registers for cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, u8 hmc_fn_id,\n+\t\t       struct irdma_dma_mem *query_fpm_mem, bool post_sq,\n+\t\t       u8 wait_type)\n+{\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tu32 tail, val, error;\n+\tenum irdma_status_code ret_code = 0;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, hmc_fn_id);\n+\tset_64bit_val(wqe, 32, query_fpm_mem->pa);\n+\n+\thdr = LS_64(IRDMA_CQP_OP_QUERY_FPM_VAL, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QUERY_FPM WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_get_cqp_reg_info(cqp, &val, &tail, &error);\n+\tif (post_sq) {\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\t\tif (wait_type == IRDMA_CQP_WAIT_POLL_REGS)\n+\t\t\tret_code = irdma_cqp_poll_registers(cqp, tail,\n+\t\t\t\t\t\t\t    cqp->dev->hw_attrs.max_done_count);\n+\t\telse if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)\n+\t\t\tret_code = irdma_sc_query_fpm_val_done(cqp);\n+\t}\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_ceq_init - initialize ceq\n+ * @ceq: ceq sc structure\n+ * @info: ceq initialization info\n+ */\n+static enum irdma_status_code\n+irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, struct irdma_ceq_init_info *info)\n+{\n+\tu32 pble_obj_cnt;\n+\n+\tif (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size ||\n+\t    info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tif (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))\n+\t\treturn IRDMA_ERR_INVALID_CEQ_ID;\n+\tpble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\n+\tif (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\tceq->size = sizeof(*ceq);\n+\tceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;\n+\tceq->ceq_id = info->ceq_id;\n+\tceq->dev = info->dev;\n+\tceq->elem_cnt = info->elem_cnt;\n+\tceq->ceq_elem_pa = info->ceqe_pa;\n+\tceq->virtual_map = info->virtual_map;\n+\tceq->itr_no_expire = info->itr_no_expire;\n+\tceq->reg_cq = info->reg_cq;\n+\tceq->reg_cq_size = 0;\n+\tspin_lock_init(&ceq->req_cq_lock);\n+\tceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);\n+\tceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);\n+\tceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);\n+\tceq->tph_en = info->tph_en;\n+\tceq->tph_val = info->tph_val;\n+\tceq->vsi = info->vsi;\n+\tceq->polarity = 1;\n+\tIRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);\n+\tceq->dev->ceq[info->ceq_id] = ceq;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_ceq_create - create ceq wqe\n+ * @ceq: ceq sc structure\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+\n+static enum irdma_status_code irdma_sc_ceq_create(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t\t  u64 scratch, bool post_sq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\tcqp = ceq->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\tset_64bit_val(wqe, 16, ceq->elem_cnt);\n+\tset_64bit_val(wqe, 32,\n+\t\t      (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));\n+\tset_64bit_val(wqe, 48,\n+\t\t      (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));\n+\tset_64bit_val(wqe, 56,\n+\t\t      LS_64(ceq->tph_val, IRDMA_CQPSQ_TPHVAL) |\n+\t\t      LS_64(ceq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));\n+\thdr = ceq->ceq_id | LS_64(IRDMA_CQP_OP_CREATE_CEQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(ceq->pbl_chunk_size, IRDMA_CQPSQ_CEQ_LPBLSIZE) |\n+\t      LS_64(ceq->virtual_map, IRDMA_CQPSQ_CEQ_VMAP) |\n+\t      LS_64(ceq->itr_no_expire, IRDMA_CQPSQ_CEQ_ITRNOEXPIRE) |\n+\t      LS_64(ceq->tph_en, IRDMA_CQPSQ_TPHEN) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CEQ_CREATE WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_cceq_create_done - poll for control ceq wqe to complete\n+ * @ceq: ceq sc structure\n+ */\n+static enum irdma_status_code\n+irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tcqp = ceq->dev->cqp;\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete\n+ * @ceq: ceq sc structure\n+ */\n+static enum irdma_status_code\n+irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tif (ceq->reg_cq)\n+\t\tirdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);\n+\n+\tcqp = ceq->dev->cqp;\n+\tcqp->process_cqp_sds = irdma_update_sds_noccq;\n+\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_cceq_create - create cceq\n+ * @ceq: ceq sc structure\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code irdma_sc_cceq_create(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t\t   u64 scratch)\n+{\n+\tenum irdma_status_code ret_code;\n+\n+\tceq->dev->ccq->vsi = ceq->vsi;\n+\tif (ceq->reg_cq) {\n+\t\tret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);\n+\t\tif (ret_code)\n+\t\t\treturn ret_code;\n+\t}\n+\n+\tret_code = irdma_sc_ceq_create(ceq, scratch, true);\n+\tif (!ret_code)\n+\t\treturn irdma_sc_cceq_create_done(ceq);\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_ceq_destroy - destroy ceq\n+ * @ceq: ceq sc structure\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t\t   u64 scratch, bool post_sq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\n+\tcqp = ceq->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16, ceq->elem_cnt);\n+\tset_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);\n+\thdr = ceq->ceq_id |\n+\t      LS_64(IRDMA_CQP_OP_DESTROY_CEQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(ceq->pbl_chunk_size, IRDMA_CQPSQ_CEQ_LPBLSIZE) |\n+\t      LS_64(ceq->virtual_map, IRDMA_CQPSQ_CEQ_VMAP) |\n+\t      LS_64(ceq->tph_en, IRDMA_CQPSQ_TPHEN) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CEQ_DESTROY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_process_ceq - process ceq\n+ * @dev: sc device struct\n+ * @ceq: ceq sc structure\n+ */\n+static void *irdma_sc_process_ceq(struct irdma_sc_dev *dev,\n+\t\t\t\t  struct irdma_sc_ceq *ceq)\n+{\n+\tu64 temp;\n+\t__le64 *ceqe;\n+\tstruct irdma_sc_cq *cq;\n+\tu8 polarity;\n+\tu32 cq_idx = 0;\n+\tunsigned long flags;\n+\n+\tdo {\n+\t\tceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);\n+\t\tget_64bit_val(ceqe, 0, &temp);\n+\t\tpolarity = (u8)RS_64(temp, IRDMA_CEQE_VALID);\n+\t\tif (polarity != ceq->polarity)\n+\t\t\treturn NULL;\n+\n+\t\tcq = (struct irdma_sc_cq *)(unsigned long)LS_64_1(temp, 1);\n+\t\tif (!cq)\n+\t\t\treturn NULL;\n+\n+\t\tif (ceq->reg_cq) {\n+\t\t\tspin_lock_irqsave(&ceq->req_cq_lock, flags);\n+\t\t\tcq_idx = irdma_sc_find_reg_cq(ceq, cq);\n+\t\t\tspin_unlock_irqrestore(&ceq->req_cq_lock, flags);\n+\t\t}\n+\n+\t\tIRDMA_RING_MOVE_TAIL(ceq->ceq_ring);\n+\t\tif (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))\n+\t\t\tceq->polarity ^= 1;\n+\t} while (cq_idx == IRDMA_INVALID_CQ_IDX);\n+\n+\tirdma_sc_cq_ack(cq);\n+\n+\treturn cq;\n+}\n+\n+/**\n+ * irdma_sc_aeq_init - initialize aeq\n+ * @aeq: aeq structure ptr\n+ * @info: aeq initialization info\n+ */\n+static enum irdma_status_code\n+irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, struct irdma_aeq_init_info *info)\n+{\n+\tu32 pble_obj_cnt;\n+\n+\tif (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size ||\n+\t    info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tpble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\n+\tif (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\taeq->size = sizeof(*aeq);\n+\taeq->polarity = 1;\n+\taeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base;\n+\taeq->dev = info->dev;\n+\taeq->elem_cnt = info->elem_cnt;\n+\taeq->aeq_elem_pa = info->aeq_elem_pa;\n+\tIRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);\n+\taeq->virtual_map = info->virtual_map;\n+\taeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);\n+\taeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);\n+\taeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);\n+\tinfo->dev->aeq = aeq;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_aeq_create - create aeq\n+ * @aeq: aeq structure ptr\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_aeq_create(struct irdma_sc_aeq *aeq,\n+\t\t\t\t\t\t  u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 hdr;\n+\n+\tcqp = aeq->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\tset_64bit_val(wqe, 16, aeq->elem_cnt);\n+\tset_64bit_val(wqe, 32,\n+\t\t      (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));\n+\tset_64bit_val(wqe, 48,\n+\t\t      (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));\n+\n+\thdr = LS_64(IRDMA_CQP_OP_CREATE_AEQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(aeq->pbl_chunk_size, IRDMA_CQPSQ_AEQ_LPBLSIZE) |\n+\t      LS_64(aeq->virtual_map, IRDMA_CQPSQ_AEQ_VMAP) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"AEQ_CREATE WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_aeq_destroy - destroy aeq during close\n+ * @aeq: aeq structure ptr\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq,\n+\t\t\t\t\t\t   u64 scratch, bool post_sq)\n+{\n+\t__le64 *wqe;\n+\tstruct irdma_sc_cqp *cqp;\n+\tstruct irdma_sc_dev *dev;\n+\tu64 hdr;\n+\n+\tdev = aeq->dev;\n+\tif (dev->is_pf)\n+\t\twr32(dev->hw, dev->hw_regs[IRDMA_PFINT_AEQCTL], 0);\n+\n+\tcqp = dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\tset_64bit_val(wqe, 16, aeq->elem_cnt);\n+\tset_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);\n+\thdr = LS_64(IRDMA_CQP_OP_DESTROY_AEQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(aeq->pbl_chunk_size, IRDMA_CQPSQ_AEQ_LPBLSIZE) |\n+\t      LS_64(aeq->virtual_map, IRDMA_CQPSQ_AEQ_VMAP) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_WQE, \"AEQ_DESTROY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tif (post_sq)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_get_next_aeqe - get next aeq entry\n+ * @aeq: aeq structure ptr\n+ * @info: aeqe info to be returned\n+ */\n+static enum irdma_status_code\n+irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, struct irdma_aeqe_info *info)\n+{\n+\tu64 temp, compl_ctx;\n+\t__le64 *aeqe;\n+\tu16 wqe_idx;\n+\tu8 ae_src;\n+\tu8 polarity;\n+\n+\taeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);\n+\tget_64bit_val(aeqe, 0, &compl_ctx);\n+\tget_64bit_val(aeqe, 8, &temp);\n+\tpolarity = (u8)RS_64(temp, IRDMA_AEQE_VALID);\n+\n+\tif (aeq->polarity != polarity)\n+\t\treturn IRDMA_ERR_Q_EMPTY;\n+\n+\tirdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, \"AEQ_ENTRY WQE\", aeqe, 16);\n+\n+\tae_src = (u8)RS_64(temp, IRDMA_AEQE_AESRC);\n+\twqe_idx = (u16)RS_64(temp, IRDMA_AEQE_WQDESCIDX);\n+\tinfo->qp_cq_id = (u32)RS_64(temp, IRDMA_AEQE_QPCQID_LOW) |\n+\t\t\t ((u32)RS_64(temp, IRDMA_AEQE_QPCQID_HI) << 18);\n+\tinfo->ae_id = (u16)RS_64(temp, IRDMA_AEQE_AECODE);\n+\tinfo->tcp_state = (u8)RS_64(temp, IRDMA_AEQE_TCPSTATE);\n+\tinfo->iwarp_state = (u8)RS_64(temp, IRDMA_AEQE_IWSTATE);\n+\tinfo->q2_data_written = (u8)RS_64(temp, IRDMA_AEQE_Q2DATA);\n+\tinfo->aeqe_overflow = (bool)RS_64(temp, IRDMA_AEQE_OVERFLOW);\n+\n+\tswitch (info->ae_id) {\n+\tcase IRDMA_AE_PRIV_OPERATION_DENIED:\n+\tcase IRDMA_AE_AMP_INVALIDATE_TYPE1_MW:\n+\tcase IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW:\n+\tcase IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG:\n+\tcase IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH:\n+\tcase IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:\n+\tcase IRDMA_AE_UDA_XMIT_BAD_PD:\n+\tcase IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:\n+\tcase IRDMA_AE_BAD_CLOSE:\n+\tcase IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:\n+\tcase IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO:\n+\tcase IRDMA_AE_STAG_ZERO_INVALID:\n+\tcase IRDMA_AE_IB_RREQ_AND_Q1_FULL:\n+\tcase IRDMA_AE_IB_INVALID_REQUEST:\n+\tcase IRDMA_AE_WQE_UNEXPECTED_OPCODE:\n+\tcase IRDMA_AE_IB_REMOTE_ACCESS_ERROR:\n+\tcase IRDMA_AE_IB_REMOTE_OP_ERROR:\n+\tcase IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:\n+\tcase IRDMA_AE_DDP_UBE_INVALID_MO:\n+\tcase IRDMA_AE_DDP_UBE_INVALID_QN:\n+\tcase IRDMA_AE_DDP_NO_L_BIT:\n+\tcase IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:\n+\tcase IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:\n+\tcase IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:\n+\tcase IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:\n+\tcase IRDMA_AE_ROCE_RSP_LENGTH_ERROR:\n+\tcase IRDMA_AE_INVALID_ARP_ENTRY:\n+\tcase IRDMA_AE_INVALID_TCP_OPTION_RCVD:\n+\tcase IRDMA_AE_STALE_ARP_ENTRY:\n+\tcase IRDMA_AE_INVALID_AH_ENTRY:\n+\tcase IRDMA_AE_LLP_CLOSE_COMPLETE:\n+\tcase IRDMA_AE_LLP_CONNECTION_RESET:\n+\tcase IRDMA_AE_LLP_FIN_RECEIVED:\n+\tcase IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:\n+\tcase IRDMA_AE_LLP_SEGMENT_TOO_SMALL:\n+\tcase IRDMA_AE_LLP_SYN_RECEIVED:\n+\tcase IRDMA_AE_LLP_TERMINATE_RECEIVED:\n+\tcase IRDMA_AE_LLP_TOO_MANY_RETRIES:\n+\tcase IRDMA_AE_LLP_DOUBT_REACHABILITY:\n+\tcase IRDMA_AE_LLP_CONNECTION_ESTABLISHED:\n+\tcase IRDMA_AE_RESET_SENT:\n+\tcase IRDMA_AE_TERMINATE_SENT:\n+\tcase IRDMA_AE_RESET_NOT_SENT:\n+\tcase IRDMA_AE_LCE_QP_CATASTROPHIC:\n+\tcase IRDMA_AE_QP_SUSPEND_COMPLETE:\n+\tcase IRDMA_AE_UDA_L4LEN_INVALID:\n+\t\tinfo->qp = true;\n+\t\tinfo->compl_ctx = compl_ctx;\n+\t\tae_src = IRDMA_AE_SOURCE_RSVD;\n+\t\tbreak;\n+\tcase IRDMA_AE_LCE_CQ_CATASTROPHIC:\n+\t\tinfo->cq = true;\n+\t\tinfo->compl_ctx = LS_64_1(compl_ctx, 1);\n+\t\tae_src = IRDMA_AE_SOURCE_RSVD;\n+\t\tbreak;\n+\tcase IRDMA_AE_ROCE_EMPTY_MCG:\n+\tcase IRDMA_AE_ROCE_BAD_MC_IP_ADDR:\n+\tcase IRDMA_AE_ROCE_BAD_MC_QPID:\n+\tcase IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH:\n+\t\tae_src = IRDMA_AE_SOURCE_RSVD;\n+\t\tbreak;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\tswitch (ae_src) {\n+\tcase IRDMA_AE_SOURCE_RQ:\n+\tcase IRDMA_AE_SOURCE_RQ_0011:\n+\t\tinfo->qp = true;\n+\t\tinfo->wqe_idx = wqe_idx;\n+\t\tinfo->compl_ctx = compl_ctx;\n+\t\tbreak;\n+\tcase IRDMA_AE_SOURCE_CQ:\n+\tcase IRDMA_AE_SOURCE_CQ_0110:\n+\tcase IRDMA_AE_SOURCE_CQ_1010:\n+\tcase IRDMA_AE_SOURCE_CQ_1110:\n+\t\tinfo->cq = true;\n+\t\tinfo->compl_ctx = LS_64_1(compl_ctx, 1);\n+\t\tbreak;\n+\tcase IRDMA_AE_SOURCE_SQ:\n+\tcase IRDMA_AE_SOURCE_SQ_0111:\n+\t\tinfo->qp = true;\n+\t\tinfo->sq = true;\n+\t\tinfo->wqe_idx = wqe_idx;\n+\t\tinfo->compl_ctx = compl_ctx;\n+\t\tbreak;\n+\tcase IRDMA_AE_SOURCE_IN_RR_WR:\n+\tcase IRDMA_AE_SOURCE_IN_RR_WR_1011:\n+\t\tinfo->qp = true;\n+\t\tinfo->compl_ctx = compl_ctx;\n+\t\tinfo->in_rdrsp_wr = true;\n+\t\tbreak;\n+\tcase IRDMA_AE_SOURCE_OUT_RR:\n+\tcase IRDMA_AE_SOURCE_OUT_RR_1111:\n+\t\tinfo->qp = true;\n+\t\tinfo->compl_ctx = compl_ctx;\n+\t\tinfo->out_rdrsp = true;\n+\t\tbreak;\n+\tcase IRDMA_AE_SOURCE_RSVD:\n+\t\t/* fallthrough */\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\tIRDMA_RING_MOVE_TAIL(aeq->aeq_ring);\n+\tif (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring))\n+\t\taeq->polarity ^= 1;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_repost_aeq_entries - repost completed aeq entries\n+ * @dev: sc device struct\n+ * @count: allocate count\n+ */\n+static enum irdma_status_code\n+irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)\n+{\n+\twr32(dev->hw, dev->hw_regs[IRDMA_AEQALLOC], count);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_aeq_create_done - create aeq\n+ * @aeq: aeq structure ptr\n+ */\n+static enum irdma_status_code irdma_sc_aeq_create_done(struct irdma_sc_aeq *aeq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tcqp = aeq->dev->cqp;\n+\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_AEQ,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_aeq_destroy_done - destroy of aeq during close\n+ * @aeq: aeq structure ptr\n+ */\n+static enum irdma_status_code\n+irdma_sc_aeq_destroy_done(struct irdma_sc_aeq *aeq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tcqp = aeq->dev->cqp;\n+\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_AEQ,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_ccq_init - initialize control cq\n+ * @cq: sc's cq ctruct\n+ * @info: info for control cq initialization\n+ */\n+static enum irdma_status_code\n+irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info)\n+{\n+\tu32 pble_obj_cnt;\n+\n+\tif (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||\n+\t    info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)\n+\t\treturn IRDMA_ERR_INVALID_SIZE;\n+\n+\tif (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))\n+\t\treturn IRDMA_ERR_INVALID_CEQ_ID;\n+\n+\tpble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;\n+\n+\tif (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)\n+\t\treturn IRDMA_ERR_INVALID_PBLE_INDEX;\n+\n+\tcq->cq_pa = info->cq_pa;\n+\tcq->cq_uk.cq_base = info->cq_base;\n+\tcq->shadow_area_pa = info->shadow_area_pa;\n+\tcq->cq_uk.shadow_area = info->shadow_area;\n+\tcq->shadow_read_threshold = info->shadow_read_threshold;\n+\tcq->dev = info->dev;\n+\tcq->ceq_id = info->ceq_id;\n+\tcq->cq_uk.cq_size = info->num_elem;\n+\tcq->cq_type = IRDMA_CQ_TYPE_CQP;\n+\tcq->ceqe_mask = info->ceqe_mask;\n+\tIRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);\n+\tcq->cq_uk.cq_id = 0; /* control cq is id 0 always */\n+\tcq->ceq_id_valid = info->ceq_id_valid;\n+\tcq->tph_en = info->tph_en;\n+\tcq->tph_val = info->tph_val;\n+\tcq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;\n+\tcq->pbl_list = info->pbl_list;\n+\tcq->virtual_map = info->virtual_map;\n+\tcq->pbl_chunk_size = info->pbl_chunk_size;\n+\tcq->first_pm_pbl_idx = info->first_pm_pbl_idx;\n+\tcq->cq_uk.polarity = true;\n+\tcq->vsi = info->vsi;\n+\tcq->cq_uk.cq_ack_db = cq->dev->cq_ack_db;\n+\n+\t/* Only applicable to CQs other than CCQ so initialize to zero */\n+\tcq->cq_uk.cqe_alloc_db = NULL;\n+\n+\tinfo->dev->ccq = cq;\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_ccq_create_done - poll cqp for ccq create\n+ * @ccq: ccq sc struct\n+ */\n+static enum irdma_status_code irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\n+\tcqp = ccq->dev->cqp;\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL);\n+}\n+\n+/**\n+ * irdma_sc_ccq_create - create control cq\n+ * @ccq: ccq sc struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @check_overflow: overlow flag for ccq\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_ccq_create(struct irdma_sc_cq *ccq,\n+\t\t\t\t\t\t  u64 scratch,\n+\t\t\t\t\t\t  bool check_overflow,\n+\t\t\t\t\t\t  bool post_sq)\n+{\n+\tenum irdma_status_code ret_code;\n+\n+\tret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq);\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\tif (post_sq) {\n+\t\tret_code = irdma_sc_ccq_create_done(ccq);\n+\t\tif (ret_code)\n+\t\t\treturn ret_code;\n+\t}\n+\tccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_sc_ccq_destroy - destroy ccq during close\n+ * @ccq: ccq sc struct\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @post_sq: flag for cqp db to ring\n+ */\n+static enum irdma_status_code irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq,\n+\t\t\t\t\t\t   u64 scratch, bool post_sq)\n+{\n+\tstruct irdma_sc_cqp *cqp;\n+\t__le64 *wqe;\n+\tu64 hdr;\n+\tenum irdma_status_code ret_code = 0;\n+\tu32 tail, val, error;\n+\n+\tcqp = ccq->dev->cqp;\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 0, ccq->cq_uk.cq_size);\n+\tset_64bit_val(wqe, 8, RS_64_1(ccq, 1));\n+\tset_64bit_val(wqe, 40, ccq->shadow_area_pa);\n+\n+\thdr = ccq->cq_uk.cq_id |\n+\t      LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0),\n+\t\t    IRDMA_CQPSQ_CQ_CEQID) |\n+\t      LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(ccq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |\n+\t      LS_64(ccq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |\n+\t      LS_64(ccq->tph_en, IRDMA_CQPSQ_TPHEN) |\n+\t      LS_64(ccq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"CCQ_DESTROY WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_get_cqp_reg_info(cqp, &val, &tail, &error);\n+\tif (error)\n+\t\treturn IRDMA_ERR_CQP_COMPL_ERROR;\n+\n+\tif (post_sq) {\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\t\tret_code = irdma_cqp_poll_registers(cqp, tail, 1000);\n+\t}\n+\n+\tcqp->process_cqp_sds = irdma_update_sds_noccq;\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info\n+ * @dev : ptr to irdma_dev struct\n+ * @hmc_fn_id: hmc function id\n+ */\n+enum irdma_status_code irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev,\n+\t\t\t\t\t    u8 hmc_fn_id)\n+{\n+\tstruct irdma_hmc_info *hmc_info;\n+\tstruct irdma_dma_mem query_fpm_mem;\n+\tenum irdma_status_code ret_code = 0;\n+\tbool poll_registers = true;\n+\tu8 wait_type;\n+\n+\tif (hmc_fn_id > dev->hw_attrs.max_hw_vf_fpm_id ||\n+\t    (dev->hmc_fn_id != hmc_fn_id &&\n+\t     hmc_fn_id < dev->hw_attrs.first_hw_vf_fpm_id))\n+\t\treturn IRDMA_ERR_INVALID_HMCFN_ID;\n+\n+\tdev_dbg(rfdev_to_dev(dev), \"HMC: hmc_fn_id %u, dev->hmc_fn_id %u\\n\",\n+\t\thmc_fn_id, dev->hmc_fn_id);\n+\tif (hmc_fn_id == dev->hmc_fn_id) {\n+\t\thmc_info = dev->hmc_info;\n+\t\tquery_fpm_mem.pa = dev->fpm_query_buf_pa;\n+\t\tquery_fpm_mem.va = dev->fpm_query_buf;\n+\t} else {\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"HMC: Bad hmc function id: hmc_fn_id %u, dev->hmc_fn_id %u\\n\",\n+\t\t\thmc_fn_id, dev->hmc_fn_id);\n+\n+\t\treturn IRDMA_ERR_INVALID_HMCFN_ID;\n+\t}\n+\thmc_info->hmc_fn_id = hmc_fn_id;\n+\twait_type = poll_registers ? (u8)IRDMA_CQP_WAIT_POLL_REGS :\n+\t\t\t\t     (u8)IRDMA_CQP_WAIT_POLL_CQ;\n+\n+\tret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,\n+\t\t\t\t\t  &query_fpm_mem, true, wait_type);\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\t/* parse the fpm_query_buf and fill hmc obj info */\n+\tret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info,\n+\t\t\t\t\t\t&dev->hmc_fpm_misc);\n+\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_HMC, \"QUERY FPM BUFFER\",\n+\t\t\tquery_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE);\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and\n+ * populates fpm base address in hmc_info\n+ * @dev : ptr to irdma_dev struct\n+ * @hmc_fn_id: hmc function id\n+ */\n+static enum irdma_status_code irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t  u8 hmc_fn_id)\n+{\n+\tstruct irdma_hmc_info *hmc_info;\n+\tstruct irdma_hmc_obj_info *obj_info;\n+\t__le64 *buf;\n+\tstruct irdma_dma_mem commit_fpm_mem;\n+\tenum irdma_status_code ret_code = 0;\n+\tbool poll_registers = true;\n+\tu8 wait_type;\n+\n+\tif (hmc_fn_id > dev->hw_attrs.max_hw_vf_fpm_id ||\n+\t    (dev->hmc_fn_id != hmc_fn_id &&\n+\t     hmc_fn_id < dev->hw_attrs.first_hw_vf_fpm_id))\n+\t\treturn IRDMA_ERR_INVALID_HMCFN_ID;\n+\n+\tif (hmc_fn_id != dev->hmc_fn_id)\n+\t\treturn IRDMA_ERR_INVALID_FPM_FUNC_ID;\n+\n+\thmc_info = dev->hmc_info;\n+\tif (!hmc_info)\n+\t\treturn IRDMA_ERR_BAD_PTR;\n+\n+\tobj_info = hmc_info->hmc_obj;\n+\tbuf = dev->fpm_commit_buf;\n+\n+\tset_64bit_val(buf, 0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt);\n+\tset_64bit_val(buf, 8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt);\n+\tset_64bit_val(buf, 16, (u64)0); /* RSRVD */\n+\tset_64bit_val(buf, 24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt);\n+\tset_64bit_val(buf, 32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt);\n+\tset_64bit_val(buf, 40, (u64)0); /* RSVD */\n+\tset_64bit_val(buf, 48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt);\n+\tset_64bit_val(buf, 56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt);\n+\tset_64bit_val(buf, 64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt);\n+\tset_64bit_val(buf, 72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt);\n+\tset_64bit_val(buf, 80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt);\n+\tset_64bit_val(buf, 88,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt);\n+\tset_64bit_val(buf, 96,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt);\n+\tset_64bit_val(buf, 104,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt);\n+\tset_64bit_val(buf, 112,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt);\n+\tset_64bit_val(buf, 120, (u64)0); /* RSVD */\n+\tset_64bit_val(buf, 128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt);\n+\tset_64bit_val(buf, 136,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt);\n+\tset_64bit_val(buf, 144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt);\n+\tset_64bit_val(buf, 152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt);\n+\tset_64bit_val(buf, 160,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);\n+\tset_64bit_val(buf, 168,\n+\t\t      (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);\n+\n+\tcommit_fpm_mem.pa = dev->fpm_commit_buf_pa;\n+\tcommit_fpm_mem.va = dev->fpm_commit_buf;\n+\n+\twait_type = poll_registers ? (u8)IRDMA_CQP_WAIT_POLL_REGS :\n+\t\t\t\t     (u8)IRDMA_CQP_WAIT_POLL_CQ;\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_HMC, \"COMMIT FPM BUFFER\",\n+\t\t\tcommit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE);\n+\tret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,\n+\t\t\t\t\t   &commit_fpm_mem, true, wait_type);\n+\tif (!ret_code)\n+\t\tret_code = irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf,\n+\t\t\t\t\t\t\t hmc_info->hmc_obj,\n+\t\t\t\t\t\t\t &hmc_info->sd_table.sd_cnt);\n+\tirdma_debug_buf(dev, IRDMA_DEBUG_HMC, \"COMMIT FPM BUFFER\",\n+\t\t\tcommit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE);\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * cqp_sds_wqe_fill - fill cqp wqe doe sd\n+ * @cqp: struct for cqp hw\n+ * @info: sd info for wqe\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code\n+cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, struct irdma_update_sds_info *info,\n+\t\t u64 scratch)\n+{\n+\tu64 data;\n+\tu64 hdr;\n+\t__le64 *wqe;\n+\tint mem_entries, wqe_entries;\n+\tstruct irdma_dma_mem *sdbuf = &cqp->sdbuf;\n+\tu64 offset;\n+\tu32 wqe_idx;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tIRDMA_CQP_INIT_WQE(wqe);\n+\twqe_entries = (info->cnt > 3) ? 3 : info->cnt;\n+\tmem_entries = info->cnt - wqe_entries;\n+\n+\tif (mem_entries) {\n+\t\toffset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE;\n+\t\tmemcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);\n+\n+\t\tdata = (u64)sdbuf->pa + offset;\n+\t} else {\n+\t\tdata = 0;\n+\t}\n+\tdata |= LS_64(info->hmc_fn_id, IRDMA_CQPSQ_UPESD_HMCFNID);\n+\tset_64bit_val(wqe, 16, data);\n+\n+\tswitch (wqe_entries) {\n+\tcase 3:\n+\t\tset_64bit_val(wqe, 48,\n+\t\t\t      (LS_64(info->entry[2].cmd, IRDMA_CQPSQ_UPESD_SDCMD) |\n+\t\t\t       LS_64(1, IRDMA_CQPSQ_UPESD_ENTRY_VALID)));\n+\n+\t\tset_64bit_val(wqe, 56, info->entry[2].data);\n+\t\t/* fallthrough */\n+\tcase 2:\n+\t\tset_64bit_val(wqe, 32,\n+\t\t\t      (LS_64(info->entry[1].cmd, IRDMA_CQPSQ_UPESD_SDCMD) |\n+\t\t\t       LS_64(1, IRDMA_CQPSQ_UPESD_ENTRY_VALID)));\n+\n+\t\tset_64bit_val(wqe, 40, info->entry[1].data);\n+\t\t/* fallthrough */\n+\tcase 1:\n+\t\tset_64bit_val(wqe, 0,\n+\t\t\t      LS_64(info->entry[0].cmd, IRDMA_CQPSQ_UPESD_SDCMD));\n+\n+\t\tset_64bit_val(wqe, 8, info->entry[0].data);\n+\t\tbreak;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\thdr = LS_64(IRDMA_CQP_OP_UPDATE_PE_SDS, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |\n+\t      LS_64(mem_entries, IRDMA_CQPSQ_UPESD_ENTRY_COUNT);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"UPDATE_PE_SDS WQE\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_update_pe_sds - cqp wqe for sd\n+ * @dev: ptr to irdma_dev struct\n+ * @info: sd info for sd's\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_update_pe_sds(struct irdma_sc_dev *dev,\n+\t\t    struct irdma_update_sds_info *info, u64 scratch)\n+{\n+\tstruct irdma_sc_cqp *cqp = dev->cqp;\n+\tenum irdma_status_code ret_code;\n+\n+\tret_code = cqp_sds_wqe_fill(cqp, info, scratch);\n+\tif (!ret_code)\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_update_sds_noccq - update sd before ccq created\n+ * @dev: sc device struct\n+ * @info: sd info for sd's\n+ */\n+enum irdma_status_code\n+irdma_update_sds_noccq(struct irdma_sc_dev *dev,\n+\t\t       struct irdma_update_sds_info *info)\n+{\n+\tu32 error, val, tail;\n+\tstruct irdma_sc_cqp *cqp = dev->cqp;\n+\tenum irdma_status_code ret_code;\n+\n+\tret_code = cqp_sds_wqe_fill(cqp, info, 0);\n+\tif (ret_code)\n+\t\treturn ret_code;\n+\n+\tirdma_get_cqp_reg_info(cqp, &val, &tail, &error);\n+\tif (error)\n+\t\treturn IRDMA_ERR_CQP_COMPL_ERROR;\n+\tirdma_sc_cqp_post_sq(cqp);\n+\treturn irdma_cqp_poll_registers(cqp, tail,\n+\t\t\t\t\tcqp->dev->hw_attrs.max_done_count);\n+}\n+\n+/**\n+ * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages\n+ * @cqp: struct for cqp hw\n+ * @scratch: u64 saved to be used during cqp completion\n+ * @hmc_fn_id: hmc function id\n+ * @post_sq: flag for cqp db to ring\n+ * @poll_registers: flag to poll register for cqp completion\n+ */\n+enum irdma_status_code\n+irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,\n+\t\t\t\t    u8 hmc_fn_id, bool post_sq,\n+\t\t\t\t    bool poll_registers)\n+{\n+\tu64 hdr;\n+\t__le64 *wqe;\n+\tu32 tail, val, error;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\tset_64bit_val(wqe, 16,\n+\t\t      LS_64(hmc_fn_id, IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID));\n+\n+\thdr = LS_64(IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED, IRDMA_CQPSQ_OPCODE) |\n+\t      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, hdr);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"SHMC_PAGES_ALLOCATED WQE\",\n+\t\t\twqe, IRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_get_cqp_reg_info(cqp, &val, &tail, &error);\n+\tif (error)\n+\t\treturn IRDMA_ERR_CQP_COMPL_ERROR;\n+\n+\tif (post_sq) {\n+\t\tirdma_sc_cqp_post_sq(cqp);\n+\t\tif (poll_registers)\n+\t\t\t/* check for cqp sq tail update */\n+\t\t\treturn irdma_cqp_poll_registers(cqp, tail, 1000);\n+\t\telse\n+\t\t\treturn irdma_sc_poll_for_cqp_op_done(cqp,\n+\t\t\t\t\t\t\t     IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED,\n+\t\t\t\t\t\t\t     NULL);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_cqp_ring_full - check if cqp ring is full\n+ * @cqp: struct for cqp hw\n+ */\n+static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)\n+{\n+\treturn IRDMA_RING_FULL_ERR(cqp->sq_ring);\n+}\n+\n+/**\n+ * irdma_est_sd - returns approximate number of SDs for HMC\n+ * @dev: sc device struct\n+ * @hmc_info: hmc structure, size and count for HMC objects\n+ */\n+static u32 irdma_est_sd(struct irdma_sc_dev *dev,\n+\t\t\tstruct irdma_hmc_info *hmc_info)\n+{\n+\tint i;\n+\tu64 size = 0;\n+\tu64 sd;\n+\n+\tfor (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)\n+\t\tif (i != IRDMA_HMC_IW_PBLE)\n+\t\t\tsize += round_up(hmc_info->hmc_obj[i].cnt *\n+\t\t\t\t\t hmc_info->hmc_obj[i].size, 512);\n+\tif (dev->is_pf)\n+\t\tsize += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *\n+\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);\n+\tif (size & 0x1FFFFF)\n+\t\tsd = (size >> 21) + 1; /* add 1 for remainder */\n+\telse\n+\t\tsd = size >> 21;\n+\tif (!dev->is_pf) {\n+\t\t/* 2MB alignment for VF PBLE HMC */\n+\t\tsize = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *\n+\t\t       hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size;\n+\t\tif (size & 0x1FFFFF)\n+\t\t\tsd += (size >> 21) + 1; /* add 1 for remainder */\n+\t\telse\n+\t\t\tsd += size >> 21;\n+\t}\n+\tif (sd > 0xFFFFFFFF) {\n+\t\tdev_dbg(rfdev_to_dev(dev), \"HMC: sd overflow[%lld]\\n\", sd);\n+\t\tsd = 0xFFFFFFFF - 1;\n+\t}\n+\n+\treturn (u32)sd;\n+}\n+\n+/**\n+ * irdma_sc_query_rdma_features_done - poll cqp for query features done\n+ * @cqp: struct for cqp hw\n+ */\n+static enum irdma_status_code\n+irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp)\n+{\n+\treturn irdma_sc_poll_for_cqp_op_done(cqp,\n+\t\t\t\t\t     IRDMA_CQP_OP_QUERY_RDMA_FEATURES,\n+\t\t\t\t\t     NULL);\n+}\n+\n+/**\n+ * irdma_sc_query_rdma_features - query RDMA features and FW ver\n+ * @cqp: struct for cqp hw\n+ * @buf: buffer to hold query info\n+ * @scratch: u64 saved to be used during cqp completion\n+ */\n+static enum irdma_status_code\n+irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,\n+\t\t\t     struct irdma_dma_mem *buf, u64 scratch)\n+{\n+\t__le64 *wqe;\n+\tu64 temp;\n+\n+\twqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);\n+\tif (!wqe)\n+\t\treturn IRDMA_ERR_RING_FULL;\n+\n+\ttemp = buf->pa;\n+\tset_64bit_val(wqe, 32, temp);\n+\n+\ttemp = LS_64(cqp->polarity, IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID) |\n+\t       LS_64(buf->size, IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN) |\n+\t       LS_64(IRDMA_CQP_OP_QUERY_RDMA_FEATURES, IRDMA_CQPSQ_UP_OP);\n+\tdma_wmb(); /* make sure WQE is written before valid bit is set */\n+\n+\tset_64bit_val(wqe, 24, temp);\n+\n+\tirdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, \"QUERY RDMA FEATURES\", wqe,\n+\t\t\tIRDMA_CQP_WQE_SIZE * 8);\n+\tirdma_sc_cqp_post_sq(cqp);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * irdma_get_rdma_features - get RDMA features\n+ * @dev: sc device struct\n+ */\n+enum irdma_status_code irdma_get_rdma_features(struct irdma_sc_dev *dev)\n+{\n+\tenum irdma_status_code ret_code;\n+\tstruct irdma_dma_mem feat_buf;\n+\tu64 temp;\n+\tu16 byte_idx, feat_type, feat_cnt;\n+\n+\tfeat_buf.size = ALIGN(IRDMA_FEATURE_BUF_SIZE,\n+\t\t\t      IRDMA_FEATURE_BUF_ALIGNMENT);\n+\tfeat_buf.va = dma_alloc_coherent(hw_to_dev(dev->hw), feat_buf.size,\n+\t\t\t\t\t &feat_buf.pa, GFP_KERNEL);\n+\tif (!feat_buf.va)\n+\t\treturn IRDMA_ERR_NO_MEMORY;\n+\n+\tret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);\n+\tif (!ret_code)\n+\t\tret_code = irdma_sc_query_rdma_features_done(dev->cqp);\n+\tif (ret_code)\n+\t\tgoto exit;\n+\n+\tget_64bit_val(feat_buf.va, 0, &temp);\n+\tfeat_cnt = (u16)RS_64(temp, IRDMA_FEATURE_CNT);\n+\tif (feat_cnt < IRDMA_MAX_FEATURES) {\n+\t\tret_code = IRDMA_ERR_INVALID_FEAT_CNT;\n+\t\tgoto exit;\n+\t} else if (feat_cnt > IRDMA_MAX_FEATURES) {\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"DEV: feature buf size insufficient\\n\");\n+\t}\n+\n+\tfor (byte_idx = 0, feat_type = 0; feat_type < IRDMA_MAX_FEATURES;\n+\t     feat_type++, byte_idx += 8) {\n+\t\tget_64bit_val(feat_buf.va, byte_idx, &temp);\n+\t\tdev->feature_info[feat_type] = RS_64(temp, IRDMA_FEATURE_INFO);\n+\t}\n+exit:\n+\tdma_free_coherent(hw_to_dev(dev->hw), feat_buf.size, feat_buf.va,\n+\t\t\t  feat_buf.pa);\n+\tfeat_buf.va = NULL;\n+\treturn ret_code;\n+}\n+\n+static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev,\n+\t\t\t\tstruct irdma_hmc_info *hmc_info, u32 qpwanted)\n+{\n+\tu32 powerof2 = 1;\n+\n+\twhile (powerof2 < dev->hw_attrs.max_hw_wqes)\n+\t\tpowerof2 *= 2;\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = powerof2 * qpwanted;\n+\n+\tpowerof2 = 1;\n+\twhile (powerof2 < dev->hw_attrs.max_hw_ird)\n+\t\tpowerof2 *= 2;\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = powerof2 * qpwanted * 2;\n+}\n+\n+static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev,\n+\t\t\t\tstruct irdma_hmc_info *hmc_info, u32 qpwanted)\n+{\n+\tstruct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc;\n+\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt =\n+\t\t2 * hmc_fpm_misc->xf_block_size * qpwanted;\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = 2 * 16 * qpwanted;\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;\n+\n+\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;\n+\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =\n+\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt /\n+\t\t\thmc_fpm_misc->rrf_block_size;\n+\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;\n+\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =\n+\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt /\n+\t\t\thmc_fpm_misc->ooiscf_block_size;\n+}\n+\n+/**\n+ * irdma_config_fpm_values - configure HMC objects\n+ * @dev: sc device struct\n+ * @qp_count: desired qp count\n+ */\n+enum irdma_status_code irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count)\n+{\n+\tstruct irdma_virt_mem virt_mem;\n+\tu32 i, mem_size;\n+\tu32 qpwanted, mrwanted, pblewanted;\n+\tu32 powerof2, hte;\n+\tu32 sd_needed;\n+\tu32 sd_diff;\n+\tu32 loop_count = 0;\n+\tstruct irdma_hmc_info *hmc_info;\n+\tstruct irdma_hmc_fpm_misc *hmc_fpm_misc;\n+\tenum irdma_status_code ret_code = 0;\n+\n+\thmc_info = dev->hmc_info;\n+\thmc_fpm_misc = &dev->hmc_fpm_misc;\n+\n+\tret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id);\n+\tif (ret_code) {\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"HMC: irdma_sc_init_iw_hmc returned error_code = %d\\n\",\n+\t\t\tret_code);\n+\t\treturn ret_code;\n+\t}\n+\n+\tfor (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)\n+\t\thmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;\n+\tsd_needed = irdma_est_sd(dev, hmc_info);\n+\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\"HMC: FW max resources sd_needed[%08d] first_sd_index[%04d]\\n\",\n+\t\tsd_needed, hmc_info->first_sd_index);\n+\tdev_dbg(rfdev_to_dev(dev), \"HMC: sd count %d where max sd is %d\\n\",\n+\t\thmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds);\n+\n+\tqpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt);\n+\n+\tpowerof2 = 1;\n+\twhile (powerof2 <= qpwanted)\n+\t\tpowerof2 *= 2;\n+\tpowerof2 /= 2;\n+\tqpwanted = powerof2;\n+\n+\tmrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt;\n+\tpblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;\n+\n+\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\"HMC: req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\\n\",\n+\t\tqp_count, hmc_fpm_misc->max_sds,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;\n+\n+\thmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;\n+\n+\tdo {\n+\t\t++loop_count;\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt =\n+\t\t\tmin(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt);\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;\n+\n+\t\thte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);\n+\t\tpowerof2 = 1;\n+\t\twhile (powerof2 < hte)\n+\t\t\tpowerof2 *= 2;\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt =\n+\t\t\tpowerof2 * hmc_fpm_misc->ht_multiplier;\n+\t\tif (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)\n+\t\t\tcfg_fpm_value_gen_1(dev, hmc_info, qpwanted);\n+\t\telse\n+\t\t\tcfg_fpm_value_gen_2(dev, hmc_info, qpwanted);\n+\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt =\n+\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =\n+\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt =\n+\t\t\t(round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket;\n+\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;\n+\t\tsd_needed = irdma_est_sd(dev, hmc_info);\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\\n\",\n+\t\t\tsd_needed, hmc_fpm_misc->max_sds, mrwanted,\n+\t\t\tpblewanted, qpwanted);\n+\n+\t\t/* Do not reduce resources further. All objects fit with max SDs */\n+\t\tif (sd_needed <= hmc_fpm_misc->max_sds)\n+\t\t\tbreak;\n+\n+\t\tsd_diff = sd_needed - hmc_fpm_misc->max_sds;\n+\t\tif (sd_diff > 128) {\n+\t\t\tif (qpwanted > 128)\n+\t\t\t\tqpwanted /= 2;\n+\t\t\tmrwanted /= 2;\n+\t\t\tpblewanted /= 2;\n+\t\t\tcontinue;\n+\t\t}\n+\t\tif (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &&\n+\t\t    pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) {\n+\t\t\tpblewanted -= 256 * FPM_MULTIPLIER * sd_diff;\n+\t\t\tcontinue;\n+\t\t} else if (pblewanted > (100 * FPM_MULTIPLIER)) {\n+\t\t\tpblewanted -= 10 * FPM_MULTIPLIER;\n+\t\t} else if (pblewanted > FPM_MULTIPLIER) {\n+\t\t\tpblewanted -= FPM_MULTIPLIER;\n+\t\t} else if (qpwanted <= 128) {\n+\t\t\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)\n+\t\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;\n+\t\t\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)\n+\t\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;\n+\t\t}\n+\t\tif (mrwanted > FPM_MULTIPLIER)\n+\t\t\tmrwanted -= FPM_MULTIPLIER;\n+\t\tif (!(loop_count % 10) && qpwanted > 128) {\n+\t\t\tqpwanted /= 2;\n+\t\t\tif (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)\n+\t\t\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;\n+\t\t}\n+\t} while (loop_count < 2000);\n+\n+\tif (sd_needed > hmc_fpm_misc->max_sds) {\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"HMC: cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\\n\",\n+\t\t\tloop_count, sd_needed, hmc_info->sd_table.sd_cnt);\n+\t\treturn IRDMA_ERR_CFG;\n+\t}\n+\n+\tif (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) {\n+\t\tpblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 *\n+\t\t\t      FPM_MULTIPLIER;\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;\n+\t\tsd_needed = irdma_est_sd(dev, hmc_info);\n+\t}\n+\n+\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\"HMC: loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\\n\",\n+\t\tloop_count, sd_needed, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,\n+\t\thmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,\n+\t\thmc_info->sd_table.sd_cnt, hmc_info->first_sd_index);\n+\n+\tret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id);\n+\tif (ret_code) {\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"HMC: cfg_iw_fpm returned error_code[x%08X]\\n\",\n+\t\t\trd32(dev->hw, dev->hw_regs[IRDMA_CQPERRCODES]));\n+\t\treturn ret_code;\n+\t}\n+\n+\tmem_size = sizeof(struct irdma_hmc_sd_entry) *\n+\t\t   (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);\n+\tvirt_mem.size = mem_size;\n+\tvirt_mem.va = kzalloc(virt_mem.size, GFP_ATOMIC);\n+\tif (!virt_mem.va) {\n+\t\tdev_dbg(rfdev_to_dev(dev),\n+\t\t\t\"HMC: failed to allocate memory for sd_entry buffer\\n\");\n+\t\treturn IRDMA_ERR_NO_MEMORY;\n+\t}\n+\thmc_info->sd_table.sd_entry = virt_mem.va;\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available\n+ * @dev: rdma device\n+ * @pcmdinfo: cqp command info\n+ */\n+static enum irdma_status_code irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t struct cqp_cmds_info *pcmdinfo)\n+{\n+\tenum irdma_status_code status;\n+\tstruct irdma_dma_mem val_mem;\n+\tbool alloc = false;\n+\n+\tdev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;\n+\tswitch (pcmdinfo->cqp_cmd) {\n+\tcase IRDMA_OP_CEQ_DESTROY:\n+\t\tstatus = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,\n+\t\t\t\t\t      pcmdinfo->in.u.ceq_destroy.scratch,\n+\t\t\t\t\t      pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_AEQ_DESTROY:\n+\t\tstatus = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,\n+\t\t\t\t\t      pcmdinfo->in.u.aeq_destroy.scratch,\n+\t\t\t\t\t      pcmdinfo->post_sq);\n+\n+\t\tbreak;\n+\tcase IRDMA_OP_CEQ_CREATE:\n+\t\tstatus = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,\n+\t\t\t\t\t     pcmdinfo->in.u.ceq_create.scratch,\n+\t\t\t\t\t     pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_AEQ_CREATE:\n+\t\tstatus = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,\n+\t\t\t\t\t     pcmdinfo->in.u.aeq_create.scratch,\n+\t\t\t\t\t     pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_QP_UPLOAD_CONTEXT:\n+\t\tstatus = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev,\n+\t\t\t\t\t\t    &pcmdinfo->in.u.qp_upload_context.info,\n+\t\t\t\t\t\t    pcmdinfo->in.u.qp_upload_context.scratch,\n+\t\t\t\t\t\t    pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_CQ_CREATE:\n+\t\tstatus = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq,\n+\t\t\t\t\t    pcmdinfo->in.u.cq_create.scratch,\n+\t\t\t\t\t    pcmdinfo->in.u.cq_create.check_overflow,\n+\t\t\t\t\t    pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_CQ_MODIFY:\n+\t\tstatus = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq,\n+\t\t\t\t\t    &pcmdinfo->in.u.cq_modify.info,\n+\t\t\t\t\t    pcmdinfo->in.u.cq_modify.scratch,\n+\t\t\t\t\t    pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_CQ_DESTROY:\n+\t\tstatus = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq,\n+\t\t\t\t\t     pcmdinfo->in.u.cq_destroy.scratch,\n+\t\t\t\t\t     pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_QP_FLUSH_WQES:\n+\t\tstatus = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,\n+\t\t\t\t\t\t&pcmdinfo->in.u.qp_flush_wqes.info,\n+\t\t\t\t\t\tpcmdinfo->in.u.qp_flush_wqes.scratch,\n+\t\t\t\t\t\tpcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_GEN_AE:\n+\t\tstatus = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,\n+\t\t\t\t\t &pcmdinfo->in.u.gen_ae.info,\n+\t\t\t\t\t pcmdinfo->in.u.gen_ae.scratch,\n+\t\t\t\t\t pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_MANAGE_PUSH_PAGE:\n+\t\tstatus = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp,\n+\t\t\t\t\t\t   &pcmdinfo->in.u.manage_push_page.info,\n+\t\t\t\t\t\t   pcmdinfo->in.u.manage_push_page.scratch,\n+\t\t\t\t\t\t   pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_UPDATE_PE_SDS:\n+\t\tstatus = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev,\n+\t\t\t\t\t     &pcmdinfo->in.u.update_pe_sds.info,\n+\t\t\t\t\t     pcmdinfo->in.u.update_pe_sds.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE:\n+\t\tstatus =\n+\t\t\tirdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp,\n+\t\t\t\t\t\t\t  pcmdinfo->in.u.manage_hmc_pm.scratch,\n+\t\t\t\t\t\t\t  (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,\n+\t\t\t\t\t\t\t  pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,\n+\t\t\t\t\t\t\t  true);\n+\t\tbreak;\n+\tcase IRDMA_OP_SUSPEND:\n+\t\tstatus = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp,\n+\t\t\t\t\t     pcmdinfo->in.u.suspend_resume.qp,\n+\t\t\t\t\t     pcmdinfo->in.u.suspend_resume.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_RESUME:\n+\t\tstatus = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp,\n+\t\t\t\t\t    pcmdinfo->in.u.suspend_resume.qp,\n+\t\t\t\t\t    pcmdinfo->in.u.suspend_resume.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_QUERY_FPM_VAL:\n+\t\tval_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa;\n+\t\tval_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va;\n+\t\tstatus = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp,\n+\t\t\t\t\t\tpcmdinfo->in.u.query_fpm_val.scratch,\n+\t\t\t\t\t\tpcmdinfo->in.u.query_fpm_val.hmc_fn_id,\n+\t\t\t\t\t\t&val_mem, true, IRDMA_CQP_WAIT_EVENT);\n+\t\tbreak;\n+\tcase IRDMA_OP_COMMIT_FPM_VAL:\n+\t\tval_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa;\n+\t\tval_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va;\n+\t\tstatus = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp,\n+\t\t\t\t\t\t pcmdinfo->in.u.commit_fpm_val.scratch,\n+\t\t\t\t\t\t pcmdinfo->in.u.commit_fpm_val.hmc_fn_id,\n+\t\t\t\t\t\t &val_mem,\n+\t\t\t\t\t\t true,\n+\t\t\t\t\t\t IRDMA_CQP_WAIT_EVENT);\n+\t\tbreak;\n+\tcase IRDMA_OP_STATS_ALLOCATE:\n+\t\talloc = true;\n+\t\t/* fall-through */\n+\tcase IRDMA_OP_STATS_FREE:\n+\t\tstatus = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp,\n+\t\t\t\t\t\t    &pcmdinfo->in.u.stats_manage.info,\n+\t\t\t\t\t\t    alloc,\n+\t\t\t\t\t\t    pcmdinfo->in.u.stats_manage.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_STATS_GATHER:\n+\t\tstatus = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp,\n+\t\t\t\t\t       &pcmdinfo->in.u.stats_gather.info,\n+\t\t\t\t\t       pcmdinfo->in.u.stats_gather.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_WS_MODIFY_NODE:\n+\t\tstatus = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,\n+\t\t\t\t\t\t &pcmdinfo->in.u.ws_node.info,\n+\t\t\t\t\t\t IRDMA_MODIFY_NODE,\n+\t\t\t\t\t\t pcmdinfo->in.u.ws_node.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_WS_DELETE_NODE:\n+\t\tstatus = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,\n+\t\t\t\t\t\t &pcmdinfo->in.u.ws_node.info,\n+\t\t\t\t\t\t IRDMA_DEL_NODE,\n+\t\t\t\t\t\t pcmdinfo->in.u.ws_node.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_WS_ADD_NODE:\n+\t\tstatus = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,\n+\t\t\t\t\t\t &pcmdinfo->in.u.ws_node.info,\n+\t\t\t\t\t\t IRDMA_ADD_NODE,\n+\t\t\t\t\t\t pcmdinfo->in.u.ws_node.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_SET_UP_MAP:\n+\t\tstatus = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp,\n+\t\t\t\t\t     &pcmdinfo->in.u.up_map.info,\n+\t\t\t\t\t     pcmdinfo->in.u.up_map.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_QUERY_RDMA_FEATURES:\n+\t\tstatus = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp,\n+\t\t\t\t\t\t      &pcmdinfo->in.u.query_rdma.query_buff_mem,\n+\t\t\t\t\t\t      pcmdinfo->in.u.query_rdma.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_DELETE_ARP_CACHE_ENTRY:\n+\t\tstatus = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp,\n+\t\t\t\t\t\t      pcmdinfo->in.u.del_arp_cache_entry.scratch,\n+\t\t\t\t\t\t      pcmdinfo->in.u.del_arp_cache_entry.arp_index,\n+\t\t\t\t\t\t      pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_MANAGE_APBVT_ENTRY:\n+\t\tstatus = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp,\n+\t\t\t\t\t\t     &pcmdinfo->in.u.manage_apbvt_entry.info,\n+\t\t\t\t\t\t     pcmdinfo->in.u.manage_apbvt_entry.scratch,\n+\t\t\t\t\t\t     pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY:\n+\t\tstatus = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp,\n+\t\t\t\t\t\t\t   &pcmdinfo->in.u.manage_qhash_table_entry.info,\n+\t\t\t\t\t\t\t   pcmdinfo->in.u.manage_qhash_table_entry.scratch,\n+\t\t\t\t\t\t\t   pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_QP_MODIFY:\n+\t\tstatus = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp,\n+\t\t\t\t\t    &pcmdinfo->in.u.qp_modify.info,\n+\t\t\t\t\t    pcmdinfo->in.u.qp_modify.scratch,\n+\t\t\t\t\t    pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_QP_CREATE:\n+\t\tstatus = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp,\n+\t\t\t\t\t    &pcmdinfo->in.u.qp_create.info,\n+\t\t\t\t\t    pcmdinfo->in.u.qp_create.scratch,\n+\t\t\t\t\t    pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_QP_DESTROY:\n+\t\tstatus = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp,\n+\t\t\t\t\t     pcmdinfo->in.u.qp_destroy.scratch,\n+\t\t\t\t\t     pcmdinfo->in.u.qp_destroy.remove_hash_idx,\n+\t\t\t\t\t     pcmdinfo->in.u.qp_destroy.ignore_mw_bnd,\n+\t\t\t\t\t     pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_ALLOC_STAG:\n+\t\tstatus = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev,\n+\t\t\t\t\t     &pcmdinfo->in.u.alloc_stag.info,\n+\t\t\t\t\t     pcmdinfo->in.u.alloc_stag.scratch,\n+\t\t\t\t\t     pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_MR_REG_NON_SHARED:\n+\t\tstatus = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev,\n+\t\t\t\t\t\t    &pcmdinfo->in.u.mr_reg_non_shared.info,\n+\t\t\t\t\t\t    pcmdinfo->in.u.mr_reg_non_shared.scratch,\n+\t\t\t\t\t\t    pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_DEALLOC_STAG:\n+\t\tstatus =\n+\t\t\tirdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev,\n+\t\t\t\t\t      &pcmdinfo->in.u.dealloc_stag.info,\n+\t\t\t\t\t      pcmdinfo->in.u.dealloc_stag.scratch,\n+\t\t\t\t\t      pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_MW_ALLOC:\n+\t\tstatus = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev,\n+\t\t\t\t\t   &pcmdinfo->in.u.mw_alloc.info,\n+\t\t\t\t\t   pcmdinfo->in.u.mw_alloc.scratch,\n+\t\t\t\t\t   pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_ADD_ARP_CACHE_ENTRY:\n+\t\tstatus = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp,\n+\t\t\t\t\t\t      &pcmdinfo->in.u.add_arp_cache_entry.info,\n+\t\t\t\t\t\t      pcmdinfo->in.u.add_arp_cache_entry.scratch,\n+\t\t\t\t\t\t      pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY:\n+\t\tstatus = dev->cqp_misc_ops->alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp,\n+\t\t\t\t\t\t\t\t  pcmdinfo->in.u.alloc_local_mac_entry.scratch,\n+\t\t\t\t\t\t\t\t  pcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_ADD_LOCAL_MAC_ENTRY:\n+\t\tstatus = dev->cqp_misc_ops->add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp,\n+\t\t\t\t\t\t\t\t&pcmdinfo->in.u.add_local_mac_entry.info,\n+\t\t\t\t\t\t\t\tpcmdinfo->in.u.add_local_mac_entry.scratch,\n+\t\t\t\t\t\t\t\tpcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_DELETE_LOCAL_MAC_ENTRY:\n+\t\tstatus = dev->cqp_misc_ops->del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp,\n+\t\t\t\t\t\t\t\tpcmdinfo->in.u.del_local_mac_entry.scratch,\n+\t\t\t\t\t\t\t\tpcmdinfo->in.u.del_local_mac_entry.entry_idx,\n+\t\t\t\t\t\t\t\tpcmdinfo->in.u.del_local_mac_entry.ignore_ref_count,\n+\t\t\t\t\t\t\t\tpcmdinfo->post_sq);\n+\t\tbreak;\n+\tcase IRDMA_OP_AH_CREATE:\n+\t\tstatus = dev->iw_uda_ops->create_ah(pcmdinfo->in.u.ah_create.cqp,\n+\t\t\t\t\t\t    &pcmdinfo->in.u.ah_create.info,\n+\t\t\t\t\t\t    pcmdinfo->in.u.ah_create.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_AH_DESTROY:\n+\t\tstatus = dev->iw_uda_ops->destroy_ah(pcmdinfo->in.u.ah_destroy.cqp,\n+\t\t\t\t\t\t     &pcmdinfo->in.u.ah_destroy.info,\n+\t\t\t\t\t\t     pcmdinfo->in.u.ah_destroy.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_MC_CREATE:\n+\t\tstatus = dev->iw_uda_ops->mcast_grp_create(pcmdinfo->in.u.mc_create.cqp,\n+\t\t\t\t\t\t\t   &pcmdinfo->in.u.mc_create.info,\n+\t\t\t\t\t\t\t   pcmdinfo->in.u.mc_create.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_MC_DESTROY:\n+\t\tstatus = dev->iw_uda_ops->mcast_grp_destroy(pcmdinfo->in.u.mc_destroy.cqp,\n+\t\t\t\t\t\t\t    &pcmdinfo->in.u.mc_destroy.info,\n+\t\t\t\t\t\t\t    pcmdinfo->in.u.mc_destroy.scratch);\n+\t\tbreak;\n+\tcase IRDMA_OP_MC_MODIFY:\n+\t\tstatus = dev->iw_uda_ops->mcast_grp_modify(pcmdinfo->in.u.mc_modify.cqp,\n+\t\t\t\t\t\t\t   &pcmdinfo->in.u.mc_modify.info,\n+\t\t\t\t\t\t\t   pcmdinfo->in.u.mc_modify.scratch);\n+\t\tbreak;\n+\tdefault:\n+\t\tstatus = IRDMA_NOT_SUPPORTED;\n+\t\tbreak;\n+\t}\n+\n+\treturn status;\n+}\n+\n+/**\n+ * irdma_process_cqp_cmd - process all cqp commands\n+ * @dev: sc device struct\n+ * @pcmdinfo: cqp command info\n+ */\n+enum irdma_status_code irdma_process_cqp_cmd(struct irdma_sc_dev *dev,\n+\t\t\t\t\t     struct cqp_cmds_info *pcmdinfo)\n+{\n+\tenum irdma_status_code status = 0;\n+\tunsigned long flags;\n+\n+\tspin_lock_irqsave(&dev->cqp_lock, flags);\n+\tif (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))\n+\t\tstatus = irdma_exec_cqp_cmd(dev, pcmdinfo);\n+\telse\n+\t\tlist_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);\n+\tspin_unlock_irqrestore(&dev->cqp_lock, flags);\n+\treturn status;\n+}\n+\n+/**\n+ * irdma_process_bh - called from tasklet for cqp list\n+ * @dev: sc device struct\n+ */\n+enum irdma_status_code irdma_process_bh(struct irdma_sc_dev *dev)\n+{\n+\tenum irdma_status_code status = 0;\n+\tstruct cqp_cmds_info *pcmdinfo;\n+\tunsigned long flags;\n+\n+\tspin_lock_irqsave(&dev->cqp_lock, flags);\n+\twhile (!list_empty(&dev->cqp_cmd_head) &&\n+\t       !irdma_cqp_ring_full(dev->cqp)) {\n+\t\tpcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev);\n+\t\tstatus = irdma_exec_cqp_cmd(dev, pcmdinfo);\n+\t\tif (status)\n+\t\t\tbreak;\n+\t}\n+\tspin_unlock_irqrestore(&dev->cqp_lock, flags);\n+\treturn status;\n+}\n+\n+/**\n+ * irdma_enable_irq - Enable interrupt\n+ * @dev: pointer to the device structure\n+ * @idx: vector index\n+ */\n+static void irdma_ena_irq(struct irdma_sc_dev *dev, u32 idx)\n+{\n+\tu32 val;\n+\n+\tval = IRDMA_GLINT_DYN_CTL_INTENA_M | IRDMA_GLINT_DYN_CTL_CLEARPBA_M |\n+\t      IRDMA_GLINT_DYN_CTL_ITR_INDX_M;\n+\tif (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1)\n+\t\twr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * idx, val);\n+\telse\n+\t\twr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1),\n+\t\t     val);\n+}\n+\n+/**\n+ * irdma_disable_irq - Disable interrupt\n+ * @dev: pointer to the device structure\n+ * @idx: vector index\n+ */\n+static void irdma_disable_irq(struct irdma_sc_dev *dev, u32 idx)\n+{\n+\tif (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1)\n+\t\twr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * idx, 0);\n+\telse\n+\t\twr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1),\n+\t\t     0);\n+}\n+\n+/**\n+ * irdma_config_ceq- Configure CEQ interrupt\n+ * @dev: pointer to the device structure\n+ * @ceq_id: Completion Event Queue ID\n+ * @idx: vector index\n+ */\n+static void irdma_cfg_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx)\n+{\n+\tu32 reg_val;\n+\n+\treg_val = (IRDMA_GLINT_CEQCTL_CAUSE_ENA_M |\n+\t\t   (idx << IRDMA_GLINT_CEQCTL_MSIX_INDX_S) |\n+\t\t   IRDMA_GLINT_CEQCTL_ITR_INDX_M);\n+\n+\twr32(dev->hw, dev->hw_regs[IRDMA_GLINT_CEQCTL] + 4 * ceq_id, reg_val);\n+}\n+\n+/**\n+ * irdma_config_aeq- Configure AEQ interrupt\n+ * @dev: pointer to the device structure\n+ * @idx: vector index\n+ */\n+static void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx)\n+{\n+\tu32 reg_val;\n+\n+\treg_val = (IRDMA_PFINT_AEQCTL_CAUSE_ENA_M |\n+\t\t   (idx << IRDMA_PFINT_AEQCTL_MSIX_INDX_S) |\n+\t\t   IRDMA_PFINT_AEQCTL_ITR_INDX_M);\n+\n+\twr32(dev->hw, dev->hw_regs[IRDMA_PFINT_AEQCTL], reg_val);\n+}\n+\n+/* iwarp pd ops */\n+static struct irdma_pd_ops iw_pd_ops = {\n+\t.pd_init = irdma_sc_pd_init\n+};\n+\n+static struct irdma_priv_qp_ops iw_priv_qp_ops = {\n+\t.iw_mr_fast_register = irdma_sc_mr_fast_register,\n+\t.qp_create = irdma_sc_qp_create,\n+\t.qp_destroy = irdma_sc_qp_destroy,\n+\t.qp_flush_wqes = irdma_sc_qp_flush_wqes,\n+\t.qp_init = irdma_sc_qp_init,\n+\t.qp_modify = irdma_sc_qp_modify,\n+\t.qp_send_lsmm = irdma_sc_send_lsmm,\n+\t.qp_send_lsmm_nostag = irdma_sc_send_lsmm_nostag,\n+\t.qp_send_rtt = irdma_sc_send_rtt,\n+\t.qp_setctx = irdma_sc_qp_setctx,\n+\t.qp_setctx_roce = irdma_sc_qp_setctx_roce,\n+\t.qp_upload_context = irdma_sc_qp_upload_context,\n+\t.update_resume_qp = irdma_sc_resume_qp,\n+\t.update_suspend_qp = irdma_sc_suspend_qp,\n+};\n+\n+static struct irdma_mr_ops iw_mr_ops = {\n+\t.alloc_stag = irdma_sc_alloc_stag,\n+\t.dealloc_stag = irdma_sc_dealloc_stag,\n+\t.mr_reg_non_shared = irdma_sc_mr_reg_non_shared,\n+\t.mr_reg_shared = irdma_sc_mr_reg_shared,\n+\t.mw_alloc = irdma_sc_mw_alloc,\n+\t.query_stag = irdma_sc_query_stag,\n+};\n+\n+static struct irdma_cqp_misc_ops iw_cqp_misc_ops = {\n+\t.add_arp_cache_entry = irdma_sc_add_arp_cache_entry,\n+\t.add_local_mac_entry = irdma_sc_add_local_mac_entry,\n+\t.alloc_local_mac_entry = irdma_sc_alloc_local_mac_entry,\n+\t.cqp_nop = irdma_sc_cqp_nop,\n+\t.del_arp_cache_entry = irdma_sc_del_arp_cache_entry,\n+\t.del_local_mac_entry = irdma_sc_del_local_mac_entry,\n+\t.gather_stats = irdma_sc_gather_stats,\n+\t.manage_apbvt_entry = irdma_sc_manage_apbvt_entry,\n+\t.manage_push_page = irdma_sc_manage_push_page,\n+\t.manage_qhash_table_entry = irdma_sc_manage_qhash_table_entry,\n+\t.manage_stats_instance = irdma_sc_manage_stats_inst,\n+\t.manage_ws_node = irdma_sc_manage_ws_node,\n+\t.query_arp_cache_entry = irdma_sc_query_arp_cache_entry,\n+\t.query_rdma_features = irdma_sc_query_rdma_features,\n+\t.set_up_map = irdma_sc_set_up_map,\n+};\n+\n+static struct irdma_irq_ops iw_irq_ops = {\n+\t.irdma_cfg_aeq = irdma_cfg_aeq,\n+\t.irdma_cfg_ceq = irdma_cfg_ceq,\n+\t.irdma_dis_irq = irdma_disable_irq,\n+\t.irdma_en_irq = irdma_ena_irq,\n+};\n+\n+static struct irdma_cqp_ops iw_cqp_ops = {\n+\t.check_cqp_progress = irdma_check_cqp_progress,\n+\t.cqp_create = irdma_sc_cqp_create,\n+\t.cqp_destroy = irdma_sc_cqp_destroy,\n+\t.cqp_get_next_send_wqe = irdma_sc_cqp_get_next_send_wqe,\n+\t.cqp_init = irdma_sc_cqp_init,\n+\t.cqp_post_sq = irdma_sc_cqp_post_sq,\n+\t.poll_for_cqp_op_done = irdma_sc_poll_for_cqp_op_done,\n+};\n+\n+static struct irdma_priv_cq_ops iw_priv_cq_ops = {\n+\t.cq_ack = irdma_sc_cq_ack,\n+\t.cq_create = irdma_sc_cq_create,\n+\t.cq_destroy = irdma_sc_cq_destroy,\n+\t.cq_init = irdma_sc_cq_init,\n+\t.cq_modify = irdma_sc_cq_modify,\n+\t.cq_resize = irdma_sc_cq_resize,\n+};\n+\n+static struct irdma_ccq_ops iw_ccq_ops = {\n+\t.ccq_arm = irdma_sc_ccq_arm,\n+\t.ccq_create = irdma_sc_ccq_create,\n+\t.ccq_create_done = irdma_sc_ccq_create_done,\n+\t.ccq_destroy = irdma_sc_ccq_destroy,\n+\t.ccq_get_cqe_info = irdma_sc_ccq_get_cqe_info,\n+\t.ccq_init = irdma_sc_ccq_init,\n+};\n+\n+static struct irdma_ceq_ops iw_ceq_ops = {\n+\t.cceq_create = irdma_sc_cceq_create,\n+\t.cceq_create_done = irdma_sc_cceq_create_done,\n+\t.cceq_destroy_done = irdma_sc_cceq_destroy_done,\n+\t.ceq_create = irdma_sc_ceq_create,\n+\t.ceq_destroy = irdma_sc_ceq_destroy,\n+\t.ceq_init = irdma_sc_ceq_init,\n+\t.process_ceq = irdma_sc_process_ceq,\n+};\n+\n+static struct irdma_aeq_ops iw_aeq_ops = {\n+\t.aeq_create = irdma_sc_aeq_create,\n+\t.aeq_create_done = irdma_sc_aeq_create_done,\n+\t.aeq_destroy = irdma_sc_aeq_destroy,\n+\t.aeq_destroy_done = irdma_sc_aeq_destroy_done,\n+\t.aeq_init = irdma_sc_aeq_init,\n+\t.get_next_aeqe = irdma_sc_get_next_aeqe,\n+\t.repost_aeq_entries = irdma_sc_repost_aeq_entries,\n+};\n+\n+static struct irdma_hmc_ops iw_hmc_ops = {\n+\t.cfg_iw_fpm = irdma_sc_cfg_iw_fpm,\n+\t.commit_fpm_val = irdma_sc_commit_fpm_val,\n+\t.commit_fpm_val_done = irdma_sc_commit_fpm_val_done,\n+\t.create_hmc_object = irdma_sc_create_hmc_obj,\n+\t.del_hmc_object = irdma_sc_del_hmc_obj,\n+\t.init_iw_hmc = irdma_sc_init_iw_hmc,\n+\t.manage_hmc_pm_func_table = irdma_sc_manage_hmc_pm_func_table,\n+\t.manage_hmc_pm_func_table_done = irdma_sc_manage_hmc_pm_func_table_done,\n+\t.parse_fpm_commit_buf = irdma_sc_parse_fpm_commit_buf,\n+\t.parse_fpm_query_buf = irdma_sc_parse_fpm_query_buf,\n+\t.pf_init_vfhmc = NULL,\n+\t.query_fpm_val = irdma_sc_query_fpm_val,\n+\t.query_fpm_val_done = irdma_sc_query_fpm_val_done,\n+\t.static_hmc_pages_allocated = irdma_sc_static_hmc_pages_allocated,\n+\t.vf_cfg_vffpm = NULL,\n+};\n+\n+/**\n+ * irdma_wait_pe_ready - Check if firmware is ready\n+ * @dev: provides access to registers\n+ */\n+static int irdma_wait_pe_ready(struct irdma_sc_dev *dev)\n+{\n+\tu32 statuscpu0;\n+\tu32 statuscpu1;\n+\tu32 statuscpu2;\n+\tu32 retrycount = 0;\n+\n+\tdo {\n+\t\tstatuscpu0 = rd32(dev->hw, dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]);\n+\t\tstatuscpu1 = rd32(dev->hw, dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]);\n+\t\tstatuscpu2 = rd32(dev->hw, dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]);\n+\t\tif (statuscpu0 == 0x80 && statuscpu1 == 0x80 &&\n+\t\t    statuscpu2 == 0x80)\n+\t\t\treturn 0;\n+\t\tmdelay(1000);\n+\t} while (retrycount++ < dev->hw_attrs.max_pe_ready_count);\n+\treturn -1;\n+}\n+\n+/**\n+ * irdma_sc_ctrl_init - Initialize control part of device\n+ * @ver: version\n+ * @dev: Device pointer\n+ * @info: Device init info\n+ */\n+enum irdma_status_code irdma_sc_ctrl_init(enum irdma_vers ver,\n+\t\t\t\t\t  struct irdma_sc_dev *dev,\n+\t\t\t\t\t  struct irdma_device_init_info *info)\n+{\n+\tu32 val;\n+\tu16 hmc_fcn = 0;\n+\tenum irdma_status_code ret_code = 0;\n+\tu8 db_size;\n+\n+\tspin_lock_init(&dev->cqp_lock);\n+\tINIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */\n+\tdev->hmc_fn_id = info->hmc_fn_id;\n+\tdev->is_pf = info->is_pf;\n+\tdev->fpm_query_buf_pa = info->fpm_query_buf_pa;\n+\tdev->fpm_query_buf = info->fpm_query_buf;\n+\tdev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;\n+\tdev->fpm_commit_buf = info->fpm_commit_buf;\n+\tdev->hw = info->hw;\n+\tdev->hw->hw_addr = info->bar0;\n+\tdev->irq_ops = &iw_irq_ops;\n+\tdev->cqp_ops = &iw_cqp_ops;\n+\tdev->ccq_ops = &iw_ccq_ops;\n+\tdev->ceq_ops = &iw_ceq_ops;\n+\tdev->aeq_ops = &iw_aeq_ops;\n+\tdev->hmc_ops = &iw_hmc_ops;\n+\tdev->iw_priv_cq_ops = &iw_priv_cq_ops;\n+\n+\t/* Setup the hardware limits, hmc may limit further */\n+\tdev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;\n+\tdev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;\n+\tdev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;\n+\tdev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;\n+\tdev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;\n+\tdev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;\n+\tdev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;\n+\tdev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;\n+\tdev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;\n+\tdev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;\n+\tdev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;\n+\tdev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;\n+\tdev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;\n+\tdev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;\n+\tdev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;\n+\tdev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;\n+\tdev->hw_attrs.max_hw_ird = IRDMA_MAX_IRD_SIZE;\n+\tdev->hw_attrs.max_hw_ord = IRDMA_MAX_ORD_SIZE;\n+\tdev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;\n+\tdev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS;\n+\n+\t//dev->hw_attrs.max_hw_sq_quanta = IRDMA_QP_SW_MAX_SQ_QUANTA;\n+\tdev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;\n+\tdev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;\n+\tdev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS;\n+\tdev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;\n+\n+\tdev->hw_attrs.max_pe_ready_count = 14;\n+\tdev->hw_attrs.max_done_count = IRDMA_DONE_COUNT;\n+\tdev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;\n+\tdev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;\n+\n+\tdev->hw_attrs.uk_attrs.hw_rev = ver;\n+\n+\tinfo->init_hw(dev);\n+\tif (dev->is_pf) {\n+\t\tif (irdma_wait_pe_ready(dev))\n+\t\t\treturn IRDMA_ERR_TIMEOUT;\n+\n+\t\tval = rd32(dev->hw, dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);\n+\t\tdb_size = (u8)RS_32(val, IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE);\n+\t\tif (db_size != IRDMA_PE_DB_SIZE_4M &&\n+\t\t    db_size != IRDMA_PE_DB_SIZE_8M) {\n+\t\t\tpr_err(\"RDMA feature not enabled! db_size=%d\\n\",\n+\t\t\t       db_size);\n+\t\t\treturn IRDMA_ERR_PE_DOORBELL_NOT_ENA;\n+\t\t}\n+\t}\n+\tdev->db_addr = dev->hw->hw_addr + dev->hw_regs[IRDMA_DB_ADDR_OFFSET];\n+\tdev->hw->hmc.hmc_fn_id = (u8)hmc_fcn;\n+\n+\treturn ret_code;\n+}\n+\n+/**\n+ * irdma_sc_rt_init - Runtime initialize device\n+ * @dev: IWARP device pointer\n+ */\n+void irdma_sc_rt_init(struct irdma_sc_dev *dev)\n+{\n+\tmutex_init(&dev->ws_mutex);\n+\tirdma_device_init_uk(&dev->dev_uk);\n+\tdev->cqp_misc_ops = &iw_cqp_misc_ops;\n+\tdev->iw_pd_ops = &iw_pd_ops;\n+\tdev->iw_priv_qp_ops = &iw_priv_qp_ops;\n+\tdev->mr_ops = &iw_mr_ops;\n+\tdev->iw_uda_ops = &irdma_uda_ops;\n+}\n+\n+/**\n+ * irdma_update_stats - Update statistics\n+ * @hw_stats: hw_stats instance to update\n+ * @gather_stats: updated stat counters\n+ * @last_gather_stats: last stat counters\n+ */\n+void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,\n+\t\t\tstruct irdma_gather_stats *gather_stats,\n+\t\t\tstruct irdma_gather_stats *last_gather_stats)\n+{\n+\tu64 *stats_val = hw_stats->stats_val_32;\n+\n+\tstats_val[IRDMA_HW_STAT_INDEX_RXVLANERR] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rxvlanerr,\n+\t\t\t\t  last_gather_stats->rxvlanerr,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4rxdiscard,\n+\t\t\t\t  last_gather_stats->ip4rxdiscard,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4rxtrunc,\n+\t\t\t\t  last_gather_stats->ip4rxtrunc,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4txnoroute,\n+\t\t\t\t  last_gather_stats->ip4txnoroute,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6rxdiscard,\n+\t\t\t\t  last_gather_stats->ip6rxdiscard,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6rxtrunc,\n+\t\t\t\t  last_gather_stats->ip6rxtrunc,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6txnoroute,\n+\t\t\t\t  last_gather_stats->ip6txnoroute,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_TCPRTXSEG] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->tcprtxseg,\n+\t\t\t\t  last_gather_stats->tcprtxseg,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->tcprxopterr,\n+\t\t\t\t  last_gather_stats->tcprxopterr,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->tcprxprotoerr,\n+\t\t\t\t  last_gather_stats->tcprxprotoerr,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rxrpcnphandled,\n+\t\t\t\t  last_gather_stats->rxrpcnphandled,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rxrpcnpignored,\n+\t\t\t\t  last_gather_stats->rxrpcnpignored,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->txnpcnpsent,\n+\t\t\t\t  last_gather_stats->txnpcnpsent,\n+\t\t\t\t  IRDMA_MAX_STATS_32);\n+\tstats_val = hw_stats->stats_val_64;\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4RXOCTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4rxocts,\n+\t\t\t\t  last_gather_stats->ip4rxocts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4RXPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4rxpkts,\n+\t\t\t\t  last_gather_stats->ip4rxpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4txfrag,\n+\t\t\t\t  last_gather_stats->ip4txfrag,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4rxmcpkts,\n+\t\t\t\t  last_gather_stats->ip4rxmcpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4TXOCTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4txocts,\n+\t\t\t\t  last_gather_stats->ip4txocts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4TXPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4txpkts,\n+\t\t\t\t  last_gather_stats->ip4txpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4txfrag,\n+\t\t\t\t  last_gather_stats->ip4txfrag,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip4txmcpkts,\n+\t\t\t\t  last_gather_stats->ip4txmcpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6RXOCTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6rxocts,\n+\t\t\t\t  last_gather_stats->ip6rxocts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6RXPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6rxpkts,\n+\t\t\t\t  last_gather_stats->ip6rxpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6txfrags,\n+\t\t\t\t  last_gather_stats->ip6txfrags,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6rxmcpkts,\n+\t\t\t\t  last_gather_stats->ip6rxmcpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6TXOCTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6txocts,\n+\t\t\t\t  last_gather_stats->ip6txocts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6TXPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6txpkts,\n+\t\t\t\t  last_gather_stats->ip6txpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6txfrags,\n+\t\t\t\t  last_gather_stats->ip6txfrags,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->ip6txmcpkts,\n+\t\t\t\t  last_gather_stats->ip6txmcpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_TCPRXSEGS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->tcprxsegs,\n+\t\t\t\t  last_gather_stats->tcprxsegs,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_TCPTXSEG] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->tcptxsegs,\n+\t\t\t\t  last_gather_stats->tcptxsegs,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMARXRDS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmarxrds,\n+\t\t\t\t  last_gather_stats->rdmarxrds,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMARXSNDS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmarxsnds,\n+\t\t\t\t  last_gather_stats->rdmarxsnds,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMARXWRS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmarxwrs,\n+\t\t\t\t  last_gather_stats->rdmarxwrs,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMATXRDS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmatxrds,\n+\t\t\t\t  last_gather_stats->rdmatxrds,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMATXSNDS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmatxsnds,\n+\t\t\t\t  last_gather_stats->rdmatxsnds,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMATXWRS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmatxwrs,\n+\t\t\t\t  last_gather_stats->rdmatxwrs,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMAVBND] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmavbn,\n+\t\t\t\t  last_gather_stats->rdmavbn,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RDMAVINV] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rdmavinv,\n+\t\t\t\t  last_gather_stats->rdmavinv,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_UDPRXPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->udprxpkts,\n+\t\t\t\t  last_gather_stats->udprxpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_UDPTXPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->udptxpkts,\n+\t\t\t\t  last_gather_stats->udptxpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tstats_val[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] +=\n+\t\tIRDMA_STATS_DELTA(gather_stats->rxnpecnmrkpkts,\n+\t\t\t\t  last_gather_stats->rxnpecnmrkpkts,\n+\t\t\t\t  IRDMA_MAX_STATS_48);\n+\tmemcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));\n+}\ndiff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h\nnew file mode 100644\nindex 0000000..089846d\n--- /dev/null\n+++ b/drivers/infiniband/hw/irdma/defs.h\n@@ -0,0 +1,2126 @@\n+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */\n+/* Copyright (c) 2019, Intel Corporation. */\n+\n+#ifndef IRDMA_DEFS_H\n+#define IRDMA_DEFS_H\n+\n+#define IRDMA_FIRST_USER_QP_ID\t3\n+\n+#define ECN_CODE_PT_VAL\t2\n+\n+#define IRDMA_PUSH_OFFSET\t\t(8 * 1024 * 1024)\n+#define IRDMA_PF_FIRST_PUSH_PAGE_INDEX\t16\n+#define IRDMA_VF_PUSH_OFFSET\t\t((8 + 64) * 1024)\n+#define IRDMA_VF_FIRST_PUSH_PAGE_INDEX\t2\n+#define IRDMA_VF_STATS_SIZE_V0\t280\n+\n+#define IRDMA_PE_DB_SIZE_4M\t1\n+#define IRDMA_PE_DB_SIZE_8M\t2\n+\n+enum irdma_protocol_used {\n+\tIRDMA_ANY_PROTOCOL = 0,\n+\tIRDMA_IWARP_PROTOCOL_ONLY = 1,\n+\tIRDMA_ROCE_PROTOCOL_ONLY = 2,\n+};\n+\n+#define IRDMA_QP_STATE_INVALID\t\t0\n+#define IRDMA_QP_STATE_IDLE\t\t1\n+#define IRDMA_QP_STATE_RTS\t\t2\n+#define IRDMA_QP_STATE_CLOSING\t\t3\n+#define IRDMA_QP_STATE_RTR\t\t4\n+#define IRDMA_QP_STATE_TERMINATE\t5\n+#define IRDMA_QP_STATE_ERROR\t\t6\n+\n+#define IRDMA_MAX_USER_PRIORITY\t\t8\n+#define IRDMA_MAX_APPS\t\t\t8\n+#define IRDMA_MAX_STATS_COUNT\t\t128\n+#define IRDMA_FIRST_NON_PF_STAT\t\t4\n+\n+#define IRDMA_MIN_MTU_IPV4\t576\n+#define IRDMA_MIN_MTU_IPV6\t1280\n+#define IRDMA_MTU_TO_MSS_IPV4\t40\n+#define IRDMA_MTU_TO_MSS_IPV6\t60\n+#define IRDMA_DEFAULT_MTU\t1500\n+\n+#define IRDMA_MAX_ENCODED_IRD_SIZE\t4\n+\n+#define Q2_FPSN_OFFSET\t\t64\n+#define TERM_DDP_LEN_TAGGED\t14\n+#define TERM_DDP_LEN_UNTAGGED\t18\n+#define TERM_RDMA_LEN\t\t28\n+#define RDMA_OPCODE_M\t\t0x0f\n+#define RDMA_READ_REQ_OPCODE\t1\n+#define Q2_BAD_FRAME_OFFSET\t72\n+#define CQE_MAJOR_DRV\t\t0x8000\n+\n+#define IRDMA_TERM_SENT\t\t1\n+#define IRDMA_TERM_RCVD\t\t2\n+#define IRDMA_TERM_DONE\t\t4\n+#define IRDMA_MAC_HLEN\t\t14\n+#define IRDMA_CQP_WAIT_POLL_REGS\t1\n+#define IRDMA_CQP_WAIT_POLL_CQ\t\t2\n+#define IRDMA_CQP_WAIT_EVENT\t\t3\n+\n+#define IRDMA_AE_SOURCE_RSVD\t\t0x0\n+#define IRDMA_AE_SOURCE_RQ\t\t0x1\n+#define IRDMA_AE_SOURCE_RQ_0011\t\t0x3\n+\n+#define IRDMA_AE_SOURCE_CQ\t\t0x2\n+#define IRDMA_AE_SOURCE_CQ_0110\t\t0x6\n+#define IRDMA_AE_SOURCE_CQ_1010\t\t0xa\n+#define IRDMA_AE_SOURCE_CQ_1110\t\t0xe\n+\n+#define IRDMA_AE_SOURCE_SQ\t\t0x5\n+#define IRDMA_AE_SOURCE_SQ_0111\t\t0x7\n+\n+#define IRDMA_AE_SOURCE_IN_RR_WR\t0x9\n+#define IRDMA_AE_SOURCE_IN_RR_WR_1011\t0xb\n+#define IRDMA_AE_SOURCE_OUT_RR\t\t0xd\n+#define IRDMA_AE_SOURCE_OUT_RR_1111\t0xf\n+\n+#define IRDMA_TCP_STATE_NON_EXISTENT\t0\n+#define IRDMA_TCP_STATE_CLOSED\t\t1\n+#define IRDMA_TCP_STATE_LISTEN\t\t2\n+#define IRDMA_STATE_SYN_SEND\t\t3\n+#define IRDMA_TCP_STATE_SYN_RECEIVED\t4\n+#define IRDMA_TCP_STATE_ESTABLISHED\t5\n+#define IRDMA_TCP_STATE_CLOSE_WAIT\t6\n+#define IRDMA_TCP_STATE_FIN_WAIT_1\t7\n+#define IRDMA_TCP_STATE_CLOSING\t\t8\n+#define IRDMA_TCP_STATE_LAST_ACK\t9\n+#define IRDMA_TCP_STATE_FIN_WAIT_2\t10\n+#define IRDMA_TCP_STATE_TIME_WAIT\t11\n+#define IRDMA_TCP_STATE_RESERVED_1\t12\n+#define IRDMA_TCP_STATE_RESERVED_2\t13\n+#define IRDMA_TCP_STATE_RESERVED_3\t14\n+#define IRDMA_TCP_STATE_RESERVED_4\t15\n+\n+#define IRDMA_CQP_SW_SQSIZE_4\t\t4\n+#define IRDMA_CQP_SW_SQSIZE_2048\t2048\n+\n+#define IRDMA_CQ_TYPE_IWARP\t1\n+#define IRDMA_CQ_TYPE_ILQ\t2\n+#define IRDMA_CQ_TYPE_IEQ\t3\n+#define IRDMA_CQ_TYPE_CQP\t4\n+/* CQP SQ WQES */\n+#define IRDMA_QP_TYPE_IWARP\t1\n+#define IRDMA_QP_TYPE_UDA\t2\n+#define IRDMA_QP_TYPE_ROCE_RC\t3\n+#define IRDMA_QP_TYPE_ROCE_UD\t4\n+\n+#define IRDMA_DONE_COUNT\t1000\n+#define IRDMA_SLEEP_COUNT\t10\n+\n+#define IRDMA_UPDATE_SD_BUFF_SIZE\t128\n+#define IRDMA_FEATURE_BUF_SIZE\t\t(8 * IRDMA_MAX_FEATURES)\n+\n+#define IRDMA_MAX_QUANTA_PER_WR\t8\n+\n+#define IRDMA_QP_SW_MAX_WQ_QUANTA\t32768\n+#define IRDMA_QP_SW_MAX_SQ_QUANTA\t32768\n+#define IRDMA_QP_SW_MAX_RQ_QUANTA\t32768\n+#define IRDMA_MAX_QP_WRS (((IRDMA_QP_SW_MAX_WQ_QUANTA - IRDMA_SQ_RSVD) / IRDMA_MAX_QUANTA_PER_WR))\n+\n+#define IRDMAQP_TERM_SEND_TERM_AND_FIN\t\t0\n+#define IRDMAQP_TERM_SEND_TERM_ONLY\t\t1\n+#define IRDMAQP_TERM_SEND_FIN_ONLY\t\t2\n+#define IRDMAQP_TERM_DONOT_SEND_TERM_OR_FIN\t3\n+\n+#define IRDMA_HW_PAGE_SIZE\t4096\n+#define IRDMA_CQE_QTYPE_RQ\t0\n+#define IRDMA_CQE_QTYPE_SQ\t1\n+\n+#define IRDMA_QP_SW_MIN_WQSIZE\t8u /* in WRs*/\n+#define IRDMA_QP_WQE_MIN_SIZE\t32\n+#define IRDMA_QP_WQE_MAX_SIZE\t256\n+#define IRDMA_QP_WQE_MIN_QUANTA 1\n+#define IRDMA_MAX_RQ_WQE_SHIFT_GEN1 2\n+\n+#define IRDMA_SQ_RSVD\t258\n+#define IRDMA_RQ_RSVD\t1\n+\n+#define IRDMA_FEATURE_RTS_AE\t\t\t1ULL\n+#define IRDMA_FEATURE_CQ_RESIZE\t\t\t2ULL\n+\n+#define IRDMAQP_OP_RDMA_WRITE\t\t\t0x00\n+#define IRDMAQP_OP_RDMA_READ\t\t\t0x01\n+#define IRDMAQP_OP_RDMA_SEND\t\t\t0x03\n+#define IRDMAQP_OP_RDMA_SEND_INV\t\t0x04\n+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT\t\t0x05\n+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT_INV\t0x06\n+#define IRDMAQP_OP_BIND_MW\t\t\t0x08\n+#define IRDMAQP_OP_FAST_REGISTER\t\t0x09\n+#define IRDMAQP_OP_LOCAL_INVALIDATE\t\t0x0a\n+#define IRDMAQP_OP_RDMA_READ_LOC_INV\t\t0x0b\n+#define IRDMAQP_OP_NOP\t\t\t\t0x0c\n+#define IRDMAQP_OP_RDMA_WRITE_SOL\t\t0x0d\n+#define IRDMAQP_OP_GEN_RTS_AE\t\t\t0x30\n+\n+#define IRDMA_OP_CEQ_DESTROY\t\t\t1\n+#define IRDMA_OP_AEQ_DESTROY\t\t\t2\n+#define IRDMA_OP_DELETE_ARP_CACHE_ENTRY\t\t3\n+#define IRDMA_OP_MANAGE_APBVT_ENTRY\t\t4\n+#define IRDMA_OP_CEQ_CREATE\t\t\t5\n+#define IRDMA_OP_AEQ_CREATE\t\t\t6\n+#define IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY\t7\n+#define IRDMA_OP_QP_MODIFY\t\t\t8\n+#define IRDMA_OP_QP_UPLOAD_CONTEXT\t\t9\n+#define IRDMA_OP_CQ_CREATE\t\t\t10\n+#define IRDMA_OP_CQ_DESTROY\t\t\t11\n+#define IRDMA_OP_QP_CREATE\t\t\t12\n+#define IRDMA_OP_QP_DESTROY\t\t\t13\n+#define IRDMA_OP_ALLOC_STAG\t\t\t14\n+#define IRDMA_OP_MR_REG_NON_SHARED\t\t15\n+#define IRDMA_OP_DEALLOC_STAG\t\t\t16\n+#define IRDMA_OP_MW_ALLOC\t\t\t17\n+#define IRDMA_OP_QP_FLUSH_WQES\t\t\t18\n+#define IRDMA_OP_ADD_ARP_CACHE_ENTRY\t\t19\n+#define IRDMA_OP_MANAGE_PUSH_PAGE\t\t20\n+#define IRDMA_OP_UPDATE_PE_SDS\t\t\t21\n+#define IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE\t22\n+#define IRDMA_OP_SUSPEND\t\t\t23\n+#define IRDMA_OP_RESUME\t\t\t\t24\n+#define IRDMA_OP_MANAGE_VF_PBLE_BP\t\t25\n+#define IRDMA_OP_QUERY_FPM_VAL\t\t\t26\n+#define IRDMA_OP_COMMIT_FPM_VAL\t\t\t27\n+#define IRDMA_OP_REQ_CMDS\t\t\t28\n+#define IRDMA_OP_CMPL_CMDS\t\t\t29\n+#define IRDMA_OP_AH_CREATE\t\t\t30\n+#define IRDMA_OP_AH_MODIFY\t\t\t31\n+#define IRDMA_OP_AH_DESTROY\t\t\t32\n+#define IRDMA_OP_MC_CREATE\t\t\t33\n+#define IRDMA_OP_MC_DESTROY\t\t\t34\n+#define IRDMA_OP_MC_MODIFY\t\t\t35\n+#define IRDMA_OP_STATS_ALLOCATE\t\t\t36\n+#define IRDMA_OP_STATS_FREE\t\t\t37\n+#define IRDMA_OP_STATS_GATHER\t\t\t38\n+#define IRDMA_OP_WS_ADD_NODE\t\t\t39\n+#define IRDMA_OP_WS_MODIFY_NODE\t\t\t40\n+#define IRDMA_OP_WS_DELETE_NODE\t\t\t41\n+#define IRDMA_OP_SET_UP_MAP\t\t\t42\n+#define IRDMA_OP_GEN_AE\t\t\t\t43\n+#define IRDMA_OP_QUERY_RDMA_FEATURES\t\t44\n+#define IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY\t\t45\n+#define IRDMA_OP_ADD_LOCAL_MAC_ENTRY\t\t46\n+#define IRDMA_OP_DELETE_LOCAL_MAC_ENTRY\t\t47\n+#define IRDMA_OP_CQ_MODIFY                      48\n+#define IRDMA_OP_SIZE_CQP_STAT_ARRAY\t\t49\n+\n+#define IRDMA_CQP_OP_CREATE_QP\t\t\t\t0\n+#define IRDMA_CQP_OP_MODIFY_QP\t\t\t\t0x1\n+#define IRDMA_CQP_OP_DESTROY_QP\t\t\t\t0x02\n+#define IRDMA_CQP_OP_CREATE_CQ\t\t\t\t0x03\n+#define IRDMA_CQP_OP_MODIFY_CQ\t\t\t\t0x04\n+#define IRDMA_CQP_OP_DESTROY_CQ\t\t\t\t0x05\n+#define IRDMA_CQP_OP_ALLOC_STAG\t\t\t\t0x09\n+#define IRDMA_CQP_OP_REG_MR\t\t\t\t0x0a\n+#define IRDMA_CQP_OP_QUERY_STAG\t\t\t\t0x0b\n+#define IRDMA_CQP_OP_REG_SMR\t\t\t\t0x0c\n+#define IRDMA_CQP_OP_DEALLOC_STAG\t\t\t0x0d\n+#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE\t\t0x0e\n+#define IRDMA_CQP_OP_MANAGE_ARP\t\t\t\t0x0f\n+#define IRDMA_CQP_OP_MANAGE_VF_PBLE_BP\t\t\t0x10\n+#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES\t\t\t0x11\n+#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES\t\t0x12\n+#define IRDMA_CQP_OP_UPLOAD_CONTEXT\t\t\t0x13\n+#define IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY\t0x14\n+#define IRDMA_CQP_OP_UPLOAD_CONTEXT\t\t\t0x13\n+#define IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE\t\t0x15\n+#define IRDMA_CQP_OP_CREATE_CEQ\t\t\t\t0x16\n+#define IRDMA_CQP_OP_DESTROY_CEQ\t\t\t0x18\n+#define IRDMA_CQP_OP_CREATE_AEQ\t\t\t\t0x19\n+#define IRDMA_CQP_OP_DESTROY_AEQ\t\t\t0x1b\n+#define IRDMA_CQP_OP_CREATE_ADDR_HANDLE\t\t\t0x1c\n+#define IRDMA_CQP_OP_MODIFY_ADDR_HANDLE\t\t\t0x1d\n+#define IRDMA_CQP_OP_DESTROY_ADDR_HANDLE\t\t0x1e\n+#define IRDMA_CQP_OP_UPDATE_PE_SDS\t\t\t0x1f\n+#define IRDMA_CQP_OP_QUERY_FPM_VAL\t\t\t0x20\n+#define IRDMA_CQP_OP_COMMIT_FPM_VAL\t\t\t0x21\n+#define IRDMA_CQP_OP_FLUSH_WQES\t\t\t\t0x22\n+/* IRDMA_CQP_OP_GEN_AE is the same value as IRDMA_CQP_OP_FLUSH_WQES */\n+#define IRDMA_CQP_OP_GEN_AE\t\t\t\t0x22\n+#define IRDMA_CQP_OP_MANAGE_APBVT\t\t\t0x23\n+#define IRDMA_CQP_OP_NOP\t\t\t\t0x24\n+#define IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY\t0x25\n+#define IRDMA_CQP_OP_CREATE_MCAST_GRP\t\t\t0x26\n+#define IRDMA_CQP_OP_MODIFY_MCAST_GRP\t\t\t0x27\n+#define IRDMA_CQP_OP_DESTROY_MCAST_GRP\t\t\t0x28\n+#define IRDMA_CQP_OP_SUSPEND_QP\t\t\t\t0x29\n+#define IRDMA_CQP_OP_RESUME_QP\t\t\t\t0x2a\n+#define IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED\t\t0x2b\n+#define IRDMA_CQP_OP_WORK_SCHED_NODE\t\t\t0x2c\n+#define IRDMA_CQP_OP_MANAGE_STATS\t\t\t0x2d\n+#define IRDMA_CQP_OP_GATHER_STATS\t\t\t0x2e\n+#define IRDMA_CQP_OP_UP_MAP\t\t\t\t0x2f\n+\n+/* Async Events codes */\n+#define IRDMA_AE_AMP_UNALLOCATED_STAG\t\t\t\t\t0x0102\n+#define IRDMA_AE_AMP_INVALID_STAG\t\t\t\t\t0x0103\n+#define IRDMA_AE_AMP_BAD_QP\t\t\t\t\t\t0x0104\n+#define IRDMA_AE_AMP_BAD_PD\t\t\t\t\t\t0x0105\n+#define IRDMA_AE_AMP_BAD_STAG_KEY\t\t\t\t\t0x0106\n+#define IRDMA_AE_AMP_BAD_STAG_INDEX\t\t\t\t\t0x0107\n+#define IRDMA_AE_AMP_BOUNDS_VIOLATION\t\t\t\t\t0x0108\n+#define IRDMA_AE_AMP_RIGHTS_VIOLATION\t\t\t\t\t0x0109\n+#define IRDMA_AE_AMP_TO_WRAP\t\t\t\t\t\t0x010a\n+#define IRDMA_AE_AMP_FASTREG_VALID_STAG\t\t\t\t\t0x010c\n+#define IRDMA_AE_AMP_FASTREG_MW_STAG\t\t\t\t\t0x010d\n+#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS\t\t\t\t0x010e\n+#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH\t\t\t\t0x0110\n+#define IRDMA_AE_AMP_INVALIDATE_SHARED\t\t\t\t\t0x0111\n+#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS\t\t\t0x0112\n+#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS\t\t\t0x0113\n+#define IRDMA_AE_AMP_MWBIND_VALID_STAG\t\t\t\t\t0x0114\n+#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG\t\t\t\t\t0x0115\n+#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG\t\t\t\t0x0116\n+#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG\t\t\t\t\t0x0117\n+#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS\t\t\t\t0x0118\n+#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS\t\t\t\t0x0119\n+#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT\t\t\t\t0x011a\n+#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED\t\t\t\t0x011b\n+#define IRDMA_AE_PRIV_OPERATION_DENIED\t\t\t\t\t0x011c\n+#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW\t\t\t\t0x011d\n+#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW\t\t\t\t0x011e\n+#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG\t\t\t0x011f\n+#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH\t\t\t\t0x0121\n+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG\t\t\t\t0x0132\n+#define IRDMA_AE_UDA_XMIT_BAD_PD\t\t\t\t\t0x0133\n+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT\t\t\t\t0x0134\n+#define IRDMA_AE_UDA_L4LEN_INVALID\t\t\t\t\t0x0135\n+#define IRDMA_AE_BAD_CLOSE\t\t\t\t\t\t0x0201\n+#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE\t\t\t\t0x0202\n+#define IRDMA_AE_CQ_OPERATION_ERROR\t\t\t\t\t0x0203\n+#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO\t\t\t\t0x0205\n+#define IRDMA_AE_STAG_ZERO_INVALID\t\t\t\t\t0x0206\n+#define IRDMA_AE_IB_RREQ_AND_Q1_FULL\t\t\t\t\t0x0207\n+#define IRDMA_AE_IB_INVALID_REQUEST\t\t\t\t\t0x0208\n+#define IRDMA_AE_WQE_UNEXPECTED_OPCODE\t\t\t\t\t0x020a\n+#define IRDMA_AE_WQE_INVALID_PARAMETER\t\t\t\t\t0x020b\n+#define IRDMA_AE_WQE_INVALID_FRAG_DATA\t\t\t\t\t0x020c\n+#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR\t\t\t\t\t0x020d\n+#define IRDMA_AE_IB_REMOTE_OP_ERROR\t\t\t\t\t0x020e\n+#define IRDMA_AE_WQE_LSMM_TOO_LONG\t\t\t\t\t0x0220\n+#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN\t\t\t\t0x0301\n+#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER\t0x0303\n+#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION\t\t\t\t0x0304\n+#define IRDMA_AE_DDP_UBE_INVALID_MO\t\t\t\t\t0x0305\n+#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE\t\t0x0306\n+#define IRDMA_AE_DDP_UBE_INVALID_QN\t\t\t\t\t0x0307\n+#define IRDMA_AE_DDP_NO_L_BIT\t\t\t\t\t\t0x0308\n+#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION\t\t\t0x0311\n+#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE\t\t\t\t0x0312\n+#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST\t\t\t\t0x0313\n+#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP\t\t\t0x0314\n+#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR\t\t\t\t\t0x0316\n+#define IRDMA_AE_ROCE_EMPTY_MCG\t\t\t\t\t\t0x0380\n+#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR\t\t\t\t\t0x0381\n+#define IRDMA_AE_ROCE_BAD_MC_QPID\t\t\t\t\t0x0382\n+#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH\t\t\t\t0x0383\n+#define IRDMA_AE_INVALID_ARP_ENTRY\t\t\t\t\t0x0401\n+#define IRDMA_AE_INVALID_TCP_OPTION_RCVD\t\t\t\t0x0402\n+#define IRDMA_AE_STALE_ARP_ENTRY\t\t\t\t\t0x0403\n+#define IRDMA_AE_INVALID_AH_ENTRY\t\t\t\t\t0x0406\n+#define IRDMA_AE_LLP_CLOSE_COMPLETE\t\t\t\t\t0x0501\n+#define IRDMA_AE_LLP_CONNECTION_RESET\t\t\t\t\t0x0502\n+#define IRDMA_AE_LLP_FIN_RECEIVED\t\t\t\t\t0x0503\n+#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR\t\t\t\t0x0505\n+#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL\t\t\t\t\t0x0507\n+#define IRDMA_AE_LLP_SYN_RECEIVED\t\t\t\t\t0x0508\n+#define IRDMA_AE_LLP_TERMINATE_RECEIVED\t\t\t\t\t0x0509\n+#define IRDMA_AE_LLP_TOO_MANY_RETRIES\t\t\t\t\t0x050a\n+#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES\t\t\t\t0x050b\n+#define IRDMA_AE_LLP_DOUBT_REACHABILITY\t\t\t\t\t0x050c\n+#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED\t\t\t\t0x050e\n+#define IRDMA_AE_RESOURCE_EXHAUSTION\t\t\t\t\t0x0520\n+#define IRDMA_AE_RESET_SENT\t\t\t\t\t\t0x0601\n+#define IRDMA_AE_TERMINATE_SENT\t\t\t\t\t\t0x0602\n+#define IRDMA_AE_RESET_NOT_SENT\t\t\t\t\t\t0x0603\n+#define IRDMA_AE_LCE_QP_CATASTROPHIC\t\t\t\t\t0x0700\n+#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC\t\t\t\t0x0701\n+#define IRDMA_AE_LCE_CQ_CATASTROPHIC\t\t\t\t\t0x0702\n+#define IRDMA_AE_QP_SUSPEND_COMPLETE\t\t\t\t\t0x0900\n+\n+#define LS_64_1(val, bits)\t((u64)(uintptr_t)(val) << (bits))\n+#define RS_64_1(val, bits)\t((u64)(uintptr_t)(val) >> (bits))\n+#define LS_32_1(val, bits)\t(u32)((val) << (bits))\n+#define RS_32_1(val, bits)\t(u32)((val) >> (bits))\n+#define LS_64(val, field)\t(((u64)(val) << field ## _S) & (field ## _M))\n+#define RS_64(val, field)\t((u64)((val) & field ## _M) >> field ## _S)\n+#define LS_32(val, field)\t(((val) << field ## _S) & (field ## _M))\n+#define RS_32(val, field)\t(((val) & field ## _M) >> field ## _S)\n+\n+#define FLD_LS_64(dev, val, field)\t\\\n+\t(((u64)(val) << (dev)->hw_shifts[field ## _S]) & (dev)->hw_masks[field ## _M])\n+#define FLD_RS_64(dev, val, field)\t\\\n+\t((u64)((val) & (dev)->hw_masks[field ## _M]) >> (dev)->hw_shifts[field ## _S])\n+#define FLD_LS_32(dev, val, field)\t\\\n+\t(((val) << (dev)->hw_shifts[field ## _S]) & (dev)->hw_masks[field ## _M])\n+#define FLD_RS_32(dev, val, field)\t\\\n+\t((u64)((val) & (dev)->hw_masks[field ## _M]) >> (dev)->hw_shifts[field ## _S])\n+\n+#define FW_MAJOR_VER(dev)\t\\\n+\t((u16)RS_64((dev)->feature_info[IRDMA_FEATURE_FW_INFO], IRDMA_FW_VER_MAJOR))\n+#define FW_MINOR_VER(dev)\t\\\n+\t((u16)RS_64((dev)->feature_info[IRDMA_FEATURE_FW_INFO], IRDMA_FW_VER_MINOR))\n+\n+#define IRDMA_STATS_DELTA(a, b, c) ((a) >= (b) ? (a) - (b) : (a) + (c) - (b))\n+#define IRDMA_MAX_STATS_32\t0xFFFFFFFFULL\n+#define IRDMA_MAX_STATS_48\t0xFFFFFFFFFFFFULL\n+\n+#define IRDMA_MAX_CQ_READ_THRESH 0x3FFFF\n+/* ILQ CQP hash table fields */\n+#define IRDMA_CQPSQ_QHASH_VLANID_S 32\n+#define IRDMA_CQPSQ_QHASH_VLANID_M \\\n+\t((u64)0xfff << IRDMA_CQPSQ_QHASH_VLANID_S)\n+\n+#define IRDMA_CQPSQ_QHASH_QPN_S 32\n+#define IRDMA_CQPSQ_QHASH_QPN_M \\\n+\t((u64)0x3ffff << IRDMA_CQPSQ_QHASH_QPN_S)\n+\n+#define IRDMA_CQPSQ_QHASH_QS_HANDLE_S 0\n+#define IRDMA_CQPSQ_QHASH_QS_HANDLE_M ((u64)0x3ff << IRDMA_CQPSQ_QHASH_QS_HANDLE_S)\n+\n+#define IRDMA_CQPSQ_QHASH_SRC_PORT_S 16\n+#define IRDMA_CQPSQ_QHASH_SRC_PORT_M \\\n+\t((u64)0xffff << IRDMA_CQPSQ_QHASH_SRC_PORT_S)\n+\n+#define IRDMA_CQPSQ_QHASH_DEST_PORT_S 0\n+#define IRDMA_CQPSQ_QHASH_DEST_PORT_M \\\n+\t((u64)0xffff << IRDMA_CQPSQ_QHASH_DEST_PORT_S)\n+\n+#define IRDMA_CQPSQ_QHASH_ADDR0_S 32\n+#define IRDMA_CQPSQ_QHASH_ADDR0_M \\\n+\t((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR0_S)\n+\n+#define IRDMA_CQPSQ_QHASH_ADDR1_S 0\n+#define IRDMA_CQPSQ_QHASH_ADDR1_M \\\n+\t((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR1_S)\n+\n+#define IRDMA_CQPSQ_QHASH_ADDR2_S 32\n+#define IRDMA_CQPSQ_QHASH_ADDR2_M \\\n+\t((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR2_S)\n+\n+#define IRDMA_CQPSQ_QHASH_ADDR3_S 0\n+#define IRDMA_CQPSQ_QHASH_ADDR3_M \\\n+\t((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR3_S)\n+\n+#define IRDMA_CQPSQ_QHASH_WQEVALID_S 63\n+#define IRDMA_CQPSQ_QHASH_WQEVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QHASH_WQEVALID_S)\n+#define IRDMA_CQPSQ_QHASH_OPCODE_S 32\n+#define IRDMA_CQPSQ_QHASH_OPCODE_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_QHASH_OPCODE_S)\n+\n+#define IRDMA_CQPSQ_QHASH_MANAGE_S 61\n+#define IRDMA_CQPSQ_QHASH_MANAGE_M \\\n+\t((u64)0x3 << IRDMA_CQPSQ_QHASH_MANAGE_S)\n+\n+#define IRDMA_CQPSQ_QHASH_IPV4VALID_S 60\n+#define IRDMA_CQPSQ_QHASH_IPV4VALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QHASH_IPV4VALID_S)\n+\n+#define IRDMA_CQPSQ_QHASH_VLANVALID_S 59\n+#define IRDMA_CQPSQ_QHASH_VLANVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QHASH_VLANVALID_S)\n+\n+#define IRDMA_CQPSQ_QHASH_ENTRYTYPE_S 42\n+#define IRDMA_CQPSQ_QHASH_ENTRYTYPE_M \\\n+\t((u64)0x7 << IRDMA_CQPSQ_QHASH_ENTRYTYPE_S)\n+\n+/* Stats */\n+#define IRDMA_CQPSQ_STATS_WQEVALID_S 63\n+#define IRDMA_CQPSQ_STATS_WQEVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STATS_WQEVALID_S)\n+\n+#define IRDMA_CQPSQ_STATS_ALLOC_INST_S 62\n+#define IRDMA_CQPSQ_STATS_ALLOC_INST_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STATS_ALLOC_INST_S)\n+\n+#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_S 60\n+#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_S)\n+\n+#define IRDMA_CQPSQ_STATS_USE_INST_S 61\n+#define IRDMA_CQPSQ_STATS_USE_INST_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STATS_USE_INST_S)\n+\n+#define IRDMA_CQPSQ_STATS_OP_S 32\n+#define IRDMA_CQPSQ_STATS_OP_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_STATS_OP_S)\n+\n+#define IRDMA_CQPSQ_STATS_INST_INDEX_S 0\n+#define IRDMA_CQPSQ_STATS_INST_INDEX_M \\\n+\t((u64)0x7f << IRDMA_CQPSQ_STATS_INST_INDEX_S)\n+\n+#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_S 0\n+#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_S)\n+\n+/* WS - Work Scheduler */\n+#define IRDMA_CQPSQ_WS_WQEVALID_S 63\n+#define IRDMA_CQPSQ_WS_WQEVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_WS_WQEVALID_S)\n+\n+#define IRDMA_CQPSQ_WS_NODEOP_S 52\n+#define IRDMA_CQPSQ_WS_NODEOP_M \\\n+\t((u64)0x3 << IRDMA_CQPSQ_WS_NODEOP_S)\n+\n+#define IRDMA_CQPSQ_WS_ENABLENODE_S 62\n+#define IRDMA_CQPSQ_WS_ENABLENODE_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_WS_ENABLENODE_S)\n+\n+#define IRDMA_CQPSQ_WS_NODETYPE_S 61\n+#define IRDMA_CQPSQ_WS_NODETYPE_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_WS_NODETYPE_S)\n+\n+#define IRDMA_CQPSQ_WS_PRIOTYPE_S 59\n+#define IRDMA_CQPSQ_WS_PRIOTYPE_M \\\n+\t((u64)0x3 << IRDMA_CQPSQ_WS_PRIOTYPE_S)\n+\n+#define IRDMA_CQPSQ_WS_TC_S 56\n+#define IRDMA_CQPSQ_WS_TC_M \\\n+\t((u64)0x7 << IRDMA_CQPSQ_WS_TC_S)\n+\n+#define IRDMA_CQPSQ_WS_VMVFTYPE_S 54\n+#define IRDMA_CQPSQ_WS_VMVFTYPE_M \\\n+\t((u64)0x3 << IRDMA_CQPSQ_WS_VMVFTYPE_S)\n+\n+#define IRDMA_CQPSQ_WS_VMVFNUM_S 42\n+#define IRDMA_CQPSQ_WS_VMVFNUM_M \\\n+\t((u64)0x3ff << IRDMA_CQPSQ_WS_VMVFNUM_S)\n+\n+#define IRDMA_CQPSQ_WS_OP_S 32\n+#define IRDMA_CQPSQ_WS_OP_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_WS_OP_S)\n+\n+#define IRDMA_CQPSQ_WS_PARENTID_S 16\n+#define IRDMA_CQPSQ_WS_PARENTID_M \\\n+\t((u64)0x3ff << IRDMA_CQPSQ_WS_PARENTID_S)\n+\n+#define IRDMA_CQPSQ_WS_NODEID_S 0\n+#define IRDMA_CQPSQ_WS_NODEID_M \\\n+\t((u64)0x3ff << IRDMA_CQPSQ_WS_NODEID_S)\n+\n+#define IRDMA_CQPSQ_WS_VSI_S 48\n+#define IRDMA_CQPSQ_WS_VSI_M \\\n+\t((u64)0x3ff << IRDMA_CQPSQ_WS_VSI_S)\n+\n+#define IRDMA_CQPSQ_WS_WEIGHT_S 32\n+#define IRDMA_CQPSQ_WS_WEIGHT_M \\\n+\t((u64)0x7f << IRDMA_CQPSQ_WS_WEIGHT_S)\n+\n+/* UP to UP mapping */\n+#define IRDMA_CQPSQ_UP_WQEVALID_S 63\n+#define IRDMA_CQPSQ_UP_WQEVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UP_WQEVALID_S)\n+\n+#define IRDMA_CQPSQ_UP_USEVLAN_S 62\n+#define IRDMA_CQPSQ_UP_USEVLAN_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UP_USEVLAN_S)\n+\n+#define IRDMA_CQPSQ_UP_USEOVERRIDE_S 61\n+#define IRDMA_CQPSQ_UP_USEOVERRIDE_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UP_USEOVERRIDE_S)\n+\n+#define IRDMA_CQPSQ_UP_OP_S 32\n+#define IRDMA_CQPSQ_UP_OP_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_UP_OP_S)\n+\n+#define IRDMA_CQPSQ_UP_HMCFCNIDX_S 0\n+#define IRDMA_CQPSQ_UP_HMCFCNIDX_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_UP_HMCFCNIDX_S)\n+\n+#define IRDMA_CQPSQ_UP_CNPOVERRIDE_S 32\n+#define IRDMA_CQPSQ_UP_CNPOVERRIDE_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_UP_CNPOVERRIDE_S)\n+\n+/* Query RDMA features*/\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_S 63\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_S)\n+\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_S 0\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_M \\\n+\t((u64)0xffffffff << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_S)\n+\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_S 32\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_M \\\n+\t((u64)0x3f << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_S)\n+\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED_S 32\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED_M \\\n+\t(0xffffULL << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED_S)\n+\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION_S 16\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION_M \\\n+\t(0xffULL << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION_S)\n+\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION_S 0\n+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION_M \\\n+\t(0xffULL << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION_S)\n+\n+/* CQP Host Context */\n+#define IRDMA_CQPHC_EN_DC_TCP_S 25\n+#define IRDMA_CQPHC_EN_DC_TCP_M BIT_ULL(IRDMA_CQPHC_EN_DC_TCP_S)\n+\n+#define IRDMA_CQPHC_SQSIZE_S 8\n+#define IRDMA_CQPHC_SQSIZE_M (0xfULL << IRDMA_CQPHC_SQSIZE_S)\n+\n+#define IRDMA_CQPHC_DISABLE_PFPDUS_S 1\n+#define IRDMA_CQPHC_DISABLE_PFPDUS_M BIT_ULL(IRDMA_CQPHC_DISABLE_PFPDUS_S)\n+\n+#define IRDMA_CQPHC_ROCEV2_RTO_POLICY_S 2\n+#define IRDMA_CQPHC_ROCEV2_RTO_POLICY_M BIT_ULL(IRDMA_CQPHC_ROCEV2_RTO_POLICY_S)\n+\n+#define IRDMA_CQPHC_PROTOCOL_USED_S 3\n+#define IRDMA_CQPHC_PROTOCOL_USED_M (0x3ULL << IRDMA_CQPHC_PROTOCOL_USED_S)\n+\n+#define IRDMA_CQPHC_HW_MINVER_S 0\n+#define IRDMA_CQPHC_HW_MINVER_M (0xffffULL << IRDMA_CQPHC_HW_MINVER_S)\n+\n+#define IRDMA_CQPHC_HW_MAJVER_S 16\n+#define IRDMA_CQPHC_HW_MAJVER_M (0xffffULL << IRDMA_CQPHC_HW_MAJVER_S)\n+\n+#define IRDMA_CQPHC_STRUCTVER_S 24\n+#define IRDMA_CQPHC_STRUCTVER_M (0xffULL << IRDMA_CQPHC_STRUCTVER_S)\n+\n+#define IRDMA_CQPHC_CEQPERVF_S 32\n+#define IRDMA_CQPHC_CEQPERVF_M (0xffULL << IRDMA_CQPHC_CEQPERVF_S)\n+\n+#define IRDMA_CQPHC_ENABLED_VFS_S 32\n+#define IRDMA_CQPHC_ENABLED_VFS_M (0x3fULL << IRDMA_CQPHC_ENABLED_VFS_S)\n+\n+#define IRDMA_CQPHC_HMC_PROFILE_S 0\n+#define IRDMA_CQPHC_HMC_PROFILE_M (0x7ULL << IRDMA_CQPHC_HMC_PROFILE_S)\n+\n+#define IRDMA_CQPHC_SVER_S 24\n+#define IRDMA_CQPHC_SVER_M (0xffULL << IRDMA_CQPHC_SVER_S)\n+\n+#define IRDMA_CQPHC_SQBASE_S 9\n+#define IRDMA_CQPHC_SQBASE_M \\\n+\t(0xfffffffffffffeULL << IRDMA_CQPHC_SQBASE_S)\n+\n+#define IRDMA_CQPHC_QPCTX_S 0\n+#define IRDMA_CQPHC_QPCTX_M \\\n+\t(0xffffffffffffffffULL << IRDMA_CQPHC_QPCTX_S)\n+\n+/* iWARP QP Doorbell shadow area */\n+#define IRDMA_QP_DBSA_HW_SQ_TAIL_S 0\n+#define IRDMA_QP_DBSA_HW_SQ_TAIL_M \\\n+\t(0x7fffULL << IRDMA_QP_DBSA_HW_SQ_TAIL_S)\n+\n+/* Completion Queue Doorbell shadow area */\n+#define IRDMA_CQ_DBSA_CQEIDX_S 0\n+#define IRDMA_CQ_DBSA_CQEIDX_M (0xfffffULL << IRDMA_CQ_DBSA_CQEIDX_S)\n+\n+#define IRDMA_CQ_DBSA_SW_CQ_SELECT_S 0\n+#define IRDMA_CQ_DBSA_SW_CQ_SELECT_M \\\n+\t(0x3fffULL << IRDMA_CQ_DBSA_SW_CQ_SELECT_S)\n+\n+#define IRDMA_CQ_DBSA_ARM_NEXT_S 14\n+#define IRDMA_CQ_DBSA_ARM_NEXT_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_S)\n+\n+#define IRDMA_CQ_DBSA_ARM_NEXT_SE_S 15\n+#define IRDMA_CQ_DBSA_ARM_NEXT_SE_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_SE_S)\n+\n+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_S 16\n+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_M \\\n+\t(0x3ULL << IRDMA_CQ_DBSA_ARM_SEQ_NUM_S)\n+\n+/* CQP and iWARP Completion Queue */\n+#define IRDMA_CQ_QPCTX_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQ_QPCTX_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMA_CCQ_OPRETVAL_S 0\n+#define IRDMA_CCQ_OPRETVAL_M (0xffffffffULL << IRDMA_CCQ_OPRETVAL_S)\n+\n+#define IRDMA_CQ_MINERR_S 0\n+#define IRDMA_CQ_MINERR_M (0xffffULL << IRDMA_CQ_MINERR_S)\n+\n+#define IRDMA_CQ_MAJERR_S 16\n+#define IRDMA_CQ_MAJERR_M (0xffffULL << IRDMA_CQ_MAJERR_S)\n+\n+#define IRDMA_CQ_WQEIDX_S 32\n+#define IRDMA_CQ_WQEIDX_M (0x7fffULL << IRDMA_CQ_WQEIDX_S)\n+\n+#define IRDMA_CQ_EXTCQE_S 50\n+#define IRDMA_CQ_EXTCQE_M BIT_ULL(IRDMA_CQ_EXTCQE_S)\n+\n+#define IRDMA_CQ_ERROR_S 55\n+#define IRDMA_CQ_ERROR_M BIT_ULL(IRDMA_CQ_ERROR_S)\n+\n+#define IRDMA_CQ_SQ_S 62\n+#define IRDMA_CQ_SQ_M BIT_ULL(IRDMA_CQ_SQ_S)\n+\n+#define IRDMA_CQ_VALID_S 63\n+#define IRDMA_CQ_VALID_M BIT_ULL(IRDMA_CQ_VALID_S)\n+\n+#define IRDMA_CQ_IMMVALID_S 62\n+#define IRDMA_CQ_IMMVALID_M BIT_ULL(IRDMA_CQ_IMMVALID_S)\n+\n+#define IRDMA_CQ_UDSMACVALID_S 61\n+#define IRDMA_CQ_UDSMACVALID_M BIT_ULL(IRDMA_CQ_UDSMACVALID_S)\n+\n+#define IRDMA_CQ_UDVLANVALID_S 60\n+#define IRDMA_CQ_UDVLANVALID_M BIT_ULL(IRDMA_CQ_UDVLANVALID_S)\n+\n+#define IRDMA_CQ_UDSMAC_S 0\n+#define IRDMA_CQ_UDSMAC_M (0xffffffffffffULL << IRDMA_CQ_UDSMAC_S)\n+\n+#define IRDMA_CQ_UDVLAN_S 48\n+#define IRDMA_CQ_UDVLAN_M (0xffffULL << IRDMA_CQ_UDVLAN_S)\n+\n+#define IRDMA_CQ_IMMDATA_S 0\n+#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S)\n+\n+#define IRDMA_CQ_IMMDATALOW32_S 0\n+#define IRDMA_CQ_IMMDATALOW32_M (0xffffffffULL << IRDMA_CQ_IMMDATALOW32_S)\n+\n+#define IRDMA_CQ_IMMDATAUP32_S 32\n+#define IRDMA_CQ_IMMDATAUP32_M (0xffffffffULL << IRDMA_CQ_IMMDATAUP32_S)\n+\n+#define IRDMACQ_PAYLDLEN_S 0\n+#define IRDMACQ_PAYLDLEN_M (0xffffffffULL << IRDMACQ_PAYLDLEN_S)\n+\n+#define IRDMACQ_TCPSEQNUMRTT_S 32\n+#define IRDMACQ_TCPSEQNUMRTT_M (0xffffffffULL << IRDMACQ_TCPSEQNUMRTT_S)\n+\n+#define IRDMACQ_INVSTAG_S 0\n+#define IRDMACQ_INVSTAG_M (0xffffffffULL << IRDMACQ_INVSTAG_S)\n+\n+#define IRDMACQ_QPID_S 32\n+#define IRDMACQ_QPID_M (0x3ffffULL << IRDMACQ_QPID_S)\n+\n+#define IRDMACQ_UDSRCQPN_S 0\n+#define IRDMACQ_UDSRCQPN_M (0xffffffffULL << IRDMACQ_UDSRCQPN_S)\n+\n+#define IRDMACQ_PSHDROP_S 51\n+#define IRDMACQ_PSHDROP_M BIT_ULL(IRDMACQ_PSHDROP_S)\n+\n+#define IRDMACQ_STAG_S 53\n+#define IRDMACQ_STAG_M BIT_ULL(IRDMACQ_STAG_S)\n+\n+#define IRDMACQ_IPV4_S 53\n+#define IRDMACQ_IPV4_M BIT_ULL(IRDMACQ_IPV4_S)\n+\n+#define IRDMACQ_SOEVENT_S 54\n+#define IRDMACQ_SOEVENT_M BIT_ULL(IRDMACQ_SOEVENT_S)\n+\n+#define IRDMACQ_OP_S 56\n+#define IRDMACQ_OP_M (0x3fULL << IRDMACQ_OP_S)\n+\n+/* CEQE format */\n+#define IRDMA_CEQE_CQCTX_S 0\n+#define IRDMA_CEQE_CQCTX_M \\\n+\t(0x7fffffffffffffffULL << IRDMA_CEQE_CQCTX_S)\n+\n+#define IRDMA_CEQE_VALID_S 63\n+#define IRDMA_CEQE_VALID_M BIT_ULL(IRDMA_CEQE_VALID_S)\n+\n+/* AEQE format */\n+#define IRDMA_AEQE_COMPCTX_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_AEQE_COMPCTX_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMA_AEQE_QPCQID_LOW_S 0\n+#define IRDMA_AEQE_QPCQID_LOW_M (0x3ffffULL << IRDMA_AEQE_QPCQID_LOW_S)\n+\n+#define IRDMA_AEQE_QPCQID_HI_S 46\n+#define IRDMA_AEQE_QPCQID_HI_M BIT_ULL(IRDMA_AEQE_QPCQID_HI_S)\n+\n+#define IRDMA_AEQE_WQDESCIDX_S 18\n+#define IRDMA_AEQE_WQDESCIDX_M (0x7fffULL << IRDMA_AEQE_WQDESCIDX_S)\n+\n+#define IRDMA_AEQE_OVERFLOW_S 33\n+#define IRDMA_AEQE_OVERFLOW_M BIT_ULL(IRDMA_AEQE_OVERFLOW_S)\n+\n+#define IRDMA_AEQE_AECODE_S 34\n+#define IRDMA_AEQE_AECODE_M (0xfffULL << IRDMA_AEQE_AECODE_S)\n+\n+#define IRDMA_AEQE_AESRC_S 50\n+#define IRDMA_AEQE_AESRC_M (0xfULL << IRDMA_AEQE_AESRC_S)\n+\n+#define IRDMA_AEQE_IWSTATE_S 54\n+#define IRDMA_AEQE_IWSTATE_M (0x7ULL << IRDMA_AEQE_IWSTATE_S)\n+\n+#define IRDMA_AEQE_TCPSTATE_S 57\n+#define IRDMA_AEQE_TCPSTATE_M (0xfULL << IRDMA_AEQE_TCPSTATE_S)\n+\n+#define IRDMA_AEQE_Q2DATA_S 61\n+#define IRDMA_AEQE_Q2DATA_M (0x3ULL << IRDMA_AEQE_Q2DATA_S)\n+\n+#define IRDMA_AEQE_VALID_S 63\n+#define IRDMA_AEQE_VALID_M BIT_ULL(IRDMA_AEQE_VALID_S)\n+\n+#define IRDMA_UDA_QPSQ_NEXT_HDR_S 16\n+#define IRDMA_UDA_QPSQ_NEXT_HDR_M ((u64)0xff << IRDMA_UDA_QPSQ_NEXT_HDR_S)\n+\n+#define IRDMA_UDA_QPSQ_OPCODE_S 32\n+#define IRDMA_UDA_QPSQ_OPCODE_M ((u64)0x3f << IRDMA_UDA_QPSQ_OPCODE_S)\n+\n+#define IRDMA_UDA_QPSQ_L4LEN_S 42\n+#define IRDMA_UDA_QPSQ_L4LEN_M ((u64)0xf << IRDMA_UDA_QPSQ_L4LEN_S)\n+\n+#define IRDMA_GEN1_UDA_QPSQ_L4LEN_S 24\n+#define IRDMA_GEN1_UDA_QPSQ_L4LEN_M ((u64)0xf << IRDMA_GEN1_UDA_QPSQ_L4LEN_S)\n+\n+#define IRDMA_UDA_QPSQ_AHIDX_S 0\n+#define IRDMA_UDA_QPSQ_AHIDX_M ((u64)0x1ffff << IRDMA_UDA_QPSQ_AHIDX_S)\n+\n+#define IRDMA_UDA_QPSQ_VALID_S 63\n+#define IRDMA_UDA_QPSQ_VALID_M \\\n+\tBIT_ULL(IRDMA_UDA_QPSQ_VALID_S)\n+\n+#define IRDMA_UDA_QPSQ_SIGCOMPL_S 62\n+#define IRDMA_UDA_QPSQ_SIGCOMPL_M BIT_ULL(IRDMA_UDA_QPSQ_SIGCOMPL_S)\n+\n+#define IRDMA_UDA_QPSQ_MACLEN_S 56\n+#define IRDMA_UDA_QPSQ_MACLEN_M \\\n+\t((u64)0x7f << IRDMA_UDA_QPSQ_MACLEN_S)\n+\n+#define IRDMA_UDA_QPSQ_IPLEN_S 48\n+#define IRDMA_UDA_QPSQ_IPLEN_M \\\n+\t((u64)0x7f << IRDMA_UDA_QPSQ_IPLEN_S)\n+\n+#define IRDMA_UDA_QPSQ_L4T_S 30\n+#define IRDMA_UDA_QPSQ_L4T_M \\\n+\t((u64)0x3 << IRDMA_UDA_QPSQ_L4T_S)\n+\n+#define IRDMA_UDA_QPSQ_IIPT_S 28\n+#define IRDMA_UDA_QPSQ_IIPT_M \\\n+\t((u64)0x3 << IRDMA_UDA_QPSQ_IIPT_S)\n+\n+#define IRDMA_UDA_PAYLOADLEN_S 0\n+#define IRDMA_UDA_PAYLOADLEN_M ((u64)0x3fff << IRDMA_UDA_PAYLOADLEN_S)\n+\n+#define IRDMA_UDA_HDRLEN_S 16\n+#define IRDMA_UDA_HDRLEN_M ((u64)0x1ff << IRDMA_UDA_HDRLEN_S)\n+\n+#define IRDMA_VLAN_TAG_VALID_S 50\n+#define IRDMA_VLAN_TAG_VALID_M BIT_ULL(IRDMA_VLAN_TAG_VALID_S)\n+\n+#define IRDMA_UDA_L3PROTO_S 0\n+#define IRDMA_UDA_L3PROTO_M ((u64)0x3 << IRDMA_UDA_L3PROTO_S)\n+\n+#define IRDMA_UDA_L4PROTO_S 16\n+#define IRDMA_UDA_L4PROTO_M ((u64)0x3 << IRDMA_UDA_L4PROTO_S)\n+\n+#define IRDMA_UDA_QPSQ_DOLOOPBACK_S 44\n+#define IRDMA_UDA_QPSQ_DOLOOPBACK_M \\\n+\tBIT_ULL(IRDMA_UDA_QPSQ_DOLOOPBACK_S)\n+\n+/* CQP SQ WQE common fields */\n+#define IRDMA_CQPSQ_BUFSIZE_S 0\n+#define IRDMA_CQPSQ_BUFSIZE_M (0xffffffffULL << IRDMA_CQPSQ_BUFSIZE_S)\n+\n+#define IRDMA_CQPSQ_OPCODE_S 32\n+#define IRDMA_CQPSQ_OPCODE_M (0x3fULL << IRDMA_CQPSQ_OPCODE_S)\n+\n+#define IRDMA_CQPSQ_WQEVALID_S 63\n+#define IRDMA_CQPSQ_WQEVALID_M BIT_ULL(IRDMA_CQPSQ_WQEVALID_S)\n+\n+#define IRDMA_CQPSQ_TPHVAL_S 0\n+#define IRDMA_CQPSQ_TPHVAL_M (0xffULL << IRDMA_CQPSQ_TPHVAL_S)\n+\n+#define IRDMA_CQPSQ_VSIIDX_S 8\n+#define IRDMA_CQPSQ_VSIIDX_M (0x3ffULL << IRDMA_CQPSQ_VSIIDX_S)\n+\n+#define IRDMA_CQPSQ_TPHEN_S 60\n+#define IRDMA_CQPSQ_TPHEN_M BIT_ULL(IRDMA_CQPSQ_TPHEN_S)\n+\n+#define IRDMA_CQPSQ_PBUFADDR_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQPSQ_PBUFADDR_M IRDMA_CQPHC_QPCTX_M\n+\n+/* Create/Modify/Destroy QP */\n+\n+#define IRDMA_CQPSQ_QP_NEWMSS_S 32\n+#define IRDMA_CQPSQ_QP_NEWMSS_M (0x3fffULL << IRDMA_CQPSQ_QP_NEWMSS_S)\n+\n+#define IRDMA_CQPSQ_QP_TERMLEN_S 48\n+#define IRDMA_CQPSQ_QP_TERMLEN_M (0xfULL << IRDMA_CQPSQ_QP_TERMLEN_S)\n+\n+#define IRDMA_CQPSQ_QP_QPCTX_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQPSQ_QP_QPCTX_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMA_CQPSQ_QP_QPID_S 0\n+#define IRDMA_CQPSQ_QP_QPID_M (0x3FFFFUL)\n+\n+#define IRDMA_CQPSQ_QP_OP_S 32\n+#define IRDMA_CQPSQ_QP_OP_M IRDMACQ_OP_M\n+\n+#define IRDMA_CQPSQ_QP_ORDVALID_S 42\n+#define IRDMA_CQPSQ_QP_ORDVALID_M BIT_ULL(IRDMA_CQPSQ_QP_ORDVALID_S)\n+\n+#define IRDMA_CQPSQ_QP_TOECTXVALID_S 43\n+#define IRDMA_CQPSQ_QP_TOECTXVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_TOECTXVALID_S)\n+\n+#define IRDMA_CQPSQ_QP_CACHEDVARVALID_S 44\n+#define IRDMA_CQPSQ_QP_CACHEDVARVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_CACHEDVARVALID_S)\n+\n+#define IRDMA_CQPSQ_QP_VQ_S 45\n+#define IRDMA_CQPSQ_QP_VQ_M BIT_ULL(IRDMA_CQPSQ_QP_VQ_S)\n+\n+#define IRDMA_CQPSQ_QP_FORCELOOPBACK_S 46\n+#define IRDMA_CQPSQ_QP_FORCELOOPBACK_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_FORCELOOPBACK_S)\n+\n+#define IRDMA_CQPSQ_QP_CQNUMVALID_S 47\n+#define IRDMA_CQPSQ_QP_CQNUMVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_CQNUMVALID_S)\n+\n+#define IRDMA_CQPSQ_QP_QPTYPE_S 48\n+#define IRDMA_CQPSQ_QP_QPTYPE_M (0x7ULL << IRDMA_CQPSQ_QP_QPTYPE_S)\n+\n+#define IRDMA_CQPSQ_QP_MACVALID_S 51\n+#define IRDMA_CQPSQ_QP_MACVALID_M BIT_ULL(IRDMA_CQPSQ_QP_MACVALID_S)\n+\n+#define IRDMA_CQPSQ_QP_MSSCHANGE_S 52\n+#define IRDMA_CQPSQ_QP_MSSCHANGE_M BIT_ULL(IRDMA_CQPSQ_QP_MSSCHANGE_S)\n+\n+#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_S 54\n+#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_IGNOREMWBOUND_S)\n+\n+#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S 55\n+#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S)\n+\n+#define IRDMA_CQPSQ_QP_TERMACT_S 56\n+#define IRDMA_CQPSQ_QP_TERMACT_M (0x3ULL << IRDMA_CQPSQ_QP_TERMACT_S)\n+\n+#define IRDMA_CQPSQ_QP_RESETCON_S 58\n+#define IRDMA_CQPSQ_QP_RESETCON_M BIT_ULL(IRDMA_CQPSQ_QP_RESETCON_S)\n+\n+#define IRDMA_CQPSQ_QP_ARPTABIDXVALID_S 59\n+#define IRDMA_CQPSQ_QP_ARPTABIDXVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_QP_ARPTABIDXVALID_S)\n+\n+#define IRDMA_CQPSQ_QP_NEXTIWSTATE_S 60\n+#define IRDMA_CQPSQ_QP_NEXTIWSTATE_M \\\n+\t(0x7ULL << IRDMA_CQPSQ_QP_NEXTIWSTATE_S)\n+\n+#define IRDMA_CQPSQ_QP_DBSHADOWADDR_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQPSQ_QP_DBSHADOWADDR_M IRDMA_CQPHC_QPCTX_M\n+\n+/* Create/Modify/Destroy CQ */\n+#define IRDMA_CQPSQ_CQ_CQSIZE_S 0\n+#define IRDMA_CQPSQ_CQ_CQSIZE_M (0x1fffffULL << IRDMA_CQPSQ_CQ_CQSIZE_S)\n+\n+#define IRDMA_CQPSQ_CQ_CQCTX_S 0\n+#define IRDMA_CQPSQ_CQ_CQCTX_M \\\n+\t(0x7fffffffffffffffULL << IRDMA_CQPSQ_CQ_CQCTX_S)\n+\n+#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_S 0\n+#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_M \\\n+\t(0x3ffff << IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_S)\n+\n+#define IRDMA_CQPSQ_CQ_OP_S 32\n+#define IRDMA_CQPSQ_CQ_OP_M (0x3fULL << IRDMA_CQPSQ_CQ_OP_S)\n+\n+#define IRDMA_CQPSQ_CQ_CQRESIZE_S 43\n+#define IRDMA_CQPSQ_CQ_CQRESIZE_M BIT_ULL(IRDMA_CQPSQ_CQ_CQRESIZE_S)\n+\n+#define IRDMA_CQPSQ_CQ_LPBLSIZE_S 44\n+#define IRDMA_CQPSQ_CQ_LPBLSIZE_M (3ULL << IRDMA_CQPSQ_CQ_LPBLSIZE_S)\n+\n+#define IRDMA_CQPSQ_CQ_CHKOVERFLOW_S 46\n+#define IRDMA_CQPSQ_CQ_CHKOVERFLOW_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_CQ_CHKOVERFLOW_S)\n+\n+#define IRDMA_CQPSQ_CQ_VIRTMAP_S 47\n+#define IRDMA_CQPSQ_CQ_VIRTMAP_M BIT_ULL(IRDMA_CQPSQ_CQ_VIRTMAP_S)\n+\n+#define IRDMA_CQPSQ_CQ_ENCEQEMASK_S 48\n+#define IRDMA_CQPSQ_CQ_ENCEQEMASK_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_CQ_ENCEQEMASK_S)\n+\n+#define IRDMA_CQPSQ_CQ_CEQIDVALID_S 49\n+#define IRDMA_CQPSQ_CQ_CEQIDVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_CQ_CEQIDVALID_S)\n+\n+#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_S 61\n+#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_S)\n+\n+#define IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX_S 0\n+#define IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX_M \\\n+\t(0xfffffffULL << IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX_S)\n+\n+/* Allocate/Register/Register Shared/Deallocate Stag */\n+#define IRDMA_CQPSQ_STAG_VA_FBO_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQPSQ_STAG_VA_FBO_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMA_CQPSQ_STAG_STAGLEN_S 0\n+#define IRDMA_CQPSQ_STAG_STAGLEN_M \\\n+\t(0x3fffffffffffULL << IRDMA_CQPSQ_STAG_STAGLEN_S)\n+\n+#define IRDMA_CQPSQ_STAG_KEY_S 0\n+#define IRDMA_CQPSQ_STAG_KEY_M (0xffULL << IRDMA_CQPSQ_STAG_KEY_S)\n+\n+#define IRDMA_CQPSQ_STAG_IDX_S 8\n+#define IRDMA_CQPSQ_STAG_IDX_M (0xffffffULL << IRDMA_CQPSQ_STAG_IDX_S)\n+\n+#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX_S 32\n+#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX_M \\\n+\t(0xffffffULL << IRDMA_CQPSQ_STAG_PARENTSTAGIDX_S)\n+\n+#define IRDMA_CQPSQ_STAG_MR_S 43\n+#define IRDMA_CQPSQ_STAG_MR_M BIT_ULL(IRDMA_CQPSQ_STAG_MR_S)\n+\n+#define IRDMA_CQPSQ_STAG_MWTYPE_S 42\n+#define IRDMA_CQPSQ_STAG_MWTYPE_M BIT_ULL(IRDMA_CQPSQ_STAG_MWTYPE_S)\n+\n+#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_S 58\n+#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_S)\n+\n+#define IRDMA_CQPSQ_STAG_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S\n+#define IRDMA_CQPSQ_STAG_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M\n+\n+#define IRDMA_CQPSQ_STAG_HPAGESIZE_S 46\n+#define IRDMA_CQPSQ_STAG_HPAGESIZE_M \\\n+\t((u64)3 << IRDMA_CQPSQ_STAG_HPAGESIZE_S)\n+\n+#define IRDMA_CQPSQ_STAG_ARIGHTS_S 48\n+#define IRDMA_CQPSQ_STAG_ARIGHTS_M \\\n+\t(0x1fULL << IRDMA_CQPSQ_STAG_ARIGHTS_S)\n+\n+#define IRDMA_CQPSQ_STAG_REMACCENABLED_S 53\n+#define IRDMA_CQPSQ_STAG_REMACCENABLED_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STAG_REMACCENABLED_S)\n+\n+#define IRDMA_CQPSQ_STAG_VABASEDTO_S 59\n+#define IRDMA_CQPSQ_STAG_VABASEDTO_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STAG_VABASEDTO_S)\n+\n+#define IRDMA_CQPSQ_STAG_USEHMCFNIDX_S 60\n+#define IRDMA_CQPSQ_STAG_USEHMCFNIDX_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STAG_USEHMCFNIDX_S)\n+\n+#define IRDMA_CQPSQ_STAG_USEPFRID_S 61\n+#define IRDMA_CQPSQ_STAG_USEPFRID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_STAG_USEPFRID_S)\n+\n+#define IRDMA_CQPSQ_STAG_PBA_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQPSQ_STAG_PBA_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMA_CQPSQ_STAG_HMCFNIDX_S 0\n+#define IRDMA_CQPSQ_STAG_HMCFNIDX_M \\\n+\t(0x3fULL << IRDMA_CQPSQ_STAG_HMCFNIDX_S)\n+\n+#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_S 0\n+#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_M \\\n+\t(0xfffffffULL << IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_S)\n+\n+#define IRDMA_CQPSQ_QUERYSTAG_IDX_S IRDMA_CQPSQ_STAG_IDX_S\n+#define IRDMA_CQPSQ_QUERYSTAG_IDX_M IRDMA_CQPSQ_STAG_IDX_M\n+\n+/* Manage Local MAC Table - MLM */\n+#define IRDMA_CQPSQ_MLM_TABLEIDX_S 0\n+#define IRDMA_CQPSQ_MLM_TABLEIDX_M \\\n+\t(0x3fULL << IRDMA_CQPSQ_MLM_TABLEIDX_S)\n+\n+#define IRDMA_CQPSQ_MLM_FREEENTRY_S 62\n+#define IRDMA_CQPSQ_MLM_FREEENTRY_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_MLM_FREEENTRY_S)\n+\n+#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_S 61\n+#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_S)\n+\n+#define IRDMA_CQPSQ_MLM_MAC0_S 0\n+#define IRDMA_CQPSQ_MLM_MAC0_M (0xffULL << IRDMA_CQPSQ_MLM_MAC0_S)\n+\n+#define IRDMA_CQPSQ_MLM_MAC1_S 8\n+#define IRDMA_CQPSQ_MLM_MAC1_M (0xffULL << IRDMA_CQPSQ_MLM_MAC1_S)\n+\n+#define IRDMA_CQPSQ_MLM_MAC2_S 16\n+#define IRDMA_CQPSQ_MLM_MAC2_M (0xffULL << IRDMA_CQPSQ_MLM_MAC2_S)\n+\n+#define IRDMA_CQPSQ_MLM_MAC3_S 24\n+#define IRDMA_CQPSQ_MLM_MAC3_M (0xffULL << IRDMA_CQPSQ_MLM_MAC3_S)\n+\n+#define IRDMA_CQPSQ_MLM_MAC4_S 32\n+#define IRDMA_CQPSQ_MLM_MAC4_M (0xffULL << IRDMA_CQPSQ_MLM_MAC4_S)\n+\n+#define IRDMA_CQPSQ_MLM_MAC5_S 40\n+#define IRDMA_CQPSQ_MLM_MAC5_M (0xffULL << IRDMA_CQPSQ_MLM_MAC5_S)\n+\n+/* Manage ARP Table  - MAT */\n+#define IRDMA_CQPSQ_MAT_REACHMAX_S 0\n+#define IRDMA_CQPSQ_MAT_REACHMAX_M \\\n+\t(0xffffffffULL << IRDMA_CQPSQ_MAT_REACHMAX_S)\n+\n+#define IRDMA_CQPSQ_MAT_MACADDR_S 0\n+#define IRDMA_CQPSQ_MAT_MACADDR_M \\\n+\t(0xffffffffffffULL << IRDMA_CQPSQ_MAT_MACADDR_S)\n+\n+#define IRDMA_CQPSQ_MAT_ARPENTRYIDX_S 0\n+#define IRDMA_CQPSQ_MAT_ARPENTRYIDX_M \\\n+\t(0xfffULL << IRDMA_CQPSQ_MAT_ARPENTRYIDX_S)\n+\n+#define IRDMA_CQPSQ_MAT_ENTRYVALID_S 42\n+#define IRDMA_CQPSQ_MAT_ENTRYVALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_MAT_ENTRYVALID_S)\n+\n+#define IRDMA_CQPSQ_MAT_PERMANENT_S 43\n+#define IRDMA_CQPSQ_MAT_PERMANENT_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_MAT_PERMANENT_S)\n+\n+#define IRDMA_CQPSQ_MAT_QUERY_S 44\n+#define IRDMA_CQPSQ_MAT_QUERY_M BIT_ULL(IRDMA_CQPSQ_MAT_QUERY_S)\n+\n+/* Manage VF PBLE Backing Pages - MVPBP*/\n+#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_S 0\n+#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_M \\\n+\t(0x3ffULL << IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_S)\n+\n+#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_S 16\n+#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_M \\\n+\t(0x1ffULL << IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_S)\n+\n+#define IRDMA_CQPSQ_MVPBP_SD_INX_S 32\n+#define IRDMA_CQPSQ_MVPBP_SD_INX_M \\\n+\t(0xfffULL << IRDMA_CQPSQ_MVPBP_SD_INX_S)\n+\n+#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT_S 62\n+#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_MVPBP_INV_PD_ENT_S)\n+\n+#define IRDMA_CQPSQ_MVPBP_PD_PLPBA_S 3\n+#define IRDMA_CQPSQ_MVPBP_PD_PLPBA_M \\\n+\t(0x1fffffffffffffffULL << IRDMA_CQPSQ_MVPBP_PD_PLPBA_S)\n+\n+/* Manage Push Page - MPP */\n+#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffff\n+\n+#define IRDMA_CQPSQ_MPP_QS_HANDLE_S 0\n+#define IRDMA_CQPSQ_MPP_QS_HANDLE_M \\\n+\t(0x3ffULL << IRDMA_CQPSQ_MPP_QS_HANDLE_S)\n+\n+#define IRDMA_CQPSQ_MPP_PPIDX_S 0\n+#define IRDMA_CQPSQ_MPP_PPIDX_M (0x3ffULL << IRDMA_CQPSQ_MPP_PPIDX_S)\n+\n+#define IRDMA_CQPSQ_MPP_PPTYPE_S 60\n+#define IRDMA_CQPSQ_MPP_PPTYPE_M (0x3ULL << IRDMA_CQPSQ_MPP_PPTYPE_S)\n+\n+#define IRDMA_CQPSQ_MPP_FREE_PAGE_S 62\n+#define IRDMA_CQPSQ_MPP_FREE_PAGE_M BIT_ULL(IRDMA_CQPSQ_MPP_FREE_PAGE_S)\n+\n+/* Upload Context - UCTX */\n+#define IRDMA_CQPSQ_UCTX_QPCTXADDR_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMA_CQPSQ_UCTX_QPCTXADDR_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMA_CQPSQ_UCTX_QPID_S 0\n+#define IRDMA_CQPSQ_UCTX_QPID_M (0x3ffffULL << IRDMA_CQPSQ_UCTX_QPID_S)\n+\n+#define IRDMA_CQPSQ_UCTX_QPTYPE_S 48\n+#define IRDMA_CQPSQ_UCTX_QPTYPE_M (0xfULL << IRDMA_CQPSQ_UCTX_QPTYPE_S)\n+\n+#define IRDMA_CQPSQ_UCTX_RAWFORMAT_S 61\n+#define IRDMA_CQPSQ_UCTX_RAWFORMAT_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UCTX_RAWFORMAT_S)\n+\n+#define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62\n+#define IRDMA_CQPSQ_UCTX_FREEZEQP_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UCTX_FREEZEQP_S)\n+\n+/* Manage HMC PM Function Table - MHMC */\n+#define IRDMA_CQPSQ_MHMC_VFIDX_S 0\n+#define IRDMA_CQPSQ_MHMC_VFIDX_M (0xffULL << IRDMA_CQPSQ_MHMC_VFIDX_S)\n+\n+#define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62\n+#define IRDMA_CQPSQ_MHMC_FREEPMFN_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_MHMC_FREEPMFN_S)\n+\n+/* Set HMC Resource Profile - SHMCRP */\n+#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_S 0\n+#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_M \\\n+\t(0x7ULL << IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_S)\n+#define IRDMA_CQPSQ_SHMCRP_VFNUM_S 32\n+#define IRDMA_CQPSQ_SHMCRP_VFNUM_M (0x3fULL << IRDMA_CQPSQ_SHMCRP_VFNUM_S)\n+\n+/* Create/Destroy CEQ */\n+#define IRDMA_CQPSQ_CEQ_CEQSIZE_S 0\n+#define IRDMA_CQPSQ_CEQ_CEQSIZE_M \\\n+\t(0x1ffffULL << IRDMA_CQPSQ_CEQ_CEQSIZE_S)\n+\n+#define IRDMA_CQPSQ_CEQ_CEQID_S 0\n+#define IRDMA_CQPSQ_CEQ_CEQID_M (0x3ffULL << IRDMA_CQPSQ_CEQ_CEQID_S)\n+\n+#define IRDMA_CQPSQ_CEQ_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S\n+#define IRDMA_CQPSQ_CEQ_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M\n+\n+#define IRDMA_CQPSQ_CEQ_VMAP_S 47\n+#define IRDMA_CQPSQ_CEQ_VMAP_M BIT_ULL(IRDMA_CQPSQ_CEQ_VMAP_S)\n+\n+#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_S 46\n+#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_M BIT_ULL(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_S)\n+\n+#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_S 0\n+#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_M \\\n+\t(0xfffffffULL << IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_S)\n+\n+/* Create/Destroy AEQ */\n+#define IRDMA_CQPSQ_AEQ_AEQECNT_S 0\n+#define IRDMA_CQPSQ_AEQ_AEQECNT_M \\\n+\t(0x7ffffULL << IRDMA_CQPSQ_AEQ_AEQECNT_S)\n+\n+#define IRDMA_CQPSQ_AEQ_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S\n+#define IRDMA_CQPSQ_AEQ_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M\n+\n+#define IRDMA_CQPSQ_AEQ_VMAP_S 47\n+#define IRDMA_CQPSQ_AEQ_VMAP_M BIT_ULL(IRDMA_CQPSQ_AEQ_VMAP_S)\n+\n+#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_S 0\n+#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_M \\\n+\t(0xfffffffULL << IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_S)\n+\n+/* Commit FPM Values - CFPM */\n+#define IRDMA_COMMIT_FPM_QPCNT_S 0\n+#define IRDMA_COMMIT_FPM_QPCNT_M (0x7ffffULL << IRDMA_COMMIT_FPM_QPCNT_S)\n+\n+#define IRDMA_COMMIT_FPM_CQCNT_S 0\n+#define IRDMA_COMMIT_FPM_CQCNT_M (0xfffffULL << IRDMA_COMMIT_FPM_CQCNT_S)\n+\n+#define IRDMA_COMMIT_FPM_BASE_S 32\n+\n+#define IRDMA_CQPSQ_CFPM_HMCFNID_S 0\n+#define IRDMA_CQPSQ_CFPM_HMCFNID_M (0x3fULL << IRDMA_CQPSQ_CFPM_HMCFNID_S)\n+\n+/* Flush WQEs - FWQE */\n+#define IRDMA_CQPSQ_FWQE_AECODE_S 0\n+#define IRDMA_CQPSQ_FWQE_AECODE_M (0xffffULL << IRDMA_CQPSQ_FWQE_AECODE_S)\n+\n+#define IRDMA_CQPSQ_FWQE_AESOURCE_S 16\n+#define IRDMA_CQPSQ_FWQE_AESOURCE_M \\\n+\t(0xfULL << IRDMA_CQPSQ_FWQE_AESOURCE_S)\n+\n+#define IRDMA_CQPSQ_FWQE_RQMNERR_S 0\n+#define IRDMA_CQPSQ_FWQE_RQMNERR_M \\\n+\t(0xffffULL << IRDMA_CQPSQ_FWQE_RQMNERR_S)\n+\n+#define IRDMA_CQPSQ_FWQE_RQMJERR_S 16\n+#define IRDMA_CQPSQ_FWQE_RQMJERR_M \\\n+\t(0xffffULL << IRDMA_CQPSQ_FWQE_RQMJERR_S)\n+\n+#define IRDMA_CQPSQ_FWQE_SQMNERR_S 32\n+#define IRDMA_CQPSQ_FWQE_SQMNERR_M \\\n+\t(0xffffULL << IRDMA_CQPSQ_FWQE_SQMNERR_S)\n+\n+#define IRDMA_CQPSQ_FWQE_SQMJERR_S 48\n+#define IRDMA_CQPSQ_FWQE_SQMJERR_M \\\n+\t(0xffffULL << IRDMA_CQPSQ_FWQE_SQMJERR_S)\n+\n+#define IRDMA_CQPSQ_FWQE_QPID_S 0\n+#define IRDMA_CQPSQ_FWQE_QPID_M (0x3ffffULL << IRDMA_CQPSQ_FWQE_QPID_S)\n+\n+#define IRDMA_CQPSQ_FWQE_GENERATE_AE_S 59\n+#define IRDMA_CQPSQ_FWQE_GENERATE_AE_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_FWQE_GENERATE_AE_S)\n+\n+#define IRDMA_CQPSQ_FWQE_USERFLCODE_S 60\n+#define IRDMA_CQPSQ_FWQE_USERFLCODE_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_FWQE_USERFLCODE_S)\n+\n+#define IRDMA_CQPSQ_FWQE_FLUSHSQ_S 61\n+#define IRDMA_CQPSQ_FWQE_FLUSHSQ_M BIT_ULL(IRDMA_CQPSQ_FWQE_FLUSHSQ_S)\n+\n+#define IRDMA_CQPSQ_FWQE_FLUSHRQ_S 62\n+#define IRDMA_CQPSQ_FWQE_FLUSHRQ_M BIT_ULL(IRDMA_CQPSQ_FWQE_FLUSHRQ_S)\n+\n+/* Manage Accelerated Port Table - MAPT */\n+#define IRDMA_CQPSQ_MAPT_PORT_S 0\n+#define IRDMA_CQPSQ_MAPT_PORT_M (0xffffULL << IRDMA_CQPSQ_MAPT_PORT_S)\n+\n+#define IRDMA_CQPSQ_MAPT_ADDPORT_S 62\n+#define IRDMA_CQPSQ_MAPT_ADDPORT_M BIT_ULL(IRDMA_CQPSQ_MAPT_ADDPORT_S)\n+\n+/* Update Protocol Engine SDs */\n+#define IRDMA_CQPSQ_UPESD_SDCMD_S 0\n+#define IRDMA_CQPSQ_UPESD_SDCMD_M (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDCMD_S)\n+\n+#define IRDMA_CQPSQ_UPESD_SDDATALOW_S 0\n+#define IRDMA_CQPSQ_UPESD_SDDATALOW_M \\\n+\t(0xffffffffULL << IRDMA_CQPSQ_UPESD_SDDATALOW_S)\n+\n+#define IRDMA_CQPSQ_UPESD_SDDATAHI_S 32\n+#define IRDMA_CQPSQ_UPESD_SDDATAHI_M \\\n+\t(0xffffffffULL << IRDMA_CQPSQ_UPESD_SDDATAHI_S)\n+#define IRDMA_CQPSQ_UPESD_HMCFNID_S 0\n+#define IRDMA_CQPSQ_UPESD_HMCFNID_M \\\n+\t(0x3fULL << IRDMA_CQPSQ_UPESD_HMCFNID_S)\n+\n+#define IRDMA_CQPSQ_UPESD_ENTRY_VALID_S 63\n+#define IRDMA_CQPSQ_UPESD_ENTRY_VALID_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UPESD_ENTRY_VALID_S)\n+\n+#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT_S 0\n+#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT_M \\\n+\t(0xfULL << IRDMA_CQPSQ_UPESD_ENTRY_COUNT_S)\n+\n+#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY_S 7\n+#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY_M \\\n+\tBIT_ULL(IRDMA_CQPSQ_UPESD_SKIP_ENTRY_S)\n+\n+/* Suspend QP */\n+#define IRDMA_CQPSQ_SUSPENDQP_QPID_S 0\n+#define IRDMA_CQPSQ_SUSPENDQP_QPID_M (0x3FFFFULL)\n+\n+/* Resume QP */\n+#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE_S 0\n+#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE_M \\\n+\t(0xffffffffULL << IRDMA_CQPSQ_RESUMEQP_QSHANDLE_S)\n+\n+#define IRDMA_CQPSQ_RESUMEQP_QPID_S 0\n+#define IRDMA_CQPSQ_RESUMEQP_QPID_M (0x3FFFFUL)\n+\n+/* IW QP Context */\n+#define IRDMAQPC_DDP_VER_S 0\n+#define IRDMAQPC_DDP_VER_M (3ULL << IRDMAQPC_DDP_VER_S)\n+\n+#define IRDMAQPC_IBRDENABLE_S 2\n+#define IRDMAQPC_IBRDENABLE_M BIT_ULL(IRDMAQPC_IBRDENABLE_S)\n+\n+#define IRDMAQPC_IPV4_S 3\n+#define IRDMAQPC_IPV4_M BIT_ULL(IRDMAQPC_IPV4_S)\n+\n+#define IRDMAQPC_NONAGLE_S 4\n+#define IRDMAQPC_NONAGLE_M BIT_ULL(IRDMAQPC_NONAGLE_S)\n+\n+#define IRDMAQPC_INSERTVLANTAG_S 5\n+#define IRDMAQPC_INSERTVLANTAG_M BIT_ULL(IRDMAQPC_INSERTVLANTAG_S)\n+\n+#define IRDMAQPC_ISQP1_S 6\n+#define IRDMAQPC_ISQP1_M BIT_ULL(IRDMAQPC_ISQP1_S)\n+\n+#define IRDMAQPC_TIMESTAMP_S 7\n+#define IRDMAQPC_TIMESTAMP_M BIT_ULL(IRDMAQPC_TIMESTAMP_S)\n+\n+#define IRDMAQPC_RQWQESIZE_S 8\n+#define IRDMAQPC_RQWQESIZE_M (3ULL << IRDMAQPC_RQWQESIZE_S)\n+\n+#define IRDMAQPC_INSERTL2TAG2_S 11\n+#define IRDMAQPC_INSERTL2TAG2_M BIT_ULL(IRDMAQPC_INSERTL2TAG2_S)\n+\n+#define IRDMAQPC_LIMIT_S 12\n+#define IRDMAQPC_LIMIT_M (3ULL << IRDMAQPC_LIMIT_S)\n+\n+#define IRDMAQPC_ECN_EN_S 14\n+#define IRDMAQPC_ECN_EN_M BIT_ULL(IRDMAQPC_ECN_EN_S)\n+\n+#define IRDMAQPC_DROPOOOSEG_S 15\n+#define IRDMAQPC_DROPOOOSEG_M BIT_ULL(IRDMAQPC_DROPOOOSEG_S)\n+\n+#define IRDMAQPC_DUPACK_THRESH_S 16\n+#define IRDMAQPC_DUPACK_THRESH_M (7ULL << IRDMAQPC_DUPACK_THRESH_S)\n+\n+#define IRDMAQPC_ERR_RQ_IDX_VALID_S 19\n+#define IRDMAQPC_ERR_RQ_IDX_VALID_M BIT_ULL(IRDMAQPC_ERR_RQ_IDX_VALID_S)\n+\n+#define IRDMAQPC_DIS_VLAN_CHECKS_S 19\n+#define IRDMAQPC_DIS_VLAN_CHECKS_M (7ULL << IRDMAQPC_DIS_VLAN_CHECKS_S)\n+\n+#define IRDMAQPC_DC_TCP_EN_S 25\n+#define IRDMAQPC_DC_TCP_EN_M BIT_ULL(IRDMAQPC_DC_TCP_EN_S)\n+\n+#define IRDMAQPC_RCVTPHEN_S 28\n+#define IRDMAQPC_RCVTPHEN_M BIT_ULL(IRDMAQPC_RCVTPHEN_S)\n+\n+#define IRDMAQPC_XMITTPHEN_S 29\n+#define IRDMAQPC_XMITTPHEN_M BIT_ULL(IRDMAQPC_XMITTPHEN_S)\n+\n+#define IRDMAQPC_RQTPHEN_S 30\n+#define IRDMAQPC_RQTPHEN_M BIT_ULL(IRDMAQPC_RQTPHEN_S)\n+\n+#define IRDMAQPC_SQTPHEN_S 31\n+#define IRDMAQPC_SQTPHEN_M BIT_ULL(IRDMAQPC_SQTPHEN_S)\n+\n+#define IRDMAQPC_PPIDX_S 32\n+#define IRDMAQPC_PPIDX_M (0x3ffULL << IRDMAQPC_PPIDX_S)\n+\n+#define IRDMAQPC_PMENA_S 47\n+#define IRDMAQPC_PMENA_M BIT_ULL(IRDMAQPC_PMENA_S)\n+\n+#define IRDMAQPC_RDMAP_VER_S 62\n+#define IRDMAQPC_RDMAP_VER_M (3ULL << IRDMAQPC_RDMAP_VER_S)\n+\n+#define IRDMAQPC_ROCE_TVER_S 60\n+#define IRDMAQPC_ROCE_TVER_M (0x0fULL << IRDMAQPC_ROCE_TVER_S)\n+#define IRDMAQPC_SQADDR_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPC_SQADDR_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMAQPC_RQADDR_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPC_RQADDR_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMAQPC_TTL_S 0\n+#define IRDMAQPC_TTL_M (0xffULL << IRDMAQPC_TTL_S)\n+\n+#define IRDMAQPC_RQSIZE_S 8\n+#define IRDMAQPC_RQSIZE_M (0xfULL << IRDMAQPC_RQSIZE_S)\n+\n+#define IRDMAQPC_SQSIZE_S 12\n+#define IRDMAQPC_SQSIZE_M (0xfULL << IRDMAQPC_SQSIZE_S)\n+\n+#define IRDMAQPC_GEN1_SRCMACADDRIDX_S 16\n+#define IRDMAQPC_GEN1_SRCMACADDRIDX_M (0x3fUL << IRDMAQPC_GEN1_SRCMACADDRIDX_S)\n+\n+#define IRDMAQPC_AVOIDSTRETCHACK_S 23\n+#define IRDMAQPC_AVOIDSTRETCHACK_M BIT_ULL(IRDMAQPC_AVOIDSTRETCHACK_S)\n+\n+#define IRDMAQPC_TOS_S 24\n+#define IRDMAQPC_TOS_M (0xffULL << IRDMAQPC_TOS_S)\n+\n+#define IRDMAQPC_SRCPORTNUM_S 32\n+#define IRDMAQPC_SRCPORTNUM_M (0xffffULL << IRDMAQPC_SRCPORTNUM_S)\n+\n+#define IRDMAQPC_DESTPORTNUM_S 48\n+#define IRDMAQPC_DESTPORTNUM_M (0xffffULL << IRDMAQPC_DESTPORTNUM_S)\n+\n+#define IRDMAQPC_DESTIPADDR0_S 32\n+#define IRDMAQPC_DESTIPADDR0_M \\\n+\t(0xffffffffULL << IRDMAQPC_DESTIPADDR0_S)\n+\n+#define IRDMAQPC_DESTIPADDR1_S 0\n+#define IRDMAQPC_DESTIPADDR1_M \\\n+\t(0xffffffffULL << IRDMAQPC_DESTIPADDR1_S)\n+\n+#define IRDMAQPC_DESTIPADDR2_S 32\n+#define IRDMAQPC_DESTIPADDR2_M \\\n+\t(0xffffffffULL << IRDMAQPC_DESTIPADDR2_S)\n+\n+#define IRDMAQPC_DESTIPADDR3_S 0\n+#define IRDMAQPC_DESTIPADDR3_M \\\n+\t(0xffffffffULL << IRDMAQPC_DESTIPADDR3_S)\n+\n+#define IRDMAQPC_SNDMSS_S 16\n+#define IRDMAQPC_SNDMSS_M (0x3fffULL << IRDMAQPC_SNDMSS_S)\n+\n+#define IRDMAQPC_SYN_RST_HANDLING_S 30\n+#define IRDMAQPC_SYN_RST_HANDLING_M (0x3ULL << IRDMAQPC_SYN_RST_HANDLING_S)\n+\n+#define IRDMAQPC_VLANTAG_S 32\n+#define IRDMAQPC_VLANTAG_M (0xffffULL << IRDMAQPC_VLANTAG_S)\n+\n+#define IRDMAQPC_ARPIDX_S 48\n+#define IRDMAQPC_ARPIDX_M (0xffffULL << IRDMAQPC_ARPIDX_S)\n+\n+#define IRDMAQPC_FLOWLABEL_S 0\n+#define IRDMAQPC_FLOWLABEL_M (0xfffffULL << IRDMAQPC_FLOWLABEL_S)\n+\n+#define IRDMAQPC_WSCALE_S 20\n+#define IRDMAQPC_WSCALE_M BIT_ULL(IRDMAQPC_WSCALE_S)\n+\n+#define IRDMAQPC_KEEPALIVE_S 21\n+#define IRDMAQPC_KEEPALIVE_M BIT_ULL(IRDMAQPC_KEEPALIVE_S)\n+\n+#define IRDMAQPC_IGNORE_TCP_OPT_S 22\n+#define IRDMAQPC_IGNORE_TCP_OPT_M BIT_ULL(IRDMAQPC_IGNORE_TCP_OPT_S)\n+\n+#define IRDMAQPC_IGNORE_TCP_UNS_OPT_S 23\n+#define IRDMAQPC_IGNORE_TCP_UNS_OPT_M \\\n+\tBIT_ULL(IRDMAQPC_IGNORE_TCP_UNS_OPT_S)\n+\n+#define IRDMAQPC_TCPSTATE_S 28\n+#define IRDMAQPC_TCPSTATE_M (0xfULL << IRDMAQPC_TCPSTATE_S)\n+\n+#define IRDMAQPC_RCVSCALE_S 32\n+#define IRDMAQPC_RCVSCALE_M (0xfULL << IRDMAQPC_RCVSCALE_S)\n+\n+#define IRDMAQPC_SNDSCALE_S 40\n+#define IRDMAQPC_SNDSCALE_M (0xfULL << IRDMAQPC_SNDSCALE_S)\n+\n+#define IRDMAQPC_PDIDX_S 48\n+#define IRDMAQPC_PDIDX_M (0xffffULL << IRDMAQPC_PDIDX_S)\n+\n+#define IRDMAQPC_PDIDXHI_S 20\n+#define IRDMAQPC_PDIDXHI_M (0x3ULL << IRDMAQPC_PDIDXHI_S)\n+\n+#define IRDMAQPC_PKEY_S 32\n+#define IRDMAQPC_PKEY_M (0xffffULL << IRDMAQPC_PKEY_S)\n+\n+#define IRDMAQPC_ACKCREDITS_S 20\n+#define IRDMAQPC_ACKCREDITS_M (0x1fULL << IRDMAQPC_ACKCREDITS_S)\n+\n+#define IRDMAQPC_QKEY_S 32\n+#define IRDMAQPC_QKEY_M (0xffffffffULL << IRDMAQPC_QKEY_S)\n+\n+#define IRDMAQPC_DESTQP_S 0\n+#define IRDMAQPC_DESTQP_M (0xffffffULL << IRDMAQPC_DESTQP_S)\n+\n+#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES_S 16\n+#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES_M \\\n+\t(0xffULL << IRDMAQPC_KALIVE_TIMER_MAX_PROBES_S)\n+\n+#define IRDMAQPC_KEEPALIVE_INTERVAL_S 24\n+#define IRDMAQPC_KEEPALIVE_INTERVAL_M \\\n+\t(0xffULL << IRDMAQPC_KEEPALIVE_INTERVAL_S)\n+\n+#define IRDMAQPC_TIMESTAMP_RECENT_S 0\n+#define IRDMAQPC_TIMESTAMP_RECENT_M \\\n+\t(0xffffffffULL << IRDMAQPC_TIMESTAMP_RECENT_S)\n+\n+#define IRDMAQPC_TIMESTAMP_AGE_S 32\n+#define IRDMAQPC_TIMESTAMP_AGE_M \\\n+\t(0xffffffffULL << IRDMAQPC_TIMESTAMP_AGE_S)\n+\n+#define IRDMAQPC_SNDNXT_S 0\n+#define IRDMAQPC_SNDNXT_M (0xffffffffULL << IRDMAQPC_SNDNXT_S)\n+\n+#define IRDMAQPC_ISN_S 32\n+#define IRDMAQPC_ISN_M (0x00ffffffULL << IRDMAQPC_ISN_S)\n+\n+#define IRDMAQPC_PSNNXT_S 0\n+#define IRDMAQPC_PSNNXT_M (0x00ffffffULL << IRDMAQPC_PSNNXT_S)\n+\n+#define IRDMAQPC_LSN_S 32\n+#define IRDMAQPC_LSN_M (0x00ffffffULL << IRDMAQPC_LSN_S)\n+\n+#define IRDMAQPC_SNDWND_S 32\n+#define IRDMAQPC_SNDWND_M (0xffffffffULL << IRDMAQPC_SNDWND_S)\n+\n+#define IRDMAQPC_RCVNXT_S 0\n+#define IRDMAQPC_RCVNXT_M (0xffffffffULL << IRDMAQPC_RCVNXT_S)\n+\n+#define IRDMAQPC_EPSN_S 0\n+#define IRDMAQPC_EPSN_M (0x00ffffffULL << IRDMAQPC_EPSN_S)\n+\n+#define IRDMAQPC_RCVWND_S 32\n+#define IRDMAQPC_RCVWND_M (0xffffffffULL << IRDMAQPC_RCVWND_S)\n+\n+#define IRDMAQPC_SNDMAX_S 0\n+#define IRDMAQPC_SNDMAX_M (0xffffffffULL << IRDMAQPC_SNDMAX_S)\n+\n+#define IRDMAQPC_SNDUNA_S 32\n+#define IRDMAQPC_SNDUNA_M (0xffffffffULL << IRDMAQPC_SNDUNA_S)\n+\n+#define IRDMAQPC_PSNMAX_S 0\n+#define IRDMAQPC_PSNMAX_M (0x00ffffffULL << IRDMAQPC_PSNMAX_S)\n+#define IRDMAQPC_PSNUNA_S 32\n+#define IRDMAQPC_PSNUNA_M (0xffffffULL << IRDMAQPC_PSNUNA_S)\n+\n+#define IRDMAQPC_SRTT_S 0\n+#define IRDMAQPC_SRTT_M (0xffffffffULL << IRDMAQPC_SRTT_S)\n+\n+#define IRDMAQPC_RTTVAR_S 32\n+#define IRDMAQPC_RTTVAR_M (0xffffffffULL << IRDMAQPC_RTTVAR_S)\n+\n+#define IRDMAQPC_SSTHRESH_S 0\n+#define IRDMAQPC_SSTHRESH_M (0xffffffffULL << IRDMAQPC_SSTHRESH_S)\n+\n+#define IRDMAQPC_CWND_S 32\n+#define IRDMAQPC_CWND_M (0xffffffffULL << IRDMAQPC_CWND_S)\n+\n+#define IRDMAQPC_CWNDROCE_S 32\n+#define IRDMAQPC_CWNDROCE_M (0xffffffULL << IRDMAQPC_CWNDROCE_S)\n+#define IRDMAQPC_SNDWL1_S 0\n+#define IRDMAQPC_SNDWL1_M (0xffffffffULL << IRDMAQPC_SNDWL1_S)\n+\n+#define IRDMAQPC_SNDWL2_S 32\n+#define IRDMAQPC_SNDWL2_M (0xffffffffULL << IRDMAQPC_SNDWL2_S)\n+\n+#define IRDMAQPC_ERR_RQ_IDX_S 32\n+#define IRDMAQPC_ERR_RQ_IDX_M (0x3fffULL << IRDMAQPC_ERR_RQ_IDX_S)\n+\n+#define IRDMAQPC_MAXSNDWND_S 0\n+#define IRDMAQPC_MAXSNDWND_M (0xffffffffULL << IRDMAQPC_MAXSNDWND_S)\n+\n+#define IRDMAQPC_REXMIT_THRESH_S 48\n+#define IRDMAQPC_REXMIT_THRESH_M (0x3fULL << IRDMAQPC_REXMIT_THRESH_S)\n+\n+#define IRDMAQPC_RNRNAK_THRESH_S 54\n+#define IRDMAQPC_RNRNAK_THRESH_M (0x7ULL << IRDMAQPC_RNRNAK_THRESH_S)\n+\n+#define IRDMAQPC_TXCQNUM_S 0\n+#define IRDMAQPC_TXCQNUM_M (0x7ffffULL << IRDMAQPC_TXCQNUM_S)\n+\n+#define IRDMAQPC_RXCQNUM_S 32\n+#define IRDMAQPC_RXCQNUM_M (0x7ffffULL << IRDMAQPC_RXCQNUM_S)\n+\n+#define IRDMAQPC_STAT_INDEX_S 0\n+#define IRDMAQPC_STAT_INDEX_M (0x7fULL << IRDMAQPC_STAT_INDEX_S)\n+\n+#define IRDMAQPC_Q2ADDR_S 8\n+#define IRDMAQPC_Q2ADDR_M (0xffffffffffffffULL << IRDMAQPC_Q2ADDR_S)\n+\n+#define IRDMAQPC_LASTBYTESENT_S 0\n+#define IRDMAQPC_LASTBYTESENT_M (0xffULL << IRDMAQPC_LASTBYTESENT_S)\n+\n+#define IRDMAQPC_MACADDRESS_S 16\n+#define IRDMAQPC_MACADDRESS_M (0xffffffffffffULL << IRDMAQPC_MACADDRESS_S)\n+\n+#define IRDMAQPC_ORDSIZE_S 0\n+#define IRDMAQPC_ORDSIZE_M (0xffULL << IRDMAQPC_ORDSIZE_S)\n+\n+#define IRDMAQPC_IRDSIZE_S 16\n+#define IRDMAQPC_IRDSIZE_M (0x7ULL << IRDMAQPC_IRDSIZE_S)\n+\n+#define IRDMAQPC_UDPRIVCQENABLE_S 19\n+#define IRDMAQPC_UDPRIVCQENABLE_M BIT_ULL(IRDMAQPC_UDPRIVCQENABLE_S)\n+\n+#define IRDMAQPC_WRRDRSPOK_S 20\n+#define IRDMAQPC_WRRDRSPOK_M BIT_ULL(IRDMAQPC_WRRDRSPOK_S)\n+\n+#define IRDMAQPC_RDOK_S 21\n+#define IRDMAQPC_RDOK_M BIT_ULL(IRDMAQPC_RDOK_S)\n+\n+#define IRDMAQPC_SNDMARKERS_S 22\n+#define IRDMAQPC_SNDMARKERS_M BIT_ULL(IRDMAQPC_SNDMARKERS_S)\n+\n+#define IRDMAQPC_DCQCNENABLE_S 22\n+#define IRDMAQPC_DCQCNENABLE_M BIT_ULL(IRDMAQPC_DCQCNENABLE_S)\n+\n+#define IRDMAQPC_FW_CC_ENABLE_S 28\n+#define IRDMAQPC_FW_CC_ENABLE_M BIT_ULL(IRDMAQPC_FW_CC_ENABLE_S)\n+\n+#define IRDMAQPC_RCVNOICRC_S 31\n+#define IRDMAQPC_RCVNOICRC_M BIT_ULL(IRDMAQPC_RCVNOICRC_S)\n+\n+#define IRDMAQPC_BINDEN_S 23\n+#define IRDMAQPC_BINDEN_M BIT_ULL(IRDMAQPC_BINDEN_S)\n+\n+#define IRDMAQPC_FASTREGEN_S 24\n+#define IRDMAQPC_FASTREGEN_M BIT_ULL(IRDMAQPC_FASTREGEN_S)\n+\n+#define IRDMAQPC_PRIVEN_S 25\n+#define IRDMAQPC_PRIVEN_M BIT_ULL(IRDMAQPC_PRIVEN_S)\n+\n+#define IRDMAQPC_TIMELYENABLE_S 27\n+#define IRDMAQPC_TIMELYENABLE_M BIT_ULL(IRDMAQPC_TIMELYENABLE_S)\n+\n+#define IRDMAQPC_THIGH_S 52\n+#define IRDMAQPC_THIGH_M ((u64)0xfff << IRDMAQPC_THIGH_S)\n+\n+#define IRDMAQPC_TLOW_S 32\n+#define IRDMAQPC_TLOW_M ((u64)0xFF << IRDMAQPC_TLOW_S)\n+\n+#define IRDMAQPC_REMENDPOINTIDX_S 0\n+#define IRDMAQPC_REMENDPOINTIDX_M ((u64)0x1FFFF << IRDMAQPC_REMENDPOINTIDX_S)\n+\n+#define IRDMAQPC_USESTATSINSTANCE_S 26\n+#define IRDMAQPC_USESTATSINSTANCE_M BIT_ULL(IRDMAQPC_USESTATSINSTANCE_S)\n+\n+#define IRDMAQPC_IWARPMODE_S 28\n+#define IRDMAQPC_IWARPMODE_M BIT_ULL(IRDMAQPC_IWARPMODE_S)\n+\n+#define IRDMAQPC_RCVMARKERS_S 29\n+#define IRDMAQPC_RCVMARKERS_M BIT_ULL(IRDMAQPC_RCVMARKERS_S)\n+\n+#define IRDMAQPC_ALIGNHDRS_S 30\n+#define IRDMAQPC_ALIGNHDRS_M BIT_ULL(IRDMAQPC_ALIGNHDRS_S)\n+\n+#define IRDMAQPC_RCVNOMPACRC_S 31\n+#define IRDMAQPC_RCVNOMPACRC_M BIT_ULL(IRDMAQPC_RCVNOMPACRC_S)\n+\n+#define IRDMAQPC_RCVMARKOFFSET_S 32\n+#define IRDMAQPC_RCVMARKOFFSET_M (0x1ffULL << IRDMAQPC_RCVMARKOFFSET_S)\n+\n+#define IRDMAQPC_SNDMARKOFFSET_S 48\n+#define IRDMAQPC_SNDMARKOFFSET_M (0x1ffULL << IRDMAQPC_SNDMARKOFFSET_S)\n+\n+#define IRDMAQPC_QPCOMPCTX_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPC_QPCOMPCTX_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMAQPC_SQTPHVAL_S 0\n+#define IRDMAQPC_SQTPHVAL_M (0xffULL << IRDMAQPC_SQTPHVAL_S)\n+\n+#define IRDMAQPC_RQTPHVAL_S 8\n+#define IRDMAQPC_RQTPHVAL_M (0xffULL << IRDMAQPC_RQTPHVAL_S)\n+\n+#define IRDMAQPC_QSHANDLE_S 16\n+#define IRDMAQPC_QSHANDLE_M (0x3ffULL << IRDMAQPC_QSHANDLE_S)\n+\n+#define IRDMAQPC_EXCEPTION_LAN_QUEUE_S 32\n+#define IRDMAQPC_EXCEPTION_LAN_QUEUE_M \\\n+\t(0xfffULL << IRDMAQPC_EXCEPTION_LAN_QUEUE_S)\n+\n+#define IRDMAQPC_LOCAL_IPADDR3_S 0\n+#define IRDMAQPC_LOCAL_IPADDR3_M \\\n+\t(0xffffffffULL << IRDMAQPC_LOCAL_IPADDR3_S)\n+\n+#define IRDMAQPC_LOCAL_IPADDR2_S 32\n+#define IRDMAQPC_LOCAL_IPADDR2_M \\\n+\t(0xffffffffULL << IRDMAQPC_LOCAL_IPADDR2_S)\n+\n+#define IRDMAQPC_LOCAL_IPADDR1_S 0\n+#define IRDMAQPC_LOCAL_IPADDR1_M \\\n+\t(0xffffffffULL << IRDMAQPC_LOCAL_IPADDR1_S)\n+\n+#define IRDMAQPC_LOCAL_IPADDR0_S 32\n+#define IRDMAQPC_LOCAL_IPADDR0_M \\\n+\t(0xffffffffULL << IRDMAQPC_LOCAL_IPADDR0_S)\n+\n+#define IRDMA_FW_VER_MINOR_S 0\n+#define IRDMA_FW_VER_MINOR_M \\\n+\t(0xffffULL << IRDMA_FW_VER_MINOR_S)\n+\n+#define IRDMA_FW_VER_MAJOR_S 16\n+#define IRDMA_FW_VER_MAJOR_M \\\n+\t(0xffffULL << IRDMA_FW_VER_MAJOR_S)\n+\n+#define IRDMA_FEATURE_INFO_S 0\n+#define IRDMA_FEATURE_INFO_M \\\n+\t(0xffffffffffULL << IRDMA_FEATURE_INFO_S)\n+\n+#define IRDMA_FEATURE_CNT_S 32\n+#define IRDMA_FEATURE_CNT_M \\\n+\t(0xffffULL << IRDMA_FEATURE_CNT_S)\n+\n+#define IRDMA_RSVD_S 41\n+#define IRDMA_RSVD_M (0x7fffULL << IRDMA_RSVD_S)\n+\n+/* iwarp QP SQ WQE common fields */\n+#define IRDMAQPSQ_OPCODE_S 32\n+#define IRDMAQPSQ_OPCODE_M (0x3fULL << IRDMAQPSQ_OPCODE_S)\n+\n+#define IRDMAQPSQ_COPY_HOST_PBL_S 43\n+#define IRDMAQPSQ_COPY_HOST_PBL_M BIT_ULL(IRDMAQPSQ_COPY_HOST_PBL_S)\n+\n+#define IRDMAQPSQ_ADDFRAGCNT_S 38\n+#define IRDMAQPSQ_ADDFRAGCNT_M (0xfULL << IRDMAQPSQ_ADDFRAGCNT_S)\n+\n+#define IRDMAQPSQ_PUSHWQE_S 56\n+#define IRDMAQPSQ_PUSHWQE_M BIT_ULL(IRDMAQPSQ_PUSHWQE_S)\n+\n+#define IRDMAQPSQ_STREAMMODE_S 58\n+#define IRDMAQPSQ_STREAMMODE_M BIT_ULL(IRDMAQPSQ_STREAMMODE_S)\n+\n+#define IRDMAQPSQ_WAITFORRCVPDU_S 59\n+#define IRDMAQPSQ_WAITFORRCVPDU_M BIT_ULL(IRDMAQPSQ_WAITFORRCVPDU_S)\n+\n+#define IRDMAQPSQ_READFENCE_S 60\n+#define IRDMAQPSQ_READFENCE_M BIT_ULL(IRDMAQPSQ_READFENCE_S)\n+\n+#define IRDMAQPSQ_LOCALFENCE_S 61\n+#define IRDMAQPSQ_LOCALFENCE_M BIT_ULL(IRDMAQPSQ_LOCALFENCE_S)\n+\n+#define IRDMAQPSQ_UDPHEADER_S 61\n+#define IRDMAQPSQ_UDPHEADER_M BIT_ULL(IRDMAQPSQ_UDPHEADER_S)\n+\n+#define IRDMAQPSQ_L4LEN_S 42\n+#define IRDMAQPSQ_L4LEN_M ((u64)0xF << IRDMAQPSQ_L4LEN_S)\n+\n+#define IRDMAQPSQ_SIGCOMPL_S 62\n+#define IRDMAQPSQ_SIGCOMPL_M BIT_ULL(IRDMAQPSQ_SIGCOMPL_S)\n+\n+#define IRDMAQPSQ_VALID_S 63\n+#define IRDMAQPSQ_VALID_M BIT_ULL(IRDMAQPSQ_VALID_S)\n+\n+#define IRDMAQPSQ_FRAG_TO_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPSQ_FRAG_TO_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMAQPSQ_FRAG_VALID_S 63\n+#define IRDMAQPSQ_FRAG_VALID_M BIT_ULL(IRDMAQPSQ_FRAG_VALID_S)\n+\n+#define IRDMAQPSQ_FRAG_LEN_S 32\n+#define IRDMAQPSQ_FRAG_LEN_M (0x7fffffffULL << IRDMAQPSQ_FRAG_LEN_S)\n+\n+#define IRDMAQPSQ_FRAG_STAG_S 0\n+#define IRDMAQPSQ_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_FRAG_STAG_S)\n+\n+#define IRDMAQPSQ_GEN1_FRAG_LEN_S 0\n+#define IRDMAQPSQ_GEN1_FRAG_LEN_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_LEN_S)\n+\n+#define IRDMAQPSQ_GEN1_FRAG_STAG_S 32\n+#define IRDMAQPSQ_GEN1_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_STAG_S)\n+\n+#define IRDMAQPSQ_REMSTAGINV_S 0\n+#define IRDMAQPSQ_REMSTAGINV_M (0xffffffffULL << IRDMAQPSQ_REMSTAGINV_S)\n+\n+#define IRDMAQPSQ_DESTQKEY_S 0\n+#define IRDMAQPSQ_DESTQKEY_M (0xffffffffULL << IRDMAQPSQ_DESTQKEY_S)\n+\n+#define IRDMAQPSQ_DESTQPN_S 32\n+#define IRDMAQPSQ_DESTQPN_M (0x00ffffffULL << IRDMAQPSQ_DESTQPN_S)\n+\n+#define IRDMAQPSQ_AHID_S 0\n+#define IRDMAQPSQ_AHID_M (0x0001ffffULL << IRDMAQPSQ_AHID_S)\n+\n+#define IRDMAQPSQ_INLINEDATAFLAG_S 57\n+#define IRDMAQPSQ_INLINEDATAFLAG_M BIT_ULL(IRDMAQPSQ_INLINEDATAFLAG_S)\n+\n+#define IRDMA_INLINE_VALID_S 7\n+\n+#define IRDMAQPSQ_INLINEDATALEN_S 48\n+#define IRDMAQPSQ_INLINEDATALEN_M \\\n+\t(0xffULL << IRDMAQPSQ_INLINEDATALEN_S)\n+#define IRDMAQPSQ_IMMDATAFLAG_S 47\n+#define IRDMAQPSQ_IMMDATAFLAG_M \\\n+\tBIT_ULL(IRDMAQPSQ_IMMDATAFLAG_S)\n+#define IRDMAQPSQ_REPORTRTT_S 46\n+#define IRDMAQPSQ_REPORTRTT_M \\\n+\tBIT_ULL(IRDMAQPSQ_REPORTRTT_S)\n+\n+#define IRDMAQPSQ_IMMDATA_S 0\n+#define IRDMAQPSQ_IMMDATA_M \\\n+\t(0xffffffffffffffffULL << IRDMAQPSQ_IMMDATA_S)\n+\n+/* rdma write */\n+#define IRDMAQPSQ_REMSTAG_S 0\n+#define IRDMAQPSQ_REMSTAG_M (0xffffffffULL << IRDMAQPSQ_REMSTAG_S)\n+\n+#define IRDMAQPSQ_REMTO_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPSQ_REMTO_M IRDMA_CQPHC_QPCTX_M\n+\n+/* memory window */\n+#define IRDMAQPSQ_STAGRIGHTS_S 48\n+#define IRDMAQPSQ_STAGRIGHTS_M (0x1fULL << IRDMAQPSQ_STAGRIGHTS_S)\n+\n+#define IRDMAQPSQ_VABASEDTO_S 53\n+#define IRDMAQPSQ_VABASEDTO_M BIT_ULL(IRDMAQPSQ_VABASEDTO_S)\n+\n+#define IRDMAQPSQ_MEMWINDOWTYPE_S 54\n+#define IRDMAQPSQ_MEMWINDOWTYPE_M BIT_ULL(IRDMAQPSQ_MEMWINDOWTYPE_S)\n+\n+#define IRDMAQPSQ_MWLEN_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPSQ_MWLEN_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMAQPSQ_PARENTMRSTAG_S 32\n+#define IRDMAQPSQ_PARENTMRSTAG_M \\\n+\t(0xffffffffULL << IRDMAQPSQ_PARENTMRSTAG_S)\n+\n+#define IRDMAQPSQ_MWSTAG_S 0\n+#define IRDMAQPSQ_MWSTAG_M (0xffffffffULL << IRDMAQPSQ_MWSTAG_S)\n+\n+#define IRDMAQPSQ_BASEVA_TO_FBO_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPSQ_BASEVA_TO_FBO_M IRDMA_CQPHC_QPCTX_M\n+\n+/* Local Invalidate */\n+#define IRDMAQPSQ_LOCSTAG_S 0\n+#define IRDMAQPSQ_LOCSTAG_M (0xffffffffULL << IRDMAQPSQ_LOCSTAG_S)\n+\n+/* Fast Register */\n+#define IRDMAQPSQ_STAGKEY_S 0\n+#define IRDMAQPSQ_STAGKEY_M (0xffULL << IRDMAQPSQ_STAGKEY_S)\n+\n+#define IRDMAQPSQ_STAGINDEX_S 8\n+#define IRDMAQPSQ_STAGINDEX_M (0xffffffULL << IRDMAQPSQ_STAGINDEX_S)\n+\n+#define IRDMAQPSQ_COPYHOSTPBLS_S 43\n+#define IRDMAQPSQ_COPYHOSTPBLS_M BIT_ULL(IRDMAQPSQ_COPYHOSTPBLS_S)\n+\n+#define IRDMAQPSQ_LPBLSIZE_S 44\n+#define IRDMAQPSQ_LPBLSIZE_M (3ULL << IRDMAQPSQ_LPBLSIZE_S)\n+\n+#define IRDMAQPSQ_HPAGESIZE_S 46\n+#define IRDMAQPSQ_HPAGESIZE_M (3ULL << IRDMAQPSQ_HPAGESIZE_S)\n+\n+#define IRDMAQPSQ_STAGLEN_S 0\n+#define IRDMAQPSQ_STAGLEN_M (0x1ffffffffffULL << IRDMAQPSQ_STAGLEN_S)\n+\n+#define IRDMAQPSQ_FIRSTPMPBLIDXLO_S 48\n+#define IRDMAQPSQ_FIRSTPMPBLIDXLO_M \\\n+\t(0xffffULL << IRDMAQPSQ_FIRSTPMPBLIDXLO_S)\n+\n+#define IRDMAQPSQ_FIRSTPMPBLIDXHI_S 0\n+#define IRDMAQPSQ_FIRSTPMPBLIDXHI_M \\\n+\t(0xfffULL << IRDMAQPSQ_FIRSTPMPBLIDXHI_S)\n+\n+#define IRDMAQPSQ_PBLADDR_S 12\n+#define IRDMAQPSQ_PBLADDR_M (0xfffffffffffffULL << IRDMAQPSQ_PBLADDR_S)\n+\n+/* iwarp QP RQ WQE common fields */\n+#define IRDMAQPRQ_ADDFRAGCNT_S IRDMAQPSQ_ADDFRAGCNT_S\n+#define IRDMAQPRQ_ADDFRAGCNT_M IRDMAQPSQ_ADDFRAGCNT_M\n+\n+#define IRDMAQPRQ_VALID_S IRDMAQPSQ_VALID_S\n+#define IRDMAQPRQ_VALID_M IRDMAQPSQ_VALID_M\n+\n+#define IRDMAQPRQ_COMPLCTX_S IRDMA_CQPHC_QPCTX_S\n+#define IRDMAQPRQ_COMPLCTX_M IRDMA_CQPHC_QPCTX_M\n+\n+#define IRDMAQPRQ_FRAG_LEN_S IRDMAQPSQ_FRAG_LEN_S\n+#define IRDMAQPRQ_FRAG_LEN_M IRDMAQPSQ_FRAG_LEN_M\n+\n+#define IRDMAQPRQ_STAG_S IRDMAQPSQ_FRAG_STAG_S\n+#define IRDMAQPRQ_STAG_M IRDMAQPSQ_FRAG_STAG_M\n+\n+#define IRDMAQPRQ_TO_S IRDMAQPSQ_FRAG_TO_S\n+#define IRDMAQPRQ_TO_M IRDMAQPSQ_FRAG_TO_M\n+\n+/* Query FPM CQP buf */\n+#define IRDMA_QUERY_FPM_MAX_QPS_S 0\n+#define IRDMA_QUERY_FPM_MAX_QPS_M \\\n+\t(0x7ffffULL << IRDMA_QUERY_FPM_MAX_QPS_S)\n+\n+#define IRDMA_QUERY_FPM_MAX_CQS_S 0\n+#define IRDMA_QUERY_FPM_MAX_CQS_M \\\n+\t(0xfffffULL << IRDMA_QUERY_FPM_MAX_CQS_S)\n+\n+#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_S 0\n+#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_M \\\n+\t(0x3fffULL << IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_S)\n+\n+#define IRDMA_QUERY_FPM_MAX_PE_SDS_S 32\n+#define IRDMA_QUERY_FPM_MAX_PE_SDS_M \\\n+\t(0x3fffULL << IRDMA_QUERY_FPM_MAX_PE_SDS_S)\n+\n+#define IRDMA_QUERY_FPM_MAX_CEQS_S 0\n+#define IRDMA_QUERY_FPM_MAX_CEQS_M \\\n+\t(0x3ffULL << IRDMA_QUERY_FPM_MAX_CEQS_S)\n+\n+#define IRDMA_QUERY_FPM_XFBLOCKSIZE_S 32\n+#define IRDMA_QUERY_FPM_XFBLOCKSIZE_M \\\n+\t(0xffffffffULL << IRDMA_QUERY_FPM_XFBLOCKSIZE_S)\n+\n+#define IRDMA_QUERY_FPM_Q1BLOCKSIZE_S 32\n+#define IRDMA_QUERY_FPM_Q1BLOCKSIZE_M \\\n+\t(0xffffffffULL << IRDMA_QUERY_FPM_Q1BLOCKSIZE_S)\n+\n+#define IRDMA_QUERY_FPM_HTMULTIPLIER_S 16\n+#define IRDMA_QUERY_FPM_HTMULTIPLIER_M \\\n+\t(0xfULL << IRDMA_QUERY_FPM_HTMULTIPLIER_S)\n+\n+#define IRDMA_QUERY_FPM_TIMERBUCKET_S 32\n+#define IRDMA_QUERY_FPM_TIMERBUCKET_M \\\n+\t(0xffFFULL << IRDMA_QUERY_FPM_TIMERBUCKET_S)\n+\n+#define IRDMA_QUERY_FPM_RRFBLOCKSIZE_S 32\n+#define IRDMA_QUERY_FPM_RRFBLOCKSIZE_M \\\n+\t(0xffffffffULL << IRDMA_QUERY_FPM_RRFBLOCKSIZE_S)\n+\n+#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_S 32\n+#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_M \\\n+\t(0xffffffffULL << IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_S)\n+\n+#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_S 32\n+#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_M \\\n+\t(0xffffffffULL << IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_S)\n+\n+/* Static HMC pages allocated buf */\n+#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_S 0\n+#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_M \\\n+\t(0x3fULL << IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_S)\n+\n+#define IRDMA_GET_CURRENT_AEQ_ELEM(_aeq) \\\n+\t( \\\n+\t\t(_aeq)->aeqe_base[IRDMA_RING_CURRENT_TAIL((_aeq)->aeq_ring)].buf \\\n+\t)\n+\n+#define IRDMA_GET_CURRENT_CEQ_ELEM(_ceq) \\\n+\t( \\\n+\t\t(_ceq)->ceqe_base[IRDMA_RING_CURRENT_TAIL((_ceq)->ceq_ring)].buf \\\n+\t)\n+\n+#define IRDMA_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)\n+\n+#define IRDMA_GET_CURRENT_CQ_ELEM(_cq) \\\n+\t( \\\n+\t\t(_cq)->cq_base[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf  \\\n+\t)\n+#define IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(_cq) \\\n+\t( \\\n+\t\t((struct irdma_extended_cqe *) \\\n+\t\t((_cq)->cq_base))[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \\\n+\t)\n+\n+#define IRDMA_RING_INIT(_ring, _size) \\\n+\t{ \\\n+\t\t(_ring).head = 0; \\\n+\t\t(_ring).tail = 0; \\\n+\t\t(_ring).size = (_size); \\\n+\t}\n+#define IRDMA_RING_SIZE(_ring) ((_ring).size)\n+#define IRDMA_RING_CURRENT_HEAD(_ring) ((_ring).head)\n+#define IRDMA_RING_CURRENT_TAIL(_ring) ((_ring).tail)\n+\n+#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \\\n+\t{ \\\n+\t\tregister u32 size; \\\n+\t\tsize = (_ring).size;  \\\n+\t\tif (!IRDMA_RING_FULL_ERR(_ring)) { \\\n+\t\t\t(_ring).head = ((_ring).head + 1) % size; \\\n+\t\t\t(_retcode) = 0; \\\n+\t\t} else { \\\n+\t\t\t(_retcode) = IRDMA_ERR_RING_FULL; \\\n+\t\t} \\\n+\t}\n+#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \\\n+\t{ \\\n+\t\tregister u32 size; \\\n+\t\tsize = (_ring).size; \\\n+\t\tif ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \\\n+\t\t\t(_ring).head = ((_ring).head + (_count)) % size; \\\n+\t\t\t(_retcode) = 0; \\\n+\t\t} else { \\\n+\t\t\t(_retcode) = IRDMA_ERR_RING_FULL; \\\n+\t\t} \\\n+\t}\n+#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \\\n+\t{ \\\n+\t\tregister u32 size; \\\n+\t\tsize = (_ring).size;  \\\n+\t\tif (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \\\n+\t\t\t(_ring).head = ((_ring).head + 1) % size; \\\n+\t\t\t(_retcode) = 0; \\\n+\t\t} else { \\\n+\t\t\t(_retcode) = IRDMA_ERR_RING_FULL; \\\n+\t\t} \\\n+\t}\n+#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \\\n+\t{ \\\n+\t\tregister u32 size; \\\n+\t\tsize = (_ring).size; \\\n+\t\tif ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \\\n+\t\t\t(_ring).head = ((_ring).head + (_count)) % size; \\\n+\t\t\t(_retcode) = 0; \\\n+\t\t} else { \\\n+\t\t\t(_retcode) = IRDMA_ERR_RING_FULL; \\\n+\t\t} \\\n+\t}\n+#define IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \\\n+\t(_ring).head = ((_ring).head + (_count)) % (_ring).size\n+\n+#define IRDMA_RING_MOVE_TAIL(_ring) \\\n+\t(_ring).tail = ((_ring).tail + 1) % (_ring).size\n+\n+#define IRDMA_RING_MOVE_HEAD_NOCHECK(_ring) \\\n+\t(_ring).head = ((_ring).head + 1) % (_ring).size\n+\n+#define IRDMA_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \\\n+\t(_ring).tail = ((_ring).tail + (_count)) % (_ring).size\n+\n+#define IRDMA_RING_SET_TAIL(_ring, _pos) \\\n+\t(_ring).tail = (_pos) % (_ring).size\n+\n+#define IRDMA_RING_FULL_ERR(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 1))  \\\n+\t)\n+\n+#define IRDMA_ERR_RING_FULL2(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 2))  \\\n+\t)\n+\n+#define IRDMA_ERR_RING_FULL3(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 3))  \\\n+\t)\n+\n+#define IRDMA_SQ_RING_FULL_ERR(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 257))  \\\n+\t)\n+\n+#define IRDMA_ERR_SQ_RING_FULL2(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 258))  \\\n+\t)\n+#define IRDMA_ERR_SQ_RING_FULL3(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 259))  \\\n+\t)\n+#define IRDMA_RING_MORE_WORK(_ring) \\\n+\t( \\\n+\t\t(IRDMA_RING_USED_QUANTA(_ring) != 0) \\\n+\t)\n+\n+#define IRDMA_RING_USED_QUANTA(_ring) \\\n+\t( \\\n+\t\t(((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \\\n+\t)\n+\n+#define IRDMA_RING_FREE_QUANTA(_ring) \\\n+\t( \\\n+\t\t((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 1) \\\n+\t)\n+\n+#define IRDMA_SQ_RING_FREE_QUANTA(_ring) \\\n+\t( \\\n+\t\t((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 257) \\\n+\t)\n+\n+#define IRDMA_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \\\n+\t{ \\\n+\t\tindex = IRDMA_RING_CURRENT_HEAD(_ring); \\\n+\t\tIRDMA_RING_MOVE_HEAD(_ring, _retcode); \\\n+\t}\n+\n+enum irdma_qp_wqe_size {\n+\tIRDMA_WQE_SIZE_32  = 32,\n+\tIRDMA_WQE_SIZE_64  = 64,\n+\tIRDMA_WQE_SIZE_96  = 96,\n+\tIRDMA_WQE_SIZE_128 = 128,\n+\tIRDMA_WQE_SIZE_256 = 256,\n+};\n+\n+enum irdma_ws_node_op {\n+\tIRDMA_ADD_NODE = 0,\n+\tIRDMA_MODIFY_NODE,\n+\tIRDMA_DEL_NODE,\n+\tIRDMA_FAILOVER_START,\n+\tIRDMA_FAILOVER_COMPLETE,\n+};\n+\n+enum {\tIRDMA_Q_ALIGNMENT_M\t\t = (128 - 1),\n+\tIRDMA_AEQ_ALIGNMENT_M\t\t = (256 - 1),\n+\tIRDMA_Q2_ALIGNMENT_M\t\t = (256 - 1),\n+\tIRDMA_CEQ_ALIGNMENT_M\t\t = (256 - 1),\n+\tIRDMA_CQ0_ALIGNMENT_M\t\t = (256 - 1),\n+\tIRDMA_HOST_CTX_ALIGNMENT_M\t = (4 - 1),\n+\tIRDMA_SHADOWAREA_M\t\t = (128 - 1),\n+\tIRDMA_FPM_QUERY_BUF_ALIGNMENT_M\t = (4 - 1),\n+\tIRDMA_FPM_COMMIT_BUF_ALIGNMENT_M = (4 - 1),\n+};\n+\n+enum irdma_alignment {\n+\tIRDMA_CQP_ALIGNMENT\t    = 0x200,\n+\tIRDMA_AEQ_ALIGNMENT\t    = 0x100,\n+\tIRDMA_CEQ_ALIGNMENT\t    = 0x100,\n+\tIRDMA_CQ0_ALIGNMENT\t    = 0x100,\n+\tIRDMA_SD_BUF_ALIGNMENT      = 0x80,\n+\tIRDMA_FEATURE_BUF_ALIGNMENT = 0x8,\n+};\n+\n+enum icrdma_protocol_used {\n+\tICRDMA_ANY_PROTOCOL\t   = 0,\n+\tICRDMA_IWARP_PROTOCOL_ONLY = 1,\n+\tICRDMA_ROCE_PROTOCOL_ONLY  = 2,\n+};\n+\n+/**\n+ * set_64bit_val - set 64 bit value to hw wqe\n+ * @wqe_words: wqe addr to write\n+ * @byte_index: index in wqe\n+ * @val: value to write\n+ **/\n+static inline void set_64bit_val(__le64 *wqe_words, u32 byte_index, u64 val)\n+{\n+\twqe_words[byte_index >> 3] = cpu_to_le64(val);\n+}\n+\n+/**\n+ * set_32bit_val - set 32 bit value to hw wqe\n+ * @wqe_words: wqe addr to write\n+ * @byte_index: index in wqe\n+ * @val: value to write\n+ **/\n+static inline void set_32bit_val(u32 *wqe_words, u32 byte_index, u32 val)\n+{\n+\twqe_words[byte_index >> 2] = val;\n+}\n+\n+/**\n+ * get_64bit_val - read 64 bit value from wqe\n+ * @wqe_words: wqe addr\n+ * @byte_index: index to read from\n+ * @val: read value\n+ **/\n+static inline void get_64bit_val(__le64 *wqe_words, u32 byte_index, u64 *val)\n+{\n+\t*val = le64_to_cpu(wqe_words[byte_index >> 3]);\n+}\n+\n+/**\n+ * get_32bit_val - read 32 bit value from wqe\n+ * @wqe_words: wqe addr\n+ * @byte_index: index to reaad from\n+ * @val: return 32 bit value\n+ **/\n+static inline void get_32bit_val(u32 *wqe_words, u32 byte_index, u32 *val)\n+{\n+\t*val = wqe_words[byte_index >> 2];\n+}\n+#endif /* IRDMA_DEFS_H */\ndiff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h\nnew file mode 100644\nindex 0000000..c112005\n--- /dev/null\n+++ b/drivers/infiniband/hw/irdma/irdma.h\n@@ -0,0 +1,191 @@\n+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */\n+/* Copyright (c) 2019, Intel Corporation. */\n+\n+#ifndef IRDMA_H\n+#define IRDMA_H\n+\n+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S\t\t20\n+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_M\t\tMAKEMASK(0xfff, IRDMA_WQEALLOC_WQE_DESC_INDEX_S)\n+\n+#define IRDMA_CQPTAIL_WQTAIL_S\t\t\t0\n+#define IRDMA_CQPTAIL_WQTAIL_M\t\t\tMAKEMASK(0x7ff, IRDMA_CQPTAIL_WQTAIL_S)\n+\n+#define IRDMA_CQPTAIL_CQP_OP_ERR_S\t\t31\n+#define IRDMA_CQPTAIL_CQP_OP_ERR_M\t\tMAKEMASK(0x1, IRDMA_CQPTAIL_CQP_OP_ERR_S)\n+\n+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_S\t0\n+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_M\tMAKEMASK(0xffff, IRDMA_CQPERRCODES_CQP_MINOR_CODE_S)\n+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S\t16\n+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_M\tMAKEMASK(0xffff, IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S)\n+\n+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S\t4\n+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_M\tMAKEMASK(0x3, IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S)\n+\n+#define IRDMA_GLINT_DYN_CTL_INTENA_S\t\t0\n+#define IRDMA_GLINT_DYN_CTL_INTENA_M\t\tMAKEMASK(0x1, IRDMA_GLINT_DYN_CTL_INTENA_S)\n+\n+#define IRDMA_GLINT_DYN_CTL_CLEARPBA_S\t\t1\n+#define IRDMA_GLINT_DYN_CTL_CLEARPBA_M\t\tMAKEMASK(0x1, IRDMA_GLINT_DYN_CTL_CLEARPBA_S)\n+\n+#define IRDMA_GLINT_DYN_CTL_ITR_INDX_S\t\t3\n+#define IRDMA_GLINT_DYN_CTL_ITR_INDX_M\t\tMAKEMASK(0x3, IRDMA_GLINT_DYN_CTL_ITR_INDX_S)\n+\n+#define IRDMA_GLINT_CEQCTL_ITR_INDX_S\t\t11\n+#define IRDMA_GLINT_CEQCTL_ITR_INDX_M\t\tMAKEMASK(0x3, IRDMA_GLINT_CEQCTL_ITR_INDX_S)\n+\n+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_S\t\t30\n+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_M\t\tMAKEMASK(0x1, IRDMA_GLINT_CEQCTL_CAUSE_ENA_S)\n+\n+#define IRDMA_GLINT_CEQCTL_MSIX_INDX_S\t\t0\n+#define IRDMA_GLINT_CEQCTL_MSIX_INDX_M\t\tMAKEMASK(0x7ff, IRDMA_GLINT_CEQCTL_MSIX_INDX_S)\n+\n+#define IRDMA_PFINT_AEQCTL_MSIX_INDX_S\t\t0\n+#define IRDMA_PFINT_AEQCTL_MSIX_INDX_M\t\tMAKEMASK(0x7ff, IRDMA_PFINT_AEQCTL_MSIX_INDX_S)\n+\n+#define IRDMA_PFINT_AEQCTL_ITR_INDX_S\t\t11\n+#define IRDMA_PFINT_AEQCTL_ITR_INDX_M\t\tMAKEMASK(0x3, IRDMA_PFINT_AEQCTL_ITR_INDX_S)\n+\n+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_S\t\t30\n+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_M\t\tMAKEMASK(0x1, IRDMA_PFINT_AEQCTL_CAUSE_ENA_S)\n+\n+#define IRDMA_PFHMC_PDINV_PMSDIDX_S\t\t0\n+#define IRDMA_PFHMC_PDINV_PMSDIDX_M\t\tMAKEMASK(0xfff, IRDMA_PFHMC_PDINV_PMSDIDX_S)\n+\n+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_S\t\t15\n+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_M\t\tMAKEMASK(0x1, IRDMA_PFHMC_PDINV_PMSDPARTSEL_S)\n+\n+#define IRDMA_PFHMC_PDINV_PMPDIDX_S\t\t16\n+#define IRDMA_PFHMC_PDINV_PMPDIDX_M\t\tMAKEMASK(0x1ff, IRDMA_PFHMC_PDINV_PMPDIDX_S)\n+\n+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_S\t0\n+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_M\tMAKEMASK(0x1, IRDMA_PFHMC_SDDATALOW_PMSDVALID_S)\n+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S\t1\n+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_M\tMAKEMASK(0x1, IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S)\n+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S\t2\n+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_M\tMAKEMASK(0x3ff, IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S)\n+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S\t12\n+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_M\tMAKEMASK(0xfffff, IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S)\n+\n+#define IRDMA_PFHMC_SDCMD_PMSDWR_S\t\t31\n+#define IRDMA_PFHMC_SDCMD_PMSDWR_M\t\tMAKEMASK(0x1, IRDMA_PFHMC_SDCMD_PMSDWR_S)\n+\n+#define IRDMA_INVALID_CQ_IDX\t\t\t0xffffffff\n+\n+/* I40IW FW VER which supports RTS AE and CQ RESIZE */\n+#define IRDMA_FW_VER_0x30010\t\t\t0x30010\n+/* IRDMA FW VER */\n+#define IRDMA_FW_VER_0x1000D\t\t\t0x1000D\n+enum irdma_registers {\n+\tIRDMA_CQPTAIL,\n+\tIRDMA_CQPDB,\n+\tIRDMA_CCQPSTATUS,\n+\tIRDMA_CCQPHIGH,\n+\tIRDMA_CCQPLOW,\n+\tIRDMA_CQARM,\n+\tIRDMA_CQACK,\n+\tIRDMA_AEQALLOC,\n+\tIRDMA_CQPERRCODES,\n+\tIRDMA_WQEALLOC,\n+\tIRDMA_GLINT_DYN_CTL,\n+\tIRDMA_DB_ADDR_OFFSET,\n+\tIRDMA_GLPCI_LBARCTRL,\n+\tIRDMA_GLPE_CPUSTATUS0,\n+\tIRDMA_GLPE_CPUSTATUS1,\n+\tIRDMA_GLPE_CPUSTATUS2,\n+\tIRDMA_PFINT_AEQCTL,\n+\tIRDMA_GLINT_CEQCTL,\n+\tIRDMA_VSIQF_PE_CTL1,\n+\tIRDMA_PFHMC_PDINV,\n+\tIRDMA_GLHMC_VFPDINV,\n+\tIRDMA_MAX_REGS, /* Must be last entry */\n+};\n+\n+enum irdma_shifts {\n+\tIRDMA_CCQPSTATUS_CCQP_DONE_S,\n+\tIRDMA_CCQPSTATUS_CCQP_ERR_S,\n+\tIRDMA_CQPSQ_STAG_PDID_S,\n+\tIRDMA_CQPSQ_CQ_CEQID_S,\n+\tIRDMA_CQPSQ_CQ_CQID_S,\n+\tIRDMA_MAX_SHIFTS,\n+};\n+\n+enum irdma_masks {\n+\tIRDMA_CCQPSTATUS_CCQP_DONE_M,\n+\tIRDMA_CCQPSTATUS_CCQP_ERR_M,\n+\tIRDMA_CQPSQ_STAG_PDID_M,\n+\tIRDMA_CQPSQ_CQ_CEQID_M,\n+\tIRDMA_CQPSQ_CQ_CQID_M,\n+\tIRDMA_MAX_MASKS, /* Must be last entry */\n+};\n+\n+#define IRDMA_MAX_MGS_PER_CTX\t8\n+\n+struct irdma_mcast_grp_ctx_entry_info {\n+\tu32 qp_id;\n+\tbool valid_entry;\n+\tu16 dest_port;\n+\tu32 use_cnt;\n+};\n+\n+struct irdma_mcast_grp_info {\n+\tu8 dest_mac_addr[ETH_ALEN];\n+\tu16 vlan_id;\n+\tu8 hmc_fcn_id;\n+\tbool ipv4_valid;\n+\tbool vlan_valid;\n+\tu16 mg_id;\n+\tu32 no_of_mgs;\n+\tu32 dest_ip_addr[4];\n+\tu16 qs_handle;\n+\tstruct irdma_dma_mem dma_mem_mc;\n+\tstruct irdma_mcast_grp_ctx_entry_info mg_ctx_info[IRDMA_MAX_MGS_PER_CTX];\n+};\n+\n+enum irdma_vers {\n+\tIRDMA_GEN_RSVD,\n+\tIRDMA_GEN_1,\n+\tIRDMA_GEN_2,\n+\tIRDMA_GEN_3,\n+};\n+\n+struct irdma_uk_attrs {\n+\tu64 feature_flags;\n+\tu32 max_hw_wq_frags;\n+\tu32 max_hw_read_sges;\n+\tu32 max_hw_inline;\n+\tu32 max_hw_rq_quanta;\n+\tu32 max_hw_wq_quanta;\n+\tu32 min_hw_cq_size;\n+\tu32 max_hw_cq_size;\n+\tu16 max_hw_sq_chunk;\n+\tu8 hw_rev;\n+};\n+\n+struct irdma_hw_attrs {\n+\tstruct irdma_uk_attrs uk_attrs;\n+\tu64 max_hw_outbound_msg_size;\n+\tu64 max_hw_inbound_msg_size;\n+\tu64 max_mr_size;\n+\tu32 min_hw_qp_id;\n+\tu32 min_hw_aeq_size;\n+\tu32 max_hw_aeq_size;\n+\tu32 min_hw_ceq_size;\n+\tu32 max_hw_ceq_size;\n+\tu32 max_hw_device_pages;\n+\tu32 max_hw_vf_fpm_id;\n+\tu32 first_hw_vf_fpm_id;\n+\tu32 max_hw_ird;\n+\tu32 max_hw_ord;\n+\tu32 max_hw_wqes;\n+\tu32 max_hw_pds;\n+\tu32 max_hw_ena_vf_count;\n+\tu32 max_qp_wr;\n+\tu32 max_pe_ready_count;\n+\tu32 max_done_count;\n+\tu32 max_sleep_count;\n+\tu32 max_cqp_compl_wait_time_ms;\n+\tu16 max_stat_inst;\n+};\n+\n+void icrdma_init_hw(struct irdma_sc_dev *dev);\n+#endif /* IRDMA_H*/\ndiff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h\nnew file mode 100644\nindex 0000000..5c5970b\n--- /dev/null\n+++ b/drivers/infiniband/hw/irdma/type.h\n@@ -0,0 +1,1701 @@\n+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */\n+/* Copyright (c) 2019, Intel Corporation. */\n+\n+#ifndef IRDMA_TYPE_H\n+#define IRDMA_TYPE_H\n+#include \"osdep.h\"\n+#include \"irdma.h\"\n+#include \"user.h\"\n+#include \"hmc.h\"\n+#include \"uda.h\"\n+\n+#define IRDMA_DEBUG_ERR\t\t\"ERR\"\n+#define IRDMA_DEBUG_INIT\t\"INIT\"\n+#define IRDMA_DEBUG_DEV\t\t\"DEV\"\n+#define IRDMA_DEBUG_CM\t\t\"CM\"\n+#define IRDMA_DEBUG_VERBS\t\"VERBS\"\n+#define IRDMA_DEBUG_PUDA\t\"PUDA\"\n+#define IRDMA_DEBUG_ILQ\t\t\"ILQ\"\n+#define IRDMA_DEBUG_IEQ\t\t\"IEQ\"\n+#define IRDMA_DEBUG_QP\t\t\"QP\"\n+#define IRDMA_DEBUG_CQ\t\t\"CQ\"\n+#define IRDMA_DEBUG_MR\t\t\"MR\"\n+#define IRDMA_DEBUG_PBLE\t\"PBLE\"\n+#define IRDMA_DEBUG_WQE\t\t\"WQE\"\n+#define IRDMA_DEBUG_AEQ\t\t\"AEQ\"\n+#define IRDMA_DEBUG_CQP\t\t\"CQP\"\n+#define IRDMA_DEBUG_HMC\t\t\"HMC\"\n+#define IRDMA_DEBUG_USER\t\"USER\"\n+#define IRDMA_DEBUG_VIRT\t\"VIRT\"\n+#define IRDMA_DEBUG_DCB\t\t\"DCB\"\n+#define\tIRDMA_DEBUG_CQE\t\t\"CQE\"\n+#define IRDMA_DEBUG_CLNT\t\"CLNT\"\n+#define IRDMA_DEBUG_WS\t\t\"WS\"\n+#define IRDMA_DEBUG_STATS\t\"STATS\"\n+\n+enum irdma_page_size {\n+\tIRDMA_PAGE_SIZE_4K = 0,\n+\tIRDMA_PAGE_SIZE_2M,\n+\tIRDMA_PAGE_SIZE_1G,\n+};\n+\n+enum irdma_hdrct_flags {\n+\tDDP_LEN_FLAG  = 0x80,\n+\tDDP_HDR_FLAG  = 0x40,\n+\tRDMA_HDR_FLAG = 0x20,\n+};\n+\n+enum irdma_term_layers {\n+\tLAYER_RDMA = 0,\n+\tLAYER_DDP  = 1,\n+\tLAYER_MPA  = 2,\n+};\n+\n+enum irdma_term_error_types {\n+\tRDMAP_REMOTE_PROT = 1,\n+\tRDMAP_REMOTE_OP   = 2,\n+\tDDP_CATASTROPHIC  = 0,\n+\tDDP_TAGGED_BUF    = 1,\n+\tDDP_UNTAGGED_BUF  = 2,\n+\tDDP_LLP\t\t  = 3,\n+};\n+\n+enum irdma_term_rdma_errors {\n+\tRDMAP_INV_STAG\t\t  = 0x00,\n+\tRDMAP_INV_BOUNDS\t  = 0x01,\n+\tRDMAP_ACCESS\t\t  = 0x02,\n+\tRDMAP_UNASSOC_STAG\t  = 0x03,\n+\tRDMAP_TO_WRAP\t\t  = 0x04,\n+\tRDMAP_INV_RDMAP_VER       = 0x05,\n+\tRDMAP_UNEXPECTED_OP       = 0x06,\n+\tRDMAP_CATASTROPHIC_LOCAL  = 0x07,\n+\tRDMAP_CATASTROPHIC_GLOBAL = 0x08,\n+\tRDMAP_CANT_INV_STAG       = 0x09,\n+\tRDMAP_UNSPECIFIED\t  = 0xff,\n+};\n+\n+enum irdma_term_ddp_errors {\n+\tDDP_CATASTROPHIC_LOCAL      = 0x00,\n+\tDDP_TAGGED_INV_STAG\t    = 0x00,\n+\tDDP_TAGGED_BOUNDS\t    = 0x01,\n+\tDDP_TAGGED_UNASSOC_STAG     = 0x02,\n+\tDDP_TAGGED_TO_WRAP\t    = 0x03,\n+\tDDP_TAGGED_INV_DDP_VER      = 0x04,\n+\tDDP_UNTAGGED_INV_QN\t    = 0x01,\n+\tDDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,\n+\tDDP_UNTAGGED_INV_MSN_RANGE  = 0x03,\n+\tDDP_UNTAGGED_INV_MO\t    = 0x04,\n+\tDDP_UNTAGGED_INV_TOO_LONG   = 0x05,\n+\tDDP_UNTAGGED_INV_DDP_VER    = 0x06,\n+};\n+\n+enum irdma_term_mpa_errors {\n+\tMPA_CLOSED  = 0x01,\n+\tMPA_CRC     = 0x02,\n+\tMPA_MARKER  = 0x03,\n+\tMPA_REQ_RSP = 0x04,\n+};\n+\n+enum irdma_flush_opcode {\n+\tFLUSH_INVALID = 0,\n+\tFLUSH_PROT_ERR,\n+\tFLUSH_REM_ACCESS_ERR,\n+\tFLUSH_LOC_QP_OP_ERR,\n+\tFLUSH_REM_OP_ERR,\n+\tFLUSH_LOC_LEN_ERR,\n+\tFLUSH_GENERAL_ERR,\n+\tFLUSH_FATAL_ERR,\n+};\n+\n+enum irdma_term_eventtypes {\n+\tTERM_EVENT_QP_FATAL,\n+\tTERM_EVENT_QP_ACCESS_ERR,\n+};\n+\n+enum irdma_hw_stats_index_32b {\n+\tIRDMA_HW_STAT_INDEX_IP4RXDISCARD\t= 0,\n+\tIRDMA_HW_STAT_INDEX_IP4RXTRUNC\t\t= 1,\n+\tIRDMA_HW_STAT_INDEX_IP4TXNOROUTE\t= 2,\n+\tIRDMA_HW_STAT_INDEX_IP6RXDISCARD\t= 3,\n+\tIRDMA_HW_STAT_INDEX_IP6RXTRUNC\t\t= 4,\n+\tIRDMA_HW_STAT_INDEX_IP6TXNOROUTE\t= 5,\n+\tIRDMA_HW_STAT_INDEX_TCPRTXSEG\t\t= 6,\n+\tIRDMA_HW_STAT_INDEX_TCPRXOPTERR\t\t= 7,\n+\tIRDMA_HW_STAT_INDEX_TCPRXPROTOERR\t= 8,\n+\tIRDMA_HW_STAT_INDEX_MAX_32_GEN_1\t= 9, /* Must be same value as next entry */\n+\tIRDMA_HW_STAT_INDEX_RXVLANERR\t\t= 9,\n+\tIRDMA_HW_STAT_INDEX_RXRPCNPHANDLED\t= 10,\n+\tIRDMA_HW_STAT_INDEX_RXRPCNPIGNORED\t= 11,\n+\tIRDMA_HW_STAT_INDEX_TXNPCNPSENT\t\t= 12,\n+\tIRDMA_HW_STAT_INDEX_MAX_32, /* Must be last entry */\n+};\n+\n+enum irdma_hw_stats_index_64b {\n+\tIRDMA_HW_STAT_INDEX_IP4RXOCTS\t= 0,\n+\tIRDMA_HW_STAT_INDEX_IP4RXPKTS\t= 1,\n+\tIRDMA_HW_STAT_INDEX_IP4RXFRAGS\t= 2,\n+\tIRDMA_HW_STAT_INDEX_IP4RXMCPKTS\t= 3,\n+\tIRDMA_HW_STAT_INDEX_IP4TXOCTS\t= 4,\n+\tIRDMA_HW_STAT_INDEX_IP4TXPKTS\t= 5,\n+\tIRDMA_HW_STAT_INDEX_IP4TXFRAGS\t= 6,\n+\tIRDMA_HW_STAT_INDEX_IP4TXMCPKTS\t= 7,\n+\tIRDMA_HW_STAT_INDEX_IP6RXOCTS\t= 8,\n+\tIRDMA_HW_STAT_INDEX_IP6RXPKTS\t= 9,\n+\tIRDMA_HW_STAT_INDEX_IP6RXFRAGS\t= 10,\n+\tIRDMA_HW_STAT_INDEX_IP6RXMCPKTS\t= 11,\n+\tIRDMA_HW_STAT_INDEX_IP6TXOCTS\t= 12,\n+\tIRDMA_HW_STAT_INDEX_IP6TXPKTS\t= 13,\n+\tIRDMA_HW_STAT_INDEX_IP6TXFRAGS\t= 14,\n+\tIRDMA_HW_STAT_INDEX_IP6TXMCPKTS\t= 15,\n+\tIRDMA_HW_STAT_INDEX_TCPRXSEGS\t= 16,\n+\tIRDMA_HW_STAT_INDEX_TCPTXSEG\t= 17,\n+\tIRDMA_HW_STAT_INDEX_RDMARXRDS\t= 18,\n+\tIRDMA_HW_STAT_INDEX_RDMARXSNDS\t= 19,\n+\tIRDMA_HW_STAT_INDEX_RDMARXWRS\t= 20,\n+\tIRDMA_HW_STAT_INDEX_RDMATXRDS\t= 21,\n+\tIRDMA_HW_STAT_INDEX_RDMATXSNDS\t= 22,\n+\tIRDMA_HW_STAT_INDEX_RDMATXWRS\t= 23,\n+\tIRDMA_HW_STAT_INDEX_RDMAVBND\t= 24,\n+\tIRDMA_HW_STAT_INDEX_RDMAVINV\t= 25,\n+\tIRDMA_HW_STAT_INDEX_MAX_64_GEN_1 = 26, /* Must be same value as next entry */\n+\tIRDMA_HW_STAT_INDEX_IP4RXMCOCTS\t= 26,\n+\tIRDMA_HW_STAT_INDEX_IP4TXMCOCTS\t= 27,\n+\tIRDMA_HW_STAT_INDEX_IP6RXMCOCTS\t= 28,\n+\tIRDMA_HW_STAT_INDEX_IP6TXMCOCTS\t= 29,\n+\tIRDMA_HW_STAT_INDEX_UDPRXPKTS\t= 30,\n+\tIRDMA_HW_STAT_INDEX_UDPTXPKTS\t= 31,\n+\tIRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 32,\n+\tIRDMA_HW_STAT_INDEX_MAX_64, /* Must be last entry */\n+};\n+\n+enum irdma_feature_type {\n+\tIRDMA_FEATURE_FW_INFO = 0,\n+\tIRDMA_HW_VERSION_INFO,\n+\tIRDMA_MAX_FEATURES, /* Must be last entry */\n+};\n+\n+enum irdma_sched_prio_type {\n+\tIRDMA_PRIO_WEIGHTED_RR     = 1,\n+\tIRDMA_PRIO_STRICT\t   = 2,\n+\tIRDMA_PRIO_WEIGHTED_STRICT = 3,\n+};\n+\n+enum irdma_vm_vf_type {\n+\tIRDMA_VF_TYPE = 0,\n+\tIRDMA_VM_TYPE,\n+\tIRDMA_PF_TYPE,\n+};\n+\n+enum irdma_cqp_hmc_profile {\n+\tIRDMA_HMC_PROFILE_DEFAULT  = 1,\n+\tIRDMA_HMC_PROFILE_FAVOR_VF = 2,\n+\tIRDMA_HMC_PROFILE_EQUAL    = 3,\n+};\n+\n+enum irdma_quad_entry_type {\n+\tIRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,\n+\tIRDMA_QHASH_TYPE_TCP_SYN,\n+\tIRDMA_QHASH_TYPE_UDP_UNICAST,\n+\tIRDMA_QHASH_TYPE_UDP_MCAST,\n+\tIRDMA_QHASH_TYPE_ROCE_MCAST,\n+\tIRDMA_QHASH_TYPE_ROCEV2_HW,\n+};\n+\n+enum irdma_quad_hash_manage_type {\n+\tIRDMA_QHASH_MANAGE_TYPE_DELETE = 0,\n+\tIRDMA_QHASH_MANAGE_TYPE_ADD,\n+\tIRDMA_QHASH_MANAGE_TYPE_MODIFY,\n+};\n+\n+enum irdma_syn_rst_handling {\n+\tIRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,\n+\tIRDMA_SYN_RST_HANDLING_HW_TCP,\n+\tIRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,\n+\tIRDMA_SYN_RST_HANDLING_FW_TCP,\n+};\n+\n+struct irdma_sc_dev;\n+struct irdma_vsi_pestat;\n+struct irdma_irq_ops;\n+struct irdma_cqp_ops;\n+struct irdma_ccq_ops;\n+struct irdma_ceq_ops;\n+struct irdma_aeq_ops;\n+struct irdma_mr_ops;\n+struct irdma_cqp_misc_ops;\n+struct irdma_pd_ops;\n+struct irdma_ah_ops;\n+struct irdma_priv_qp_ops;\n+struct irdma_priv_cq_ops;\n+struct irdma_hmc_ops;\n+\n+struct irdma_cqp_init_info {\n+\tu64 cqp_compl_ctx;\n+\tu64 host_ctx_pa;\n+\tu64 sq_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tstruct irdma_cqp_quanta *sq;\n+\t__le64 *host_ctx;\n+\tu64 *scratch_array;\n+\tu32 sq_size;\n+\tu16 hw_maj_ver;\n+\tu16 hw_min_ver;\n+\tu8 struct_ver;\n+\tbool en_datacenter_tcp;\n+\tu8 hmc_profile;\n+\tu8 ena_vf_count;\n+\tu8 ceqs_per_vf;\n+\tbool disable_packed;\n+\tbool rocev2_rto_policy;\n+\tenum irdma_protocol_used protocol_used;\n+};\n+\n+struct irdma_terminate_hdr {\n+\tu8 layer_etype;\n+\tu8 error_code;\n+\tu8 hdrct;\n+\tu8 rsvd;\n+};\n+\n+struct irdma_cqp_sq_wqe {\n+\t__le64 buf[IRDMA_CQP_WQE_SIZE];\n+};\n+\n+struct irdma_sc_aeqe {\n+\t__le64 buf[IRDMA_AEQE_SIZE];\n+};\n+\n+struct irdma_ceqe {\n+\t__le64 buf[IRDMA_CEQE_SIZE];\n+};\n+\n+struct irdma_cqp_ctx {\n+\t__le64 buf[IRDMA_CQP_CTX_SIZE];\n+};\n+\n+struct irdma_cq_shadow_area {\n+\t__le64 buf[IRDMA_SHADOW_AREA_SIZE];\n+};\n+\n+struct irdma_dev_hw_stats_offsets {\n+\tu32 stats_offset_32[IRDMA_HW_STAT_INDEX_MAX_32];\n+\tu32 stats_offset_64[IRDMA_HW_STAT_INDEX_MAX_64];\n+};\n+\n+struct irdma_dev_hw_stats {\n+\tu64 stats_val_32[IRDMA_HW_STAT_INDEX_MAX_32];\n+\tu64 stats_val_64[IRDMA_HW_STAT_INDEX_MAX_64];\n+};\n+\n+struct irdma_gather_stats {\n+\tu32 rsvd1;\n+\tu32 rxvlanerr;\n+\tu64 ip4rxocts;\n+\tu64 ip4rxpkts;\n+\tu32 ip4rxtrunc;\n+\tu32 ip4rxdiscard;\n+\tu64 ip4rxfrags;\n+\tu64 ip4rxmcocts;\n+\tu64 ip4rxmcpkts;\n+\tu64 ip6rxocts;\n+\tu64 ip6rxpkts;\n+\tu32 ip6rxtrunc;\n+\tu32 ip6rxdiscard;\n+\tu64 ip6rxfrags;\n+\tu64 ip6rxmcocts;\n+\tu64 ip6rxmcpkts;\n+\tu64 ip4txocts;\n+\tu64 ip4txpkts;\n+\tu64 ip4txfrag;\n+\tu64 ip4txmcocts;\n+\tu64 ip4txmcpkts;\n+\tu64 ip6txocts;\n+\tu64 ip6txpkts;\n+\tu64 ip6txfrags;\n+\tu64 ip6txmcocts;\n+\tu64 ip6txmcpkts;\n+\tu32 ip6txnoroute;\n+\tu32 ip4txnoroute;\n+\tu64 tcprxsegs;\n+\tu32 tcprxprotoerr;\n+\tu32 tcprxopterr;\n+\tu64 tcptxsegs;\n+\tu32 rsvd2;\n+\tu32 tcprtxseg;\n+\tu64 udprxpkts;\n+\tu64 udptxpkts;\n+\tu64 rdmarxwrs;\n+\tu64 rdmarxrds;\n+\tu64 rdmarxsnds;\n+\tu64 rdmatxwrs;\n+\tu64 rdmatxrds;\n+\tu64 rdmatxsnds;\n+\tu64 rdmavbn;\n+\tu64 rdmavinv;\n+\tu64 rxnpecnmrkpkts;\n+\tu32 rxrpcnphandled;\n+\tu32 rxrpcnpignored;\n+\tu32 txnpcnpsent;\n+\tu32 rsvd3[88];\n+};\n+\n+struct irdma_stats_gather_info {\n+\tbool use_hmc_fcn_index;\n+\tbool use_stats_inst;\n+\tu8 hmc_fcn_index;\n+\tu8 stats_inst_index;\n+\tstruct irdma_dma_mem stats_buff_mem;\n+\tstruct irdma_gather_stats *gather_stats;\n+\tstruct irdma_gather_stats *last_gather_stats;\n+};\n+\n+struct irdma_vsi_pestat {\n+\tstruct irdma_hw *hw;\n+\tstruct irdma_dev_hw_stats hw_stats;\n+\tstruct irdma_stats_gather_info gather_info;\n+\tstruct timer_list stats_timer;\n+\tstruct irdma_sc_vsi *vsi;\n+\tstruct irdma_dev_hw_stats last_hw_stats;\n+\tspinlock_t lock; /* rdma stats lock */\n+};\n+\n+struct irdma_hw {\n+\tu8 __iomem *hw_addr;\n+\tstruct pci_dev *pdev;\n+\tstruct irdma_hmc_info hmc;\n+};\n+\n+struct irdma_pfpdu {\n+\tstruct list_head rxlist;\n+\tu32 rcv_nxt;\n+\tu32 fps;\n+\tu32 max_fpdu_data;\n+\tu32 nextseqnum;\n+\tbool mode;\n+\tbool mpa_crc_err;\n+\tu64 total_ieq_bufs;\n+\tu64 fpdu_processed;\n+\tu64 bad_seq_num;\n+\tu64 crc_err;\n+\tu64 no_tx_bufs;\n+\tu64 tx_err;\n+\tu64 out_of_order;\n+\tu64 pmode_count;\n+\tstruct irdma_sc_ah *ah;\n+\tstruct irdma_puda_buf *ah_buf;\n+\tspinlock_t lock; /* fpdu processing lock */\n+\tstruct irdma_puda_buf *lastrcv_buf;\n+};\n+\n+struct irdma_sc_pd {\n+\tstruct irdma_sc_dev *dev;\n+\tu32 pd_id;\n+\tint abi_ver;\n+};\n+\n+struct irdma_cqp_quanta {\n+\t__le64 elem[IRDMA_CQP_WQE_SIZE];\n+};\n+\n+struct irdma_sc_cqp {\n+\tu32 size;\n+\tu64 sq_pa;\n+\tu64 host_ctx_pa;\n+\tvoid *back_cqp;\n+\tstruct irdma_sc_dev *dev;\n+\tenum irdma_status_code (*process_cqp_sds)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t  struct irdma_update_sds_info *info);\n+\tstruct irdma_dma_mem sdbuf;\n+\tstruct irdma_ring sq_ring;\n+\tstruct irdma_cqp_quanta *sq_base;\n+\t__le64 *host_ctx;\n+\tu64 *scratch_array;\n+\tu32 cqp_id;\n+\tu32 sq_size;\n+\tu32 hw_sq_size;\n+\tu16 hw_maj_ver;\n+\tu16 hw_min_ver;\n+\tu8 struct_ver;\n+\tu8 polarity;\n+\tu8 hmc_profile;\n+\tu8 ena_vf_count;\n+\tu8 timeout_count;\n+\tu8 ceqs_per_vf;\n+\tbool en_datacenter_tcp;\n+\tbool disable_packed;\n+\tbool rocev2_rto_policy;\n+\tenum irdma_protocol_used protocol_used;\n+};\n+\n+struct irdma_sc_aeq {\n+\tu32 size;\n+\tu64 aeq_elem_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tstruct irdma_sc_aeqe *aeqe_base;\n+\tvoid *pbl_list;\n+\tu32 elem_cnt;\n+\tstruct irdma_ring aeq_ring;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tu32 first_pm_pbl_idx;\n+\tu8 polarity;\n+};\n+\n+struct irdma_sc_ceq {\n+\tu32 size;\n+\tu64 ceq_elem_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tstruct irdma_ceqe *ceqe_base;\n+\tvoid *pbl_list;\n+\tu32 ceq_id;\n+\tu32 elem_cnt;\n+\tstruct irdma_ring ceq_ring;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tbool tph_en;\n+\tu8 tph_val;\n+\tu32 first_pm_pbl_idx;\n+\tu8 polarity;\n+\tbool itr_no_expire;\n+\tstruct irdma_sc_vsi *vsi;\n+\tstruct irdma_sc_cq **reg_cq;\n+\tu32 reg_cq_size;\n+\tspinlock_t req_cq_lock; /* protect access to reg_cq array */\n+};\n+\n+struct irdma_sc_cq {\n+\tstruct irdma_cq_uk cq_uk;\n+\tu64 cq_pa;\n+\tu64 shadow_area_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tstruct irdma_sc_vsi *vsi;\n+\tvoid *pbl_list;\n+\tvoid *back_cq;\n+\tu32 ceq_id;\n+\tu32 shadow_read_threshold;\n+\tbool ceqe_mask;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tu8 cq_type;\n+\tbool ceq_id_valid;\n+\tbool tph_en;\n+\tu8 tph_val;\n+\tu32 first_pm_pbl_idx;\n+\tbool check_overflow;\n+};\n+\n+struct irdma_sc_qp {\n+\tstruct irdma_qp_uk qp_uk;\n+\tu64 sq_pa;\n+\tu64 rq_pa;\n+\tu64 hw_host_ctx_pa;\n+\tu64 shadow_area_pa;\n+\tu64 q2_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tstruct irdma_sc_vsi *vsi;\n+\tstruct irdma_sc_pd *pd;\n+\t__le64 *hw_host_ctx;\n+\tvoid *llp_stream_handle;\n+\tstruct irdma_pfpdu pfpdu;\n+\tu32 ieq_qp;\n+\tu8 *q2_buf;\n+\tu64 qp_compl_ctx;\n+\tu16 qs_handle;\n+\tu16 push_idx;\n+\tu16 push_offset;\n+\tu8 flush_wqes_count;\n+\tu8 sq_tph_val;\n+\tu8 rq_tph_val;\n+\tu8 qp_state;\n+\tu8 qp_type;\n+\tu8 hw_sq_size;\n+\tu8 hw_rq_size;\n+\tu8 src_mac_addr_idx;\n+\tbool sq_tph_en;\n+\tbool rq_tph_en;\n+\tbool rcv_tph_en;\n+\tbool xmit_tph_en;\n+\tbool virtual_map;\n+\tbool flush_sq;\n+\tbool flush_rq;\n+\tu8 user_pri;\n+\tstruct list_head list;\n+\tbool on_qoslist;\n+\tbool sq_flush;\n+\tenum irdma_flush_opcode flush_code;\n+\tenum irdma_term_eventtypes eventtype;\n+\tu8 term_flags;\n+};\n+\n+struct irdma_stats_inst_info {\n+\tbool use_hmc_fcn_index;\n+\tu8 hmc_fn_id;\n+\tu8 stats_idx;\n+};\n+\n+struct irdma_up_info {\n+\tu8 map[8];\n+\tu8 cnp_up_override;\n+\tu8 hmc_fcn_idx;\n+\tbool use_vlan;\n+\tbool use_cnp_up_override;\n+};\n+\n+#define IRDMA_MAX_WS_NODES\t0x3FF\n+#define IRDMA_WS_NODE_INVALID\t0xFFFF\n+\n+struct irdma_ws_node_info {\n+\tu16 id;\n+\tu16 vsi;\n+\tu16 parent_id;\n+\tu16 qs_handle;\n+\tbool type_leaf;\n+\tbool enable;\n+\tu8 prio_type;\n+\tu8 tc;\n+\tu8 weight;\n+};\n+\n+struct irdma_hmc_fpm_misc {\n+\tu32 max_ceqs;\n+\tu32 max_sds;\n+\tu32 xf_block_size;\n+\tu32 q1_block_size;\n+\tu32 ht_multiplier;\n+\tu32 timer_bucket;\n+\tu32 rrf_block_size;\n+\tu32 ooiscf_block_size;\n+};\n+\n+#define IRDMA_LEAF_DEFAULT_REL_BW\t\t64\n+#define IRDMA_PARENT_DEFAULT_REL_BW\t\t1\n+\n+struct irdma_qos {\n+\tstruct list_head qplist;\n+\tspinlock_t lock; /* protect qos list */\n+\tu64 lan_qos_handle;\n+\tu32 l2_sched_node_id;\n+\tu16 qs_handle;\n+\tu8 traffic_class;\n+\tu8 rel_bw;\n+\tu8 prio_type;\n+};\n+\n+#define IRDMA_INVALID_FCN_ID 0xff\n+struct irdma_sc_vsi {\n+\tu16 vsi_idx;\n+\tstruct irdma_sc_dev *dev;\n+\tvoid *back_vsi;\n+\tu32 ilq_count;\n+\tstruct irdma_virt_mem ilq_mem;\n+\tstruct irdma_puda_rsrc *ilq;\n+\tu32 ieq_count;\n+\tstruct irdma_virt_mem ieq_mem;\n+\tstruct irdma_puda_rsrc *ieq;\n+\tu32 exception_lan_q;\n+\tu16 mtu;\n+\tu16 vm_id;\n+\tu8 fcn_id;\n+\tenum irdma_vm_vf_type vm_vf_type;\n+\tbool stats_fcn_id_alloc;\n+\tstruct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];\n+\tstruct irdma_vsi_pestat *pestat;\n+\tatomic_t qp_suspend_reqs;\n+\tbool tc_change_pending;\n+\tu8 qos_rel_bw;\n+\tu8 qos_prio_type;\n+};\n+\n+struct irdma_sc_dev {\n+\tstruct list_head cqp_cmd_head; /* head of the CQP command list */\n+\tspinlock_t cqp_lock; /* protect CQP list access */\n+\tstruct irdma_dev_uk dev_uk;\n+\tbool fcn_id_array[IRDMA_MAX_STATS_COUNT];\n+\tstruct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];\n+\tu64 fpm_query_buf_pa;\n+\tu64 fpm_commit_buf_pa;\n+\t__le64 *fpm_query_buf;\n+\t__le64 *fpm_commit_buf;\n+\tvoid *back_dev;\n+\tstruct irdma_hw *hw;\n+\tu8 __iomem *db_addr;\n+\tu32 __iomem *wqe_alloc_db;\n+\tu32 __iomem *cq_arm_db;\n+\tu32 __iomem *aeq_alloc_db;\n+\tu32 __iomem *cqp_db;\n+\tu32 __iomem *cq_ack_db;\n+\tu32 __iomem *ceq_itr_mask_db;\n+\tu32 __iomem *aeq_itr_mask_db;\n+\tu32 hw_regs[IRDMA_MAX_REGS];\n+\tu64 hw_masks[IRDMA_MAX_MASKS];\n+\tu64 hw_shifts[IRDMA_MAX_SHIFTS];\n+\tu64 hw_stats_regs_32[IRDMA_HW_STAT_INDEX_MAX_32];\n+\tu64 hw_stats_regs_64[IRDMA_HW_STAT_INDEX_MAX_64];\n+\tu64 feature_info[IRDMA_MAX_FEATURES];\n+\tu64 cqp_cmd_stats[IRDMA_OP_SIZE_CQP_STAT_ARRAY];\n+\tstruct irdma_hw_attrs hw_attrs;\n+\tstruct irdma_hmc_info *hmc_info;\n+\tstruct irdma_vfdev *vf_dev[IRDMA_MAX_PE_ENA_VF_COUNT];\n+\tstruct irdma_sc_cqp *cqp;\n+\tstruct irdma_sc_aeq *aeq;\n+\tstruct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];\n+\tstruct irdma_sc_cq *ccq;\n+\tstruct irdma_irq_ops *irq_ops;\n+\tstruct irdma_cqp_ops *cqp_ops;\n+\tstruct irdma_ccq_ops *ccq_ops;\n+\tstruct irdma_ceq_ops *ceq_ops;\n+\tstruct irdma_aeq_ops *aeq_ops;\n+\tstruct irdma_pd_ops *iw_pd_ops;\n+\tstruct irdma_ah_ops *iw_ah_ops;\n+\tstruct irdma_priv_qp_ops *iw_priv_qp_ops;\n+\tstruct irdma_priv_cq_ops *iw_priv_cq_ops;\n+\tstruct irdma_mr_ops *mr_ops;\n+\tstruct irdma_cqp_misc_ops *cqp_misc_ops;\n+\tstruct irdma_hmc_ops *hmc_ops;\n+\tstruct irdma_uda_ops *iw_uda_ops;\n+\tstruct irdma_hmc_fpm_misc hmc_fpm_misc;\n+\tstruct irdma_ws_node *ws_tree_root;\n+\tstruct mutex ws_mutex; /* ws tree mutex */\n+\tu16 num_vfs;\n+\tu8 hmc_fn_id;\n+\tu8 vf_id;\n+\tbool is_pf;\n+\tbool vchnl_up;\n+\tbool ceq_valid;\n+\tu8 pci_rev;\n+\tenum irdma_status_code (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);\n+\tvoid (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);\n+\tvoid (*ws_reset)(struct irdma_sc_vsi *vsi);\n+};\n+\n+struct irdma_modify_cq_info {\n+\tu64 cq_pa;\n+\tstruct irdma_cqe *cq_base;\n+\tu32 ceq_id;\n+\tu32 cq_size;\n+\tu32 shadow_read_threshold;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tbool check_overflow;\n+\tbool cq_resize;\n+\tu32 first_pm_pbl_idx;\n+\tbool ceq_valid;\n+};\n+\n+struct irdma_create_qp_info {\n+\tbool ord_valid;\n+\tbool tcp_ctx_valid;\n+\tbool cq_num_valid;\n+\tbool arp_cache_idx_valid;\n+\tbool mac_valid;\n+\tbool force_lpb;\n+\tu8 next_iwarp_state;\n+};\n+\n+struct irdma_modify_qp_info {\n+\tu64 rx_win0;\n+\tu64 rx_win1;\n+\tu16 new_mss;\n+\tu8 next_iwarp_state;\n+\tu8 curr_iwarp_state;\n+\tu8 termlen;\n+\tbool ord_valid;\n+\tbool tcp_ctx_valid;\n+\tbool udp_ctx_valid;\n+\tbool cq_num_valid;\n+\tbool arp_cache_idx_valid;\n+\tbool reset_tcp_conn;\n+\tbool remove_hash_idx;\n+\tbool dont_send_term;\n+\tbool dont_send_fin;\n+\tbool cached_var_valid;\n+\tbool mss_change;\n+\tbool force_lpb;\n+\tbool mac_valid;\n+};\n+\n+struct irdma_ccq_cqe_info {\n+\tstruct irdma_sc_cqp *cqp;\n+\tu64 scratch;\n+\tu32 op_ret_val;\n+\tu16 maj_err_code;\n+\tu16 min_err_code;\n+\tu8 op_code;\n+\tbool error;\n+};\n+\n+struct irdma_dcb_app_info {\n+\tu8 priority;\n+\tu8 selector;\n+\tu16 prot_id;\n+};\n+\n+struct irdma_qos_tc_info {\n+\tu64 tc_ctx;\n+\tu8 rel_bw;\n+\tu8 prio_type;\n+\tu8 egress_virt_up;\n+\tu8 ingress_virt_up;\n+};\n+\n+struct irdma_l2params {\n+\tstruct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];\n+\tstruct irdma_dcb_app_info apps[IRDMA_MAX_APPS];\n+\tu32 num_apps;\n+\tu16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];\n+\tu16 mtu;\n+\tu8 up2tc[IRDMA_MAX_USER_PRIORITY];\n+\tu8 num_tc;\n+\tu8 vsi_rel_bw;\n+\tu8 vsi_prio_type;\n+\tbool mtu_changed;\n+\tbool tc_changed;\n+};\n+\n+struct irdma_vsi_init_info {\n+\tstruct irdma_sc_dev *dev;\n+\tvoid *back_vsi;\n+\tstruct irdma_l2params *params;\n+\tu16 exception_lan_q;\n+\tu16 pf_data_vsi_num;\n+\tenum irdma_vm_vf_type vm_vf_type;\n+\tu16 vm_id;\n+};\n+\n+struct irdma_vsi_stats_info {\n+\tstruct irdma_vsi_pestat *pestat;\n+\tu8 fcn_id;\n+\tbool alloc_fcn_id;\n+};\n+\n+struct irdma_device_init_info {\n+\tu64 fpm_query_buf_pa;\n+\tu64 fpm_commit_buf_pa;\n+\t__le64 *fpm_query_buf;\n+\t__le64 *fpm_commit_buf;\n+\tstruct irdma_hw *hw;\n+\tvoid __iomem *bar0;\n+\tenum irdma_status_code (*vchnl_send)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t     u32 vf_id, u8 *msg, u16 len);\n+\tvoid (*init_hw)(struct irdma_sc_dev *dev);\n+\tu8 hmc_fn_id;\n+\tbool is_pf;\n+};\n+\n+struct irdma_ceq_init_info {\n+\tu64 ceqe_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tu64 *ceqe_base;\n+\tvoid *pbl_list;\n+\tu32 elem_cnt;\n+\tu32 ceq_id;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tbool tph_en;\n+\tu8 tph_val;\n+\tu32 first_pm_pbl_idx;\n+\tbool itr_no_expire;\n+\tstruct irdma_sc_vsi *vsi;\n+\tstruct irdma_sc_cq **reg_cq;\n+\tu32 reg_cq_idx;\n+};\n+\n+struct irdma_aeq_init_info {\n+\tu64 aeq_elem_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tu32 *aeqe_base;\n+\tvoid *pbl_list;\n+\tu32 elem_cnt;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tu32 first_pm_pbl_idx;\n+};\n+\n+struct irdma_ccq_init_info {\n+\tu64 cq_pa;\n+\tu64 shadow_area_pa;\n+\tstruct irdma_sc_dev *dev;\n+\tstruct irdma_cqe *cq_base;\n+\t__le64 *shadow_area;\n+\tvoid *pbl_list;\n+\tu32 num_elem;\n+\tu32 ceq_id;\n+\tu32 shadow_read_threshold;\n+\tbool ceqe_mask;\n+\tbool ceq_id_valid;\n+\tbool tph_en;\n+\tu8 tph_val;\n+\tbool avoid_mem_cflct;\n+\tbool virtual_map;\n+\tu8 pbl_chunk_size;\n+\tu32 first_pm_pbl_idx;\n+\tstruct irdma_sc_vsi *vsi;\n+};\n+\n+struct irdma_udp_offload_info {\n+\tbool ipv4;\n+\tbool insert_vlan_tag;\n+\tu8 ttl;\n+\tu8 tos;\n+\tu16 src_port;\n+\tu16 dst_port;\n+\tu32 dest_ip_addr0;\n+\tu32 dest_ip_addr1;\n+\tu32 dest_ip_addr2;\n+\tu32 dest_ip_addr3;\n+\tu32 snd_mss;\n+\tu16 vlan_tag;\n+\tu16 arp_idx;\n+\tu32 flow_label;\n+\tu8 udp_state;\n+\tu32 psn_nxt;\n+\tu32 lsn;\n+\tu32 epsn;\n+\tu32 psn_max;\n+\tu32 psn_una;\n+\tu32 local_ipaddr0;\n+\tu32 local_ipaddr1;\n+\tu32 local_ipaddr2;\n+\tu32 local_ipaddr3;\n+\tu32 cwnd;\n+\tu8 rexmit_thresh;\n+\tu8 rnr_nak_thresh;\n+};\n+\n+struct irdma_roce_offload_info {\n+\tu16 p_key;\n+\tu16 err_rq_idx;\n+\tu32 qkey;\n+\tu32 dest_qp;\n+\tu32 local_qp;\n+\tbool is_qp1;\n+\tbool udprivcq_en;\n+\tu8 roce_tver;\n+\tu8 ack_credits;\n+\tu8 err_rq_idx_valid;\n+\tu32 pd_id;\n+\tu8 ord_size;\n+\tu8 ird_size;\n+\tbool dcqcn_en;\n+\tbool rcv_no_icrc;\n+\tbool wr_rdresp_en;\n+\tbool bind_en;\n+\tbool fast_reg_en;\n+\tbool priv_mode_en;\n+\tbool rd_en;\n+\tbool timely_en;\n+\tu16 t_high;\n+\tu16 t_low;\n+\tbool use_stats_inst;\n+\tu8 last_byte_sent;\n+\tu8 mac_addr[ETH_ALEN];\n+\tbool ecn_en;\n+\tbool dctcp_en;\n+\tbool fw_cc_enable;\n+};\n+\n+struct irdma_iwarp_offload_info {\n+\tu16 rcv_mark_offset;\n+\tu16 snd_mark_offset;\n+\tu8 ddp_ver;\n+\tu8 rdmap_ver;\n+\tbool snd_mark_en;\n+\tbool rcv_mark_en;\n+\tbool ib_rd_en;\n+\tu8 iwarp_mode;\n+\tbool align_hdrs;\n+\tbool rcv_no_mpa_crc;\n+\n+\tbool err_rq_idx_valid;\n+\tu16 err_rq_idx;\n+\tu32 pd_id;\n+\tu8 ord_size;\n+\tu8 ird_size;\n+\tbool wr_rdresp_en;\n+\tbool bind_en;\n+\tbool fast_reg_en;\n+\tbool priv_mode_en;\n+\tbool rd_en;\n+\tbool timely_en;\n+\tu16 t_high;\n+\tu16 t_low;\n+\tbool use_stats_inst;\n+\tu8 last_byte_sent;\n+\tu8 mac_addr[ETH_ALEN];\n+\tbool ecn_en;\n+\tbool dctcp_en;\n+};\n+\n+struct irdma_tcp_offload_info {\n+\tbool ipv4;\n+\tbool no_nagle;\n+\tbool insert_vlan_tag;\n+\tbool time_stamp;\n+\tu8 cwnd_inc_limit;\n+\tbool drop_ooo_seg;\n+\tu8 dup_ack_thresh;\n+\tu8 ttl;\n+\tu8 src_mac_addr_idx;\n+\tbool avoid_stretch_ack;\n+\tu8 tos;\n+\tu16 src_port;\n+\tu16 dst_port;\n+\tu32 dest_ip_addr0;\n+\tu32 dest_ip_addr1;\n+\tu32 dest_ip_addr2;\n+\tu32 dest_ip_addr3;\n+\tu32 snd_mss;\n+\tu16 syn_rst_handling;\n+\tu16 vlan_tag;\n+\tu16 arp_idx;\n+\tu32 flow_label;\n+\tbool wscale;\n+\tu8 tcp_state;\n+\tu8 snd_wscale;\n+\tu8 rcv_wscale;\n+\tu32 time_stamp_recent;\n+\tu32 time_stamp_age;\n+\tu32 snd_nxt;\n+\tu32 snd_wnd;\n+\tu32 rcv_nxt;\n+\tu32 rcv_wnd;\n+\tu32 snd_max;\n+\tu32 snd_una;\n+\tu32 srtt;\n+\tu32 rtt_var;\n+\tu32 ss_thresh;\n+\tu32 cwnd;\n+\tu32 snd_wl1;\n+\tu32 snd_wl2;\n+\tu32 max_snd_window;\n+\tu8 rexmit_thresh;\n+\tu32 local_ipaddr0;\n+\tu32 local_ipaddr1;\n+\tu32 local_ipaddr2;\n+\tu32 local_ipaddr3;\n+\tbool ignore_tcp_opt;\n+\tbool ignore_tcp_uns_opt;\n+};\n+\n+struct irdma_qp_host_ctx_info {\n+\tu64 qp_compl_ctx;\n+\tunion {\n+\t\tstruct irdma_tcp_offload_info *tcp_info;\n+\t\tstruct irdma_udp_offload_info *udp_info;\n+\t};\n+\tunion {\n+\t\tstruct irdma_iwarp_offload_info *iwarp_info;\n+\t\tstruct irdma_roce_offload_info *roce_info;\n+\t};\n+\tu32 send_cq_num;\n+\tu32 rcv_cq_num;\n+\tu32 rem_endpoint_idx;\n+\tu8 stats_idx;\n+\tbool tcp_info_valid;\n+\tbool iwarp_info_valid;\n+\tbool stats_idx_valid;\n+\tbool add_to_qoslist;\n+\tu8 user_pri;\n+};\n+\n+struct irdma_aeqe_info {\n+\tu64 compl_ctx;\n+\tu32 qp_cq_id;\n+\tu16 ae_id;\n+\tu16 wqe_idx;\n+\tu8 tcp_state;\n+\tu8 iwarp_state;\n+\tbool qp;\n+\tbool cq;\n+\tbool sq;\n+\tbool in_rdrsp_wr;\n+\tbool out_rdrsp;\n+\tu8 q2_data_written;\n+\tbool aeqe_overflow;\n+};\n+\n+struct irdma_allocate_stag_info {\n+\tu64 total_len;\n+\tu64 first_pm_pbl_idx;\n+\tu32 chunk_size;\n+\tu32 stag_idx;\n+\tu32 page_size;\n+\tu32 pd_id;\n+\tu16 access_rights;\n+\tbool remote_access;\n+\tbool use_hmc_fcn_index;\n+\tu8 hmc_fcn_index;\n+\tbool use_pf_rid;\n+};\n+\n+struct irdma_mw_alloc_info {\n+\tu32 mw_stag_index;\n+\tu32 page_size;\n+\tu32 pd_id;\n+\tbool remote_access;\n+\tbool mw_wide;\n+\tbool mw1_bind_dont_vldt_key;\n+};\n+\n+struct irdma_reg_ns_stag_info {\n+\tu64 reg_addr_pa;\n+\tu64 fbo;\n+\tvoid *va;\n+\tu64 total_len;\n+\tu32 page_size;\n+\tu32 chunk_size;\n+\tu32 first_pm_pbl_index;\n+\tenum irdma_addressing_type addr_type;\n+\tirdma_stag_index stag_idx;\n+\tu16 access_rights;\n+\tu32 pd_id;\n+\tirdma_stag_key stag_key;\n+\tbool use_hmc_fcn_index;\n+\tu8 hmc_fcn_index;\n+\tbool use_pf_rid;\n+};\n+\n+struct irdma_fast_reg_stag_info {\n+\tu64 wr_id;\n+\tu64 reg_addr_pa;\n+\tu64 fbo;\n+\tvoid *va;\n+\tu64 total_len;\n+\tu32 page_size;\n+\tu32 chunk_size;\n+\tu32 first_pm_pbl_index;\n+\tenum irdma_addressing_type addr_type;\n+\tirdma_stag_index stag_idx;\n+\tu16 access_rights;\n+\tu32 pd_id;\n+\tirdma_stag_key stag_key;\n+\tbool local_fence;\n+\tbool read_fence;\n+\tbool signaled;\n+\tbool push_wqe;\n+\tbool use_hmc_fcn_index;\n+\tu8 hmc_fcn_index;\n+\tbool use_pf_rid;\n+\tbool defer_flag;\n+};\n+\n+struct irdma_dealloc_stag_info {\n+\tu32 stag_idx;\n+\tu32 pd_id;\n+\tbool mr;\n+\tbool dealloc_pbl;\n+};\n+\n+struct irdma_register_shared_stag {\n+\tvoid *va;\n+\tenum irdma_addressing_type addr_type;\n+\tirdma_stag_index new_stag_idx;\n+\tirdma_stag_index parent_stag_idx;\n+\tu32 access_rights;\n+\tu32 pd_id;\n+\tirdma_stag_key new_stag_key;\n+};\n+\n+struct irdma_qp_init_info {\n+\tstruct irdma_qp_uk_init_info qp_uk_init_info;\n+\tstruct irdma_sc_pd *pd;\n+\tstruct irdma_sc_vsi *vsi;\n+\t__le64 *host_ctx;\n+\tu8 *q2;\n+\tu64 sq_pa;\n+\tu64 rq_pa;\n+\tu64 host_ctx_pa;\n+\tu64 q2_pa;\n+\tu64 shadow_area_pa;\n+\tu8 sq_tph_val;\n+\tu8 rq_tph_val;\n+\tu8 type;\n+\tbool sq_tph_en;\n+\tbool rq_tph_en;\n+\tbool rcv_tph_en;\n+\tbool xmit_tph_en;\n+\tbool virtual_map;\n+};\n+\n+struct irdma_cq_init_info {\n+\tstruct irdma_sc_dev *dev;\n+\tu64 cq_base_pa;\n+\tu64 shadow_area_pa;\n+\tu32 ceq_id;\n+\tu32 shadow_read_threshold;\n+\tbool virtual_map;\n+\tbool ceqe_mask;\n+\tu8 pbl_chunk_size;\n+\tu32 first_pm_pbl_idx;\n+\tbool ceq_id_valid;\n+\tbool tph_en;\n+\tu8 tph_val;\n+\tu8 type;\n+\tstruct irdma_cq_uk_init_info cq_uk_init_info;\n+\tstruct irdma_sc_vsi *vsi;\n+};\n+\n+struct irdma_upload_context_info {\n+\tu64 buf_pa;\n+\tbool freeze_qp;\n+\tbool raw_format;\n+\tu32 qp_id;\n+\tu8 qp_type;\n+};\n+\n+struct irdma_local_mac_entry_info {\n+\tu8 mac_addr[6];\n+\tu16 entry_idx;\n+};\n+\n+struct irdma_add_arp_cache_entry_info {\n+\tu8 mac_addr[ETH_ALEN];\n+\tu32 reach_max;\n+\tu16 arp_index;\n+\tbool permanent;\n+};\n+\n+struct irdma_apbvt_info {\n+\tu16 port;\n+\tbool add;\n+};\n+\n+struct irdma_qhash_table_info {\n+\tstruct irdma_sc_vsi *vsi;\n+\tenum irdma_quad_hash_manage_type manage;\n+\tenum irdma_quad_entry_type entry_type;\n+\tbool vlan_valid;\n+\tbool ipv4_valid;\n+\tu8 mac_addr[ETH_ALEN];\n+\tu16 vlan_id;\n+\tu8 user_pri;\n+\tu32 qp_num;\n+\tu32 dest_ip[4];\n+\tu32 src_ip[4];\n+\tu16 dest_port;\n+\tu16 src_port;\n+};\n+\n+struct irdma_cqp_manage_push_page_info {\n+\tu32 push_idx;\n+\tu16 qs_handle;\n+\tu8 free_page;\n+\tu8 push_page_type;\n+};\n+\n+struct irdma_qp_flush_info {\n+\tu16 sq_minor_code;\n+\tu16 sq_major_code;\n+\tu16 rq_minor_code;\n+\tu16 rq_major_code;\n+\tu16 ae_code;\n+\tu8 ae_src;\n+\tbool sq;\n+\tbool rq;\n+\tbool userflushcode;\n+\tbool generate_ae;\n+};\n+\n+struct irdma_gen_ae_info {\n+\tu16 ae_code;\n+\tu8 ae_src;\n+};\n+\n+struct irdma_cqp_timeout {\n+\tu64 compl_cqp_cmds;\n+\tu32 count;\n+};\n+\n+struct irdma_irq_ops {\n+\tvoid (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx);\n+\tvoid (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx);\n+\tvoid (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);\n+\tvoid (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);\n+};\n+\n+struct irdma_cqp_ops {\n+\tvoid (*check_cqp_progress)(struct irdma_cqp_timeout *cqp_timeout,\n+\t\t\t\t   struct irdma_sc_dev *dev);\n+\tenum irdma_status_code (*cqp_create)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t     u16 *maj_err, u16 *min_err);\n+\tenum irdma_status_code (*cqp_destroy)(struct irdma_sc_cqp *cqp);\n+\t__le64 *(*cqp_get_next_send_wqe)(struct irdma_sc_cqp *cqp, u64 scratch);\n+\tenum irdma_status_code (*cqp_init)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t   struct irdma_cqp_init_info *info);\n+\tvoid (*cqp_post_sq)(struct irdma_sc_cqp *cqp);\n+\tenum irdma_status_code (*poll_for_cqp_op_done)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t       u8 opcode,\n+\t\t\t\t\t\t       struct irdma_ccq_cqe_info *cmpl_info);\n+};\n+\n+struct irdma_ccq_ops {\n+\tvoid (*ccq_arm)(struct irdma_sc_cq *ccq);\n+\tenum irdma_status_code (*ccq_create)(struct irdma_sc_cq *ccq,\n+\t\t\t\t\t     u64 scratch, bool check_overflow,\n+\t\t\t\t\t     bool post_sq);\n+\tenum irdma_status_code (*ccq_create_done)(struct irdma_sc_cq *ccq);\n+\tenum irdma_status_code (*ccq_destroy)(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*ccq_get_cqe_info)(struct irdma_sc_cq *ccq,\n+\t\t\t\t\t\t   struct irdma_ccq_cqe_info *info);\n+\tenum irdma_status_code (*ccq_init)(struct irdma_sc_cq *ccq,\n+\t\t\t\t\t   struct irdma_ccq_init_info *info);\n+};\n+\n+struct irdma_ceq_ops {\n+\tenum irdma_status_code (*ceq_create)(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t     u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*cceq_create_done)(struct irdma_sc_ceq *ceq);\n+\tenum irdma_status_code (*cceq_destroy_done)(struct irdma_sc_ceq *ceq);\n+\tenum irdma_status_code (*cceq_create)(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t      u64 scratch);\n+\tenum irdma_status_code (*ceq_destroy)(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t      u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*ceq_init)(struct irdma_sc_ceq *ceq,\n+\t\t\t\t\t   struct irdma_ceq_init_info *info);\n+\tvoid *(*process_ceq)(struct irdma_sc_dev *dev,\n+\t\t\t     struct irdma_sc_ceq *ceq);\n+};\n+\n+struct irdma_aeq_ops {\n+\tenum irdma_status_code (*aeq_init)(struct irdma_sc_aeq *aeq,\n+\t\t\t\t\t   struct irdma_aeq_init_info *info);\n+\tenum irdma_status_code (*aeq_create)(struct irdma_sc_aeq *aeq,\n+\t\t\t\t\t     u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*aeq_destroy)(struct irdma_sc_aeq *aeq,\n+\t\t\t\t\t      u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*get_next_aeqe)(struct irdma_sc_aeq *aeq,\n+\t\t\t\t\t\tstruct irdma_aeqe_info *info);\n+\tenum irdma_status_code (*repost_aeq_entries)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t     u32 count);\n+\tenum irdma_status_code (*aeq_create_done)(struct irdma_sc_aeq *aeq);\n+\tenum irdma_status_code (*aeq_destroy_done)(struct irdma_sc_aeq *aeq);\n+};\n+\n+struct irdma_pd_ops {\n+\tvoid (*pd_init)(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd,\n+\t\t\tu32 pd_id, int abi_ver);\n+};\n+\n+struct irdma_priv_qp_ops {\n+\tenum irdma_status_code (*iw_mr_fast_register)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t\t      struct irdma_fast_reg_stag_info *info,\n+\t\t\t\t\t\t      bool post_sq);\n+\tenum irdma_status_code (*qp_create)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t    struct irdma_create_qp_info *info,\n+\t\t\t\t\t    u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*qp_destroy)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t     u64 scratch, bool remove_hash_idx,\n+\t\t\t\t\t     bool ignore_mw_bnd, bool post_sq);\n+\tenum irdma_status_code (*qp_flush_wqes)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t\tstruct irdma_qp_flush_info *info,\n+\t\t\t\t\t\tu64 scratch, bool post_sq);\n+\tenum irdma_status_code (*qp_init)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t  struct irdma_qp_init_info *info);\n+\tenum irdma_status_code (*qp_modify)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t    struct irdma_modify_qp_info *info,\n+\t\t\t\t\t    u64 scratch, bool post_sq);\n+\tvoid (*qp_send_lsmm)(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,\n+\t\t\t     irdma_stag stag);\n+\tvoid (*qp_send_lsmm_nostag)(struct irdma_sc_qp *qp, void *lsmm_buf,\n+\t\t\t\t    u32 size);\n+\tvoid (*qp_send_rtt)(struct irdma_sc_qp *qp, bool read);\n+\tenum irdma_status_code (*qp_setctx)(struct irdma_sc_qp *qp,\n+\t\t\t\t\t    __le64 *qp_ctx,\n+\t\t\t\t\t    struct irdma_qp_host_ctx_info *info);\n+\tenum irdma_status_code (*qp_setctx_roce)(struct irdma_sc_qp *qp, __le64 *qp_ctx,\n+\t\t\t\t\t\t struct irdma_qp_host_ctx_info *info);\n+\tenum irdma_status_code (*qp_upload_context)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t    struct irdma_upload_context_info *info,\n+\t\t\t\t\t\t    u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*update_suspend_qp)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t    struct irdma_sc_qp *qp,\n+\t\t\t\t\t\t    u64 scratch);\n+\tenum irdma_status_code (*update_resume_qp)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t   struct irdma_sc_qp *qp,\n+\t\t\t\t\t\t   u64 scratch);\n+};\n+\n+struct irdma_priv_cq_ops {\n+\tvoid (*cq_ack)(struct irdma_sc_cq *cq);\n+\tenum irdma_status_code (*cq_create)(struct irdma_sc_cq *cq, u64 scratch,\n+\t\t\t\t\t    bool check_overflow, bool post_sq);\n+\tenum irdma_status_code (*cq_destroy)(struct irdma_sc_cq *cq,\n+\t\t\t\t\t     u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*cq_init)(struct irdma_sc_cq *cq,\n+\t\t\t\t\t  struct irdma_cq_init_info *info);\n+\tenum irdma_status_code (*cq_modify)(struct irdma_sc_cq *cq,\n+\t\t\t\t\t    struct irdma_modify_cq_info *info,\n+\t\t\t\t\t    u64 scratch, bool post_sq);\n+\tvoid (*cq_resize)(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);\n+};\n+\n+struct irdma_mr_ops {\n+\tenum irdma_status_code (*alloc_stag)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t     struct irdma_allocate_stag_info *info,\n+\t\t\t\t\t     u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*dealloc_stag)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t       struct irdma_dealloc_stag_info *info,\n+\t\t\t\t\t       u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*mr_reg_non_shared)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t    struct irdma_reg_ns_stag_info *info,\n+\t\t\t\t\t\t    u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*mr_reg_shared)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\tstruct irdma_register_shared_stag *stag,\n+\t\t\t\t\t\tu64 scratch, bool post_sq);\n+\tenum irdma_status_code (*mw_alloc)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t   struct irdma_mw_alloc_info *info,\n+\t\t\t\t\t   u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*query_stag)(struct irdma_sc_dev *dev, u64 scratch,\n+\t\t\t\t\t     u32 stag_index, bool post_sq);\n+};\n+\n+struct irdma_cqp_misc_ops {\n+\tenum irdma_status_code (*add_arp_cache_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t      struct irdma_add_arp_cache_entry_info *info,\n+\t\t\t\t\t\t      u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*add_local_mac_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t      struct irdma_local_mac_entry_info *info,\n+\t\t\t\t\t\t      u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*alloc_local_mac_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t\tu64 scratch,\n+\t\t\t\t\t\t\tbool post_sq);\n+\tenum irdma_status_code (*cqp_nop)(struct irdma_sc_cqp *cqp, u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*del_arp_cache_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t      u64 scratch,\n+\t\t\t\t\t\t      u16 arp_index,\n+\t\t\t\t\t\t      bool post_sq);\n+\tenum irdma_status_code (*del_local_mac_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t      u64 scratch,\n+\t\t\t\t\t\t      u16 entry_idx,\n+\t\t\t\t\t\t      u8 ignore_ref_count,\n+\t\t\t\t\t\t      bool post_sq);\n+\tenum irdma_status_code (*gather_stats)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t       struct irdma_stats_gather_info *info,\n+\t\t\t\t\t       u64 scratch);\n+\tenum irdma_status_code (*manage_apbvt_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t     struct irdma_apbvt_info *info,\n+\t\t\t\t\t\t     u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*manage_push_page)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t   struct irdma_cqp_manage_push_page_info *info,\n+\t\t\t\t\t\t   u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*manage_qhash_table_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t\t   struct irdma_qhash_table_info *info,\n+\t\t\t\t\t\t\t   u64 scratch, bool post_sq);\n+\tenum irdma_status_code (*manage_stats_instance)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t\tstruct irdma_stats_inst_info *info,\n+\t\t\t\t\t\t\tbool alloc, u64 scratch);\n+\tenum irdma_status_code (*manage_ws_node)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t struct irdma_ws_node_info *info,\n+\t\t\t\t\t\t enum irdma_ws_node_op node_op,\n+\t\t\t\t\t\t u64 scratch);\n+\tenum irdma_status_code (*query_arp_cache_entry)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t\tu64 scratch, u16 arp_index, bool post_sq);\n+\tenum irdma_status_code (*query_rdma_features)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t      struct irdma_dma_mem *buf,\n+\t\t\t\t\t\t      u64 scratch);\n+\tenum irdma_status_code (*set_up_map)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t     struct irdma_up_info *info,\n+\t\t\t\t\t     u64 scratch);\n+};\n+\n+struct irdma_hmc_ops {\n+\tenum irdma_status_code (*cfg_iw_fpm)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t     u8 hmc_fn_id);\n+\tenum irdma_status_code (*commit_fpm_val)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t u64 scratch, u8 hmc_fn_id,\n+\t\t\t\t\t\t struct irdma_dma_mem *commit_fpm_mem,\n+\t\t\t\t\t\t bool post_sq, u8 wait_type);\n+\tenum irdma_status_code (*commit_fpm_val_done)(struct irdma_sc_cqp *cqp);\n+\tenum irdma_status_code (*create_hmc_object)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t    struct irdma_hmc_create_obj_info *info);\n+\tenum irdma_status_code (*del_hmc_object)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t struct irdma_hmc_del_obj_info *info,\n+\t\t\t\t\t\t bool reset);\n+\tenum irdma_status_code (*init_iw_hmc)(struct irdma_sc_dev *dev, u8 hmc_fn_id);\n+\tenum irdma_status_code (*manage_hmc_pm_func_table)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t\t   u64 scratch,\n+\t\t\t\t\t\t\t   u8 vf_index,\n+\t\t\t\t\t\t\t   bool free_pm_fcn,\n+\t\t\t\t\t\t\t   bool post_sq);\n+\tenum irdma_status_code (*manage_hmc_pm_func_table_done)(struct irdma_sc_cqp *cqp);\n+\tenum irdma_status_code (*parse_fpm_commit_buf)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t       __le64 *buf,\n+\t\t\t\t\t\t       struct irdma_hmc_obj_info *info,\n+\t\t\t\t\t\t       u32 *sd);\n+\tenum irdma_status_code (*parse_fpm_query_buf)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\t      __le64 *buf,\n+\t\t\t\t\t\t      struct irdma_hmc_info *hmc_info,\n+\t\t\t\t\t\t      struct irdma_hmc_fpm_misc *hmc_fpm_misc);\n+\tenum irdma_status_code (*pf_init_vfhmc)(struct irdma_sc_dev *dev,\n+\t\t\t\t\t\tu8 vf_hmc_fn_id,\n+\t\t\t\t\t\tu32 *vf_cnt_array);\n+\tenum irdma_status_code (*query_fpm_val)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\tu64 scratch,\n+\t\t\t\t\t\tu8 hmc_fn_id,\n+\t\t\t\t\t\tstruct irdma_dma_mem *query_fpm_mem,\n+\t\t\t\t\t\tbool post_sq, u8 wait_type);\n+\tenum irdma_status_code (*query_fpm_val_done)(struct irdma_sc_cqp *cqp);\n+\tenum irdma_status_code (*static_hmc_pages_allocated)(struct irdma_sc_cqp *cqp,\n+\t\t\t\t\t\t\t     u64 scratch,\n+\t\t\t\t\t\t\t     u8 hmc_fn_id,\n+\t\t\t\t\t\t\t     bool post_sq,\n+\t\t\t\t\t\t\t     bool poll_registers);\n+\tenum irdma_status_code (*vf_cfg_vffpm)(struct irdma_sc_dev *dev, u32 *vf_cnt_array);\n+};\n+\n+struct cqp_info {\n+\tunion {\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_qp *qp;\n+\t\t\tstruct irdma_create_qp_info info;\n+\t\t\tu64 scratch;\n+\t\t} qp_create;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_qp *qp;\n+\t\t\tstruct irdma_modify_qp_info info;\n+\t\t\tu64 scratch;\n+\t\t} qp_modify;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_qp *qp;\n+\t\t\tu64 scratch;\n+\t\t\tbool remove_hash_idx;\n+\t\t\tbool ignore_mw_bnd;\n+\t\t} qp_destroy;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cq *cq;\n+\t\t\tu64 scratch;\n+\t\t\tbool check_overflow;\n+\t\t} cq_create;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cq *cq;\n+\t\t\tstruct irdma_modify_cq_info info;\n+\t\t\tu64 scratch;\n+\t\t} cq_modify;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cq *cq;\n+\t\t\tu64 scratch;\n+\t\t} cq_destroy;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_allocate_stag_info info;\n+\t\t\tu64 scratch;\n+\t\t} alloc_stag;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_mw_alloc_info info;\n+\t\t\tu64 scratch;\n+\t\t} mw_alloc;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_reg_ns_stag_info info;\n+\t\t\tu64 scratch;\n+\t\t} mr_reg_non_shared;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_dealloc_stag_info info;\n+\t\t\tu64 scratch;\n+\t\t} dealloc_stag;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_add_arp_cache_entry_info info;\n+\t\t\tu64 scratch;\n+\t\t} add_arp_cache_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tu64 scratch;\n+\t\t\tu16 arp_index;\n+\t\t} del_arp_cache_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_local_mac_entry_info info;\n+\t\t\tu64 scratch;\n+\t\t} add_local_mac_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tu64 scratch;\n+\t\t\tu8 entry_idx;\n+\t\t\tu8 ignore_ref_count;\n+\t\t} del_local_mac_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tu64 scratch;\n+\t\t} alloc_local_mac_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_cqp_manage_push_page_info info;\n+\t\t\tu64 scratch;\n+\t\t} manage_push_page;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_upload_context_info info;\n+\t\t\tu64 scratch;\n+\t\t} qp_upload_context;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_hmc_fcn_info info;\n+\t\t\tu64 scratch;\n+\t\t} manage_hmc_pm;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_ceq *ceq;\n+\t\t\tu64 scratch;\n+\t\t} ceq_create;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_ceq *ceq;\n+\t\t\tu64 scratch;\n+\t\t} ceq_destroy;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_aeq *aeq;\n+\t\t\tu64 scratch;\n+\t\t} aeq_create;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_aeq *aeq;\n+\t\t\tu64 scratch;\n+\t\t} aeq_destroy;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_qp *qp;\n+\t\t\tstruct irdma_qp_flush_info info;\n+\t\t\tu64 scratch;\n+\t\t} qp_flush_wqes;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_qp *qp;\n+\t\t\tstruct irdma_gen_ae_info info;\n+\t\t\tu64 scratch;\n+\t\t} gen_ae;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tvoid *fpm_val_va;\n+\t\t\tu64 fpm_val_pa;\n+\t\t\tu8 hmc_fn_id;\n+\t\t\tu64 scratch;\n+\t\t} query_fpm_val;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tvoid *fpm_val_va;\n+\t\t\tu64 fpm_val_pa;\n+\t\t\tu8 hmc_fn_id;\n+\t\t\tu64 scratch;\n+\t\t} commit_fpm_val;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_apbvt_info info;\n+\t\t\tu64 scratch;\n+\t\t} manage_apbvt_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_qhash_table_info info;\n+\t\t\tu64 scratch;\n+\t\t} manage_qhash_table_entry;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_dev *dev;\n+\t\t\tstruct irdma_update_sds_info info;\n+\t\t\tu64 scratch;\n+\t\t} update_pe_sds;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_sc_qp *qp;\n+\t\t\tu64 scratch;\n+\t\t} suspend_resume;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_ah_info info;\n+\t\t\tu64 scratch;\n+\t\t} ah_create;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_ah_info info;\n+\t\t\tu64 scratch;\n+\t\t} ah_destroy;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_mcast_grp_info info;\n+\t\t\tu64 scratch;\n+\t\t} mc_create;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_mcast_grp_info info;\n+\t\t\tu64 scratch;\n+\t\t} mc_destroy;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_mcast_grp_info info;\n+\t\t\tu64 scratch;\n+\t\t} mc_modify;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_stats_inst_info info;\n+\t\t\tu64 scratch;\n+\t\t} stats_manage;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_stats_gather_info info;\n+\t\t\tu64 scratch;\n+\t\t} stats_gather;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_ws_node_info info;\n+\t\t\tu64 scratch;\n+\t\t} ws_node;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_up_info info;\n+\t\t\tu64 scratch;\n+\t\t} up_map;\n+\n+\t\tstruct {\n+\t\t\tstruct irdma_sc_cqp *cqp;\n+\t\t\tstruct irdma_dma_mem query_buff_mem;\n+\t\t\tu64 scratch;\n+\t\t} query_rdma;\n+\t} u;\n+};\n+\n+struct cqp_cmds_info {\n+\tstruct list_head cqp_cmd_entry;\n+\tu8 cqp_cmd;\n+\tu8 post_sq;\n+\tstruct cqp_info in;\n+};\n+\n+struct irdma_virtchnl_work_info {\n+\tvoid (*callback_fcn)(void *vf_dev);\n+\tvoid *worker_vf_dev;\n+};\n+#endif /* IRDMA_TYPE_H */\n",
    "prefixes": [
        "rdma-next",
        "03/17"
    ]
}