[{"id":3675454,"web_url":"http://patchwork.ozlabs.org/comment/3675454/","msgid":"<20260409123053.55c407fd@shazbot.org>","list_archive_url":null,"date":"2026-04-09T18:30:53","subject":"Re: [PATCH v2 1/1] vfio/nvgrace-gpu: Add Blackwell-Next GPU\n readiness check via CXL DVSEC","submitter":{"id":91887,"url":"http://patchwork.ozlabs.org/api/people/91887/","name":"Alex Williamson","email":"alex@shazbot.org"},"content":"On Thu, 9 Apr 2026 13:36:51 +0000\nAnkit Agrawal wrote:\n\n> Add a CXL DVSEC-based readiness check for Blackwell-Next GPUs alongside\n> the existing legacy BAR0 polling path. On probe and after reset, the\n> driver reads the CXL Device DVSEC capability to determine whether the\n> GPU memory is valid. This is checked by polling on the Memory_Active bit\n> based on the Memory_Active_Timeout.\n> \n> A static inline wrapper dispatches to the appropriate readiness check\n> based on whether the CXL DVSEC capability is present.\n> \n> Suggested-by: Alex Williamson \n> Signed-off-by: Ankit Agrawal \n> ---\n> drivers/vfio/pci/nvgrace-gpu/main.c | 75 ++++++++++++++++++++++++++---\n> include/uapi/linux/pci_regs.h | 1 +\n> 2 files changed, 68 insertions(+), 8 deletions(-)\n> \n> diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c\n> index fa056b69f899..52f7e3a3054a 100644\n> --- a/drivers/vfio/pci/nvgrace-gpu/main.c\n> +++ b/drivers/vfio/pci/nvgrace-gpu/main.c\n> @@ -64,6 +64,8 @@ struct nvgrace_gpu_pci_core_device {\n> \tbool has_mig_hw_bug;\n> \t/* GPU has just been reset */\n> \tbool reset_done;\n> +\t/* CXL Device DVSEC offset; 0 if not present (legacy GB path) */\n> +\tint cxl_dvsec;\n> };\n> \n> static void nvgrace_gpu_init_fake_bar_emu_regs(struct vfio_device *core_vdev)\n> @@ -242,7 +244,7 @@ static void nvgrace_gpu_close_device(struct vfio_device *core_vdev)\n> \tvfio_pci_core_close_device(core_vdev);\n> }\n> \n> -static int nvgrace_gpu_wait_device_ready(void __iomem *io)\n> +static int nvgrace_gpu_wait_device_ready_legacy(void __iomem *io)\n> {\n> \tunsigned long timeout = jiffies + msecs_to_jiffies(POLL_TIMEOUT_MS);\n> \n> @@ -256,6 +258,59 @@ static int nvgrace_gpu_wait_device_ready(void __iomem *io)\n> \treturn -ETIME;\n> }\n> \n> +/*\n> + * Decode the 3-bit Memory_Active_Timeout field from CXL DVSEC Range 1 Low\n> + * (bits 15:13) into milliseconds. Encoding per CXL spec r4.0 sec 8.1.3.8.2:\n> + * 000b = 1s, 001b = 4s, 010b = 16s, 011b = 64s, 100b = 256s,\n> + * 101b-111b = reserved (clamped to 256s).\n> + */\n> +static inline unsigned long nvgrace_gpu_cxl_mem_active_timeout_ms(u8 timeout)\n> +{\n> +\treturn 1000UL << (2 * min_t(u8, timeout, 4));\n> +}\n> +\n> +static int nvgrace_gpu_wait_device_ready_bw_next(struct nvgrace_gpu_pci_core_device *nvdev)\n> +{\n> +\tstruct pci_dev *pdev = nvdev->core_device.pdev;\n> +\tint pcie_dvsec = nvdev->cxl_dvsec;\n> +\tunsigned long timeout;\n> +\tu32 dvsec_memory_status;\n> +\tu8 mem_active_timeout;\n> +\n> +\tpci_read_config_dword(pdev, pcie_dvsec + PCI_DVSEC_CXL_RANGE_SIZE_LOW(0),\n> +\t\t\t &dvsec_memory_status);\n> +\n> +\tif (!(dvsec_memory_status & PCI_DVSEC_CXL_MEM_INFO_VALID))\n> +\t\treturn -ENODEV;\n\nNit, if MEM_ACTIVE is already set, we still read it twice rather than\nexit here:\n\n\tif (dvsec_memory_status & PCI_DVSEC_CXL_MEM_ACTIVE)\n\t\treturn 0;\n\n\n> +\n> +\tmem_active_timeout = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE_TIMEOUT,\n> +\t\t\t\t dvsec_memory_status);\n> +\n> +\ttimeout = jiffies +\n> +\t\t msecs_to_jiffies(nvgrace_gpu_cxl_mem_active_timeout_ms(mem_active_timeout));\n> +\n> +\tdo {\n> +\t\tpci_read_config_dword(pdev,\n> +\t\t\t\t pcie_dvsec + PCI_DVSEC_CXL_RANGE_SIZE_LOW(0),\n> +\t\t\t\t &dvsec_memory_status);\n> +\n> +\t\tif (dvsec_memory_status & PCI_DVSEC_CXL_MEM_ACTIVE)\n> +\t\t\treturn 0;\n\nDo we need to monitor PCI_DVSEC_CXL_MEM_INFO_VALID in the loop too?\n\n> +\n> +\t\tmsleep(POLL_QUANTUM_MS);\n> +\t} while (!time_after(jiffies, timeout));\n> +\n> +\treturn -ETIME;\n> +}\n> +\n> +static inline int nvgrace_gpu_wait_device_ready(struct nvgrace_gpu_pci_core_device *nvdev,\n> +\t\t\t\t\t\tvoid __iomem *io)\n> +{\n> +\treturn nvdev->cxl_dvsec ?\n> +\t\tnvgrace_gpu_wait_device_ready_bw_next(nvdev) :\n> +\t\tnvgrace_gpu_wait_device_ready_legacy(io);\n> +}\n> +\n> /*\n> * If the GPU memory is accessed by the CPU while the GPU is not ready\n> * after reset, it can cause harmless corrected RAS events to be logged.\n> @@ -275,7 +330,7 @@ nvgrace_gpu_check_device_ready(struct nvgrace_gpu_pci_core_device *nvdev)\n> \tif (!__vfio_pci_memory_enabled(vdev))\n> \t\treturn -EIO;\n> \n> -\tret = nvgrace_gpu_wait_device_ready(vdev->barmap[0]);\n> +\tret = nvgrace_gpu_wait_device_ready(nvdev, vdev->barmap[0]);\n> \tif (ret)\n> \t\treturn ret;\n> \n> @@ -1146,8 +1201,9 @@ static bool nvgrace_gpu_has_mig_hw_bug(struct pci_dev *pdev)\n> * Ensure that the BAR0 region is enabled before accessing the\n> * registers.\n> */\n> -static int nvgrace_gpu_probe_check_device_ready(struct pci_dev *pdev)\n> +static int nvgrace_gpu_probe_check_device_ready(struct nvgrace_gpu_pci_core_device *nvdev)\n> {\n> +\tstruct pci_dev *pdev = nvdev->core_device.pdev;\n> \tvoid __iomem *io;\n> \tint ret;\n> \n> @@ -1165,7 +1221,7 @@ static int nvgrace_gpu_probe_check_device_ready(struct pci_dev *pdev)\n> \t\tgoto iomap_exit;\n> \t}\n> \n> -\tret = nvgrace_gpu_wait_device_ready(io);\n> +\tret = nvgrace_gpu_wait_device_ready(nvdev, io);\n> \n> \tpci_iounmap(pdev, io);\n> iomap_exit:\n> @@ -1183,10 +1239,6 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev,\n> \tu64 memphys, memlength;\n> \tint ret;\n> \n> -\tret = nvgrace_gpu_probe_check_device_ready(pdev);\n> -\tif (ret)\n> -\t\treturn ret;\n> -\n> \tret = nvgrace_gpu_fetch_memory_property(pdev, &memphys, &memlength);\n> \tif (!ret)\n> \t\tops = &nvgrace_gpu_pci_ops;\n> @@ -1198,6 +1250,13 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev,\n> \n> \tdev_set_drvdata(&pdev->dev, &nvdev->core_device);\n> \n> +\tnvdev->cxl_dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,\n> +\t\t\t\t\t\t PCI_DVSEC_CXL_DEVICE);\n> +\n> +\tret = nvgrace_gpu_probe_check_device_ready(nvdev);\n> +\tif (ret)\n> +\t\tgoto out_put_vdev;\n> +\n> \tif (ops == &nvgrace_gpu_pci_ops) {\n> \t\tnvdev->has_mig_hw_bug = nvgrace_gpu_has_mig_hw_bug(pdev);\n> \n> diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h\n> index 14f634ab9350..718fb630f5bb 100644\n> --- a/include/uapi/linux/pci_regs.h\n> +++ b/include/uapi/linux/pci_regs.h\n> @@ -1357,6 +1357,7 @@\n> #define PCI_DVSEC_CXL_RANGE_SIZE_LOW(i)\t\t(0x1C + (i * 0x10))\n> #define PCI_DVSEC_CXL_MEM_INFO_VALID\t\t\t_BITUL(0)\n> #define PCI_DVSEC_CXL_MEM_ACTIVE\t\t\t_BITUL(1)\n> +#define PCI_DVSEC_CXL_MEM_ACTIVE_TIMEOUT\t\t__GENMASK(15, 13)\n\nBjorn, please ack if this is ok to go through vfio. Thanks,\n\nAlex\n\n> #define PCI_DVSEC_CXL_MEM_SIZE_LOW\t\t\t__GENMASK(31, 28)\n> #define PCI_DVSEC_CXL_RANGE_BASE_HIGH(i)\t\t(0x20 + (i * 0x10))\n> #define PCI_DVSEC_CXL_RANGE_BASE_LOW(i)\t\t(0x24 + (i * 0x10))","headers":{"Return-Path":"\n ","X-Original-To":["incoming@patchwork.ozlabs.org","linux-pci@vger.kernel.org"],"Delivered-To":"patchwork-incoming@legolas.ozlabs.org","Authentication-Results":["legolas.ozlabs.org;\n\tdkim=pass (2048-bit key;\n unprotected) header.d=shazbot.org header.i=@shazbot.org header.a=rsa-sha256\n header.s=fm1 header.b=ms7M6ePs;\n\tdkim=pass (2048-bit key;\n unprotected) header.d=messagingengine.com header.i=@messagingengine.com\n header.a=rsa-sha256 header.s=fm2 header.b=LIPBni0N;\n\tdkim-atps=neutral","legolas.ozlabs.org;\n spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org\n (client-ip=2600:3c04:e001:36c::12fc:5321; helo=tor.lore.kernel.org;\n envelope-from=linux-pci+bounces-52223-incoming=patchwork.ozlabs.org@vger.kernel.org;\n receiver=patchwork.ozlabs.org)","smtp.subspace.kernel.org;\n\tdkim=pass (2048-bit key) header.d=shazbot.org header.i=@shazbot.org\n header.b=\"ms7M6ePs\";\n\tdkim=pass (2048-bit key) header.d=messagingengine.com\n header.i=@messagingengine.com header.b=\"LIPBni0N\"","smtp.subspace.kernel.org;\n arc=none smtp.client-ip=103.168.172.156","smtp.subspace.kernel.org;\n dmarc=pass (p=none dis=none) header.from=shazbot.org","smtp.subspace.kernel.org;\n spf=pass smtp.mailfrom=shazbot.org"],"Received":["from tor.lore.kernel.org (tor.lore.kernel.org\n [IPv6:2600:3c04:e001:36c::12fc:5321])\n\t(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)\n\t key-exchange x25519)\n\t(No client certificate requested)\n\tby legolas.ozlabs.org (Postfix) with ESMTPS id 4fs7mR3Jmlz1xy1\n\tfor ; Fri, 10 Apr 2026 04:31:03 +1000 (AEST)","from smtp.subspace.kernel.org (conduit.subspace.kernel.org\n [100.90.174.1])\n\tby tor.lore.kernel.org (Postfix) with ESMTP id F3617300DD66\n\tfor ; Thu, 9 Apr 2026 18:31:00 +0000 (UTC)","from localhost.localdomain (localhost.localdomain [127.0.0.1])\n\tby smtp.subspace.kernel.org (Postfix) with ESMTP id 1A81C2D0C8F;\n\tThu, 9 Apr 2026 18:31:00 +0000 (UTC)","from fhigh-a5-smtp.messagingengine.com\n (fhigh-a5-smtp.messagingengine.com [103.168.172.156])\n\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))\n\t(No client certificate requested)\n\tby smtp.subspace.kernel.org (Postfix) with ESMTPS id 69173296BAF;\n\tThu, 9 Apr 2026 18:30:57 +0000 (UTC)","from phl-compute-02.internal (phl-compute-02.internal [10.202.2.42])\n\tby mailfhigh.phl.internal (Postfix) with ESMTP id 8E49814002AB;\n\tThu, 9 Apr 2026 14:30:56 -0400 (EDT)","from phl-frontend-04 ([10.202.2.163])\n by phl-compute-02.internal (MEProxy); Thu, 09 Apr 2026 14:30:56 -0400","by mail.messagingengine.com (Postfix) with ESMTPA; Thu,\n 9 Apr 2026 14:30:54 -0400 (EDT)"],"ARC-Seal":"i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116;\n\tt=1775759460; cv=none;\n b=H1e7Ep3YHx7TFsEhfa5mrHdFb3Wiuw6RMVySOXmrTO79/CIKlgX1VxA23n0jTdq0pDp7p80lbbYBf4/lz5tNF/jOyA0+QkL0poIEKK4QxKtfjAX+d4ck8r0GBtE5Xt/Jut7HLRmVrEgT4ohVrQcFiVGdPY73jFQhyrY7yWO/zO4=","ARC-Message-Signature":"i=1; a=rsa-sha256; d=subspace.kernel.org;\n\ts=arc-20240116; t=1775759460; c=relaxed/simple;\n\tbh=s21/iICOfgS4PQH3e9VfF/zirOs9dwtgCi0+dUzZy9U=;\n\th=Date:From:To:Cc:Subject:Message-ID:In-Reply-To:References:\n\t MIME-Version:Content-Type;\n b=IL9v/tOcA4pMB/FZrq12U+S/Lc4Cc5Kzc5aJHGliRG6PgOM5UvcLC4Q3mmo7Y3Z1mqoVpDovKEVeyuji27kfT1LUf5gk2J81cNIVsIZw7xRR5Ubm7Jc/h3p2WZjGLNrDHMvbD8rdgc0lv+Fu2V9/CLeZxqGnR64hgp/7Nhd/U+A=","ARC-Authentication-Results":"i=1; smtp.subspace.kernel.org;\n dmarc=pass (p=none dis=none) header.from=shazbot.org;\n spf=pass smtp.mailfrom=shazbot.org;\n dkim=pass (2048-bit key) header.d=shazbot.org header.i=@shazbot.org\n header.b=ms7M6ePs;\n dkim=pass (2048-bit key) header.d=messagingengine.com header.i=@messagingengine.com\n header.b=LIPBni0N; arc=none smtp.client-ip=103.168.172.156","DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=shazbot.org; h=\n\tcc:cc:content-transfer-encoding:content-type:content-type:date\n\t:date:from:from:in-reply-to:in-reply-to:message-id:mime-version\n\t:references:reply-to:subject:subject:to:to; s=fm1; t=1775759456;\n\t x=1775845856; bh=ABOAmFwOdQ1RvMMRK07TWzUTed6xOOr5FsPWYBl2G4Q=; b=\n\tms7M6ePsFxTWfknoQwAI/xlJS49cBkwGUhV+s8IAOfNmBd/dIIk0EvsGaTMduh2J\n\tk6jbvjigPk8J833VVkpB14a+o3xv5maxy1vgVHh+jrMCELgRsIB45rdNDrQytyne\n\t4w7tirQQL9UEzefjmA9ywYXR5fWlsUwl9VDAOOe0FH1xWgJRvZwiJodMTlcVKnuU\n\tQ71RTvMtZs4UVdvVeyQ5Nx8RlpYWekPAvfIKSJ+bwaS7Ljrm2bp8OIJUhB0HAEtz\n\toiwYbTZvgmu+2pmGbzcnJ7dl5Ttm87X2aYQQbacuCcUWiCWrvFrTlLbYGij008Wx\n\twfOaOZx/+AkPgkJkXz0UdA==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:cc:content-transfer-encoding\n\t:content-type:content-type:date:date:feedback-id:feedback-id\n\t:from:from:in-reply-to:in-reply-to:message-id:mime-version\n\t:references:reply-to:subject:subject:to:to:x-me-proxy\n\t:x-me-sender:x-me-sender:x-sasl-enc; s=fm2; t=1775759456; x=\n\t1775845856; bh=ABOAmFwOdQ1RvMMRK07TWzUTed6xOOr5FsPWYBl2G4Q=; b=L\n\tIPBni0NlfFPMqU2ToDEU8fC92IbZO2K7HuAvqmDmkUQNDASfM/LDnqaGqI4YcmYa\n\tO6+ZGjlRAUMtE/EHrDSTHv3vFl5xSPDx1ij5wui989HH7WZZ8AAjhdAsGHnVVC9M\n\tWmjJE0hgbxaOyjqDpkb+SIO3MrHsF+DMHn1bwCQSOYcuzC4dgvS7i5SPvNU6DgmK\n\te3s4LsSue3AXCm/3bMArMKz0V4CQMqkyUz3ZKgxE1p1hvo5WH4CgczsNzrcppQ/n\n\tjIn3PM6iu9/809czxoNxFepAvtmSWt4CuDVIeRU4ZU86k/BLwi7nvkZrOilvT2l4\n\tMqTbmYGLohtqWjeFNfgnQ=="],"X-ME-Sender":"\n ","X-ME-Received":"\n ","X-ME-Proxy-Cause":"\n gggruggvucftvghtrhhoucdtuddrgeefhedrtddtgddvjedvudcutefuodetggdotefrod\n ftvfcurfhrohhfihhlvgemucfhrghsthforghilhdpuffrtefokffrpgfnqfghnecuuegr\n ihhlohhuthemuceftddtnecusecvtfgvtghiphhivghnthhsucdlqddutddtmdenucfjug\n hrpeffhffvvefukfgjfhfogggtgfesthejredtredtvdenucfhrhhomheptehlvgigucgh\n ihhllhhirghmshhonhcuoegrlhgvgiesshhhrgiisghothdrohhrgheqnecuggftrfgrth\n htvghrnhepvdekfeejkedvudfhudfhteekudfgudeiteetvdeukedvheetvdekgfdugeev\n ueeunecuvehluhhsthgvrhfuihiivgeptdenucfrrghrrghmpehmrghilhhfrhhomheprg\n hlvgigsehshhgriigsohhtrdhorhhgpdhnsggprhgtphhtthhopedutddpmhhouggvpehs\n mhhtphhouhhtpdhrtghpthhtoheprghnkhhithgrsehnvhhiughirgdrtghomhdprhgtph\n htthhopehkvhhmsehvghgvrhdrkhgvrhhnvghlrdhorhhgpdhrtghpthhtohepjhhgghes\n iihivghpvgdrtggrpdhrtghpthhtohephihishhhrghihhesnhhvihguihgrrdgtohhmpd\n hrtghpthhtohepshhkohhlohhthhhumhhthhhosehnvhhiughirgdrtghomhdprhgtphht\n thhopehkvghvihhnrdhtihgrnhesihhnthgvlhdrtghomhdprhgtphhtthhopegshhgvlh\n hgrggrshesghhoohhglhgvrdgtohhmpdhrtghpthhtoheplhhinhhugidqkhgvrhhnvghl\n sehvghgvrhdrkhgvrhhnvghlrdhorhhgpdhrtghpthhtoheplhhinhhugidqphgtihesvh\n hgvghrrdhkvghrnhgvlhdrohhrgh","X-ME-Proxy":"\n \n \n \n ","Feedback-ID":"i03f14258:Fastmail","Date":"Thu, 9 Apr 2026 12:30:53 -0600","From":"Alex Williamson ","To":"Ankit Agrawal , ","Cc":", , ,\n , ,\n , ,\n alex@shazbot.org","Subject":"Re: [PATCH v2 1/1] vfio/nvgrace-gpu: Add Blackwell-Next GPU\n readiness check via CXL DVSEC","Message-ID":"<20260409123053.55c407fd@shazbot.org>","In-Reply-To":"<20260409133651.92580-1-ankita@nvidia.com>","References":"<20260409133651.92580-1-ankita@nvidia.com>","X-Mailer":"Claws Mail 4.3.1 (GTK 3.24.51; x86_64-pc-linux-gnu)","Precedence":"bulk","X-Mailing-List":"linux-pci@vger.kernel.org","List-Id":"","List-Subscribe":"","List-Unsubscribe":"","MIME-Version":"1.0","Content-Type":"text/plain; charset=US-ASCII","Content-Transfer-Encoding":"7bit"}}]