diff mbox series

[SRU,R,3/4] iommu/amd: Add PerfOpt IOMMU performance optimization support

Message ID 20261029091652.361681-4-Leo.Lin@amd.com
State New
Headers show
Series Enable IOMMU Performance Optimization support | expand

Commit Message

Yo-Jung Leo Lin \(AMD\) Oct. 29, 2026, 9:16 a.m. UTC
From: Mario Limonciello <mario.limonciello@amd.com>

BugLink: https://bugs.launchpad.net/bugs/2169668

Add support for the AMD IOMMU Performance Optimization (PerfOpt)
feature as defined in the AMD I/O Virtualization Technology (IOMMU)
Specification, Section 3.4.9 (MMIO Offset 016Ch).

This feature allows privileged integrated I/O devices (GPUs) to bypass
the IOMMU when directly accessing system memory.  The IOMMU only
enforces the IR/IW permission bits without GPA->SPA translations.

amd_iommu_enable_perfopt() performs a detach/reattach cycle to rehome
devices already on the identity domain with ATS/PRI/PASID/GCR3
disabled (skip_caps path).  amd_iommu_disable_perfopt() restores those
capabilities.  The per-device dev_data->perfopt flag tracks state.

PERF_OPT_EN is a single control bit per IOMMU, shared by every device
behind that IOMMU, while enablement is requested per device.  It is
therefore reference counted (amd_iommu->perfopt_refcount): armed on the
first requesting device and cleared on the last, so one device's
teardown never clears the bit while a peer behind the same IOMMU still
needs it.

The per-device flag is cleared on every teardown path
(blocked_domain_attach, release_device, and amd_iommu_disable_perfopt),
dropping the reference with it, so a reused dev_data never carries stale
PerfOpt state onto its next bind.

On suspend/resume the hardware is reprogrammed from scratch:
amd_iommu_perfopt_clear() forces the bit off without touching the
reference count, and amd_iommu_perfopt_restore() re-asserts it from the
count after early_enable_iommu(), so armed devices keep the optimization
across resume without relying on each consumer driver to re-arm.

The exported amd_iommu_enable_perfopt()/amd_iommu_disable_perfopt() run
only from a consumer driver's bind/unbind path.  group->mutex is not
exposed to drivers, but a device bound to its native driver cannot have
its IOMMU domain changed concurrently by the core, which serializes the
detach/attach pair against core-driven attach.

PerfOpt is opt-in -- only enabled when explicitly requested by a driver.

Tested-by: Derek J. Clark <derekjohn.clark@gmail.com>
Co-developed-by: Jatin Kataria <jkataria@netflix.com>
Signed-off-by: Jatin Kataria <jkataria@netflix.com>
Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>
Tested-by: Boqun Feng (Netflix) <boqun@kernel.org>
Acked-by: Alex Deucher <alexander.deucher@amd.com>
[joerg.roedel@amd.com: Fixed build failure on i386]
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
(cherry picked from commit 6e87850d1a8f32c1d91cce32c0d92b20f86689ed)
Signed-off-by: Yo-Jung Leo Lin (AMD) <Leo.Lin@amd.com>
---
 drivers/iommu/amd/amd_iommu.h       |   3 +
 drivers/iommu/amd/amd_iommu_types.h |   7 +
 drivers/iommu/amd/init.c            |  43 ++++++
 drivers/iommu/amd/iommu.c           | 214 ++++++++++++++++++++++++++++
 include/linux/amd-iommu.h           |  13 +-
 5 files changed, 279 insertions(+), 1 deletion(-)
diff mbox series

Patch

diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 834d8fabfba3..39e4b1aeb744 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -48,6 +48,9 @@  extern u8 amd_iommu_hpt_level;
 extern unsigned long amd_iommu_pgsize_bitmap;
 extern bool amd_iommu_hatdis;
 
+int amd_iommu_perfopt_clear(struct amd_iommu *iommu);
+int amd_iommu_perfopt_restore(struct amd_iommu *iommu);
+
 /* Protection domain ops */
 void amd_iommu_init_identity_domain(void);
 struct protection_domain *protection_domain_alloc(void);
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index c726d115939a..8b34aadb5798 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -69,6 +69,7 @@ 
 #define MMIO_MSI_ADDR_LO_OFFSET	0x015C
 #define MMIO_MSI_ADDR_HI_OFFSET	0x0160
 #define MMIO_MSI_DATA_OFFSET	0x0164
+#define MMIO_PERF_OPT_OFFSET	0x016C
 #define MMIO_INTCAPXT_EVT_OFFSET	0x0170
 #define MMIO_INTCAPXT_PPR_OFFSET	0x0178
 #define MMIO_INTCAPXT_GALOG_OFFSET	0x0180
@@ -103,6 +104,8 @@ 
 #define FEATURE_GLX		GENMASK_ULL(15, 14)
 #define FEATURE_GAM_VAPIC	BIT_ULL(21)
 #define FEATURE_PASMAX		GENMASK_ULL(36, 32)
+#define FEATURE_PERF_OPT	BIT_ULL(45)
+#define PERF_OPT_EN		BIT(13)
 #define FEATURE_GIOSUP		BIT_ULL(48)
 #define FEATURE_HASUP		BIT_ULL(49)
 #define FEATURE_EPHSUP		BIT_ULL(50)
@@ -672,6 +675,9 @@  struct amd_iommu {
 	/* Extended features 2 */
 	u64 features2;
 
+	/* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */
+	int perfopt_refcount;
+
 	/* PCI device id of the IOMMU device */
 	u16 devid;
 
@@ -838,6 +844,7 @@  struct iommu_dev_data {
 	u8 ppr          :1;		  /* Enable device PPR support */
 	bool use_vapic;			  /* Enable device to use vapic mode */
 	bool defer_attach;
+	bool perfopt;
 
 	struct ratelimit_state rs;        /* Ratelimit IOPF messages */
 };
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index f11e93c2fce4..e5bdaf7e7c02 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -1967,6 +1967,9 @@  static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
 	if (!iommu->mmio_base)
 		return -ENOMEM;
 
+	if (amd_iommu_perfopt_clear(iommu))
+		pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index);
+
 	return init_iommu_from_acpi(iommu, h);
 }
 
@@ -3056,10 +3059,46 @@  static void enable_iommus_vapic(void)
 #endif
 }
 
+static int clear_perfopt_all(void)
+{
+	struct amd_iommu *iommu;
+	int err, ret = 0;
+
+	for_each_iommu(iommu) {
+		err = amd_iommu_perfopt_clear(iommu);
+		if (err)
+			ret = err;
+	}
+
+	return ret;
+}
+
+static int restore_perfopt_all(void)
+{
+	struct amd_iommu *iommu;
+	int err, ret = 0;
+
+	for_each_iommu(iommu) {
+		err = amd_iommu_perfopt_restore(iommu);
+		if (err)
+			ret = err;
+	}
+
+	return ret;
+}
+
 static void disable_iommus(void)
 {
 	struct amd_iommu *iommu;
 
+	/*
+	 * PerfOpt is an optional performance bit, so a failure to clear it must
+	 * not skip the mandatory disable below. This also runs from the void
+	 * amd_iommu_disable() shutdown/kexec path, which cannot report an error.
+	 */
+	if (clear_perfopt_all())
+		pr_err("Failed to clear PerfOpt while disabling IOMMUs\n");
+
 	for_each_iommu(iommu)
 		iommu_disable(iommu);
 
@@ -3085,6 +3124,10 @@  static void amd_iommu_resume(void *data)
 	for_each_iommu(iommu)
 		early_enable_iommu(iommu);
 
+	/* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */
+	if (restore_perfopt_all())
+		pr_err("Failed to restore PerfOpt after IOMMU resume\n");
+
 	iommu_enable_event_buffer();
 	amd_iommu_enable_interrupts();
 }
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 96232dda1f84..31d469a0d9ed 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2351,6 +2351,9 @@  static int attach_device(struct device *dev,
 	if (ret)
 		goto out;
 
+	if (dev_data->perfopt)
+		goto skip_caps;
+
 	/* Setup GCR3 table */
 	if (pdom_is_sva_capable(domain)) {
 		ret = init_gcr3_table(dev_data, domain);
@@ -2375,6 +2378,7 @@  static int attach_device(struct device *dev,
 		pdev_enable_cap_ats(pdev);
 	}
 
+skip_caps:
 	/* Update data structures */
 	dev_data->domain = domain;
 	spin_lock_irqsave(&domain->lock, flags);
@@ -2481,6 +2485,192 @@  static void iommu_init_device_caps(struct iommu_dev_data *dev_data,
 	pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
 }
 
+/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */
+static int __perfopt_write(struct amd_iommu *iommu, bool enable)
+{
+	u32 old, val, readback;
+
+	if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT))
+		return enable ? -ENODEV : 0;
+
+	old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
+	if (old == U32_MAX)
+		return -EIO;
+
+	val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN;
+	if (val != old)
+		writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
+	readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
+	if (readback == U32_MAX ||
+	    (readback & PERF_OPT_EN) != (val & PERF_OPT_EN))
+		return -EIO;
+	return 0;
+}
+
+/*
+ * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is
+ * reference counted: armed on the first requesting device, cleared on the last.
+ */
+static int perfopt_get(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	if (iommu->perfopt_refcount == 0) {
+		ret = __perfopt_write(iommu, true);
+		if (ret)
+			goto out;
+	}
+	iommu->perfopt_refcount++;
+out:
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+static int perfopt_put(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0)
+		ret = __perfopt_write(iommu, false);
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+/*
+ * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown,
+ * and suspend). The count is preserved so amd_iommu_perfopt_restore() can
+ * re-arm on resume.
+ */
+int amd_iommu_perfopt_clear(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	ret = __perfopt_write(iommu, false);
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+/*
+ * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on
+ * resume, so devices armed before suspend keep the optimization without each
+ * consumer driver re-arming.
+ */
+int amd_iommu_perfopt_restore(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0);
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+int amd_iommu_enable_perfopt(struct pci_dev *pdev)
+{
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
+	struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
+	struct protection_domain *domain;
+	int ret;
+
+	if (!iommu || !dev_data)
+		return -ENODEV;
+
+	if (!(iommu->features & FEATURE_PERF_OPT))
+		return -ENODEV;
+
+	domain = dev_data->domain;
+	if (!domain)
+		return -ENODEV;
+
+	/* Already armed for this device (e.g. re-entry on resume). */
+	if (dev_data->perfopt)
+		return 0;
+
+	/*
+	 * The bit is only architecturally valid while the device is untranslated:
+	 * identity domain with ATS/PRI/PASID off. The identity domain is
+	 * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3
+	 * table. Re-home the device onto the same identity domain with
+	 * perfopt set, so the attach_device() skip_caps path leaves
+	 * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach
+	 * pattern used by amd_iommu_attach_device().
+	 *
+	 * Locking: this and amd_iommu_disable_perfopt() run only from the
+	 * consumer driver's bind/unbind path. group->mutex is not exposed to
+	 * drivers, but a device bound to its native driver cannot have its domain
+	 * changed concurrently by the core (VFIO ownership is mutually exclusive;
+	 * sysfs domain changes require an unused group), so the detach/attach pair
+	 * is serialized without it.
+	 */
+	dev_data->perfopt = true;
+	detach_device(&pdev->dev);
+	ret = attach_device(&pdev->dev, domain);
+	if (ret)
+		goto err_restore;
+
+	ret = perfopt_get(iommu);
+	if (ret)
+		goto err_rearm;
+
+	dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index);
+	return 0;
+
+err_rearm:
+	detach_device(&pdev->dev);
+err_restore:
+	dev_data->perfopt = false;
+	if (attach_device(&pdev->dev, domain))
+		pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n");
+	dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n",
+		     iommu->index, ret);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt);
+
+void amd_iommu_disable_perfopt(struct pci_dev *pdev)
+{
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
+	struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
+	struct protection_domain *domain;
+
+	if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain)
+		return;
+
+	if (WARN_ON(perfopt_put(iommu)))
+		pci_err(pdev, "failed to clear PerfOpt\n");
+
+	/*
+	 * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its
+	 * identity domain with the flag cleared, so a later bind without PerfOpt
+	 * sees a normally-capable device. See the locking note in
+	 * amd_iommu_enable_perfopt().
+	 */
+	domain = dev_data->domain;
+	dev_data->perfopt = false;
+	detach_device(&pdev->dev);
+	if (attach_device(&pdev->dev, domain))
+		pci_err(pdev, "failed to restore caps after PerfOpt disable\n");
+}
+EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt);
 static struct iommu_device *amd_iommu_probe_device(struct device *dev)
 {
 	struct iommu_device *iommu_dev;
@@ -2524,6 +2714,14 @@  static struct iommu_device *amd_iommu_probe_device(struct device *dev)
 static void amd_iommu_release_device(struct device *dev)
 {
 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
+
+	if (dev_data->perfopt) {
+		if (WARN_ON(perfopt_put(iommu)))
+			dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n",
+				iommu->index);
+		dev_data->perfopt = false;
+	}
 
 	WARN_ON(dev_data->domain);
 
@@ -2897,6 +3095,19 @@  static int blocked_domain_attach_device(struct iommu_domain *domain,
 					struct iommu_domain *old)
 {
 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
+
+	/*
+	 * blocked_domain is also the .release_domain, so this is the normal
+	 * teardown path: drop the reference and clear the flag here too, and
+	 * don't fail teardown if the WARN-guarded write doesn't stick.
+	 */
+	if (dev_data->perfopt) {
+		if (WARN_ON(perfopt_put(iommu)))
+			dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n",
+				iommu->index);
+		dev_data->perfopt = false;
+	}
 
 	if (dev_data->domain)
 		detach_device(dev);
@@ -2965,6 +3176,9 @@  static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev,
 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
 	int ret;
 
+	if (dev_data->perfopt && !pdom_is_in_pt_mode(domain))
+		return -EBUSY;
+
 	/*
 	 * Skip attach device to domain if new domain is same as
 	 * devices current domain
diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h
index edcee9f5335a..104817e8d897 100644
--- a/include/linux/amd-iommu.h
+++ b/include/linux/amd-iommu.h
@@ -11,11 +11,11 @@ 
 #include <linux/types.h>
 
 struct amd_iommu;
+struct pci_dev;
 
 #ifdef CONFIG_AMD_IOMMU
 
 struct task_struct;
-struct pci_dev;
 
 extern void amd_iommu_detect(void);
 
@@ -76,4 +76,15 @@  static inline int amd_iommu_snp_disable(void) { return 0; }
 static inline bool amd_iommu_sev_tio_supported(void) { return false; }
 #endif
 
+#ifdef CONFIG_AMD_IOMMU
+int amd_iommu_enable_perfopt(struct pci_dev *pdev);
+void amd_iommu_disable_perfopt(struct pci_dev *pdev);
+#else
+static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev)
+{
+	return 0;
+}
+static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { }
+#endif
+
 #endif /* _ASM_X86_AMD_IOMMU_H */