| Message ID | 20261001092227.3004747-17-smadhavan@nvidia.com |
|---|---|
| State | New |
| Headers | show |
| Series | PCI/CXL: Add CXL reset support for Type 2 devices | expand |
> CXL bus reset can clear HDM decoder programming and CXL Device DVSEC > control state. Restore the cached state after a successful cxl_bus reset > while IOMMU exclusion remains active. > > Restore PCI configuration first so HDM MMIO is accessible, and preserve a > disabled state if restoration fails. > > Tested-by: Richard Cheng <icheng@nvidia.com> > Signed-off-by: Srirangan Madhavan <smadhavan@nvidia.com> Sashiko has reviewed this patch and found no issues. It looks great!
On Thu, Oct 01, 2026 at 09:22:27AM +0000, Srirangan Madhavan wrote: > CXL bus reset can clear HDM decoder programming and CXL Device DVSEC > control state. Restore the cached state after a successful cxl_bus reset > while IOMMU exclusion remains active. > > Restore PCI configuration first so HDM MMIO is accessible, and preserve a > disabled state if restoration fails. > > Tested-by: Richard Cheng <icheng@nvidia.com> > Signed-off-by: Srirangan Madhavan <smadhavan@nvidia.com> Acked-by: Bjorn Helgaas <bhelgaas@google.com> # pci/pci.c > --- > drivers/cxl/core/resource.c | 23 +++++++++++++++++++++-- > drivers/pci/pci.c | 8 ++++++++ > include/cxl/cxl.h | 17 +++++++++++++++++ > 3 files changed, 46 insertions(+), 2 deletions(-) > > diff --git a/drivers/cxl/core/resource.c b/drivers/cxl/core/resource.c > index efbab341b30f..3422139ae3ab 100644 > --- a/drivers/cxl/core/resource.c > +++ b/drivers/cxl/core/resource.c > @@ -625,12 +625,12 @@ static int cxl_restore_hdm_decoder(struct pci_dev *pdev, void __iomem *hdm, > return 0; > } > > -static int cxl_snapshot_hdm(struct pci_dev *pdev, > - struct cxl_hdm_info **snapshot) > +int cxl_snapshot_hdm(struct pci_dev *pdev, struct cxl_hdm_info **snapshot) > { > struct cxl_hdm_info *info; > size_t size; > > + device_lock_assert(&pdev->dev); > guard(rwsem_read)(&cxl_rwsem.dpa); > info = pdev->hdm; > if (!info) > @@ -798,6 +798,25 @@ static int cxl_reset_save_restored_state(struct pci_dev *pdev, u16 command) > return rc; > } > > +int cxl_restore_state_after_pci_reset(struct pci_dev *pdev, > + struct cxl_hdm_info *snapshot) > +{ > + u16 command; > + int rc; > + > + device_lock_assert(&pdev->dev); > + guard(rwsem_write)(&cxl_rwsem.region); > + > + cxl_restore_pci_state_for_hdm_restore(pdev, &command); > + rc = cxl_restore_state(pdev, snapshot); > + if (rc) { > + cxl_reset_save_disabled_state(pdev); > + return rc; > + } > + > + return cxl_reset_save_restored_state(pdev, command); > +} > + > /* > * CXL r4.0 sec 9.7.2 defines the reset completion timeout encodings. > * Sec 9.7.3 leaves config-space access behavior undefined for 100 ms after > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c > index dc57a202c91f..6928c99aa785 100644 > --- a/drivers/pci/pci.c > +++ b/drivers/pci/pci.c > @@ -5015,6 +5015,7 @@ static int pci_reset_bus_function(struct pci_dev *dev, bool probe) > > static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) > { > + struct cxl_hdm_info *snapshot __free(kfree) = NULL; > struct pci_dev *bridge; > u16 dvsec, reg, val; > int rc; > @@ -5037,6 +5038,10 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) > if (rc) > return -ENOTTY; > > + rc = cxl_snapshot_hdm(dev, &snapshot); > + if (rc && rc != -ENXIO) > + return rc; > + > rc = pci_dev_reset_iommu_prepare(dev); > if (rc) { > pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", rc); > @@ -5057,6 +5062,9 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) > pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, > reg); > > + if (!rc && snapshot) > + rc = cxl_restore_state_after_pci_reset(dev, snapshot); > + > pci_dev_reset_iommu_done(dev); > return rc; > } > diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h > index 9904eaed5af1..b83cfaed618f 100644 > --- a/include/cxl/cxl.h > +++ b/include/cxl/cxl.h > @@ -207,6 +207,9 @@ struct cxl_register_map { > #ifdef CONFIG_CXL_RESET > void pci_cxl_hdm_cache_init(struct pci_dev *pdev); > void pci_cxl_hdm_cache_release(struct pci_dev *pdev); > +int cxl_snapshot_hdm(struct pci_dev *pdev, struct cxl_hdm_info **snapshot); > +int cxl_restore_state_after_pci_reset(struct pci_dev *pdev, > + struct cxl_hdm_info *snapshot); > int cxl_reset_function(struct pci_dev *pdev, bool probe); > #else > static inline void pci_cxl_hdm_cache_init(struct pci_dev *pdev) > @@ -217,6 +220,20 @@ static inline void pci_cxl_hdm_cache_release(struct pci_dev *pdev) > { > } > > +static inline int cxl_snapshot_hdm(struct pci_dev *pdev, > + struct cxl_hdm_info **snapshot) > +{ > + *snapshot = NULL; > + return -ENXIO; > +} > + > +static inline int > +cxl_restore_state_after_pci_reset(struct pci_dev *pdev, > + struct cxl_hdm_info *snapshot) > +{ > + return 0; > +} > + > static inline int cxl_reset_function(struct pci_dev *pdev, bool probe) > { > return -ENOTTY; > -- > 2.43.0 >
diff --git a/drivers/cxl/core/resource.c b/drivers/cxl/core/resource.c index efbab341b30f..3422139ae3ab 100644 --- a/drivers/cxl/core/resource.c +++ b/drivers/cxl/core/resource.c @@ -625,12 +625,12 @@ static int cxl_restore_hdm_decoder(struct pci_dev *pdev, void __iomem *hdm, return 0; } -static int cxl_snapshot_hdm(struct pci_dev *pdev, - struct cxl_hdm_info **snapshot) +int cxl_snapshot_hdm(struct pci_dev *pdev, struct cxl_hdm_info **snapshot) { struct cxl_hdm_info *info; size_t size; + device_lock_assert(&pdev->dev); guard(rwsem_read)(&cxl_rwsem.dpa); info = pdev->hdm; if (!info) @@ -798,6 +798,25 @@ static int cxl_reset_save_restored_state(struct pci_dev *pdev, u16 command) return rc; } +int cxl_restore_state_after_pci_reset(struct pci_dev *pdev, + struct cxl_hdm_info *snapshot) +{ + u16 command; + int rc; + + device_lock_assert(&pdev->dev); + guard(rwsem_write)(&cxl_rwsem.region); + + cxl_restore_pci_state_for_hdm_restore(pdev, &command); + rc = cxl_restore_state(pdev, snapshot); + if (rc) { + cxl_reset_save_disabled_state(pdev); + return rc; + } + + return cxl_reset_save_restored_state(pdev, command); +} + /* * CXL r4.0 sec 9.7.2 defines the reset completion timeout encodings. * Sec 9.7.3 leaves config-space access behavior undefined for 100 ms after diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index dc57a202c91f..6928c99aa785 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -5015,6 +5015,7 @@ static int pci_reset_bus_function(struct pci_dev *dev, bool probe) static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) { + struct cxl_hdm_info *snapshot __free(kfree) = NULL; struct pci_dev *bridge; u16 dvsec, reg, val; int rc; @@ -5037,6 +5038,10 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) if (rc) return -ENOTTY; + rc = cxl_snapshot_hdm(dev, &snapshot); + if (rc && rc != -ENXIO) + return rc; + rc = pci_dev_reset_iommu_prepare(dev); if (rc) { pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", rc); @@ -5057,6 +5062,9 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, reg); + if (!rc && snapshot) + rc = cxl_restore_state_after_pci_reset(dev, snapshot); + pci_dev_reset_iommu_done(dev); return rc; } diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h index 9904eaed5af1..b83cfaed618f 100644 --- a/include/cxl/cxl.h +++ b/include/cxl/cxl.h @@ -207,6 +207,9 @@ struct cxl_register_map { #ifdef CONFIG_CXL_RESET void pci_cxl_hdm_cache_init(struct pci_dev *pdev); void pci_cxl_hdm_cache_release(struct pci_dev *pdev); +int cxl_snapshot_hdm(struct pci_dev *pdev, struct cxl_hdm_info **snapshot); +int cxl_restore_state_after_pci_reset(struct pci_dev *pdev, + struct cxl_hdm_info *snapshot); int cxl_reset_function(struct pci_dev *pdev, bool probe); #else static inline void pci_cxl_hdm_cache_init(struct pci_dev *pdev) @@ -217,6 +220,20 @@ static inline void pci_cxl_hdm_cache_release(struct pci_dev *pdev) { } +static inline int cxl_snapshot_hdm(struct pci_dev *pdev, + struct cxl_hdm_info **snapshot) +{ + *snapshot = NULL; + return -ENXIO; +} + +static inline int +cxl_restore_state_after_pci_reset(struct pci_dev *pdev, + struct cxl_hdm_info *snapshot) +{ + return 0; +} + static inline int cxl_reset_function(struct pci_dev *pdev, bool probe) { return -ENOTTY;