[PATCH v16 15/18] PCI/CXL: Restore CXL state after PCI reset

[Srirangan Madhavan <smadhavan@nvidia.com>]

Restore the PCI configuration needed for HDM access, followed by cached
HDM decoder and CXL Device DVSEC state. Set Lock On Commit before
recommitting a cleared locked decoder; leave surviving locked state
unchanged.

Stop at the first restoration failure and preserve a disabled PCI command
snapshot. Keep IOMMU exclusion active through restoration and post-reset
cache invalidation.

Program the decoders from built-in CXL code that takes only a mapped
register block, and keep PCI state handling in drivers/pci/cxl.c.

The restore helper structure and diagnostics follow Dave Jiang's
reference implementation on his cxl-type2-reset branch.

Signed-off-by: Srirangan Madhavan <smadhavan@nvidia.com>
Assisted-by: LLM

---
Reference branch: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/commit/?id=81fff5893a3140361cac625b09ada798afb92e03
---
 drivers/cxl/core/core.h     |   2 +-
 drivers/cxl/core/hdm.c      |   2 +-
 drivers/cxl/core/hdm_regs.c | 153 ++++++++++++++++++++++++++---
 drivers/pci/cxl.c           | 185 ++++++++++++++++++++++++++++++++++--
 include/cxl/hdm.h           |   2 +
 5 files changed, 325 insertions(+), 19 deletions(-)

diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index e34d574f6594..9fc9fd6c771e 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -145,7 +145,7 @@ void cxl_commit_start_endpoint(const struct cxl_endpoint_decoder_settings *setti
 			       void __iomem *hdm);
 void cxl_commit_start_switch(const struct cxl_decoder_config *config,
 			     u64 targets, void __iomem *hdm);
-int cxld_await_commit(void __iomem *hdm, int id);
+int cxld_await_commit(void __iomem *hdm, int id, bool committed);
 
 int cxl_hdm_unpack_decoder(struct cxl_decoder_config *config, int id,
 			   u32 ctrl, u64 base, u64 size);
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index ec387070d7ad..ad8cb1e76afb 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -815,7 +815,7 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld)
 	scoped_guard(rwsem_read, &cxl_rwsem.dpa)
 		setup_hw_decoder(cxld, hdm);
 
-	rc = cxld_await_commit(hdm, cxld->config.id);
+	rc = cxld_await_commit(hdm, cxld->config.id, true);
 	if (rc) {
 		dev_dbg(&port->dev, "%s: error %d committing decoder\n",
 			dev_name(&cxld->dev), rc);
diff --git a/drivers/cxl/core/hdm_regs.c b/drivers/cxl/core/hdm_regs.c
index 9409beb26478..fe5e65a9ddb1 100644
--- a/drivers/cxl/core/hdm_regs.c
+++ b/drivers/cxl/core/hdm_regs.c
@@ -52,7 +52,7 @@ static void cxld_set_type(const struct cxl_decoder_config *config, u32 *ctrl)
  * clock skew and other marginal behavior
  */
 #define COMMIT_TIMEOUT_MS 20
-int cxld_await_commit(void __iomem *hdm, int id)
+int cxld_await_commit(void __iomem *hdm, int id, bool committed)
 {
 	u32 ctrl;
 	int i;
@@ -64,7 +64,7 @@ int cxld_await_commit(void __iomem *hdm, int id)
 			writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 			return -EIO;
 		}
-		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
+		if (!!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) == committed)
 			return 0;
 		fsleep(1000);
 	}
@@ -220,8 +220,7 @@ int cxl_hdm_decoder_count_mapped(struct device *dev, void __iomem *hdm,
 }
 
 /* Read one endpoint decoder; a disabled decoder keeps only its id. */
-static int cxl_hdm_read_decoder_settings(struct device *dev,
-					 struct cxl_endpoint_decoder_settings *settings,
+static int cxl_hdm_read_decoder_settings(struct cxl_endpoint_decoder_settings *settings,
 					 void __iomem *hdm, int id)
 {
 	u64 lo, hi;
@@ -230,12 +229,8 @@ static int cxl_hdm_read_decoder_settings(struct device *dev,
 
 	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 	rc = cxl_hdm_read_decoder(&settings->config, hdm, id, ctrl);
-	if (rc) {
-		dev_err(dev,
-			"CXL HDM decoder %d has invalid configuration: %d\n",
-			id, rc);
+	if (rc)
 		return rc;
-	}
 
 	if (!(settings->config.flags & CXL_DECODER_F_ENABLE))
 		*settings = (struct cxl_endpoint_decoder_settings) {
@@ -267,10 +262,13 @@ int cxl_hdm_read_state(struct device *dev, void __iomem *hdm,
 
 	info->global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
 	for (int i = 0; i < info->decoder_count; i++) {
-		rc = cxl_hdm_read_decoder_settings(dev, &info->settings[i],
+		rc = cxl_hdm_read_decoder_settings(&info->settings[i],
 						    hdm, i);
-		if (rc)
+		if (rc) {
+			dev_err(dev, "CXL HDM decoder %d has invalid configuration: %d\n",
+				i, rc);
 			return rc;
+		}
 	}
 
 	/*
@@ -286,3 +284,136 @@ int cxl_hdm_read_state(struct device *dev, void __iomem *hdm,
 
 	return 0;
 }
+
+static int cxl_hdm_decoder_uncommit(void __iomem *hdm, int id, bool *retained)
+{
+	u32 ctrl;
+
+	*retained = false;
+	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
+	if (ctrl == U32_MAX)
+		return -EIO;
+	if (ctrl & CXL_HDM_DECODER0_CTRL_COMMIT_ERROR)
+		return -EIO;
+
+	if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) {
+		if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) {
+			*retained = true;
+			return 0;
+		}
+
+		return -EIO;
+	}
+
+	if (!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED))
+		return 0;
+
+	ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
+	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
+
+	return cxld_await_commit(hdm, id, false);
+}
+
+static bool
+cxl_hdm_decoder_settings_match(const struct cxl_endpoint_decoder_settings *saved,
+			       const struct cxl_endpoint_decoder_settings *active)
+{
+	const struct cxl_decoder_config *a = &saved->config;
+	const struct cxl_decoder_config *b = &active->config;
+
+	return a->id == b->id &&
+	       a->hpa_range.start == b->hpa_range.start &&
+	       a->hpa_range.end == b->hpa_range.end &&
+	       a->interleave_ways == b->interleave_ways &&
+	       a->interleave_granularity == b->interleave_granularity &&
+	       a->target_type == b->target_type &&
+	       (a->flags & CXL_DECODER_F_RESET_MASK) ==
+		(b->flags & CXL_DECODER_F_RESET_MASK) &&
+	       saved->skip == active->skip;
+}
+
+static int
+cxl_hdm_restore_decoder(void __iomem *hdm,
+			const struct cxl_endpoint_decoder_settings *settings)
+{
+	const struct cxl_decoder_config *config = &settings->config;
+	bool retained;
+	u32 ctrl;
+	int rc;
+
+	rc = cxl_hdm_decoder_uncommit(hdm, config->id, &retained);
+	if (rc)
+		return rc;
+
+	if (retained) {
+		struct cxl_endpoint_decoder_settings active;
+
+		rc = cxl_hdm_read_decoder_settings(&active, hdm, config->id);
+		if (rc)
+			return rc;
+		if (!cxl_hdm_decoder_settings_match(settings, &active))
+			return -EIO;
+		return 0;
+	}
+
+	if (!(config->flags & CXL_DECODER_F_ENABLE))
+		return 0;
+
+	if (config->flags & CXL_DECODER_F_LOCK) {
+		ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(config->id));
+		if (ctrl == U32_MAX)
+			return -EIO;
+		ctrl |= CXL_HDM_DECODER0_CTRL_LOCK;
+		writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(config->id));
+	}
+
+	cxl_commit_start_endpoint(settings, hdm);
+	rc = cxld_await_commit(hdm, config->id, true);
+	if (rc)
+		return rc;
+
+	if (!(config->flags & CXL_DECODER_F_LOCK))
+		return 0;
+
+	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(config->id));
+	if (ctrl == U32_MAX || !(ctrl & CXL_HDM_DECODER0_CTRL_LOCK))
+		return -EIO;
+
+	return 0;
+}
+
+/**
+ * cxl_hdm_restore_state() - reprogram endpoint HDM decoders from a snapshot
+ * @hdm: mapped HDM decoder capability
+ * @snapshot: state from cxl_hdm_cache_snapshot()
+ *
+ * Restore global control, then recommit each enabled decoder in order. Leave
+ * a locked decoder that kept its programming across the reset unchanged,
+ * after checking that it still matches @snapshot.
+ *
+ * Return: 0 on success, or the first failure.
+ */
+int cxl_hdm_restore_state(void __iomem *hdm,
+			  const struct cxl_hdm_info *snapshot)
+{
+	int count;
+	int rc;
+
+	count = cxl_hdm_decoder_count(readl(hdm + CXL_HDM_DECODER_CAP_OFFSET));
+	if (count != snapshot->decoder_count)
+		return -ENXIO;
+
+	/* Restore global HDM control before committing individual decoders. */
+	writel(snapshot->global_ctrl, hdm + CXL_HDM_DECODER_CTRL_OFFSET);
+
+	for (int i = 0; i < snapshot->decoder_count; i++) {
+		rc = cxl_hdm_restore_decoder(hdm, &snapshot->settings[i]);
+		if (rc)
+			return rc;
+	}
+
+	/* Flush posted HDM writes before restoring protocol enables. */
+	readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
+
+	return 0;
+}
diff --git a/drivers/pci/cxl.c b/drivers/pci/cxl.c
index 9a298e90586d..58bbd5b47539 100644
--- a/drivers/pci/cxl.c
+++ b/drivers/pci/cxl.c
@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright (c) 2026 NVIDIA Corporation & Affiliates */
 /*
- * CXL reset support in the PCI core. Cache the HDM decoder state of a CXL
- * memory device at enumeration, so it can be restored after a reset without
- * a CXL driver, and implement CXL Device Reset. HDM register formats and
+ * CXL reset support in the PCI core: cache the HDM decoder state of a CXL
+ * memory device at enumeration, implement CXL Device Reset, and restore the
+ * cached state afterwards without a CXL driver. HDM register formats and
  * programming live in drivers/cxl.
  */
 #include <linux/bitfield.h>
@@ -16,6 +16,7 @@
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/memregion.h>
+#include <linux/overflow.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
 
@@ -232,6 +233,162 @@ void pci_cxl_hdm_cache_release(struct pci_dev *pdev)
 	cxl_hdm_cache_release(&pdev->hdm);
 }
 
+static void __iomem *cxl_pci_hdm_ioremap_current(struct pci_dev *pdev,
+						 int bar,
+						 resource_size_t offset,
+						 resource_size_t size)
+{
+	struct resource *res;
+	resource_size_t hdm_start;
+	void __iomem *hdm;
+
+	if (bar < 0 || bar >= PCI_STD_NUM_BARS || !size)
+		return IOMEM_ERR_PTR(-EINVAL);
+	if (!cxl_pci_bar_assigned(pdev, bar))
+		return IOMEM_ERR_PTR(-ENODEV);
+
+	res = &pdev->resource[bar];
+	if (offset > resource_size(res) || size > resource_size(res) - offset)
+		return IOMEM_ERR_PTR(-ENODEV);
+	if (check_add_overflow(res->start, offset, &hdm_start))
+		return IOMEM_ERR_PTR(-EOVERFLOW);
+
+	hdm = ioremap(hdm_start, size);
+	if (!hdm) {
+		pci_err(pdev, "failed to remap CXL HDM decoder registers\n");
+		return IOMEM_ERR_PTR(-ENOMEM);
+	}
+
+	return hdm;
+}
+
+static void cxl_restore_pci_state_for_hdm_restore(struct pci_dev *pdev,
+						  u16 *command)
+{
+	u32 saved_config = pdev->saved_config_space[PCI_COMMAND / 4];
+	u32 restore_config;
+
+	device_lock_assert(&pdev->dev);
+	*command = saved_config & 0xffff;
+
+	/*
+	 * Restore BARs and the related PCI state needed to reach HDM MMIO.
+	 * Temporarily alter the saved command value so that
+	 * pci_restore_state() enables MSE but cannot enable BME or INTx before
+	 * CXL state restoration completes. Preserve the original saved value
+	 * for the outer PCI reset path.
+	 */
+	restore_config = saved_config | PCI_COMMAND_MEMORY |
+			 PCI_COMMAND_INTX_DISABLE;
+	restore_config &= ~PCI_COMMAND_MASTER;
+
+	pdev->saved_config_space[PCI_COMMAND / 4] = restore_config;
+	pci_restore_state(pdev);
+	pdev->saved_config_space[PCI_COMMAND / 4] = saved_config;
+}
+
+static int cxl_restore_dvsec_ctrl(struct pci_dev *pdev,
+				  const struct cxl_hdm_info *snapshot)
+{
+	int rc;
+
+	if (!snapshot->dvsec_ctrl_valid) {
+		pci_err(pdev, "CXL DVSEC control snapshot is invalid\n");
+		return -ENXIO;
+	}
+
+	rc = pci_write_config_word(pdev, snapshot->dvsec + PCI_DVSEC_CXL_CTRL,
+				   snapshot->dvsec_ctrl);
+	if (rc) {
+		rc = pcibios_err_to_errno(rc);
+		pci_err(pdev, "failed to restore CXL DVSEC control: %d\n", rc);
+	}
+
+	return rc;
+}
+
+static int cxl_restore_state(struct pci_dev *pdev,
+			     const struct cxl_hdm_info *snapshot)
+{
+	void __iomem *hdm;
+	int rc;
+
+	hdm = cxl_pci_hdm_ioremap_current(pdev, snapshot->hdm_bar,
+					  snapshot->hdm_offset,
+					  snapshot->hdm_size);
+	if (IS_ERR(hdm)) {
+		pci_err(pdev, "failed to map CXL HDM state for restore: %ld\n",
+			PTR_ERR(hdm));
+		return PTR_ERR(hdm);
+	}
+
+	rc = cxl_hdm_restore_state(hdm, snapshot);
+	iounmap(hdm);
+	if (rc) {
+		pci_err(pdev, "failed to restore CXL HDM state: %d\n", rc);
+		return rc;
+	}
+
+	return cxl_restore_dvsec_ctrl(pdev, snapshot);
+}
+
+static void cxl_reset_save_disabled_state(struct pci_dev *pdev)
+{
+	u32 *saved_command = &pdev->saved_config_space[PCI_COMMAND / 4];
+	int rc;
+
+	rc = pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
+	if (rc)
+		pci_err(pdev,
+			"failed to disable device after CXL reset failure: %d\n",
+			pcibios_err_to_errno(rc));
+
+	/* Preserve the pre-reset snapshot if PCI state has not been restored. */
+	if (!pdev->state_saved) {
+		rc = pci_save_state(pdev);
+		if (rc)
+			pci_err(pdev,
+				"failed to save disabled state after CXL reset failure: %d\n",
+				rc);
+	}
+
+	/* Ensure the outer PCI restore cannot re-enable MSE or BME. */
+	*saved_command &= GENMASK(31, 16);
+	*saved_command |= PCI_COMMAND_INTX_DISABLE;
+}
+
+static int cxl_reset_save_restored_state(struct pci_dev *pdev, u16 command)
+{
+	int rc;
+
+	rc = pci_write_config_word(pdev, PCI_COMMAND, command);
+	if (rc) {
+		rc = pcibios_err_to_errno(rc);
+		pci_err(pdev, "failed to restore PCI command state: %d\n", rc);
+		cxl_reset_save_disabled_state(pdev);
+		return rc;
+	}
+
+	rc = pci_save_state(pdev);
+	if (rc) {
+		pci_err(pdev, "failed to save restored state after CXL reset: %d\n",
+			rc);
+		cxl_reset_save_disabled_state(pdev);
+	}
+
+	return rc;
+}
+
+/* Caller holds the region lock and keeps device traffic isolated. */
+static int pci_cxl_restore_state(struct pci_dev *pdev,
+			  const struct cxl_hdm_info *snapshot, u16 *command)
+	__must_hold(cxl_region_quiesce)
+{
+	device_lock_assert(&pdev->dev);
+	cxl_restore_pci_state_for_hdm_restore(pdev, command);
+	return cxl_restore_state(pdev, snapshot);
+}
+
 /*
  * CXL r4.0 sec 9.7.2 defines the reset completion timeout encodings.
  * Sec 9.7.3 leaves config-space access behavior undefined for 100 ms after
@@ -412,7 +569,9 @@ static int cxl_clear_memory(struct pci_dev *pdev, int dvsec, bool initiate)
 }
 
 static int __cxl_reset_execute(struct pci_dev *pdev, int dvsec, u16 cap,
-			      struct cxl_hdm_quiesce *q)
+			      const struct cxl_hdm_info *snapshot,
+			      struct cxl_hdm_quiesce *q,
+			      u16 *command)
 {
 	int rc, rc2;
 
@@ -437,10 +596,15 @@ static int __cxl_reset_execute(struct pci_dev *pdev, int dvsec, u16 cap,
 		pci_err(pdev, "failed to clear CXL Reset Memory Clear: %d\n", rc2);
 	rc = rc ?: rc2;
 
+	if (!rc)
+		rc = pci_cxl_restore_state(pdev, snapshot, command);
+
 	rc2 = cxl_hdm_quiesce_flush(q);
 	if (rc2)
 		pci_err(pdev, "failed to invalidate CPU cache after CXL reset: %d\n", rc2);
 	rc = rc ?: rc2;
+	if (rc)
+		cxl_reset_save_disabled_state(pdev);
 
 	pci_dev_reset_iommu_done(pdev);
 	return rc;
@@ -451,6 +615,7 @@ static int cxl_reset_quiesced(struct pci_dev *pdev, int dvsec, u16 cap)
 {
 	struct cxl_hdm_info *snapshot __free(kfree) = NULL;
 	struct cxl_pci_cache cache = {};
+	u16 command;
 	int rc, rc2;
 
 	rc = pci_read_config_word(pdev, dvsec + PCI_DVSEC_CXL_CTRL2,
@@ -474,11 +639,19 @@ static int cxl_reset_quiesced(struct pci_dev *pdev, int dvsec, u16 cap)
 		return rc;
 	}
 	if (!rc)
-		rc = __cxl_reset_execute(pdev, dvsec, cap, &cache.quiesce);
+		rc = __cxl_reset_execute(pdev, dvsec, cap, snapshot, &cache.quiesce,
+					 &command);
 	/* Restore cache policy after any attempt to disable caching. */
 	rc2 = cxl_reset_restore_cache_policy(pdev, dvsec, cache.saved_ctrl2);
 	cxl_hdm_quiesce_destroy(&cache.quiesce);
-	return rc ?: rc2;
+	if (rc)
+		return rc;
+	if (rc2) {
+		cxl_reset_save_disabled_state(pdev);
+		return rc2;
+	}
+
+	return cxl_reset_save_restored_state(pdev, command);
 }
 
 static int cxl_reset_execute(struct pci_dev *pdev, int dvsec, u16 cap)
diff --git a/include/cxl/hdm.h b/include/cxl/hdm.h
index 7919d872a34d..80d6ebb9f4aa 100644
--- a/include/cxl/hdm.h
+++ b/include/cxl/hdm.h
@@ -54,6 +54,8 @@ int cxl_hdm_decoder_count_mapped(struct device *dev, void __iomem *hdm,
 				 resource_size_t size);
 int cxl_hdm_read_state(struct device *dev, void __iomem *hdm,
 		       struct cxl_hdm_info *info);
+int cxl_hdm_restore_state(void __iomem *hdm,
+			  const struct cxl_hdm_info *snapshot);
 bool cxl_hdm_cache_present(struct cxl_hdm_info *const *slot);
 void cxl_hdm_cache_set_dvsec_ctrl(struct pci_dev *pdev, u16 ctrl);
 void cxl_hdm_cache_refresh_decoder(struct pci_dev *pdev, void __iomem *hdm,
-- 
2.43.0