[PATCH v6 13/31] vfio/pci: Bracket PCI resets with the CXL reset hooks

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

A CXL Type-2 device must quiesce its coherent link before any reset and
restore its HDM decoder state afterward, or a reset can leave the device
mastering the bus over decoders that are no longer valid. This has to
happen around every reset the PCI core drives for the device, not only
the ones VFIO issues directly, and around AER and DPC recovery, which
can reset the link without the reset callbacks.

Add:
- reset_prepare
- reset_done
- an optional error_resume to the CXL ops

Export:
- vfio_pci_core_reset_prepare()
- vfio_pci_core_reset_done()
- vfio_pci_core_error_detected()
- vfio_pci_core_error_resume()
which take the vfio device.

vfio-pci calls them from its own pci_error_handlers, which resolve the
driver data it owns. A variant driver that binds CXL devices calls them
from its own handlers the same way; a driver without reset_prepare,
reset_done, error_detected and resume callbacks gets no CXL support.

The provider hooks run with memory_lock held for write: vfio-pci-core
already holds it across its own hot reset, and
vfio_pci_core_reset_prepare() takes it for a reset started anywhere else
until vfio_pci_core_reset_done(). The hot reset records its task, so a
reset that another task starts meanwhile waits for memory_lock instead
of skipping it. pci_dev_lock() blocks user config access, which a
PCI_COMMAND write can be waiting for while it holds memory_lock, so
reset_prepare lets config access through while it waits. reset_done
returns an error when the provider could not restore its device, and
VFIO_DEVICE_PCI_HOT_RESET returns that error; for any other reset the
provider logs it.

Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/vfio_pci.c      |  39 +++++++-
 drivers/vfio/pci/vfio_pci_core.c | 154 +++++++++++++++++++++++++++++++
 include/linux/vfio_pci_core.h    |  22 +++++
 3 files changed, 214 insertions(+), 1 deletion(-)

diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 830369ff878d..286244059569 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -223,13 +223,50 @@ static const struct pci_device_id vfio_pci_table[] = {
 
 MODULE_DEVICE_TABLE(pci, vfio_pci_table);
 
+static void vfio_pci_reset_prepare(struct pci_dev *pdev)
+{
+	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
+	vfio_pci_core_reset_prepare(vdev);
+}
+
+static void vfio_pci_reset_done(struct pci_dev *pdev)
+{
+	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
+	vfio_pci_core_reset_done(vdev);
+}
+
+static pci_ers_result_t vfio_pci_error_detected(struct pci_dev *pdev,
+						pci_channel_state_t state)
+{
+	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
+	vfio_pci_core_error_detected(vdev);
+	return vfio_pci_core_aer_err_detected(pdev, state);
+}
+
+static void vfio_pci_error_resume(struct pci_dev *pdev)
+{
+	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
+	vfio_pci_core_error_resume(vdev);
+}
+
+static const struct pci_error_handlers vfio_pci_err_handlers = {
+	.error_detected = vfio_pci_error_detected,
+	.reset_prepare = vfio_pci_reset_prepare,
+	.reset_done = vfio_pci_reset_done,
+	.resume = vfio_pci_error_resume,
+};
+
 static struct pci_driver vfio_pci_driver = {
 	.name			= "vfio-pci",
 	.id_table		= vfio_pci_table,
 	.probe			= vfio_pci_probe,
 	.remove			= vfio_pci_remove,
 	.sriov_configure	= vfio_pci_sriov_configure,
-	.err_handler		= &vfio_pci_core_err_handlers,
+	.err_handler		= &vfio_pci_err_handlers,
 	.driver_managed_dma	= true,
 };
 
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 3d4e01eea7b1..f1e1ec36eec7 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -2462,6 +2462,7 @@ static bool vfio_pci_is_cxl_type2(struct pci_dev *pdev)
  */
 static int vfio_pci_core_cxl_init(struct vfio_pci_core_device *vdev)
 {
+	const struct pci_error_handlers *err_handler;
 	const struct vfio_cxl_ops *ops;
 	int ret;
 
@@ -2473,6 +2474,23 @@ static int vfio_pci_core_cxl_init(struct vfio_pci_core_device *vdev)
 	if (!ops)
 		return IS_BUILTIN(CONFIG_VFIO_CXL) ? -EPROBE_DEFER : 0;
 
+	/*
+	 * The provider must see every PCI core reset of the device and every
+	 * AER or DPC recovery. A driver without reset_prepare, reset_done,
+	 * error_detected and resume callbacks gets no CXL support; the driver's
+	 * callbacks must call vfio_pci_core_reset_prepare(), _reset_done(),
+	 * _error_detected() and _error_resume().
+	 */
+	err_handler = vdev->pdev->driver ? vdev->pdev->driver->err_handler : NULL;
+	if (!err_handler || !err_handler->reset_prepare ||
+	    !err_handler->reset_done || !err_handler->error_detected ||
+	    !err_handler->resume) {
+		pci_info(vdev->pdev,
+			 "driver lacks PCI reset or error recovery hooks, continuing without CXL support\n");
+		vfio_pci_put_cxl_ops(ops);
+		return 0;
+	}
+
 	ret = ops->init(vdev);
 	/*
 	 * A provider that is not ready yet (for example its CXL port has not
@@ -2759,6 +2777,128 @@ int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
 
+/*
+ * Take memory_lock for a reset that the PCI core runs on a locked device.
+ * pci_dev_lock() also blocks user config access, and vfio_basic_config_write()
+ * holds memory_lock while its pci_user_*_config() calls wait for that block
+ * to clear. Waiting for memory_lock with config access blocked would hang
+ * both tasks, so let user config access through while waiting: the PCI core
+ * calls reset_prepare before it saves or touches the device. The caller holds
+ * the device lock, so a block found set is the caller's own.
+ */
+static void vfio_pci_reset_lock_memory(struct vfio_pci_core_device *vdev)
+{
+	struct pci_dev *pdev = vdev->pdev;
+
+	if (down_write_trylock(&vdev->memory_lock))
+		return;
+
+	/* The caller did not block config access: nothing to release. */
+	if (pci_cfg_access_trylock(pdev)) {
+		pci_cfg_access_unlock(pdev);
+		down_write(&vdev->memory_lock);
+		return;
+	}
+
+	pci_cfg_access_unlock(pdev);
+	down_write(&vdev->memory_lock);
+	pci_cfg_access_lock(pdev);
+}
+
+/**
+ * vfio_pci_core_reset_prepare() - prepare a device for a PCI core reset
+ * @vdev: vfio PCI device about to be reset
+ *
+ * Call from the driver's pci_error_handlers.reset_prepare callback.
+ */
+void vfio_pci_core_reset_prepare(struct vfio_pci_core_device *vdev)
+{
+	if (!vdev->cxl_ops)
+		return;
+
+	/*
+	 * vfio-pci-core holds memory_lock across its own hot reset. Hold it
+	 * across a reset started anywhere else too, until reset_done. Test
+	 * the owner, not just a flag: another task can reset the device while
+	 * a hot reset holds memory_lock and has not yet locked the device.
+	 */
+	if (READ_ONCE(vdev->hot_reset_task) != current)
+		vfio_pci_reset_lock_memory(vdev);
+
+	vdev->cxl_ops->reset_prepare(vdev);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_reset_prepare);
+
+/**
+ * vfio_pci_core_reset_done() - restore a device after a PCI core reset
+ * @vdev: vfio PCI device that was reset
+ *
+ * Call from the driver's pci_error_handlers.reset_done callback.
+ */
+void vfio_pci_core_reset_done(struct vfio_pci_core_device *vdev)
+{
+	int ret;
+
+	if (!vdev->cxl_ops)
+		return;
+
+	ret = vdev->cxl_ops->reset_done(vdev);
+
+	/*
+	 * hot_reset_task cannot change since reset_prepare: setting it needs
+	 * memory_lock, which this task or the hot reset task holds.
+	 */
+	if (READ_ONCE(vdev->hot_reset_task) == current) {
+		/* VFIO_DEVICE_PCI_HOT_RESET returns the error. */
+		vdev->reset_done_err = ret;
+		return;
+	}
+
+	/*
+	 * Nobody waits for the result of a reset started elsewhere. A provider
+	 * that failed has logged it and keeps its device memory out of use
+	 * until a later reset restores the device.
+	 */
+	up_write(&vdev->memory_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_reset_done);
+
+/**
+ * vfio_pci_core_error_detected() - prepare a device for AER or DPC recovery
+ * @vdev: vfio PCI device the recovery reports an error on
+ *
+ * Call from the driver's pci_error_handlers.error_detected callback. The
+ * recovery may reset the device without reset_prepare and reset_done, so a
+ * provider takes its device memory out of use here.
+ */
+void vfio_pci_core_error_detected(struct vfio_pci_core_device *vdev)
+{
+	if (!vdev->cxl_ops)
+		return;
+
+	guard(rwsem_write)(&vdev->memory_lock);
+	vdev->cxl_ops->reset_prepare(vdev);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_error_detected);
+
+/**
+ * vfio_pci_core_error_resume() - resume a device after AER or DPC recovery
+ * @vdev: vfio PCI device the recovery resumes
+ *
+ * Call from the driver's pci_error_handlers.resume callback. A provider puts
+ * its device memory back if the device still decodes it; otherwise the memory
+ * stays out of use until the user resets the device.
+ */
+void vfio_pci_core_error_resume(struct vfio_pci_core_device *vdev)
+{
+	if (!vdev->cxl_ops || !vdev->cxl_ops->error_resume)
+		return;
+
+	guard(rwsem_write)(&vdev->memory_lock);
+	vdev->cxl_ops->error_resume(vdev);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_error_resume);
+
 const struct pci_error_handlers vfio_pci_core_err_handlers = {
 	.error_detected = vfio_pci_core_aer_err_detected,
 };
@@ -2915,6 +3055,8 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 			ret = -EBUSY;
 			break;
 		}
+		WRITE_ONCE(vdev->hot_reset_task, current);
+		vdev->reset_done_err = 0;
 
 		vfio_pci_dma_buf_move(vdev, true);
 		vfio_pci_zap_bars(vdev);
@@ -2940,6 +3082,17 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 
 	ret = pci_reset_bus(pdev);
 
+	/* A provider that could not restore its device fails the hot reset. */
+	if (!ret) {
+		list_for_each_entry(vdev, &dev_set->device_list,
+				    vdev.dev_set_list) {
+			if (vdev->reset_done_err) {
+				ret = vdev->reset_done_err;
+				break;
+			}
+		}
+	}
+
 	vdev = list_last_entry(&dev_set->device_list,
 			       struct vfio_pci_core_device, vdev.dev_set_list);
 
@@ -2948,6 +3101,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 					 vdev.dev_set_list) {
 		if (vdev->vdev.open_count && __vfio_pci_memory_enabled(vdev))
 			vfio_pci_dma_buf_move(vdev, false);
+		WRITE_ONCE(vdev->hot_reset_task, NULL);
 		up_write(&vdev->memory_lock);
 	}
 
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 6520a2dcfd4a..881e51c42414 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -72,6 +72,21 @@ struct vfio_cxl_ops {
 	void	(*release)(struct vfio_pci_core_device *vdev);
 	int	(*open_device)(struct vfio_pci_core_device *vdev);
 	void	(*close_device)(struct vfio_pci_core_device *vdev);
+	/*
+	 * Optional. Called with memory_lock held for write when AER or DPC
+	 * recovery resumes the device, after reset_prepare was called for the
+	 * error. Config space is as the recovery left it.
+	 */
+	void	(*error_resume)(struct vfio_pci_core_device *vdev);
+	/*
+	 * Called around a PCI core reset of the device with memory_lock held
+	 * for write; reset_prepare also when AER or DPC recovery reports an
+	 * error. reset_done returns an error if the provider could not
+	 * restore the device; VFIO_DEVICE_PCI_HOT_RESET returns it, and for
+	 * any other reset the provider logs it.
+	 */
+	void	(*reset_prepare)(struct vfio_pci_core_device *vdev);
+	int	(*reset_done)(struct vfio_pci_core_device *vdev);
 	/* Pinned per bound CXL device so vfio-cxl cannot unload under usage */
 	struct module *owner;
 };
@@ -166,6 +181,9 @@ struct vfio_pci_core_device {
 	struct vfio_pci_core_device	*sriov_pf_core_dev;
 	struct notifier_block	nb;
 	struct rw_semaphore	memory_lock;
+	/* Task running VFIO_DEVICE_PCI_HOT_RESET while it holds memory_lock */
+	struct task_struct	*hot_reset_task;
+	int			reset_done_err;
 	struct list_head	dmabufs;
 	/*
 	 * Excluded BAR sub-ranges, one list per BAR, sorted by start and not
@@ -207,6 +225,10 @@ void vfio_pci_core_release_dev(struct vfio_device *core_vdev);
 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev);
 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev);
 extern const struct pci_error_handlers vfio_pci_core_err_handlers;
+void vfio_pci_core_reset_prepare(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_reset_done(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_error_detected(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_error_resume(struct vfio_pci_core_device *vdev);
 int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
 				  int nr_virtfn);
 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
-- 
2.53.0