[PATCH v6 17/31] vfio/pci: Call the device provider across low-power transitions

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

A provider may serve state that does not survive low power. A CXL Type-2
device loses its committed HDM decoder in D3hot or D3cold, and the HDM
memory region must stay out of use until the host restores the decoder.
Without a hook the provider can only keep the device in D0.

Add optional power_down and power_up ops. vfio-pci-core calls power_down
with memory_lock held for write before the device leaves D0: in
vfio_pci_set_power_state() for a lower state, and in runtime suspend
before the PCI core moves the device to low power. It calls power_up
with memory_lock held after the device is back in D0 with its config
space restored: after a guest PM control write returns the device to D0,
and in runtime resume. If a request for a lower state leaves the device
in D0, because the transition failed or a quirk keeps it there,
vfio-pci-core calls power_up at once.

The other paths that return a device to D0 either reset it afterwards
(close, VFIO_DEVICE_RESET and the hot reset ioctl), where the provider
restores its state, or tear it down (unregister).

Suggested-by: Alex Williamson <alex@shazbot.org>
Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/vfio_pci_config.c |  4 +++
 drivers/vfio/pci/vfio_pci_core.c   | 40 +++++++++++++++++++++++++++++-
 drivers/vfio/pci/vfio_pci_priv.h   |  1 +
 include/linux/vfio_pci_core.h      |  7 ++++++
 4 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index 6fbece95cb9f..ec556921ecd4 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -712,6 +712,8 @@ static int __init init_pci_cap_basic_perm(struct perm_bits *perm)
 static void vfio_lock_and_set_power_state(struct vfio_pci_core_device *vdev,
 					  pci_power_t state)
 {
+	pci_power_t prev = vdev->pdev->current_state;
+
 	if (state >= PCI_D3hot) {
 		vfio_pci_zap_and_down_write_memory_lock(vdev);
 		vfio_pci_dma_buf_move(vdev, true);
@@ -720,6 +722,8 @@ static void vfio_lock_and_set_power_state(struct vfio_pci_core_device *vdev,
 	}
 
 	vfio_pci_set_power_state(vdev, state);
+	if (state == PCI_D0 && prev > PCI_D0)
+		vfio_pci_provider_power_up(vdev);
 	if (__vfio_pci_memory_enabled(vdev))
 		vfio_pci_dma_buf_move(vdev, false);
 	up_write(&vdev->memory_lock);
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 9d0c22a4f3c6..be2acab625c0 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -318,10 +318,31 @@ static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
  * by PM capability emulation and separately from pci_dev internal saved state
  * to avoid it being overwritten and consumed around other resets.
  */
+/*
+ * A provider whose device state does not survive low power is told before
+ * the device leaves D0 and after it is back in D0 with its config space
+ * restored. Both run with memory_lock held for write.
+ */
+static void vfio_pci_provider_power_down(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	if (vdev->cxl_ops && vdev->cxl_ops->power_down)
+		vdev->cxl_ops->power_down(vdev);
+}
+
+void vfio_pci_provider_power_up(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	if (vdev->cxl_ops && vdev->cxl_ops->power_up)
+		vdev->cxl_ops->power_up(vdev);
+}
+
 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)
 {
 	struct pci_dev *pdev = vdev->pdev;
-	bool needs_restore = false, needs_save = false;
+	bool needs_restore = false, needs_save = false, powered_down = false;
 	int ret;
 
 	/* Prevent changing power state for PFs with VFs enabled */
@@ -329,6 +350,11 @@ int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t stat
 		lockdep_assert_held_write(&vdev->memory_lock);
 		if (vdev->sriov_active)
 			return -EBUSY;
+
+		if (pdev->current_state == PCI_D0) {
+			vfio_pci_provider_power_down(vdev);
+			powered_down = true;
+		}
 	}
 
 	if (vdev->needs_pm_restore) {
@@ -366,6 +392,14 @@ int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t stat
 		}
 	}
 
+	/*
+	 * The device stayed in D0: the transition failed, or a quirk keeps it
+	 * there. No later D0 transition will call power_up, so undo power_down
+	 * now.
+	 */
+	if (powered_down && pdev->current_state == PCI_D0)
+		vfio_pci_provider_power_up(vdev);
+
 	return ret;
 }
 
@@ -502,6 +536,8 @@ static int vfio_pci_core_runtime_suspend(struct device *dev)
 	 * (saved locally in 'vdev->pm_save').
 	 */
 	vfio_pci_set_power_state(vdev, PCI_D0);
+	/* The PCI core moves the device to low power once this returns. */
+	vfio_pci_provider_power_down(vdev);
 	up_write(&vdev->memory_lock);
 
 	/*
@@ -526,6 +562,8 @@ static int vfio_pci_core_runtime_resume(struct device *dev)
 	 * low power mode.
 	 */
 	down_write(&vdev->memory_lock);
+	/* The PCI core has restored the config space of the device. */
+	vfio_pci_provider_power_up(vdev);
 	if (vdev->pm_wake_eventfd_ctx) {
 		eventfd_signal(vdev->pm_wake_eventfd_ctx);
 		__vfio_pci_runtime_pm_exit(vdev);
diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
index 23bccfbb7eb7..411adc2ba379 100644
--- a/drivers/vfio/pci/vfio_pci_priv.h
+++ b/drivers/vfio/pci/vfio_pci_priv.h
@@ -78,6 +78,7 @@ void vfio_config_free(struct vfio_pci_core_device *vdev);
 
 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,
 			     pci_power_t state);
+void vfio_pci_provider_power_up(struct vfio_pci_core_device *vdev);
 
 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev);
 int vfio_pci_try_reset_function(struct vfio_pci_core_device *vdev);
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 7dd93ef103cb..ec1660e0d6d2 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -94,6 +94,13 @@ struct vfio_cxl_ops {
 	 * saves it first, the close path keeps the open-time copy.
 	 */
 	int	(*reset)(struct vfio_pci_core_device *vdev);
+	/*
+	 * Optional. Called with memory_lock held for write before the device
+	 * leaves D0, and after it returns to D0 with its config space
+	 * restored. power_down may be called again while already powered down.
+	 */
+	void	(*power_down)(struct vfio_pci_core_device *vdev);
+	void	(*power_up)(struct vfio_pci_core_device *vdev);
 	/* Pinned per bound CXL device so vfio-cxl cannot unload under usage */
 	struct module *owner;
 };
-- 
2.53.0