[RFC PATCH 12/21] vfio/pci: Skip clearing bus master on live update restored device

[Vipin Sharma <vipinsh@google.com>]

Store the restored serialized data in struct vfio_pci_core_device{}.
Skip clearing the bus master bit on the restored VFIO devices when
opened for the first time after live update reboot.

In the live update finish, clean up the pointer to the restored KHO
data. Warn if the device open count is 0, which indicates that userspace
might not have opened and restored the device.

Signed-off-by: Vipin Sharma <vipinsh@google.com>
---
 drivers/vfio/pci/vfio_pci_core.c       |  8 ++++++--
 drivers/vfio/pci/vfio_pci_liveupdate.c | 19 ++++++++++++++-----
 include/linux/vfio_pci_core.h          |  1 +
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 0894673a9262..29236b015242 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -475,8 +475,12 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
 			return ret;
 	}
 
-	/* Don't allow our initial saved state to include busmaster */
-	pci_clear_master(pdev);
+	/*
+	 * Don't allow our initial saved state to include busmaster. However, if
+	 * device is participating in liveupdate then don't change this bit.
+	 */
+	if (!vdev->liveupdate_restore)
+		pci_clear_master(pdev);
 
 	ret = pci_enable_device(pdev);
 	if (ret)
diff --git a/drivers/vfio/pci/vfio_pci_liveupdate.c b/drivers/vfio/pci/vfio_pci_liveupdate.c
index 789b52665e35..6cc94d9a0386 100644
--- a/drivers/vfio/pci/vfio_pci_liveupdate.c
+++ b/drivers/vfio/pci/vfio_pci_liveupdate.c
@@ -96,12 +96,10 @@ static void vfio_pci_liveupdate_finish(struct liveupdate_file_handler *handler,
 	struct vfio_device *device;
 	struct folio *folio;
 
-	if (reclaimed) {
+	if (reclaimed)
 		folio = virt_to_folio(phys_to_virt(data));
-		goto out_folio_put;
-	} else {
+	else
 		folio = kho_restore_folio(data);
-	}
 
 	if (!folio)
 		return;
@@ -113,7 +111,14 @@ static void vfio_pci_liveupdate_finish(struct liveupdate_file_handler *handler,
 		goto out_folio_put;
 
 	vdev = container_of(device, struct vfio_pci_core_device, vdev);
-	pci_try_reset_function(vdev->pdev);
+	if (reclaimed) {
+		guard(mutex)(&device->dev_set->lock);
+		if (!vfio_device_cdev_opened(device))
+			pci_err(vdev->pdev, "Open count is 0, userspace might not have restored the device.\n");
+		vdev->liveupdate_restore = NULL;
+	} else {
+		pci_try_reset_function(vdev->pdev);
+	}
 	put_device(&device->device);
 
 out_folio_put:
@@ -124,6 +129,7 @@ static int vfio_pci_liveupdate_retrieve(struct liveupdate_file_handler *handler,
 					u64 data, struct file **file)
 {
 	struct vfio_pci_core_device_ser *ser;
+	struct vfio_pci_core_device *vdev;
 	struct vfio_device_file *df;
 	struct vfio_device *device;
 	struct folio *folio;
@@ -167,6 +173,9 @@ static int vfio_pci_liveupdate_retrieve(struct liveupdate_file_handler *handler,
 	 */
 	filep->f_mapping = device->inode->i_mapping;
 	*file = filep;
+	vdev = container_of(device, struct vfio_pci_core_device, vdev);
+	guard(mutex)(&device->dev_set->lock);
+	vdev->liveupdate_restore = ser;
 
 	return 0;
 
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index f541044e42a2..8c3fe2db7eb3 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -94,6 +94,7 @@ struct vfio_pci_core_device {
 	struct vfio_pci_core_device	*sriov_pf_core_dev;
 	struct notifier_block	nb;
 	struct rw_semaphore	memory_lock;
+	void			*liveupdate_restore;
 };
 
 /* Will be exported for vfio pci drivers usage */
-- 
2.51.0.858.gf9c4a03a3a-goog