[RFC PATCH 13/21] vfio/pci: Preserve VFIO PCI config space through live update

[Vipin Sharma <vipinsh@google.com>]

Save and restore vconfig, pci_config_map, and rbar members of the struct
vfio_pci_core_device{} during live update. Use the max size of PCI
config space i.e. 4096 bytes for storing vconfig and pci_config_map
irrespective of the exact size. Store the current config size which is
present in the struct pci_dev{} also, to know how much actual data is
present in the vconfig and the pci_config_map.

vconfig represents virtual PCI config used by VFIO to virtualize certain
bits of the config space in the PCI device. This should be preserved as
those virtualized bits cannot be retrieved from reading hardware.

pci_config_map is used to identify starting point of a capability. This
is not strictly needed to be preserved and can be recreated after kexec
but saving it in kHO reduces the code change.  Currently, pci_config_map
is populated in the same code where vconfig gets initialized. If
pci_config_map is not saved then a separate flow need to be added for
just populating pci_config_map.

rbar is used to restore BARs after a reset. This value needs to be
preserved as reset will lose this information.

Signed-off-by: Vipin Sharma <vipinsh@google.com>
---
 drivers/vfio/pci/vfio_pci_config.c     | 17 ++++++++++++
 drivers/vfio/pci/vfio_pci_liveupdate.c | 38 ++++++++++++++++++++++++++
 drivers/vfio/pci/vfio_pci_priv.h       |  5 ++++
 3 files changed, 60 insertions(+)

diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index 8f02f236b5b4..36a71fc3d526 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -1756,6 +1756,23 @@ int vfio_config_init(struct vfio_pci_core_device *vdev)
 	vdev->pci_config_map = map;
 	vdev->vconfig = vconfig;
 
+	if (vdev->liveupdate_restore) {
+		ret = vfio_pci_liveupdate_restore_config(vdev);
+		if (ret)
+			goto out;
+		/*
+		 * Liveupdate might have started after userspace writes to BARs
+		 * but before VFIO sanitizes them which happens when BARs are
+		 * read next time.
+		 *
+		 * Assume BARs are dirty so that VFIO will sanitize them
+		 * unconditionally next time and avoid giving userspace wrong
+		 * value.
+		 */
+		vdev->bardirty = true;
+		return 0;
+	}
+
 	memset(map, PCI_CAP_ID_BASIC, PCI_STD_HEADER_SIZEOF);
 	memset(map + PCI_STD_HEADER_SIZEOF, PCI_CAP_ID_INVALID,
 	       pdev->cfg_size - PCI_STD_HEADER_SIZEOF);
diff --git a/drivers/vfio/pci/vfio_pci_liveupdate.c b/drivers/vfio/pci/vfio_pci_liveupdate.c
index 6cc94d9a0386..824dba2750fe 100644
--- a/drivers/vfio/pci/vfio_pci_liveupdate.c
+++ b/drivers/vfio/pci/vfio_pci_liveupdate.c
@@ -18,12 +18,43 @@
 
 struct vfio_pci_core_device_ser {
 	u16 bdf;
+	u32 cfg_size;
+	u8 pci_config_map[PCI_CFG_SPACE_EXP_SIZE];
+	u8 vconfig[PCI_CFG_SPACE_EXP_SIZE];
+	u32 rbar[7];
 } __packed;
 
+static int vfio_pci_liveupdate_deserialize_config(struct vfio_pci_core_device *vdev,
+						  struct vfio_pci_core_device_ser *ser)
+{
+	struct pci_dev *pdev = vdev->pdev;
+
+	if (WARN_ON_ONCE(pdev->cfg_size != ser->cfg_size)) {
+		dev_err(&pdev->dev, "Config size in serialized (%d) not matching the one pci_dev (%d)",
+			ser->cfg_size, pdev->cfg_size);
+		return -EINVAL;
+	}
+
+	memcpy(vdev->pci_config_map, ser->pci_config_map, ser->cfg_size);
+	memcpy(vdev->vconfig, ser->vconfig, ser->cfg_size);
+	memcpy(vdev->rbar, ser->rbar, sizeof(vdev->rbar));
+	return 0;
+}
+
+static void vfio_pci_liveupdate_serialize_config(struct vfio_pci_core_device *vdev,
+						 struct vfio_pci_core_device_ser *ser)
+{
+	ser->cfg_size = vdev->pdev->cfg_size;
+	memcpy(ser->pci_config_map, vdev->pci_config_map, ser->cfg_size);
+	memcpy(ser->vconfig, vdev->vconfig, ser->cfg_size);
+	memcpy(ser->rbar, vdev->rbar, sizeof(vdev->rbar));
+}
+
 static int vfio_pci_lu_serialize(struct vfio_pci_core_device *vdev,
 				 struct vfio_pci_core_device_ser *ser)
 {
 	ser->bdf = pci_dev_id(vdev->pdev);
+	vfio_pci_liveupdate_serialize_config(vdev, ser);
 	return 0;
 }
 
@@ -221,3 +252,10 @@ void __init vfio_pci_liveupdate_init(void)
 	if (err)
 		pr_err("VFIO PCI liveupdate file handler register failed, error %d.\n", err);
 }
+
+int vfio_pci_liveupdate_restore_config(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_pci_core_device_ser *ser = vdev->liveupdate_restore;
+
+	return vfio_pci_liveupdate_deserialize_config(vdev, ser);
+}
diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
index 7779fd744ff5..0d5aca6c2471 100644
--- a/drivers/vfio/pci/vfio_pci_priv.h
+++ b/drivers/vfio/pci/vfio_pci_priv.h
@@ -109,8 +109,13 @@ static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
 
 #ifdef CONFIG_LIVEUPDATE
 void vfio_pci_liveupdate_init(void);
+int vfio_pci_liveupdate_restore_config(struct vfio_pci_core_device *vdev);
 #else
 static inline void vfio_pci_liveupdate_init(void) { }
+int vfio_pci_liveupdate_restore_config(struct vfio_pci_core_device *vdev)
+{
+	return -EINVAL;
+}
 #endif /* CONFIG_LIVEUPDATE */
 
 #endif
-- 
2.51.0.858.gf9c4a03a3a-goog