[PATCH v6 16/31] vfio/pci: Revoke and re-arm one device-specific region

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

The BARs of a vfio PCI device follow the memory enable bit: clearing it
zaps their mmaps and revokes their dma-bufs, and setting it re-arms the
dma-bufs. A device-specific region does not have to follow that bit. The
HDM memory region of a CXL Type-2 device follows the committed HDM
decoder, which changes across reset, bus reset and low power, not with
Memory Space Enable. Other device-specific regions, such as nvgrace
coherent memory or the IGD OpRegion, do not depend on it either.

Record the region index of each dma-buf at export. vfio_pci_dma_buf_move()
now moves only the BAR dma-bufs, and vfio_pci_dma_buf_cleanup() still
drains every dma-buf of the device.

Export vfio_pci_core_revoke_region(), which zaps the user mmaps of one
device-specific region and revokes its dma-bufs, and
vfio_pci_core_rearm_region(), which re-arms them. A provider calls them
under memory_lock when the state its region depends on changes. Also
export __vfio_pci_vmf_insert_pfn(), the pfn insert of
vfio_pci_vmf_insert_pfn() without its memory enable and runtime PM
checks, for the fault handler of such a region. Export
vfio_pci_core_clear_memory_enable() for a provider whose failed
operation left Memory Space off in the device: it zaps the BAR mmaps,
revokes the BAR dma-bufs and clears vfio's record of the bit.

Suggested-by: Alex Williamson <alex@shazbot.org>
Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/vfio_pci_core.c   | 109 +++++++++++++++++++++++++++--
 drivers/vfio/pci/vfio_pci_dmabuf.c |  98 +++++++++++++++++---------
 drivers/vfio/pci/vfio_pci_priv.h   |   7 ++
 include/linux/vfio_pci_core.h      |   8 +++
 4 files changed, 180 insertions(+), 42 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 6676c561465b..9d0c22a4f3c6 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -1982,6 +1982,77 @@ static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev)
 	unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true);
 }
 
+/**
+ * vfio_pci_core_revoke_region() - take a device-specific region out of use
+ * @vdev: vfio PCI device, with memory_lock held for write
+ * @index: vfio region index of a device-specific region
+ *
+ * Zap the user mmaps of the region and revoke its dma-bufs, for a provider
+ * whose region depends on device state that is about to change. The BARs
+ * follow the memory enable bit instead; a device-specific region may not.
+ * A later fault on the region goes back to the region's own fault handler.
+ */
+void vfio_pci_core_revoke_region(struct vfio_pci_core_device *vdev,
+				 unsigned int index)
+{
+	struct vfio_pci_region *region;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	if (WARN_ON(index < VFIO_PCI_NUM_REGIONS ||
+		    index - VFIO_PCI_NUM_REGIONS >= vdev->num_regions))
+		return;
+
+	region = &vdev->region[index - VFIO_PCI_NUM_REGIONS];
+	unmap_mapping_range(vdev->vdev.inode->i_mapping,
+			    VFIO_PCI_INDEX_TO_OFFSET(index), region->size, true);
+	vfio_pci_dma_buf_move_region(vdev, index, true);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_revoke_region);
+
+/**
+ * vfio_pci_core_rearm_region() - put a revoked device-specific region back
+ * @vdev: vfio PCI device, with memory_lock held for write
+ * @index: vfio region index passed to vfio_pci_core_revoke_region()
+ */
+void vfio_pci_core_rearm_region(struct vfio_pci_core_device *vdev,
+				unsigned int index)
+{
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	vfio_pci_dma_buf_move_region(vdev, index, false);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_rearm_region);
+
+/**
+ * vfio_pci_core_clear_memory_enable() - follow a device that lost Memory Space
+ * @vdev: vfio PCI device, with memory_lock held for write
+ *
+ * For a provider whose failed operation left Memory Space disabled in the
+ * device. Zap the BAR mmaps, revoke the BAR dma-bufs and clear vfio's record
+ * of the Memory Space bit, as a user write that clears the bit would. The
+ * user's next Memory Space write enables it again.
+ */
+void vfio_pci_core_clear_memory_enable(struct vfio_pci_core_device *vdev)
+{
+	__le16 *virt_cmd;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	/* Closed, or not open yet: vfio_config_init() reads the device. */
+	if (!vdev->vconfig)
+		return;
+
+	virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND];
+	if (!(*virt_cmd & cpu_to_le16(PCI_COMMAND_MEMORY)))
+		return;
+
+	vfio_pci_zap_bars(vdev);
+	vfio_pci_dma_buf_move(vdev, true);
+	*virt_cmd &= cpu_to_le16(~PCI_COMMAND_MEMORY);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_clear_memory_enable);
+
 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
 {
 	down_write(&vdev->memory_lock);
@@ -2019,16 +2090,26 @@ static unsigned long vma_to_pfn(struct vm_area_struct *vma)
 	return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff;
 }
 
-vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
-				   struct vm_fault *vmf,
-				   unsigned long pfn,
-				   unsigned int order)
+/**
+ * __vfio_pci_vmf_insert_pfn() - insert a pfn without the memory enable and
+ *	runtime PM checks
+ * @vdev: vfio PCI device, with memory_lock held for read
+ * @vmf: the fault being served
+ * @pfn: first pfn to insert
+ * @order: page order of the mapping
+ *
+ * For a device-specific region that does not follow the memory enable bit.
+ * The caller checks the state its region depends on under memory_lock, and
+ * revokes the region with vfio_pci_core_revoke_region() when it changes or
+ * before the device leaves D0.
+ */
+vm_fault_t __vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
+				     struct vm_fault *vmf,
+				     unsigned long pfn,
+				     unsigned int order)
 {
 	lockdep_assert_held_read(&vdev->memory_lock);
 
-	if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev))
-		return VM_FAULT_SIGBUS;
-
 	if (!order)
 		return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
 
@@ -2040,6 +2121,20 @@ vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
 
 	return VM_FAULT_FALLBACK;
 }
+EXPORT_SYMBOL_GPL(__vfio_pci_vmf_insert_pfn);
+
+vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
+				   struct vm_fault *vmf,
+				   unsigned long pfn,
+				   unsigned int order)
+{
+	lockdep_assert_held_read(&vdev->memory_lock);
+
+	if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev))
+		return VM_FAULT_SIGBUS;
+
+	return __vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order);
+}
 EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn);
 
 static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
index 8606c15ab2fe..d0f7ec5487de 100644
--- a/drivers/vfio/pci/vfio_pci_dmabuf.c
+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
@@ -19,6 +19,7 @@ struct vfio_pci_dma_buf {
 	u32 nr_ranges;
 	struct kref kref;
 	struct completion comp;
+	u32 region_index;
 	u8 revoked : 1;
 };
 
@@ -288,6 +289,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	priv->vdev = vdev;
 	priv->nr_ranges = get_dma_buf.nr_ranges;
 	priv->size = length;
+	priv->region_index = get_dma_buf.region_index;
 	ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider,
 					     get_dma_buf.region_index,
 					     priv->phys_vec, dma_ranges,
@@ -347,6 +349,49 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	return ret;
 }
 
+static void vfio_pci_dma_buf_move_one(struct vfio_pci_dma_buf *priv,
+				      bool revoked)
+{
+	if (!get_file_active(&priv->dmabuf->file))
+		return;
+
+	if (priv->revoked != revoked) {
+		dma_resv_lock(priv->dmabuf->resv, NULL);
+		if (revoked)
+			priv->revoked = true;
+		dma_buf_invalidate_mappings(priv->dmabuf);
+		dma_resv_wait_timeout(priv->dmabuf->resv,
+				      DMA_RESV_USAGE_BOOKKEEP, false,
+				      MAX_SCHEDULE_TIMEOUT);
+		dma_resv_unlock(priv->dmabuf->resv);
+		if (revoked) {
+			kref_put(&priv->kref, vfio_pci_dma_buf_done);
+			wait_for_completion(&priv->comp);
+			/*
+			 * Re-arm the registered kref reference and the
+			 * completion so the post-revoke state matches the
+			 * post-creation state.  An un-revoke followed by a
+			 * new mapping needs the kref to be non-zero before
+			 * kref_get(), and vfio_pci_dma_buf_cleanup()
+			 * delegates its drain back through this revoke
+			 * path on a possibly-already-revoked dma-buf.
+			 */
+			kref_init(&priv->kref);
+			reinit_completion(&priv->comp);
+		} else {
+			dma_resv_lock(priv->dmabuf->resv, NULL);
+			priv->revoked = false;
+			dma_resv_unlock(priv->dmabuf->resv);
+		}
+	}
+	fput(priv->dmabuf->file);
+}
+
+/*
+ * Revoke or re-arm the dma-bufs of the BARs, which follow the memory enable
+ * bit. A device-specific region is moved by its provider through
+ * vfio_pci_dma_buf_move_region().
+ */
 void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
 {
 	struct vfio_pci_dma_buf *priv;
@@ -354,41 +399,23 @@ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
 
 	lockdep_assert_held_write(&vdev->memory_lock);
 
-	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
-		if (!get_file_active(&priv->dmabuf->file))
-			continue;
+	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm)
+		if (priv->region_index < VFIO_PCI_NUM_REGIONS)
+			vfio_pci_dma_buf_move_one(priv, revoked);
+}
 
-		if (priv->revoked != revoked) {
-			dma_resv_lock(priv->dmabuf->resv, NULL);
-			if (revoked)
-				priv->revoked = true;
-			dma_buf_invalidate_mappings(priv->dmabuf);
-			dma_resv_wait_timeout(priv->dmabuf->resv,
-					      DMA_RESV_USAGE_BOOKKEEP, false,
-					      MAX_SCHEDULE_TIMEOUT);
-			dma_resv_unlock(priv->dmabuf->resv);
-			if (revoked) {
-				kref_put(&priv->kref, vfio_pci_dma_buf_done);
-				wait_for_completion(&priv->comp);
-				/*
-				 * Re-arm the registered kref reference and the
-				 * completion so the post-revoke state matches the
-				 * post-creation state.  An un-revoke followed by a
-				 * new mapping needs the kref to be non-zero before
-				 * kref_get(), and vfio_pci_dma_buf_cleanup()
-				 * delegates its drain back through this revoke
-				 * path on a possibly-already-revoked dma-buf.
-				 */
-				kref_init(&priv->kref);
-				reinit_completion(&priv->comp);
-			} else {
-				dma_resv_lock(priv->dmabuf->resv, NULL);
-				priv->revoked = false;
-				dma_resv_unlock(priv->dmabuf->resv);
-			}
-		}
-		fput(priv->dmabuf->file);
-	}
+/* Revoke or re-arm the dma-bufs exported from one device-specific region. */
+void vfio_pci_dma_buf_move_region(struct vfio_pci_core_device *vdev,
+				  unsigned int index, bool revoked)
+{
+	struct vfio_pci_dma_buf *priv;
+	struct vfio_pci_dma_buf *tmp;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm)
+		if (priv->region_index == index)
+			vfio_pci_dma_buf_move_one(priv, revoked);
 }
 
 void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
@@ -405,7 +432,8 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
 	 * ready, so cleanup itself does not need to participate in kref
 	 * bookkeeping.
 	 */
-	vfio_pci_dma_buf_move(vdev, true);
+	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm)
+		vfio_pci_dma_buf_move_one(priv, true);
 
 	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
 		if (!get_file_active(&priv->dmabuf->file))
diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
index dbede6e24fa9..23bccfbb7eb7 100644
--- a/drivers/vfio/pci/vfio_pci_priv.h
+++ b/drivers/vfio/pci/vfio_pci_priv.h
@@ -141,6 +141,8 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 				  size_t argsz);
 void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev);
 void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked);
+void vfio_pci_dma_buf_move_region(struct vfio_pci_core_device *vdev,
+				  unsigned int index, bool revoked);
 #else
 static inline int
 vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
@@ -156,6 +158,11 @@ static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev,
 					 bool revoked)
 {
 }
+static inline void
+vfio_pci_dma_buf_move_region(struct vfio_pci_core_device *vdev,
+			     unsigned int index, bool revoked)
+{
+}
 #endif
 
 #endif
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index cb9ba92b696a..7dd93ef103cb 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -227,6 +227,11 @@ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
 				      unsigned int type, unsigned int subtype,
 				      const struct vfio_pci_regops *ops,
 				      size_t size, u32 flags, void *data);
+void vfio_pci_core_revoke_region(struct vfio_pci_core_device *vdev,
+				 unsigned int index);
+void vfio_pci_core_rearm_region(struct vfio_pci_core_device *vdev,
+				unsigned int index);
+void vfio_pci_core_clear_memory_enable(struct vfio_pci_core_device *vdev);
 void vfio_pci_core_close_device(struct vfio_device *core_vdev);
 int vfio_pci_core_init_dev(struct vfio_device *core_vdev);
 void vfio_pci_core_release_dev(struct vfio_device *core_vdev);
@@ -250,6 +255,9 @@ ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
 		size_t count, loff_t *ppos);
 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
 		size_t count, loff_t *ppos);
+vm_fault_t __vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
+				     struct vm_fault *vmf, unsigned long pfn,
+				     unsigned int order);
 vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
 				   struct vm_fault *vmf, unsigned long pfn,
 				   unsigned int order);
-- 
2.53.0