[RFC PATCH v2 10/13] vfio/pci: Detach guest_memfd access before teardown

["Aneesh Kumar K.V (Arm)" <aneesh.kumar@kernel.org>]

Implement the guest_memfd detach callback and forward it through the VFIO
filesystem provider. Require the backing driver to supply the callback.

Mark the provider detached under the vDEVICE MMIO mutex and VFIO memory
lock, draining in-flight access and invalidation callbacks. Reject further
host accesses, PFN lookups, bindings and private conversions while allowing
shared cleanup.

Cc: Alex Williamson <alex@shazbot.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: kvm@vger.kernel.org
Cc: patches@lists.linux.dev
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 drivers/vfio/pci/vfio_pci_gmem.c        | 25 +++++++++++++++++++++++++
 drivers/vfio/pci/vfio_pci_gmem.h        |  6 ++++++
 drivers/vfio/pci/vfio_pci_gmem_access.c |  7 +++++--
 drivers/vfio/vfio_main.c                | 10 ++++++++--
 4 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_gmem.c b/drivers/vfio/pci/vfio_pci_gmem.c
index 8cb5f0cb3bb8..1ab65e43eb1d 100644
--- a/drivers/vfio/pci/vfio_pci_gmem.c
+++ b/drivers/vfio/pci/vfio_pci_gmem.c
@@ -176,6 +176,8 @@ static int vfio_gmem_bind(void *data, u64 offset, u64 size,
 	int ret;
 
 	guard(iommufd_vdevice_mmio)(ctx->ivdev);
+	if (ctx->callbacks_detached)
+		return -ENODEV;
 	ret = vfio_gmem_resolve_bar_range(ctx, offset, size, &bar, &off, &pa);
 	if (ret)
 		return ret;
@@ -219,6 +221,8 @@ static int vfio_gmem_get_pfn(void *data, u64 offset, unsigned long *pfn)
 	int ret;
 
 	guard(rwsem_read)(&vdev->memory_lock);
+	if (ctx->callbacks_detached)
+		return -ENODEV;
 	if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev))
 		return -EAGAIN;
 
@@ -248,6 +252,10 @@ static int vfio_gmem_prepare_conversion(void *context,
 	struct vfio_pci_gmem *ctx = context;
 
 	iommufd_vdevice_mmio_lock(ctx->ivdev);
+	if (req && ctx->callbacks_detached) {
+		iommufd_vdevice_mmio_unlock(ctx->ivdev);
+		return -ENODEV;
+	}
 	if (req && req->vdev_id != ctx->ivdev->virt_id) {
 		iommufd_vdevice_mmio_unlock(ctx->ivdev);
 		return -EPERM;
@@ -287,6 +295,22 @@ static void vfio_gmem_finish_conversion(void *context,
 	iommufd_vdevice_mmio_unlock(ctx->ivdev);
 }
 
+/**
+ * vfio_gmem_detach() - stop invalidation callbacks before core teardown
+ * @data: attached provider context
+ *
+ * Mark the provider detached under the VFIO memory lock. VFIO userspace
+ * mappings remain excluded until release clears the provider association.
+ */
+static void vfio_gmem_detach(void *data)
+{
+	struct vfio_pci_gmem *ctx = data;
+
+	guard(iommufd_vdevice_mmio)(ctx->ivdev);
+	scoped_guard(rwsem_write, &ctx->vdev->memory_lock)
+		ctx->callbacks_detached = true;
+}
+
 /**
  * vfio_gmem_release() - release a provider after successful core cleanup
  * @data: provider context to release
@@ -318,6 +342,7 @@ const struct guest_memfd_provider_operations vfio_pci_gmem_provider_ops = {
 	.get_pfn = vfio_gmem_get_pfn,
 	.prepare_conversion = vfio_gmem_prepare_conversion,
 	.finish_conversion = vfio_gmem_finish_conversion,
+	.detach = vfio_gmem_detach,
 	.release = vfio_gmem_release,
 };
 EXPORT_SYMBOL_GPL(vfio_pci_gmem_provider_ops);
diff --git a/drivers/vfio/pci/vfio_pci_gmem.h b/drivers/vfio/pci/vfio_pci_gmem.h
index b4ca831196fd..76a2fbe1472a 100644
--- a/drivers/vfio/pci/vfio_pci_gmem.h
+++ b/drivers/vfio/pci/vfio_pci_gmem.h
@@ -13,6 +13,12 @@ struct vfio_pci_gmem {
 	struct iommufd_vdevice *ivdev;
 	struct file *resource;
 	unsigned long owner;
+	/*
+	 * Disables callbacks and new bind/private requests. Set once with
+	 * both the vDEVICE MMIO and VFIO memory locks held; readers hold
+	 * either lock. Shared cleanup remains allowed after detachment.
+	 */
+	bool callbacks_detached;
 };
 
 #endif /* VFIO_PCI_GMEM_H */
diff --git a/drivers/vfio/pci/vfio_pci_gmem_access.c b/drivers/vfio/pci/vfio_pci_gmem_access.c
index 40219f05345b..b723a0877ff3 100644
--- a/drivers/vfio/pci/vfio_pci_gmem_access.c
+++ b/drivers/vfio/pci/vfio_pci_gmem_access.c
@@ -14,7 +14,7 @@
  * Called by vfio_pci_zap_bars() after it removes host BAR mappings. Notify
  * guest_memfd synchronously so KVM removes the corresponding shared guest
  * mappings. This helper does not remove host mappings or change attributes.
- * Skip notification if no provider is attached.
+ * Skip notification if no provider is attached or its callbacks are detached.
  *
  */
 void vfio_pci_gmem_invalidate(struct vfio_pci_core_device *vdev)
@@ -22,7 +22,7 @@ void vfio_pci_gmem_invalidate(struct vfio_pci_core_device *vdev)
 	struct vfio_pci_gmem *ctx = vdev->gmem;
 
 	lockdep_assert_held_write(&vdev->memory_lock);
-	if (ctx)
+	if (ctx && !ctx->callbacks_detached)
 		ctx->gmem->invalidate(ctx->gmem);
 }
 
@@ -35,6 +35,7 @@ void vfio_pci_gmem_invalidate(struct vfio_pci_core_device *vdev)
  * Consult completed guest_memfd attributes for the requested interval.
  * Private mappings in another interval do not block this access. Try the
  * inode lock instead of waiting in the reverse order from conversion.
+ * Detachment excludes access during teardown.
  *
  * Return: true when the requested interval is accessible.
  */
@@ -51,6 +52,8 @@ bool vfio_pci_gmem_access_allowed(struct vfio_pci_core_device *vdev,
 	if (!ctx->ivdev)
 		return false;
 
+	if (ctx->callbacks_detached)
+		return false;
 	return ctx->gmem->host_accessible(ctx->gmem,
 					  offset, size);
 }
diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c
index 2051e14ac616..d6ea08c08e03 100644
--- a/drivers/vfio/vfio_main.c
+++ b/drivers/vfio/vfio_main.c
@@ -274,9 +274,9 @@ static void *vfio_gmem_provider_attach(struct file *file,
 	if (!ops || !ops->attach || !ops->release)
 		return ERR_PTR(-EOPNOTSUPP);
 
-	/* VFIO requires PFN lookup, binding and conversion. */
+	/* VFIO requires PFN lookup, binding, conversion and detachment. */
 	if (!ops->get_pfn || !ops->bind || !ops->prepare_conversion ||
-	    !ops->finish_conversion)
+	    !ops->finish_conversion || !ops->detach)
 		return ERR_PTR(-EOPNOTSUPP);
 
 	/* VFIO provides PFNs, not folios. */
@@ -340,7 +340,12 @@ static void vfio_gmem_provider_finish_conversion(
 	provider->ops->finish_conversion(provider->data, req);
 }
 
+static void vfio_gmem_provider_detach(void *data)
+{
+	struct vfio_gmem_provider *provider = data;
 
+	provider->ops->detach(provider->data);
+}
 
 static const struct guest_memfd_provider_operations vfio_gmem_provider_ops = {
 	.attach = vfio_gmem_provider_attach,
@@ -349,6 +354,7 @@ static const struct guest_memfd_provider_operations vfio_gmem_provider_ops = {
 	.get_pfn = vfio_gmem_provider_get_pfn,
 	.prepare_conversion = vfio_gmem_provider_prepare_conversion,
 	.finish_conversion = vfio_gmem_provider_finish_conversion,
+	.detach = vfio_gmem_provider_detach,
 };
 
 static const struct super_operations vfio_fs_super_ops = {
-- 
2.43.0