[RFC PATCH 03/14] KVM: guest_memfd: Allow other subsystems to map guest_memfd files

[Fred Griffoul <griffoul@gmail.com>]

From: Fred Griffoul <fgriffo@amazon.co.uk>

A device assigned to a guest must map the guest's shared memory and
lose a page whenever guest_memfd takes it away. iommufd cannot do that:
it pins what it maps, which guest_memfd does not allow, and a device
cannot fault.

Add an export interface. A subsystem attaches to a guest_memfd file,
looks up the frame behind each shared page without taking a reference,
and gets an update() callback when what a lookup returns for a range
may have changed. The old frames stay valid until update() returns, so
a device that never faults can switch each mapping in one step.

update() is called on conversions and memory failure; punch holes, PFN
mode and pages that get memory follow. The functions are exported for
symbol_get(), since KVM may be a module while the consumer is built in.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: Fred Griffoul <fgriffo@amazon.co.uk>
---
 arch/x86/kvm/Makefile       |   3 +-
 include/linux/guest_memfd.h |  55 +++++++++++++
 virt/kvm/guest_memfd.c      | 149 ++++++++++++++++++++++++++++++++++++
 3 files changed, 206 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 5fd01bd19d1c..25fb3d6e2032 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -62,7 +62,8 @@ exports_grep_trailer := --include='*.[ch]' -nrw $(srctree)/virt/kvm $(srctree)/a
 				-e kvm_write_track_add_gfn \
 				-e kvm_write_track_remove_gfn \
 				-e kvm_file_to_kvm_fn \
-				-e kvm_gmem_revoke
+				-e kvm_gmem_revoke \
+				-e kvm_gmem_export
 
 # Force grep to emit a goofy group separator that can in turn be replaced with
 # the above newline macro (newlines in Make are a nightmare).  Note, grep only
diff --git a/include/linux/guest_memfd.h b/include/linux/guest_memfd.h
index 9d8764b93af2..fdf7f8404d6c 100644
--- a/include/linux/guest_memfd.h
+++ b/include/linux/guest_memfd.h
@@ -3,6 +3,7 @@
 #define _LINUX_GUEST_MEMFD_H
 
 #include <linux/bits.h>
+#include <linux/list.h>
 #include <linux/types.h>
 
 struct file;
@@ -82,4 +83,58 @@ static inline int kvm_gmem_revoke(struct inode *inode, pgoff_t start, size_t nr_
 }
 #endif
 
+/*
+ * Mapping guest_memfd files in other subsystems.
+ *
+ * A subsystem that maps guest_memfd pages, such as iommufd, can neither pin
+ * nor fault them.  It attaches to the file, looks up the frame behind each
+ * shared page without taking a reference, and guest_memfd calls its update()
+ * whenever what a lookup returns for a range may have changed: conversions and
+ * memory failure.  update() makes the consumer's mappings of the range match
+ * what a lookup returns now: a frame, or a hole for a private page or a page
+ * without a frame.  It may be called for pages that did not change, and for
+ * ranges that do not follow the blocks that lookups report: the consumer may
+ * then remap whole blocks, also outside the range, e.g. to merge them into
+ * larger mappings.  A consumer that can fault may instead only remove its
+ * mappings of the range.
+ *
+ * Until update() returns, the frames it replaces stay valid, so a consumer
+ * may switch each mapping to its new target in one step; once it returns,
+ * the consumer no longer accesses them.  update() cannot fail: a consumer
+ * that needs memory to apply one, e.g. to split a large mapping, reserves it
+ * beforehand.
+ *
+ * guest_memfd calls update() with its invalidate lock held, exclusive when
+ * pages convert and shared on memory failure, possibly with a folio of the
+ * file locked; so update() may only call kvm_gmem_export_lookup_pfn().  Under
+ * the shared lock, updates may run at the same time, also for overlapping
+ * ranges.  The consumer holds a lock across a lookup and the mapping it
+ * installs, which update() also takes.  Lock order:
+ *
+ *   guest_memfd invalidate lock
+ *     consumer lock
+ *
+ * A consumer attaches and detaches outside its lock.  KVM may be a module
+ * while the consumer is built in: consumers reach these with symbol_get().
+ */
+struct guest_memfd_export;
+
+struct guest_memfd_export_ops {
+	void (*update)(struct guest_memfd_export *e, pgoff_t start,
+		       pgoff_t nr_pages);
+};
+
+struct guest_memfd_export {
+	const struct guest_memfd_export_ops *ops;
+
+	/* Private to guest_memfd. */
+	struct list_head node;
+	struct file *file;
+};
+
+int kvm_gmem_export_attach(struct file *file, struct guest_memfd_export *e);
+void kvm_gmem_export_detach(struct guest_memfd_export *e);
+int kvm_gmem_export_lookup_pfn(struct guest_memfd_export *e, pgoff_t index,
+			       unsigned long *pfn, int *max_order, u32 *attrs);
+
 #endif /* _LINUX_GUEST_MEMFD_H */
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 1554a17906da..81031483a1f1 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -38,6 +38,8 @@ struct gmem_inode {
 
 	void *provider;
 	const struct guest_memfd_provider_operations *provider_ops;
+	/* Subsystems that map the file, see include/linux/guest_memfd.h. */
+	struct list_head exports;
 
 	u64 flags;
 	/*
@@ -420,6 +422,21 @@ static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start,
 		__kvm_gmem_invalidate_end(f, start, end);
 }
 
+/*
+ * What a lookup returns for [start, end) may have changed: exclusive lock when
+ * pages converted, shared on memory failure.
+ */
+static void kvm_gmem_update_exports(struct inode *inode, pgoff_t start,
+				    pgoff_t end)
+{
+	struct guest_memfd_export *e;
+
+	lockdep_assert_held(&inode->i_mapping->invalidate_lock);
+
+	list_for_each_entry(e, &GMEM_I(inode)->exports, node)
+		e->ops->update(e, start, end - start);
+}
+
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
 static void kvm_gmem_pfn_reclaim(struct gmem_inode *gi, pgoff_t start, pgoff_t end)
 {
@@ -1130,6 +1147,9 @@ static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start,
 
 	mas_store_prealloc(&mas, xa_mk_value(attrs));
 
+	/* After the store, before a vCPU can fault a private page in. */
+	kvm_gmem_update_exports(inode, start, end);
+
 	kvm_gmem_invalidate_end(inode, start, end);
 out:
 	filemap_invalidate_unlock(mapping);
@@ -1204,6 +1224,131 @@ static struct file_operations kvm_gmem_fops = {
 	.unlocked_ioctl	= kvm_gmem_ioctl,
 };
 
+/**
+ * kvm_gmem_export_attach - Map a guest_memfd file in another subsystem.
+ * @file: The guest_memfd file.
+ * @e:    The export, with @e->ops set.
+ *
+ * guest_memfd calls @e->ops->update() from now on, and holds a reference to
+ * @file until kvm_gmem_export_detach().  May sleep.
+ *
+ * Return: 0, -ENODEV if @file is not a guest_memfd file, or -EOPNOTSUPP in
+ * PFN mode.
+ */
+int kvm_gmem_export_attach(struct file *file, struct guest_memfd_export *e)
+{
+	struct inode *inode = file_inode(file);
+
+	if (file->f_op != &kvm_gmem_fops)
+		return -ENODEV;
+	if (kvm_gmem_is_pfn_mode(GMEM_I(inode)))
+		return -EOPNOTSUPP;
+
+	e->file = get_file(file);
+	filemap_invalidate_lock(inode->i_mapping);
+	list_add_tail(&e->node, &GMEM_I(inode)->exports);
+	filemap_invalidate_unlock(inode->i_mapping);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_gmem_export_attach);
+
+/**
+ * kvm_gmem_export_detach - Stop mapping a guest_memfd file.
+ * @e: The export.
+ *
+ * Waits for an update() in progress.  May sleep.
+ */
+void kvm_gmem_export_detach(struct guest_memfd_export *e)
+{
+	struct inode *inode = file_inode(e->file);
+
+	filemap_invalidate_lock(inode->i_mapping);
+	list_del(&e->node);
+	filemap_invalidate_unlock(inode->i_mapping);
+	fput(e->file);
+}
+EXPORT_SYMBOL_GPL(kvm_gmem_export_detach);
+
+/* Are [index, index + nr) all shared?  Without the invalidate lock. */
+static bool kvm_gmem_range_is_shared(struct inode *inode, pgoff_t index,
+				     unsigned long nr)
+{
+	unsigned long first = index;
+	void *entry;
+
+	mt_for_each(&GMEM_I(inode)->attributes, entry, first, index + nr - 1) {
+		if (kvm_gmem_get_attributes(inode, entry) &
+		    KVM_MEMORY_ATTRIBUTE_PRIVATE)
+			return false;
+	}
+	return true;
+}
+
+/* What kvm_gmem_export_lookup_pfn() returns for a page inside the file. */
+static int __kvm_gmem_export_lookup(struct inode *inode, pgoff_t index,
+				    unsigned long *pfn, int *max_order,
+				    u32 *attrs)
+{
+	struct folio *folio;
+
+	if (!kvm_gmem_range_is_shared(inode, index, 1))
+		return -ENOENT;
+
+	/* A folio that was never zeroed is a hole. */
+	folio = filemap_get_folio(inode->i_mapping, index);
+	if (IS_ERR(folio))
+		return -ENOENT;
+	if (folio_test_hwpoison(folio) || !folio_test_uptodate(folio)) {
+		folio_put(folio);
+		return -ENOENT;
+	}
+	*pfn = folio_file_pfn(folio, index);
+	*max_order = 0;
+	*attrs = 0;
+	folio_put(folio);
+
+	/* Nor may the block hold a private page. */
+	while (*max_order &&
+	       !kvm_gmem_range_is_shared(inode,
+					 round_down(index, 1UL << *max_order),
+					 1UL << *max_order))
+		(*max_order)--;
+	return 0;
+}
+
+/**
+ * kvm_gmem_export_lookup_pfn - The frame behind a shared page.
+ * @e:         The export.
+ * @index:     The page index in the file.
+ * @pfn:       The frame.
+ * @max_order: In: the largest order the caller can map.  Out: the order of
+ *             the naturally aligned block around @index that is contiguous,
+ *             shared and of the same @attrs; 0 for -ENOENT.
+ * @attrs:     The GUEST_MEMFD_PFN_* attributes of the block.
+ *
+ * Takes no reference and allocates nothing, and may be called from the
+ * export's update(): the answer holds until the update() that changes it.
+ *
+ * Context: May sleep.
+ * Return: 0, -ENOENT for a private page or a page without a frame, -EINVAL
+ * beyond the end of the file, or another error from the provider.
+ */
+int kvm_gmem_export_lookup_pfn(struct guest_memfd_export *e, pgoff_t index,
+			       unsigned long *pfn, int *max_order, u32 *attrs)
+{
+	struct inode *inode = file_inode(e->file);
+	int r;
+
+	if (index >= i_size_read(inode) >> PAGE_SHIFT)
+		return -EINVAL;
+
+	r = __kvm_gmem_export_lookup(inode, index, pfn, max_order, attrs);
+	if (r == -ENOENT)
+		*max_order = 0;
+	return r;
+}
+EXPORT_SYMBOL_GPL(kvm_gmem_export_lookup_pfn);
+
 static int kvm_gmem_migrate_folio(struct address_space *mapping,
 				  struct folio *dst, struct folio *src,
 				  enum migrate_mode mode)
@@ -1241,6 +1386,9 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol
 	filter = kvm_gmem_get_all_gfns_filter(inode);
 	kvm_gmem_invalidate_start(inode, start, end, filter);
 
+	/* Lookups see the poisoned folio as a hole: take it from the exports. */
+	kvm_gmem_update_exports(inode, start, end);
+
 	/*
 	 * Do not truncate the range, what action is taken in response to the
 	 * error is userspace's decision (assuming the architecture supports
@@ -1867,6 +2015,7 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb)
 	gi->provider = NULL;
 	gi->provider_ops = NULL;
 	INIT_LIST_HEAD(&gi->gmem_file_list);
+	INIT_LIST_HEAD(&gi->exports);
 	return &gi->vfs_inode;
 }
 
-- 
2.47.3