[RFC PATCH v2 01/11] KVM: guest_memfd: Introduce guest memory ops and route native gmem through them

[Fred Griffoul <griffoul@gmail.com>]

From: David Woodhouse <dwmw@amazon.co.uk>

Guest_memfd today has exactly one backing implementation, hard-wired
into virt/kvm/guest_memfd.c.  To let other subsystems supply guest
memory from an alternate source -- for example a physically contiguous
range without struct page, or memory shared with a peer device --
lift the operations into a small vtable and route the built-in gmem
through it.

Introduce struct kvm_gmem_ops in <linux/kvm_host.h>: bind / unbind /
get_pfn / populate / invalidate for the memory-side operations, plus
release / mmap / fallocate for the file-level ones.  Every file that
can back guest memory is created with one shared struct
file_operations, kvm_gmem_fops, and file->private_data points at a
struct kvm_gmem_backing whose ->ops field is the per-instance vtable.
This mirrors the dma-buf / socket / iommufd pattern: a canonical fops
identifies the file (is_kvm_gmem_file() is a plain fops-equality
check), and per-instance behaviour is reached via private_data.

Every callback on kvm_gmem_fops -- release, mmap, fallocate, and the
memory-side dispatchers already present -- reaches through the ops
table, so foreign implementations get a private hook for each and
KVM code never has to distinguish "native vs foreign".  Native gmem
becomes one implementation of the ops (kvm_gmem_native_ops), with the
current release/mmap/fallocate bodies moved into
kvm_gmem_native_release/_mmap/_fallocate.  struct gmem_file gains a
struct kvm_gmem_backing as its first member so file->private_data can
be reinterpreted uniformly across implementations.

.populate is deliberately left NULL for the built-in path;
kvm_gmem_populate() keeps an inline fast path because the current
folio-locked + attribute-checked shape of post_populate() does not fit
the ops signature cleanly.  Wiring that flow through ops is a separate
cleanup.  .invalidate is likewise NULL for the built-in path, because
its invalidations run through inode->i_mapping (truncate, hwpoison,
free_folio) rather than an out-of-band notification.

kvm_gmem_unbind() gains a struct kvm * parameter so the dispatcher can
pass it to ops->unbind() without keeping a duplicate copy anywhere.
The three existing callers in kvm_main.c already have @kvm in scope.

Based on a proof-of-concept by Connor Williamson <connordw@amazon.com>.

Signed-off-by: David Woodhouse (Kiro) <dwmw@amazon.co.uk>
---
 arch/x86/kvm/Makefile    |   5 +-
 include/linux/kvm_host.h |  82 ++++++++++
 virt/kvm/guest_memfd.c   | 339 +++++++++++++++++++++++++++++++--------
 virt/kvm/kvm_main.c      |   8 +-
 virt/kvm/kvm_mm.h        |   4 +-
 5 files changed, 365 insertions(+), 73 deletions(-)

diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 77337c37324b..36a319f382b7 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -63,7 +63,10 @@ exports_grep_trailer := --include='*.[ch]' -nrw $(srctree)/virt/kvm $(srctree)/a
 				-e kvm_write_track_remove_gfn \
 				-e kvm_get_kvm \
 				-e kvm_get_kvm_safe \
-				-e kvm_put_kvm
+				-e kvm_put_kvm \
+				-e kvm_gmem_invalidate_range \
+				-e kvm_gmem_fops \
+				-e file_is_kvm
 
 # Force grep to emit a goofy group separator that can in turn be replaced with
 # the above newline macro (newlines in Make are a nightmare).  Note, grep only
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index ab8cfaec82d3..b33d2f475042 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -621,6 +621,88 @@ static inline bool kvm_slot_has_gmem(const struct kvm_memory_slot *slot)
 	return slot && (slot->flags & KVM_MEM_GUEST_MEMFD);
 }
 
+#ifdef CONFIG_KVM_GUEST_MEMFD
+/*
+ * struct kvm_gmem_ops - vtable a guest_memfd backing implementation exposes.
+ *
+ * Every file that can back KVM guest memory -- KVM's own guest_memfd (see
+ * kvm_gmem_create()) and any other subsystem opting in -- is created with
+ * the shared kvm_gmem_fops (see is_kvm_gmem_file()).  file->private_data on
+ * such a file points at a struct kvm_gmem_backing whose ->ops field is the
+ * per-instance vtable below.  KVM never looks any further; the rest of the
+ * per-file state (allocator, refcount, private xarray, ...) is up to the
+ * implementation and is reached via container_of() from the backing.
+ *
+ * The memory-side callbacks (bind, unbind, get_pfn, populate, invalidate)
+ * are the reason the interface exists.  The file-level callbacks (release,
+ * mmap, fallocate, ...) are dispatched by kvm_gmem_fops so that every
+ * implementation gets a private hook -- see the dispatchers in
+ * virt/kvm/guest_memfd.c.
+ *
+ * get_pfn() contract:
+ *  - *pfn is the host PFN backing @gfn.  It need not have a struct page.
+ *  - *max_order is the largest order KVM may use to map starting at @gfn.
+ *    The implementation MUST clamp it to the physical contiguity and the
+ *    HPA/GPA alignment of the backing, so KVM never builds a huge SPTE that
+ *    spans discontiguous physical memory.
+ *  - If *page is non-NULL on return the caller owns that page reference and
+ *    is responsible for put_page() after use.  If *page is left NULL the PFN
+ *    is treated as non-refcounted, and its lifetime is owned by the
+ *    implementation across bind()/unbind().
+ *
+ * Memory intended to back guest RAM MUST be reported as E820_TYPE_RAM by the
+ * host so KVM maps it write-back (and applies the memory-encryption bit on
+ * encrypted hosts) rather than treating it as MMIO.
+ */
+struct kvm_gmem_ops {
+	/* Memory operations. */
+	int (*bind)(struct file *file, struct kvm *kvm,
+		    struct kvm_memory_slot *slot, loff_t offset);
+	void (*unbind)(struct file *file, struct kvm *kvm,
+		       struct kvm_memory_slot *slot);
+	int (*get_pfn)(struct file *file, struct kvm *kvm,
+		       struct kvm_memory_slot *slot, gfn_t gfn,
+		       kvm_pfn_t *pfn, struct page **page, int *max_order);
+	int (*populate)(struct file *file, struct kvm *kvm,
+			struct kvm_memory_slot *slot, gfn_t gfn,
+			kvm_pfn_t *pfn, struct page *src_page, int order);
+	void (*invalidate)(struct file *file, struct kvm *kvm,
+			   struct kvm_memory_slot *slot,
+			   gfn_t start, gfn_t end);
+
+	/*
+	 * File-level operations, dispatched by kvm_gmem_fops.  These are
+	 * per-instance analogues of the like-named file_operations members,
+	 * with the same semantics.  A NULL entry causes the dispatcher to
+	 * fail the file-level operation with an appropriate errno.
+	 */
+	void (*release)(struct file *file);
+	int (*mmap)(struct file *file, struct vm_area_struct *vma);
+	long (*fallocate)(struct file *file, int mode, loff_t offset,
+			  loff_t len);
+	long (*ioctl)(struct file *file, unsigned int cmd, unsigned long arg);
+};
+
+/*
+ * Anchor placed at file->private_data of every guest_memfd-backing file.
+ * KVM's own gmem embeds it as the first member of struct gmem_file; other
+ * implementations do the same in their own per-fd state.  is_kvm_gmem_file()
+ * (fops-identity check) proves the reinterpretation is safe.
+ */
+struct kvm_gmem_backing {
+	const struct kvm_gmem_ops *ops;
+};
+
+/* Defined in guest_memfd.c; single canonical fops for every gmem file. */
+extern struct file_operations kvm_gmem_fops;
+
+static inline bool is_kvm_gmem_file(struct file *file)
+{
+	return file && file->f_op == &kvm_gmem_fops;
+}
+
+#endif /* CONFIG_KVM_GUEST_MEMFD */
+
 static inline bool kvm_slot_dirty_track_enabled(const struct kvm_memory_slot *slot)
 {
 	return slot->flags & KVM_MEM_LOG_DIRTY_PAGES;
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index db57c5766ab6..057f90eb835a 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -22,11 +22,22 @@ static struct vfsmount *kvm_gmem_mnt;
  * specific to its associated VM, e.g. memslots=>gmem bindings.
  */
 struct gmem_file {
+	/*
+	 * MUST be first: file->private_data points here.
+	 * is_kvm_gmem_file(file) proves this reinterpretation is safe.
+	 */
+	struct kvm_gmem_backing backing;
+
 	struct kvm *kvm;
 	struct xarray bindings;
 	struct list_head entry;
 };
 
+static inline struct gmem_file *gmem_file_of(struct file *file)
+{
+	return container_of(file->private_data, struct gmem_file, backing);
+}
+
 struct gmem_inode {
 	struct shared_policy policy;
 	struct inode vfs_inode;
@@ -296,7 +307,7 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
 	return r;
 }
 
-static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
+static long kvm_gmem_native_fallocate(struct file *file, int mode, loff_t offset,
 			       loff_t len)
 {
 	int ret;
@@ -320,9 +331,10 @@ static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
 	return ret;
 }
 
-static int kvm_gmem_release(struct inode *inode, struct file *file)
+static void kvm_gmem_native_release(struct file *file)
 {
-	struct gmem_file *f = file->private_data;
+	struct inode *inode = file_inode(file);
+	struct gmem_file *f = gmem_file_of(file);
 	struct kvm_memory_slot *slot;
 	struct kvm *kvm = f->kvm;
 	unsigned long index;
@@ -367,7 +379,6 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
 
 	kvm_put_kvm(kvm);
 
-	return 0;
 }
 
 static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot)
@@ -466,7 +477,7 @@ static const struct vm_operations_struct kvm_gmem_vm_ops = {
 #endif
 };
 
-static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma)
+static int kvm_gmem_native_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	if (!kvm_gmem_supports_mmap(file_inode(file)))
 		return -ENODEV;
@@ -481,12 +492,61 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma)
 	return 0;
 }
 
-static struct file_operations kvm_gmem_fops = {
-	.mmap		= kvm_gmem_mmap,
+/* File-op dispatchers — thin: they all go through the backing's ops table. */
+
+static int kvm_gmem_fops_release(struct inode *inode, struct file *file)
+{
+	struct kvm_gmem_backing *b = file->private_data;
+
+	if (b && b->ops && b->ops->release)
+		b->ops->release(file);
+	return 0;
+}
+
+static int kvm_gmem_fops_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct kvm_gmem_backing *b = file->private_data;
+
+	if (!b || !b->ops || !b->ops->mmap)
+		return -ENODEV;
+	return b->ops->mmap(file, vma);
+}
+
+static long kvm_gmem_fops_fallocate(struct file *file, int mode,
+				    loff_t offset, loff_t len)
+{
+	struct kvm_gmem_backing *b = file->private_data;
+
+	if (!b || !b->ops || !b->ops->fallocate)
+		return -EOPNOTSUPP;
+	return b->ops->fallocate(file, mode, offset, len);
+}
+
+static long kvm_gmem_fops_ioctl(struct file *file, unsigned int cmd,
+				unsigned long arg)
+{
+	struct kvm_gmem_backing *b = file->private_data;
+
+	if (!b || !b->ops || !b->ops->ioctl)
+		return -ENOTTY;
+	return b->ops->ioctl(file, cmd, arg);
+}
+
+/*
+ * A single struct file_operations for every guest_memfd-backing file --
+ * KVM's built-in gmem and any subsystem-supplied backing alike.  All
+ * callbacks that can vary per implementation dispatch through the
+ * kvm_gmem_backing at file->private_data.
+ */
+struct file_operations kvm_gmem_fops = {
 	.open		= generic_file_open,
-	.release	= kvm_gmem_release,
-	.fallocate	= kvm_gmem_fallocate,
+	.release	= kvm_gmem_fops_release,
+	.mmap		= kvm_gmem_fops_mmap,
+	.fallocate	= kvm_gmem_fops_fallocate,
+	.unlocked_ioctl	= kvm_gmem_fops_ioctl,
+	.compat_ioctl	= kvm_gmem_fops_ioctl,
 };
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_fops);
 
 static int kvm_gmem_migrate_folio(struct address_space *mapping,
 				  struct folio *dst, struct folio *src,
@@ -557,6 +617,43 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
 	return true;
 }
 
+static int  kvm_gmem_native_bind(struct file *file, struct kvm *kvm,
+				 struct kvm_memory_slot *slot, loff_t offset);
+static void kvm_gmem_native_unbind(struct file *slot_file, struct kvm *kvm,
+				   struct kvm_memory_slot *slot);
+static int  kvm_gmem_native_get_pfn(struct file *file, struct kvm *kvm,
+				    struct kvm_memory_slot *slot, gfn_t gfn,
+				    kvm_pfn_t *pfn, struct page **page,
+				    int *max_order);
+static void kvm_gmem_native_release(struct file *file);
+static int  kvm_gmem_native_mmap(struct file *file,
+				 struct vm_area_struct *vma);
+static long kvm_gmem_native_fallocate(struct file *file, int mode,
+				      loff_t offset, loff_t len);
+
+static const struct kvm_gmem_ops kvm_gmem_native_ops = {
+	/* Memory operations. */
+	.bind      = kvm_gmem_native_bind,
+	.unbind    = kvm_gmem_native_unbind,
+	.get_pfn   = kvm_gmem_native_get_pfn,
+	/*
+	 * .populate is deliberately NULL: kvm_gmem_populate() has an inline
+	 * fast path that drives post_populate() with the folio locked and
+	 * the memory-attribute check in the right place; that flow doesn't
+	 * fit the current ops signature cleanly.  Wiring it through ops
+	 * is a separate cleanup.
+	 *
+	 * .invalidate is deliberately NULL: invalidations for the built-in
+	 * implementation flow through inode->i_mapping (truncate, hwpoison,
+	 * free_folio) and don't need an out-of-band callback from KVM.
+	 */
+
+	/* File-level operations (dispatched by kvm_gmem_fops). */
+	.release   = kvm_gmem_native_release,
+	.mmap      = kvm_gmem_native_mmap,
+	.fallocate = kvm_gmem_native_fallocate,
+};
+
 static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
 {
 	static const char *name = "[kvm-gmem]";
@@ -605,7 +702,8 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
 	}
 
 	file->f_flags |= O_LARGEFILE;
-	file->private_data = f;
+	file->private_data = &f->backing;
+	f->backing.ops = &kvm_gmem_native_ops;
 
 	kvm_get_kvm(kvm);
 	f->kvm = kvm;
@@ -640,34 +738,27 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
 	return __kvm_gmem_create(kvm, size, flags);
 }
 
-int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
-		  unsigned int fd, uoff_t offset)
+static const struct kvm_gmem_ops *kvm_gmem_get_ops(struct file *file)
+{
+	if (!is_kvm_gmem_file(file))
+		return NULL;
+	return ((struct kvm_gmem_backing *)file->private_data)->ops;
+}
+
+static int kvm_gmem_native_bind(struct file *file, struct kvm *kvm,
+				struct kvm_memory_slot *slot, loff_t offset)
 {
 	uoff_t size = slot->npages << PAGE_SHIFT;
+	struct gmem_file *f = gmem_file_of(file);
+	struct inode *inode = file_inode(file);
 	unsigned long start, end;
-	struct gmem_file *f;
-	struct inode *inode;
-	struct file *file;
 	int r = -EINVAL;
 
-	BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset));
-	BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));
-
-	file = fget(fd);
-	if (!file)
-		return -EBADF;
-
-	if (file->f_op != &kvm_gmem_fops)
-		goto err;
-
-	f = file->private_data;
 	if (f->kvm != kvm)
-		goto err;
-
-	inode = file_inode(file);
+		return -EINVAL;
 
-	if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode))
-		goto err;
+	if (offset < 0 || !PAGE_ALIGNED(offset) || offset + size > i_size_read(inode))
+		return -EINVAL;
 
 	filemap_invalidate_lock(inode->i_mapping);
 
@@ -677,14 +768,13 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
 	if (!xa_empty(&f->bindings) &&
 	    xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) {
 		r = -EEXIST;
-		filemap_invalidate_unlock(inode->i_mapping);
-		goto err;
+		goto out;
 	}
 
 	/*
-	 * memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so
-	 * kvm_gmem_bind() must occur on a new memslot.  Because the memslot
-	 * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file.
+	 * memslots of flag KVM_MEM_GUEST_MEMFD are immutable, so bind must
+	 * occur on a new memslot.  Because the memslot is not visible yet,
+	 * kvm_gmem_get_pfn() is guaranteed to see the file.
 	 */
 	WRITE_ONCE(slot->gmem.file, file);
 	slot->gmem.pgoff = start;
@@ -692,15 +782,51 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
 		slot->flags |= KVM_MEMSLOT_GMEM_ONLY;
 
 	xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL);
+	r = 0;
+out:
 	filemap_invalidate_unlock(inode->i_mapping);
+	return r;
+}
+
+int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
+		  unsigned int fd, uoff_t offset)
+{
+	const struct kvm_gmem_ops *ops;
+	struct file *file;
+	int r;
+
+	BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset));
+	BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));
+
+	file = fget(fd);
+	if (!file)
+		return -EBADF;
+
+	ops = kvm_gmem_get_ops(file);
+	if (!ops || !ops->bind || !ops->get_pfn) {
+		r = -EINVAL;
+		goto out;
+	}
+
+	if (!PAGE_ALIGNED(offset)) {
+		r = -EINVAL;
+		goto out;
+	}
 
 	/*
-	 * Drop the reference to the file, even on success.  The file pins KVM,
-	 * not the other way 'round.  Active bindings are invalidated if the
-	 * file is closed before memslots are destroyed.
+	 * On success .bind() associates the fd with @kvm and (if the
+	 * implementation needs it) records the reference that makes the fd
+	 * pin the VM.  A .bind() that returns success is trusted to have
+	 * done so; there is no separate ownership check here.
+	 */
+	r = ops->bind(file, kvm, slot, offset);
+out:
+	/*
+	 * Drop the reference to the file even on success.  The file pins
+	 * KVM, not the other way 'round: it is the implementation's job to
+	 * invalidate its bindings if the fd is closed before the memslots
+	 * are destroyed.
 	 */
-	r = 0;
-err:
 	fput(file);
 	return r;
 }
@@ -719,21 +845,15 @@ static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f)
 	WRITE_ONCE(slot->gmem.file, NULL);
 }
 
-void kvm_gmem_unbind(struct kvm_memory_slot *slot)
+static void kvm_gmem_native_unbind(struct file *slot_file, struct kvm *kvm,
+				   struct kvm_memory_slot *slot)
 {
-	/*
-	 * Nothing to do if the underlying file was _already_ closed, as
-	 * kvm_gmem_release() invalidates and nullifies all bindings.
-	 */
-	if (!slot->gmem.file)
-		return;
-
 	CLASS(gmem_get_file, file)(slot);
 
 	/*
-	 * However, if the file is _being_ closed, then the bindings need to be
-	 * removed as kvm_gmem_release() might not run until after the memslot
-	 * is freed.  Note, modifying the bindings is safe even though the file
+	 * If the file is _being_ closed, then the bindings need to be removed
+	 * as kvm_gmem_release() might not run until after the memslot is
+	 * freed.  Note, modifying the bindings is safe even though the file
 	 * is dying as kvm_gmem_release() nullifies slot->gmem.file under
 	 * slots_lock, and only puts its reference to KVM after destroying all
 	 * bindings.  I.e. reaching this point means kvm_gmem_release() hasn't
@@ -741,15 +861,41 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot)
 	 * until the caller drops slots_lock.
 	 */
 	if (!file) {
-		__kvm_gmem_unbind(slot, slot->gmem.file->private_data);
+		__kvm_gmem_unbind(slot, gmem_file_of(slot_file));
 		return;
 	}
 
 	filemap_invalidate_lock(file->f_mapping);
-	__kvm_gmem_unbind(slot, file->private_data);
+	__kvm_gmem_unbind(slot, gmem_file_of(file));
 	filemap_invalidate_unlock(file->f_mapping);
 }
 
+void kvm_gmem_unbind(struct kvm *kvm, struct kvm_memory_slot *slot)
+{
+	const struct kvm_gmem_ops *ops;
+	struct file *file;
+
+	/*
+	 * Nothing to do if the underlying file was _already_ closed, as
+	 * kvm_gmem_release() invalidates and nullifies all bindings.
+	 */
+	file = READ_ONCE(slot->gmem.file);
+	if (!file)
+		return;
+
+	ops = kvm_gmem_get_ops(file);
+	if (ops && ops->unbind)
+		ops->unbind(file, kvm, slot);
+
+	/*
+	 * .unbind() may clear slot->gmem.file itself (as the built-in gmem
+	 * ops do, under filemap_invalidate_lock, to serialise against
+	 * release/invalidate); catch the ones that don't.
+	 */
+	if (READ_ONCE(slot->gmem.file))
+		WRITE_ONCE(slot->gmem.file, NULL);
+}
+
 /* Returns a locked folio on success.  */
 static struct folio *__kvm_gmem_get_pfn(struct file *file,
 					struct kvm_memory_slot *slot,
@@ -757,7 +903,7 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file,
 					int *max_order)
 {
 	struct file *slot_file = READ_ONCE(slot->gmem.file);
-	struct gmem_file *f = file->private_data;
+	struct gmem_file *f = gmem_file_of(file);
 	struct folio *folio;
 
 	if (file != slot_file) {
@@ -787,17 +933,14 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file,
 	return folio;
 }
 
-int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
-		     gfn_t gfn, kvm_pfn_t *pfn, struct page **page,
-		     int *max_order)
+static int kvm_gmem_native_get_pfn(struct file *file, struct kvm *kvm,
+				   struct kvm_memory_slot *slot, gfn_t gfn,
+				   kvm_pfn_t *pfn, struct page **page,
+				   int *max_order)
 {
 	pgoff_t index = kvm_gmem_get_index(slot, gfn);
 	struct folio *folio;
-	int r = 0;
-
-	CLASS(gmem_get_file, file)(slot);
-	if (!file)
-		return -EFAULT;
+	int r;
 
 	folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order);
 	if (IS_ERR(folio))
@@ -809,7 +952,6 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
 	}
 
 	r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);
-
 	folio_unlock(folio);
 
 	if (!r)
@@ -819,6 +961,24 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
 
 	return r;
 }
+
+int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
+		     gfn_t gfn, kvm_pfn_t *pfn, struct page **page,
+		     int *max_order)
+{
+	const struct kvm_gmem_ops *ops;
+
+	CLASS(gmem_get_file, file)(slot);
+	if (!file)
+		return -EFAULT;
+
+	ops = kvm_gmem_get_ops(file);
+	if (!ops || !ops->get_pfn)
+		return -EFAULT;
+
+	*page = NULL;
+	return ops->get_pfn(file, kvm, slot, gfn, pfn, page, max_order);
+}
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn);
 
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE
@@ -860,11 +1020,51 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot,
 	return ret;
 }
 
+static int kvm_gmem_populate_one(const struct kvm_gmem_ops *ops,
+				      struct file *file, struct kvm *kvm,
+				      struct kvm_memory_slot *slot, gfn_t gfn,
+				      struct page *src_page,
+				      kvm_gmem_populate_cb post_populate,
+				      void *opaque)
+{
+	struct page *ignored_page = NULL;
+	kvm_pfn_t pfn;
+	int ret;
+
+	/*
+	 * Resolve the PFN via the ops.  A populate() op may also copy
+	 * @src_page into the backing before the arch post_populate() step
+	 * (e.g. SNP LAUNCH_UPDATE); otherwise fall back to get_pfn().
+	 */
+	if (ops->populate)
+		ret = ops->populate(file, kvm, slot, gfn, &pfn,
+				    src_page, 0);
+	else
+		ret = ops->get_pfn(file, kvm, slot, gfn, &pfn,
+				   &ignored_page, NULL);
+	if (ret)
+		return ret;
+
+	ret = post_populate(kvm, gfn, pfn, src_page, opaque);
+
+	/*
+	 * Populate does not consume a refcount on the destination page.
+	 * Drop @page (if any) AFTER post_populate() so the PFN cannot be
+	 * freed and reallocated behind an arch hook (e.g. RMPUPDATE) that
+	 * is still operating on it.
+	 */
+	if (ignored_page)
+		put_page(ignored_page);
+
+	return ret;
+}
+
 long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
 		       long npages, bool may_writeback_src,
 		       kvm_gmem_populate_cb post_populate, void *opaque)
 {
 	struct kvm_memory_slot *slot;
+	const struct kvm_gmem_ops *ops;
 	int ret = 0;
 	long i;
 
@@ -884,6 +1084,8 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
 	if (!file)
 		return -EFAULT;
 
+	ops = kvm_gmem_get_ops(file);
+
 	npages = min_t(ulong, slot->npages - (start_gfn - slot->base_gfn), npages);
 	for (i = 0; i < npages; i++) {
 		struct page *src_page = NULL;
@@ -906,8 +1108,13 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
 			}
 		}
 
-		ret = __kvm_gmem_populate(kvm, slot, file, start_gfn + i, src_page,
-					  post_populate, opaque);
+		if (ops && (ops->populate || ops->get_pfn != kvm_gmem_native_get_pfn))
+			ret = kvm_gmem_populate_one(ops, file, kvm, slot,
+						    start_gfn + i, src_page,
+						    post_populate, opaque);
+		else
+			ret = __kvm_gmem_populate(kvm, slot, file, start_gfn + i,
+						  src_page, post_populate, opaque);
 
 		if (src_page)
 			put_page(src_page);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index e44c20c04961..3af5270d2a90 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -935,7 +935,7 @@ static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
 {
 	if (slot->flags & KVM_MEM_GUEST_MEMFD)
-		kvm_gmem_unbind(slot);
+		kvm_gmem_unbind(kvm, slot);
 
 	kvm_destroy_dirty_bitmap(slot);
 
@@ -1747,7 +1747,7 @@ static void kvm_commit_memory_region(struct kvm *kvm,
 		 * flags-only changes on guest_memfd slots should be impossible.
 		 */
 		if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD))
-			kvm_gmem_unbind(old);
+			kvm_gmem_unbind(kvm, old);
 
 		/*
 		 * The final quirk.  Free the detached, old slot, but only its
@@ -2118,7 +2118,7 @@ static int kvm_set_memory_region(struct kvm *kvm,
 
 out_unbind:
 	if (mem->flags & KVM_MEM_GUEST_MEMFD)
-		kvm_gmem_unbind(new);
+		kvm_gmem_unbind(kvm, new);
 out:
 	kfree(new);
 	return r;
@@ -5475,7 +5475,7 @@ bool file_is_kvm(struct file *file)
 {
 	return file && file->f_op == &kvm_vm_fops;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm);
+EXPORT_SYMBOL_GPL(file_is_kvm);
 
 static int kvm_dev_ioctl_create_vm(unsigned long type)
 {
diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h
index 7510ca915dd1..ea0fe74e89d4 100644
--- a/virt/kvm/kvm_mm.h
+++ b/virt/kvm/kvm_mm.h
@@ -76,7 +76,7 @@ void kvm_gmem_exit(void);
 int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
 int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
 		  unsigned int fd, uoff_t offset);
-void kvm_gmem_unbind(struct kvm_memory_slot *slot);
+void kvm_gmem_unbind(struct kvm *kvm, struct kvm_memory_slot *slot);
 #else
 static inline int kvm_gmem_init(struct module *module)
 {
@@ -91,7 +91,7 @@ static inline int kvm_gmem_bind(struct kvm *kvm,
 	return -EIO;
 }
 
-static inline void kvm_gmem_unbind(struct kvm_memory_slot *slot)
+static inline void kvm_gmem_unbind(struct kvm *kvm, struct kvm_memory_slot *slot)
 {
 	WARN_ON_ONCE(1);
 }
-- 
2.47.3