[RFC PATCH 05/14] KVM: guest_memfd: Export PFN mode files

[Fred Griffoul <griffoul@gmail.com>]

From: Fred Griffoul <fgriffo@amazon.co.uk>

PFN mode files cannot be attached yet, though a provider's frames are
the memory that most needs a device mapping that follows its revokes.

Allow them. A lookup asks the provider with lookup_pfn_range(), which
allocates nothing, then get_pfn() for the block and its attributes. A
revoke updates the exports before the frames are reclaimed. A punch hole
must remove the frames from the exports before the provider gets them
back, while the provider still backs them: lookups report the range as
holes until then, and the exports are updated again afterwards.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: Fred Griffoul <fgriffo@amazon.co.uk>
---
 include/linux/guest_memfd.h |  20 ++++---
 virt/kvm/guest_memfd.c      | 111 ++++++++++++++++++++++++++++++------
 2 files changed, 105 insertions(+), 26 deletions(-)

diff --git a/include/linux/guest_memfd.h b/include/linux/guest_memfd.h
index ba3eebb1ed1a..937693ae9573 100644
--- a/include/linux/guest_memfd.h
+++ b/include/linux/guest_memfd.h
@@ -90,13 +90,13 @@ static inline int kvm_gmem_revoke(struct inode *inode, pgoff_t start, size_t nr_
  * nor fault them.  It attaches to the file, looks up the frame behind each
  * shared page without taking a reference, and guest_memfd calls its update()
  * whenever what a lookup returns for a range may have changed: conversions,
- * punch holes and memory failure.  update() makes the consumer's mappings of
- * the range match what a lookup returns now: a frame, or a hole for a private
- * page or a page without a frame.  It may be called for pages that did not
- * change, and for ranges that do not follow the blocks that lookups report:
- * the consumer may then remap whole blocks, also outside the range, e.g. to
- * merge them into larger mappings.  A consumer that can fault may instead only
- * remove its mappings of the range.
+ * punch holes, memory failure and a provider's kvm_gmem_revoke().  update()
+ * makes the consumer's mappings of the range match what a lookup returns now:
+ * a frame, or a hole for a private page or a page without a frame.  It may be
+ * called for pages that did not change, and for ranges that do not follow the
+ * blocks that lookups report: the consumer may then remap whole blocks, also
+ * outside the range, e.g. to merge them into larger mappings.  A consumer that
+ * can fault may instead only remove its mappings of the range.
  *
  * Until update() returns, the frames it replaces stay valid, so a consumer
  * may switch each mapping to its new target in one step; once it returns,
@@ -111,8 +111,10 @@ static inline int kvm_gmem_revoke(struct inode *inode, pgoff_t start, size_t nr_
  * same time, also for overlapping ranges.  The consumer holds a lock across a
  * lookup and the mapping it installs, which update() also takes.  Lock order:
  *
- *   guest_memfd invalidate lock
- *     consumer lock
+ *   PFN mode provider lock (across kvm_gmem_revoke())
+ *     guest_memfd invalidate lock
+ *       consumer lock
+ *         provider leaf lock (get_pfn(), lookup_pfn_range())
  *
  * A consumer attaches and detaches outside its lock.  KVM may be a module
  * while the consumer is built in: consumers reach these with symbol_get().
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 65a951fa49c2..015f6657c386 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -40,6 +40,12 @@ struct gmem_inode {
 	const struct guest_memfd_provider_operations *provider_ops;
 	/* Subsystems that map the file, see include/linux/guest_memfd.h. */
 	struct list_head exports;
+	/*
+	 * A PFN mode punch hole in progress: lookups find holes in
+	 * [punch_start, punch_end) until the provider has the frames back.
+	 * Written under the invalidate lock, read without it.
+	 */
+	pgoff_t punch_start, punch_end;
 
 	u64 flags;
 	/*
@@ -515,15 +521,32 @@ static int __kvm_gmem_revoke(struct inode *inode, pgoff_t start,
 	kvm_gmem_invalidate_start(inode, start, end, filter);
 
 	if (kvm_gmem_is_pfn_mode(GMEM_I(inode))) {
+		struct gmem_inode *gi = GMEM_I(inode);
+
 		unmap_mapping_range(inode->i_mapping, offset, len, 0);
+		/*
+		 * A punch hole gives the frames back only after the reclaim:
+		 * until then, lookups find holes.  Afterwards, the exports map
+		 * what the provider backs the range with.
+		 */
+		if (put_pfn_range) {
+			WRITE_ONCE(gi->punch_start, start);
+			/* Pairs with the smp_rmb() in kvm_gmem_export_pfn(). */
+			smp_wmb();
+			WRITE_ONCE(gi->punch_end, end);
+		}
+		kvm_gmem_update_exports(inode, start, end);
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
 		if (pfn)
 			kvm_arch_gmem_reclaim(*pfn, nr_pages);
 		else
-			kvm_gmem_pfn_reclaim(GMEM_I(inode), start, end);
+			kvm_gmem_pfn_reclaim(gi, start, end);
 #endif
-		if (put_pfn_range)
-			gmem_provider_put_pfn_range(GMEM_I(inode), start, nr_pages);
+		if (put_pfn_range) {
+			gmem_provider_put_pfn_range(gi, start, nr_pages);
+			WRITE_ONCE(gi->punch_end, 0);
+			kvm_gmem_update_exports(inode, start, end);
+		}
 	} else {
 		kvm_gmem_truncate(inode, start, end);
 	}
@@ -1258,8 +1281,7 @@ static struct file_operations kvm_gmem_fops = {
  * guest_memfd calls @e->ops->update() from now on, and holds a reference to
  * @file until kvm_gmem_export_detach().  May sleep.
  *
- * Return: 0, -ENODEV if @file is not a guest_memfd file, or -EOPNOTSUPP in
- * PFN mode.
+ * Return: 0, or -ENODEV if @file is not a guest_memfd file.
  */
 int kvm_gmem_export_attach(struct file *file, struct guest_memfd_export *e)
 {
@@ -1267,8 +1289,6 @@ int kvm_gmem_export_attach(struct file *file, struct guest_memfd_export *e)
 
 	if (file->f_op != &kvm_gmem_fops)
 		return -ENODEV;
-	if (kvm_gmem_is_pfn_mode(GMEM_I(inode)))
-		return -EOPNOTSUPP;
 
 	e->file = get_file(file);
 	filemap_invalidate_lock(inode->i_mapping);
@@ -1310,28 +1330,83 @@ static bool kvm_gmem_range_is_shared(struct inode *inode, pgoff_t index,
 	return true;
 }
 
+/*
+ * PFN mode: lookup_pfn_range() first, so that nothing is allocated, then
+ * get_pfn() for the order and attributes.
+ */
+static int kvm_gmem_export_pfn(struct gmem_inode *gi, pgoff_t index,
+			       unsigned long *pfn, int *max_order, u32 *attrs)
+{
+	pgoff_t punch_start, punch_end;
+	int order = *max_order, block;
+	unsigned long raw_pfn;
+	size_t nr_pages;
+	kvm_pfn_t frame;
+	int r;
+
+	/*
+	 * The end first: a new end comes with its start, and an old end of 0
+	 * makes the window empty.
+	 */
+	punch_end = READ_ONCE(gi->punch_end);
+	/* Pairs with the smp_wmb() in __kvm_gmem_revoke(). */
+	smp_rmb();
+	punch_start = READ_ONCE(gi->punch_start);
+	if (index >= punch_start && index < punch_end)
+		return -ENOENT;
+
+	r = gi->provider_ops->lookup_pfn_range(gi->provider, index, &raw_pfn,
+					       &nr_pages);
+	if (r)
+		return r;
+
+	r = gmem_provider_get_pfn(gi, index, &frame, &block, attrs);
+	if (r)
+		return r == -EFAULT ? -ENOENT : r;
+
+	order = min(order, block);
+	if (frame ^ index)
+		order = min_t(int, order, __ffs64(frame ^ index));
+	/* Nor may the block reach into a punch hole. */
+	while (order &&
+	       round_down(index, 1UL << order) < punch_end &&
+	       round_up(index + 1, 1UL << order) > punch_start)
+		order--;
+	*pfn = frame;
+	*max_order = order;
+	return 0;
+}
+
 /* What kvm_gmem_export_lookup_pfn() returns for a page inside the file. */
 static int __kvm_gmem_export_lookup(struct inode *inode, pgoff_t index,
 				    unsigned long *pfn, int *max_order,
 				    u32 *attrs)
 {
+	struct gmem_inode *gi = GMEM_I(inode);
 	struct folio *folio;
+	int r;
 
 	if (!kvm_gmem_range_is_shared(inode, index, 1))
 		return -ENOENT;
 
-	/* A folio that was never zeroed is a hole. */
-	folio = filemap_get_folio(inode->i_mapping, index);
-	if (IS_ERR(folio))
-		return -ENOENT;
-	if (folio_test_hwpoison(folio) || !folio_test_uptodate(folio)) {
+	if (kvm_gmem_is_pfn_mode(gi)) {
+		r = kvm_gmem_export_pfn(gi, index, pfn, max_order, attrs);
+		if (r)
+			return r;
+	} else {
+		/* A folio that was never zeroed is a hole. */
+		folio = filemap_get_folio(inode->i_mapping, index);
+		if (IS_ERR(folio))
+			return -ENOENT;
+		if (folio_test_hwpoison(folio) || !folio_test_uptodate(folio)) {
+			folio_put(folio);
+			return -ENOENT;
+		}
+		*pfn = folio_file_pfn(folio, index);
+		*max_order = 0;
+		*attrs = 0;
 		folio_put(folio);
-		return -ENOENT;
 	}
-	*pfn = folio_file_pfn(folio, index);
-	*max_order = 0;
-	*attrs = 0;
-	folio_put(folio);
 
 	/* Nor may the block hold a private page. */
 	while (*max_order &&
@@ -2042,6 +2117,8 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb)
 	gi->provider_ops = NULL;
 	INIT_LIST_HEAD(&gi->gmem_file_list);
 	INIT_LIST_HEAD(&gi->exports);
+	gi->punch_start = 0;
+	gi->punch_end = 0;
 	return &gi->vfs_inode;
 }
 
-- 
2.47.3