[PATCH RFC v4 05/12] mm/gup: add get_user_page_vma() to fault in a page under a held lock

[Rik van Riel <riel@surriel.com>]

__access_remote_vm() needs a single page from a VMA it has already
looked up and locked, faulting it in when necessary, under either the
mmap lock or the per-VMA lock. get_user_pages_remote() does not fit: it
hard codes the mmap lock and re-looks-up the VMA, neither of which is
wanted here.

Add get_user_page_vma(), a simplified __get_user_pages() that walks the
page tables with follow_page_mask(), faults a missing page in with
faultin_page(), and on success returns it with a reference and the
caller's lock still held. Like __get_user_pages() it runs
check_vma_flags(), so callers need not pre-check the VMA.

A VM_IO/VM_PFNMAP VMA is the exception to that check: it can still hold
COWed pages that have a struct page, so follow_page_mask() is allowed to
look for one. Memory with no struct page -- a raw PFN, or a present PFN
reported as -EEXIST -- is returned as -EFAULT, so the caller can reach it
through vma->vm_ops->access().

The caller sets FOLL_VMA_LOCK when it holds the per-VMA lock rather than
the mmap lock, which reaches the fault code as FAULT_FLAG_VMA_LOCK.

Anything that cannot complete under the per-VMA lock -- a dropped fault,
a userfaultfd VMA (uffd assumes current is the faulting task), a hard
error, or ->access() memory -- releases the lock and returns -EAGAIN,
so the caller retries under the mmap lock.

faultin_page() reports these retries as -EAGAIN for both lock types;
only the mmap caller records the dropped lock in *locked.

A VM_FAULT_ERROR decoding to no errno now returns -EFAULT instead of
BUG(), warning under CONFIG_DEBUG_VM. This relaxes the same case on the
existing __get_user_pages() path too.

Assisted-by: Claude:claude-opus-4.8
Signed-off-by: Rik van Riel <riel@surriel.com>
---
 mm/gup.c      | 176 +++++++++++++++++++++++++++++++++++++++++++-------
 mm/internal.h |   8 ++-
 2 files changed, 159 insertions(+), 25 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c
index dcece7b5253c..cb7245b7542f 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1080,9 +1080,16 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
 }
 
 /*
- * mmap_lock must be held on entry.  If @flags has FOLL_UNLOCKABLE but not
- * FOLL_NOWAIT, the mmap_lock may be released.  If it is, *@locked will be set
- * to 0 and -EBUSY returned.
+ * The caller holds either the mmap lock, or the per-VMA lock (with
+ * FOLL_VMA_LOCK), on entry. If @flags has FOLL_UNLOCKABLE but not FOLL_NOWAIT,
+ * the mmap_lock may be released. If it is, *@locked will be set to 0 and
+ * -EAGAIN returned.
+ *
+ * The return value does not depend on the lock type: a fault that made
+ * progress but needs a retry (VM_FAULT_RETRY / VM_FAULT_COMPLETED) is reported
+ * as -EAGAIN for both the mmap lock and the per-VMA lock (FOLL_VMA_LOCK). Only
+ * the *@locked side effect is lock-type specific, as the per-VMA lock path has
+ * no unlockable mmap_lock to drop.
  */
 static int faultin_page(struct vm_area_struct *vma,
 		unsigned long address, unsigned int flags, bool unshare,
@@ -1097,6 +1104,8 @@ static int faultin_page(struct vm_area_struct *vma,
 		fault_flags |= FAULT_FLAG_WRITE;
 	if (flags & FOLL_REMOTE)
 		fault_flags |= FAULT_FLAG_REMOTE;
+	if (flags & FOLL_VMA_LOCK)
+		fault_flags |= FAULT_FLAG_VMA_LOCK;
 	if (flags & FOLL_UNLOCKABLE) {
 		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 		/*
@@ -1125,41 +1134,160 @@ static int faultin_page(struct vm_area_struct *vma,
 
 	ret = handle_mm_fault(vma, address, fault_flags, NULL);
 
+	/*
+	 * A fully completed fault (VM_FAULT_COMPLETED) or one that needs a retry
+	 * (VM_FAULT_RETRY) has released the lock it was holding. Report both as
+	 * -EAGAIN so the caller retries: the mmap lock caller retakes it here,
+	 * the per-VMA lock caller (FOLL_VMA_LOCK) falls back to the mmap lock.
+	 *
+	 * Dropping the mmap lock is recorded in *@locked. There is no such lock
+	 * to drop under the per-VMA lock, where @locked is not used, so leave it
+	 * alone in that case.
+	 */
 	if (ret & VM_FAULT_COMPLETED) {
-		/*
-		 * With FAULT_FLAG_RETRY_NOWAIT we'll never release the
-		 * mmap lock in the page fault handler. Sanity check this.
-		 */
-		WARN_ON_ONCE(fault_flags & FAULT_FLAG_RETRY_NOWAIT);
-		*locked = 0;
-
-		/*
-		 * We should do the same as VM_FAULT_RETRY, but let's not
-		 * return -EBUSY since that's not reflecting the reality of
-		 * what has happened - we've just fully completed a page
-		 * fault, with the mmap lock released.  Use -EAGAIN to show
-		 * that we want to take the mmap lock _again_.
-		 */
+		if (!(flags & FOLL_VMA_LOCK)) {
+			/*
+			 * With FAULT_FLAG_RETRY_NOWAIT we'll never release the
+			 * mmap lock in the page fault handler. Sanity check this.
+			 */
+			WARN_ON_ONCE(fault_flags & FAULT_FLAG_RETRY_NOWAIT);
+			*locked = 0;
+		}
 		return -EAGAIN;
 	}
 
 	if (ret & VM_FAULT_ERROR) {
 		int err = vm_fault_to_errno(ret, flags);
 
-		if (err)
-			return err;
-		BUG();
+		/*
+		 * VM_FAULT_ERROR always decodes to an errno; a zero here would
+		 * mean handle_mm_fault() returned an unexpected combination.
+		 * Report -EFAULT rather than crash: under the per-VMA lock the
+		 * mmap lock retry produces the definitive result.
+		 */
+		VM_WARN_ON_ONCE(!err);
+		return err ? err : -EFAULT;
 	}
 
 	if (ret & VM_FAULT_RETRY) {
-		if (!(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
+		if (!(flags & FOLL_VMA_LOCK) &&
+		    !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
 			*locked = 0;
-		return -EBUSY;
+		return -EAGAIN;
 	}
 
 	return 0;
 }
 
+/*
+ * get_user_page_vma - get one page from @vma, whose lock the caller already
+ * holds: the mmap lock, or (with FOLL_VMA_LOCK) the per-VMA lock. Walks the
+ * page tables, faulting the page in if needed, and on success returns it with
+ * a reference and the lock still held.
+ *
+ * Runs check_vma_flags() like __get_user_pages(), so callers need not pre-check
+ * the VMA; most rejections are returned as their error. A VM_IO/VM_PFNMAP VMA
+ * is the exception: a COWed page with a struct page is returned, while a raw
+ * PFN has none and yields -EFAULT, to be reached via vma->vm_ops->access().
+ *
+ * Under FOLL_VMA_LOCK, anything that cannot be finished under the per-VMA lock
+ * (a dropped fault, userfaultfd, a hard error, or ->access() memory) releases
+ * the lock and returns -EAGAIN, so the caller retries under the mmap lock.
+ */
+struct page *get_user_page_vma(struct vm_area_struct *vma, unsigned long addr,
+			       unsigned int gup_flags)
+{
+	bool vma_locked = gup_flags & FOLL_VMA_LOCK;
+	unsigned long page_mask;
+	struct page *page;
+	int locked = 1;
+	bool pfnmap;
+	int ret;
+
+	/*
+	 * Two lock modes are supported: the mmap lock, with neither flag, or
+	 * the per-VMA lock, with both FOLL_VMA_LOCK and FOLL_UNLOCKABLE. An
+	 * unlockable mmap fault would drop the lock and report it in *locked,
+	 * which this function does not relay, so reject a lone flag.
+	 */
+	VM_WARN_ON_ONCE(!(gup_flags & FOLL_VMA_LOCK) != !(gup_flags & FOLL_UNLOCKABLE));
+
+	/*
+	 * Validate the VMA up front, like __get_user_pages(). A VM_IO/VM_PFNMAP
+	 * VMA is not rejected outright: it can hold COWed pages that have a
+	 * struct page; let follow_page_mask() look for them, and treat only
+	 * its struct-page-less PFNs as unreachable. Other rejections
+	 * (secretmem, permissions, etc) result in immediate failure.
+	 */
+	ret = check_vma_flags(vma, gup_flags, VM_IO | VM_PFNMAP);
+	if (ret)
+		goto fail;
+	pfnmap = vma->vm_flags & (VM_IO | VM_PFNMAP);
+
+	for (;;) {
+		if (fatal_signal_pending(current)) {
+			ret = -EINTR;
+			goto fail;
+		}
+		cond_resched();
+
+		/* follow_page_mask() requires @page_mask; it is unused here. */
+		page = follow_page_mask(vma, addr,
+					gup_flags | FOLL_TOUCH | FOLL_GET,
+					&page_mask);
+		if (!IS_ERR_OR_NULL(page)) {
+			/* Match __get_user_pages(): flush for VIVT/aliasing caches. */
+			flush_anon_page(vma, page, addr);
+			flush_dcache_page(page);
+			return page;
+		}
+
+		/*
+		 * No struct page: a raw PFN of a VM_IO/VM_PFNMAP VMA, whether
+		 * seen by the up-front check (@pfnmap) or reported as -EEXIST
+		 * for a present PFN. Return -EFAULT so the caller reaches it
+		 * through vma->vm_ops->access().
+		 */
+		if (pfnmap || PTR_ERR(page) == -EEXIST) {
+			ret = -EFAULT;
+			goto fail;
+		}
+		/* A hard error from the walk itself. */
+		if (page && PTR_ERR(page) != -EMLINK) {
+			ret = PTR_ERR(page);
+			goto fail;
+		}
+
+		/*
+		 * The page is not present, or needs unsharing. A remote fault
+		 * under the per-VMA lock cannot deliver userfaultfd (which
+		 * assumes current is the faulting task), so fall back for those.
+		 */
+		if (vma_locked && userfaultfd_armed(vma)) {
+			ret = -EAGAIN;
+			goto fail;
+		}
+		ret = faultin_page(vma, addr, gup_flags | FOLL_REMOTE | FOLL_GET,
+				   PTR_ERR(page) == -EMLINK, &locked);
+		if (ret == -EAGAIN)
+			return ERR_PTR(-EAGAIN);	/* fault released the per-VMA lock */
+		if (ret)
+			goto fail;
+	}
+
+fail:
+	/*
+	 * Under the per-VMA lock the caller cannot reach ->access() or act on a
+	 * hard error (both need the mmap lock), so release the lock and have it
+	 * retry there; the mmap-lock pass produces the definitive error.
+	 */
+	if (vma_locked) {
+		vma_end_read(vma);
+		return ERR_PTR(-EAGAIN);
+	}
+	return ERR_PTR(ret);
+}
+
 /*
  * Writing to file-backed mappings which require folio dirty tracking using GUP
  * is a fundamentally broken operation, as kernel write access to GUP mappings
@@ -1197,8 +1325,8 @@ static bool writable_file_mapping_allowed(struct vm_area_struct *vma,
 	return !vma_needs_dirty_tracking(vma);
 }
 
-static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags,
-			   vm_flags_t ignore_flags)
+int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags,
+		    vm_flags_t ignore_flags)
 {
 	vm_flags_t vm_flags = vma->vm_flags;
 	int write = (gup_flags & FOLL_WRITE);
diff --git a/mm/internal.h b/mm/internal.h
index 181e79f1d6a2..706f7f08fd18 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1595,6 +1595,10 @@ struct vm_struct *__get_vm_area_node(unsigned long size,
  */
 int __must_check try_grab_folio(struct folio *folio, int refs,
 				unsigned int flags);
+int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags,
+		    vm_flags_t ignore_flags);
+struct page *get_user_page_vma(struct vm_area_struct *vma, unsigned long addr,
+			       unsigned int gup_flags);
 
 /*
  * mm/huge_memory.c
@@ -1641,11 +1645,13 @@ enum {
 	FOLL_UNLOCKABLE = 1 << 21,
 	/* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */
 	FOLL_MADV_POPULATE = 1 << 22,
+	/* caller holds the per-VMA lock, not the mmap lock */
+	FOLL_VMA_LOCK = 1 << 23,
 };
 
 #define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \
 			    FOLL_FAST_ONLY | FOLL_UNLOCKABLE | \
-			    FOLL_MADV_POPULATE)
+			    FOLL_MADV_POPULATE | FOLL_VMA_LOCK)
 
 /*
  * Indicates for which pages that are write-protected in the page table,
-- 
2.53.0-Meta