From: Rik van Riel <riel@surriel.com>
To: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org,
kernel-team@meta.com, Dave Hansen <dave.hansen@linux.intel.com>,
Peter Zijlstra <peterz@infradead.org>,
Suren Baghdasaryan <surenb@google.com>,
Lorenzo Stoakes <ljs@kernel.org>,
Vlastimil Babka <vbabka@kernel.org>,
David Hildenbrand <david@kernel.org>,
"Liam R. Howlett" <liam@infradead.org>,
Mike Rapoport <rppt@kernel.org>, Michal Hocko <mhocko@suse.com>,
Jason Gunthorpe <jgg@ziepe.ca>,
John Hubbard <jhubbard@nvidia.com>, Peter Xu <peterx@redhat.com>,
Matthew Wilcox <willy@infradead.org>,
Usama Arif <usamaarif642@gmail.com>,
Rik van Riel <riel@surriel.com>
Subject: [PATCH RFC v4 10/12] mm/gup: pass an end address to follow_page_mask() and return a page count
Date: Fri, 24 Jul 2026 18:29:32 -0400 [thread overview]
Message-ID: <20260724222934.1463812-11-riel@surriel.com> (raw)
In-Reply-To: <20260724222934.1463812-1-riel@surriel.com>
follow_page_mask() reports how many pages the returned page covers with a
page_mask: a bitmask of the enclosing naturally-aligned huge page. Callers
turn that into a stride, which assumes the run is a power-of-two block
aligned to its own size.
That form cannot describe an arbitrary contiguous run, which a later patch
needs to batch PTE-mapped large folios.
Replace the page_mask output with nr_pages, the number of contiguous pages
the returned page covers starting at @address, and add an @end argument
bounding how far the walk looks, so a small request does not scan an entire
large folio.
The huge PMD and PUD paths report the same stride as before, now as a count
clamped to @end. __get_user_pages() uses the count directly.
get_user_page_vma() passes @address + PAGE_SIZE and still returns one page.
No functional change intended.
Assisted-by: Claude:claude-opus-4.8
Signed-off-by: Rik van Riel <riel@surriel.com>
---
mm/gup.c | 102 ++++++++++++++++++++++++++++++-------------------------
1 file changed, 56 insertions(+), 46 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index 7f4107f7ff10..96b00ddf7e04 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -647,8 +647,9 @@ static inline bool can_follow_write_pud(pud_t pud, struct page *page,
}
static struct page *follow_huge_pud(struct vm_area_struct *vma,
- unsigned long addr, pud_t *pudp,
- int flags, unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pud_t *pudp, int flags,
+ unsigned long *nr_pages)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
@@ -672,10 +673,13 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
return ERR_PTR(-EMLINK);
ret = try_grab_folio(page_folio(page), 1, flags);
- if (ret)
+ if (ret) {
page = ERR_PTR(ret);
- else
- *page_mask = HPAGE_PUD_NR - 1;
+ } else {
+ unsigned long off = (addr & ~PUD_MASK) >> PAGE_SHIFT;
+
+ *nr_pages = min(HPAGE_PUD_NR - off, (end - addr) >> PAGE_SHIFT);
+ }
return page;
}
@@ -699,9 +703,9 @@ static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
}
static struct page *follow_huge_pmd(struct vm_area_struct *vma,
- unsigned long addr, pmd_t *pmd,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pmd_t *pmd, unsigned int flags,
+ unsigned long *nr_pages)
{
struct mm_struct *mm = vma->vm_mm;
pmd_t pmdval = *pmd;
@@ -738,23 +742,25 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma,
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
- *page_mask = HPAGE_PMD_NR - 1;
+ *nr_pages = min(HPAGE_PMD_NR - ((addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT),
+ (end - addr) >> PAGE_SHIFT);
return page;
}
#else /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */
static struct page *follow_huge_pud(struct vm_area_struct *vma,
- unsigned long addr, pud_t *pudp,
- int flags, unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pud_t *pudp, int flags,
+ unsigned long *nr_pages)
{
return NULL;
}
static struct page *follow_huge_pmd(struct vm_area_struct *vma,
- unsigned long addr, pmd_t *pmd,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long addr, unsigned long end,
+ pmd_t *pmd, unsigned int flags,
+ unsigned long *nr_pages)
{
return NULL;
}
@@ -800,7 +806,8 @@ static inline bool can_follow_write_pte(pte_t pte, struct page *page,
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
- unsigned long address, pmd_t *pmd, unsigned int flags)
+ unsigned long address, unsigned long end, pmd_t *pmd,
+ unsigned int flags, unsigned long *nr_pages)
{
struct mm_struct *mm = vma->vm_mm;
struct folio *folio;
@@ -885,6 +892,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
*/
folio_mark_accessed(folio);
}
+
out:
pte_unmap_unlock(ptep, ptl);
return page;
@@ -896,9 +904,9 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
}
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
- unsigned long address, pud_t *pudp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ pud_t *pudp, unsigned int flags,
+ unsigned long *nr_pages)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
@@ -912,7 +920,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
if (!pmd_present(pmdval))
return no_page_table(vma, flags, address);
if (likely(!pmd_leaf(pmdval)))
- return follow_page_pte(vma, address, pmd, flags);
+ return follow_page_pte(vma, address, end, pmd, flags, nr_pages);
if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags))
return no_page_table(vma, flags, address);
@@ -925,24 +933,24 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
}
if (unlikely(!pmd_leaf(pmdval))) {
spin_unlock(ptl);
- return follow_page_pte(vma, address, pmd, flags);
+ return follow_page_pte(vma, address, end, pmd, flags, nr_pages);
}
if (pmd_trans_huge(pmdval) && (flags & FOLL_SPLIT_PMD)) {
spin_unlock(ptl);
split_huge_pmd(vma, pmd, address);
/* If pmd was left empty, stuff a page table in there quickly */
return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) :
- follow_page_pte(vma, address, pmd, flags);
+ follow_page_pte(vma, address, end, pmd, flags, nr_pages);
}
- page = follow_huge_pmd(vma, address, pmd, flags, page_mask);
+ page = follow_huge_pmd(vma, address, end, pmd, flags, nr_pages);
spin_unlock(ptl);
return page;
}
static struct page *follow_pud_mask(struct vm_area_struct *vma,
- unsigned long address, p4d_t *p4dp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ p4d_t *p4dp, unsigned int flags,
+ unsigned long *nr_pages)
{
pud_t *pudp, pud;
spinlock_t *ptl;
@@ -955,7 +963,7 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
return no_page_table(vma, flags, address);
if (pud_leaf(pud)) {
ptl = pud_lock(mm, pudp);
- page = follow_huge_pud(vma, address, pudp, flags, page_mask);
+ page = follow_huge_pud(vma, address, end, pudp, flags, nr_pages);
spin_unlock(ptl);
if (page)
return page;
@@ -964,13 +972,13 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
if (unlikely(pud_bad(pud)))
return no_page_table(vma, flags, address);
- return follow_pmd_mask(vma, address, pudp, flags, page_mask);
+ return follow_pmd_mask(vma, address, end, pudp, flags, nr_pages);
}
static struct page *follow_p4d_mask(struct vm_area_struct *vma,
- unsigned long address, pgd_t *pgdp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ pgd_t *pgdp, unsigned int flags,
+ unsigned long *nr_pages)
{
p4d_t *p4dp, p4d;
@@ -981,15 +989,16 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
if (!p4d_present(p4d) || p4d_bad(p4d))
return no_page_table(vma, flags, address);
- return follow_pud_mask(vma, address, p4dp, flags, page_mask);
+ return follow_pud_mask(vma, address, end, p4dp, flags, nr_pages);
}
/**
* follow_page_mask - look up a page descriptor from a user-virtual address
* @vma: vm_area_struct mapping @address
* @address: virtual address to look up
+ * @end: virtual address at which to stop batching contiguous pages
* @flags: flags modifying lookup behaviour
- * @page_mask: a pointer to output page_mask
+ * @nr_pages: output; number of contiguous pages the caller can read
*
* @flags can have FOLL_ flags set, defined in <linux/mm.h>
*
@@ -998,15 +1007,17 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
* trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only
* relevant with FOLL_PIN and !FOLL_WRITE.
*
- * On output, @page_mask is set according to the size of the page.
+ * On output, @nr_pages holds how many contiguous pages the folio that includes
+ * the returned page has mapped into this process, so the caller can advance
+ * over a large folio in one step.
*
* Return: the mapped (struct page *), %NULL if no mapping exists, or
* an error pointer if there is a mapping to something not represented
* by a page descriptor (see also vm_normal_page()).
*/
static struct page *follow_page_mask(struct vm_area_struct *vma,
- unsigned long address, unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, unsigned long end,
+ unsigned int flags, unsigned long *nr_pages)
{
pgd_t *pgd;
struct mm_struct *mm = vma->vm_mm;
@@ -1014,13 +1025,13 @@ static struct page *follow_page_mask(struct vm_area_struct *vma,
vma_pgtable_walk_begin(vma);
- *page_mask = 0;
+ *nr_pages = 1;
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
page = no_page_table(vma, flags, address);
else
- page = follow_p4d_mask(vma, address, pgd, flags, page_mask);
+ page = follow_p4d_mask(vma, address, end, pgd, flags, nr_pages);
vma_pgtable_walk_end(vma);
@@ -1198,7 +1209,7 @@ struct page *get_user_page_vma(struct vm_area_struct *vma, unsigned long addr,
unsigned int gup_flags)
{
bool vma_locked = gup_flags & FOLL_VMA_LOCK;
- unsigned long page_mask;
+ unsigned long nr_pages;
struct page *page;
int locked = 1;
bool pfnmap;
@@ -1231,10 +1242,10 @@ struct page *get_user_page_vma(struct vm_area_struct *vma, unsigned long addr,
}
cond_resched();
- /* follow_page_mask() requires @page_mask; it is unused here. */
- page = follow_page_mask(vma, addr,
+ /* This helper hands back a single page; cap the batch at one. */
+ page = follow_page_mask(vma, addr, addr + PAGE_SIZE,
gup_flags | FOLL_TOUCH | FOLL_GET,
- &page_mask);
+ &nr_pages);
if (!IS_ERR_OR_NULL(page)) {
/* Match __get_user_pages(): flush for VIVT/aliasing caches. */
flush_anon_page(vma, page, addr);
@@ -1529,7 +1540,6 @@ static long __get_user_pages(struct mm_struct *mm,
{
long ret = 0, i = 0;
struct vm_area_struct *vma = NULL;
- unsigned long page_mask = 0;
if (!nr_pages)
return 0;
@@ -1544,7 +1554,7 @@ static long __get_user_pages(struct mm_struct *mm,
do {
struct page *page;
- unsigned int page_increm;
+ unsigned long page_increm;
/* first iteration or cross vma bound */
if (!vma || start >= vma->vm_end) {
@@ -1571,7 +1581,7 @@ static long __get_user_pages(struct mm_struct *mm,
pages ? &page : NULL);
if (ret)
goto out;
- page_mask = 0;
+ page_increm = 1;
goto next_page;
}
@@ -1594,7 +1604,8 @@ static long __get_user_pages(struct mm_struct *mm,
}
cond_resched();
- page = follow_page_mask(vma, start, gup_flags, &page_mask);
+ page = follow_page_mask(vma, start, start + nr_pages * PAGE_SIZE,
+ gup_flags, &page_increm);
if (!page || PTR_ERR(page) == -EMLINK) {
ret = faultin_page(vma, start, gup_flags,
PTR_ERR(page) == -EMLINK, locked);
@@ -1627,7 +1638,6 @@ static long __get_user_pages(struct mm_struct *mm,
goto out;
}
next_page:
- page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
if (page_increm > nr_pages)
page_increm = nr_pages;
--
2.53.0-Meta
next prev parent reply other threads:[~2026-07-24 22:30 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-07-24 22:29 [PATCH RFC v4 0/12] mm: use per-VMA lock in __access_remote_vm for improved monitoring reliability Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 01/12] x86/mm: add untagged_addr_remote_unlocked() Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 02/12] riscv/mm: " Rik van Riel
2026-07-24 22:29 ` Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 03/12] mm: rename get_user_page_vma_remote() to get_user_page_lookup_vma() Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 04/12] mm/gup: let check_vma_flags() ignore selected VMA flags Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 05/12] mm/gup: add get_user_page_vma() to fault in a page under a held lock Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 06/12] mm: use per-VMA lock in __access_remote_vm() for single-VMA accesses Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 07/12] mm: read remote strings under the per-VMA lock Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 08/12] selftests/mm: cover /proc/pid/mem access to VM_PFNMAP memory Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 09/12] mm/gup: build get_user_page_lookup_vma() on get_user_page_vma() Rik van Riel
2026-07-24 22:29 ` Rik van Riel [this message]
2026-07-24 22:29 ` [PATCH RFC v4 11/12] mm/gup: batch contiguous PTE-mapped large folios in follow_page_mask() Rik van Riel
2026-07-24 22:29 ` [PATCH RFC v4 12/12] selftests/mm: add a slow-GUP content and COW test for mTHP Rik van Riel
2026-07-26 11:56 ` Mike Rapoport
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260724222934.1463812-11-riel@surriel.com \
--to=riel@surriel.com \
--cc=akpm@linux-foundation.org \
--cc=dave.hansen@linux.intel.com \
--cc=david@kernel.org \
--cc=jgg@ziepe.ca \
--cc=jhubbard@nvidia.com \
--cc=kernel-team@meta.com \
--cc=liam@infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=mhocko@suse.com \
--cc=peterx@redhat.com \
--cc=peterz@infradead.org \
--cc=rppt@kernel.org \
--cc=surenb@google.com \
--cc=usamaarif642@gmail.com \
--cc=vbabka@kernel.org \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.