From: Swarna Prabhu <sw.prabhu6@gmail.com>
To: akpm@linux-foundation.org, david@kernel.org, jgg@ziepe.ca,
jhubbard@nvidia.com, viro@zeniv.linux.org.uk
Cc: linux-mm@kvack.org, linux-block@vger.kernel.org,
linux-kernel@vger.kernel.org, dave@stgolabs.net,
riel@surriel.com, yeoreum.yun@arm.com, hch@lst.de,
Swarna Prabhu <s.prabhu@samsung.com>
Subject: [PATCH 3/5] mm/gup: make the GUP-fast walkers record into struct gup_out
Date: Wed, 18 Nov 2026 13:34:56 -0800 [thread overview]
Message-ID: <20261118213458.377258-4-sw.prabhu6@gmail.com> (raw)
In-Reply-To: <20261118213458.377258-1-sw.prabhu6@gmail.com>
From: Swarna Prabhu <s.prabhu@samsung.com>
Pass 'struct gup_out' down the GUP-fast walkers and record acquired
pages through 'gup_out_add_range()' replacing pages array indexing and
pointer advancement. Record each existing PTE batch with one append.
Retain the walker's page-count return value so that callers can detect
short walks. Have gup_fast() obtain its result from the descriptor and
reset the count after releasing pins when a concurrent fork is detected.
Move fallback handling into gup_fast_fallback_out(), sharing the
descriptor with the slow path so recording continues after the pages
acquired by GUP-fast. Keep gup_fast_fallback() as an array-based wrapper
that initializes an empty descriptor.
No functional change.
Signed-off-by: Swarna Prabhu <s.prabhu@samsung.com>
Assisted-by: LLM
---
mm/gup.c | 94 +++++++++++++++++++++++++++++++++-----------------------
1 file changed, 56 insertions(+), 38 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index 0b8dd3fde06b..43b92058a213 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -3146,7 +3146,7 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags)
*/
static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
unsigned long addr, unsigned long end,
- unsigned int flags, struct page **pages)
+ unsigned int flags, struct gup_out *out)
{
unsigned long nr_pages = 0, nr_batch;
struct folio *folio = NULL;
@@ -3158,7 +3158,6 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
do {
pte_t pte = ptep_get_lockless(ptep);
struct page *page;
- unsigned long i;
nr_batch = 1;
@@ -3229,8 +3228,7 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
nr_batch = 1;
folio_set_referenced(folio);
- for (i = 0; i < nr_batch; i++)
- pages[nr_pages + i] = page + i;
+ gup_out_add_range(out, folio, page, nr_batch);
nr_pages += nr_batch;
folio = NULL;
} while (ptep += nr_batch, addr += nr_batch * PAGE_SIZE, addr != end);
@@ -3254,7 +3252,7 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
*/
static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
unsigned long addr, unsigned long end,
- unsigned int flags, struct page **pages)
+ unsigned int flags, struct gup_out *out)
{
return 0;
}
@@ -3262,11 +3260,11 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp,
unsigned long addr, unsigned long end,
- unsigned int flags, struct page **pages)
+ unsigned int flags, struct gup_out *out)
{
struct page *mapped_page, *page;
struct folio *folio;
- unsigned long nr_pages, i;
+ unsigned long nr_pages;
/* See gup_fast_pte_range() */
if (pmd_protnone(orig))
@@ -3301,19 +3299,18 @@ static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp,
return 0;
}
- for (i = 0; i < nr_pages; i++)
- *(pages++) = page++;
+ gup_out_add_range(out, folio, page, nr_pages);
folio_set_referenced(folio);
return nr_pages;
}
static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp,
unsigned long addr, unsigned long end,
- unsigned int flags, struct page **pages)
+ unsigned int flags, struct gup_out *out)
{
struct page *mapped_page, *page;
struct folio *folio;
- unsigned long nr_pages, i;
+ unsigned long nr_pages;
if (!pud_access_permitted(orig, flags & FOLL_WRITE))
return 0;
@@ -3345,15 +3342,14 @@ static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp,
return 0;
}
- for (i = 0; i < nr_pages; i++)
- *(pages++) = page++;
+ gup_out_add_range(out, folio, page, nr_pages);
folio_set_referenced(folio);
return nr_pages;
}
static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud,
unsigned long addr, unsigned long end,
- unsigned int flags, struct page **pages)
+ unsigned int flags, struct gup_out *out)
{
unsigned long next, nr_pages = 0, chunk_nr_pages;
pmd_t *pmdp;
@@ -3369,12 +3365,12 @@ static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud,
if (unlikely(pmd_leaf(pmd))) {
chunk_nr_pages = gup_fast_pmd_leaf(pmd, pmdp, addr,
next, flags,
- &pages[nr_pages]);
+ out);
} else
chunk_nr_pages = gup_fast_pte_range(pmd, pmdp, addr,
next, flags,
- &pages[nr_pages]);
+ out);
nr_pages += chunk_nr_pages;
if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
break;
@@ -3385,7 +3381,7 @@ static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud,
static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d,
unsigned long addr, unsigned long end,
- unsigned int flags, struct page **pages)
+ unsigned int flags, struct gup_out *out)
{
unsigned long next, nr_pages = 0, chunk_nr_pages;
pud_t *pudp;
@@ -3400,11 +3396,11 @@ static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d,
if (unlikely(pud_leaf(pud)))
chunk_nr_pages = gup_fast_pud_leaf(pud, pudp, addr,
next, flags,
- &pages[nr_pages]);
+ out);
else
chunk_nr_pages = gup_fast_pmd_range(pudp, pud, addr,
next, flags,
- &pages[nr_pages]);
+ out);
nr_pages += chunk_nr_pages;
if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
break;
@@ -3414,7 +3410,7 @@ static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d,
}
static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
- unsigned long end, unsigned int flags, struct page **pages)
+ unsigned long end, unsigned int flags, struct gup_out *out)
{
unsigned long next, nr_pages = 0, chunk_nr_pages;
p4d_t *p4dp;
@@ -3428,7 +3424,7 @@ static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long ad
break;
BUILD_BUG_ON(p4d_leaf(p4d));
chunk_nr_pages = gup_fast_pud_range(p4dp, p4d, addr, next,
- flags, &pages[nr_pages]);
+ flags, out);
nr_pages += chunk_nr_pages;
if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
break;
@@ -3439,7 +3435,7 @@ static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long ad
static unsigned long gup_fast_pgd_range(unsigned long addr,
unsigned long end, unsigned int flags,
- struct page **pages)
+ struct gup_out *out)
{
unsigned long next, nr_pages = 0, chunk_nr_pages;
pgd_t *pgdp;
@@ -3453,7 +3449,7 @@ static unsigned long gup_fast_pgd_range(unsigned long addr,
break;
BUILD_BUG_ON(pgd_leaf(pgd));
chunk_nr_pages = gup_fast_p4d_range(pgdp, pgd, addr, next,
- flags, &pages[nr_pages]);
+ flags, out);
nr_pages += chunk_nr_pages;
if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
break;
@@ -3464,7 +3460,7 @@ static unsigned long gup_fast_pgd_range(unsigned long addr,
#else
static inline unsigned long gup_fast_pgd_range(unsigned long addr,
unsigned long end, unsigned int flags,
- struct page **pages)
+ struct gup_out *out)
{
return 0;
}
@@ -3481,12 +3477,27 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end)
}
#endif
+/*
+ * Undo what GUP-fast recorded in @out.
+ *
+ * Don't perform any sanity checks because we might have raced with fork() and
+ * some anonymous pages might now actually be shared -- which is why we're
+ * unpinning after all.
+ */
+static void gup_fast_unpin(struct gup_out *out)
+{
+ gup_fast_unpin_user_pages(out->pages, out->nr_pages);
+ out->nr_pages = 0;
+}
+
static unsigned long gup_fast(unsigned long start, unsigned long end,
- unsigned int gup_flags, struct page **pages)
+ unsigned int gup_flags, struct gup_out *out)
{
- unsigned long flags, nr_pinned;
+ unsigned long flags;
unsigned seq;
+ VM_WARN_ON_ONCE(out->nr_pages);
+
if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) ||
!gup_fast_permitted(start, end))
return 0;
@@ -3508,7 +3519,7 @@ static unsigned long gup_fast(unsigned long start, unsigned long end,
* that come from callers of tlb_remove_table_sync_one().
*/
local_irq_save(flags);
- nr_pinned = gup_fast_pgd_range(start, end, gup_flags, pages);
+ gup_fast_pgd_range(start, end, gup_flags, out);
local_irq_restore(flags);
/*
@@ -3517,22 +3528,22 @@ static unsigned long gup_fast(unsigned long start, unsigned long end,
*/
if (gup_flags & FOLL_PIN) {
if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) {
- gup_fast_unpin_user_pages(pages, nr_pinned);
+ gup_fast_unpin(out);
return 0;
} else {
- sanity_check_pinned_pages(pages, nr_pinned);
+ sanity_check_pinned_pages(out->pages, out->nr_pages);
}
}
- return nr_pinned;
+ return out->nr_pages;
}
-static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
- unsigned int gup_flags, struct page **pages)
+static long gup_fast_fallback_out(unsigned long start, unsigned long nr_pages,
+ unsigned int gup_flags, struct gup_out *out)
{
unsigned long len, end;
unsigned long nr_pinned;
int locked = 0;
- int ret;
+ long ret;
if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
FOLL_FORCE | FOLL_PIN | FOLL_GET |
@@ -3553,16 +3564,15 @@ static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
if (end > TASK_SIZE_MAX)
return -EFAULT;
- nr_pinned = gup_fast(start, end, gup_flags, pages);
+ nr_pinned = gup_fast(start, end, gup_flags, out);
if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
return nr_pinned;
/* Slow path: try to get the remaining pages with get_user_pages */
start += nr_pinned << PAGE_SHIFT;
- pages += nr_pinned;
- ret = __gup_longterm_locked(current->mm, start, nr_pages - nr_pinned,
- pages, &locked,
- gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE);
+ ret = __gup_longterm_locked_out(current->mm, start,
+ nr_pages - nr_pinned, out, &locked,
+ gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE);
if (ret < 0) {
/*
* The caller has to unpin the pages we already pinned so
@@ -3575,6 +3585,14 @@ static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
return ret + nr_pinned;
}
+static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
+ unsigned int gup_flags, struct page **pages)
+{
+ struct gup_out out = GUP_OUT_PAGES(pages);
+
+ return gup_fast_fallback_out(start, nr_pages, gup_flags, &out);
+}
+
/**
* get_user_pages_fast_only() - pin user pages in memory
* @start: starting user address
--
2.53.0
next prev parent reply other threads:[~2026-11-18 21:37 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-18 21:34 [RFC PATCH 0/5] gup: introduce folio-range interface for pinning user memory Swarna Prabhu
2026-11-18 21:34 ` [PATCH 1/5] mm/gup: add struct gup_out output descriptor Swarna Prabhu
2026-11-18 21:34 ` [PATCH 2/5] mm/gup: make the slow path walkers record into struct gup_out Swarna Prabhu
2026-11-18 21:34 ` Swarna Prabhu [this message]
2026-11-18 21:34 ` [PATCH 4/5] mm/gup: return pinned memory as folio ranges Swarna Prabhu
2026-11-18 21:34 ` [PATCH 5/5] iov_iter: extract bvecs of user-backed iters from " Swarna Prabhu
2026-11-18 23:28 ` [RFC PATCH 0/5] gup: introduce folio-range interface for pinning user memory David Hildenbrand (Arm)
2026-11-19 0:19 ` John Hubbard
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261118213458.377258-4-sw.prabhu6@gmail.com \
--to=sw.prabhu6@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=dave@stgolabs.net \
--cc=david@kernel.org \
--cc=hch@lst.de \
--cc=jgg@ziepe.ca \
--cc=jhubbard@nvidia.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=riel@surriel.com \
--cc=s.prabhu@samsung.com \
--cc=viro@zeniv.linux.org.uk \
--cc=yeoreum.yun@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox