* [PATCH 6.12.y 2/4] mm/gup: remove pXX_devmap usage from get_user_pages()
2026-11-12 2:34 ` [PATCH 6.12.y 1/4] mm/gup: remove redundant check for PCI P2PDMA page Sasha Levin
@ 2026-11-12 2:34 ` Sasha Levin
2026-11-12 2:34 ` [PATCH 6.12.y 3/4] mm/gup: remove dead pgmap refcounting code Sasha Levin
2026-11-12 2:34 ` [PATCH 6.12.y 4/4] mm/gup: recheck the PUD under PTL in follow_pud_mask() Sasha Levin
2 siblings, 0 replies; 5+ messages in thread
From: Sasha Levin @ 2026-11-12 2:34 UTC (permalink / raw)
To: stable
Cc: Alistair Popple, Jason Gunthorpe, Dan Williams, Balbir Singh,
Björn Töpel, Björn Töpel, Christoph Hellwig,
Chunyan Zhang, David Hildenbrand, Deepak Gupta, Gerald Schaefer,
Inki Dae, John Groves, John Hubbard, Lorenzo Stoakes,
Matthew Wilcox (Oracle), Will Deacon, Andrew Morton, Sasha Levin
From: Alistair Popple <apopple@nvidia.com>
[ Upstream commit fd2825b0760a10a9626986cca64f5664302ffdfc ]
GUP uses pXX_devmap() calls to see if it needs to a get a reference on the
associated pgmap data structure to ensure the pages won't go away.
However it's a driver responsibility to ensure that if pages are mapped
(ie. discoverable by GUP) that they are not offlined or removed from the
memmap so there is no need to hold a reference on the pgmap data structure
to ensure this.
Furthermore mappings with PFN_DEV are no longer created, hence this
effectively dead code anyway so can be removed.
Link: https://lkml.kernel.org/r/708b2be76876659ec5261fe5d059b07268b98b36.1750323463.git-series.apopple@nvidia.com
Signed-off-by: Alistair Popple <apopple@nvidia.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Cc: Balbir Singh <balbirs@nvidia.com>
Cc: Björn Töpel <bjorn@kernel.org>
Cc: Björn Töpel <bjorn@rivosinc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Chunyan Zhang <zhang.lyra@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Deepak Gupta <debug@rivosinc.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Inki Dae <m.szyprowski@samsung.com>
Cc: John Groves <john@groves.net>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: 79cb51d538ab ("mm/gup: recheck the PUD under PTL in follow_pud_mask()")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
include/linux/huge_mm.h | 3 -
mm/gup.c | 160 ++--------------------------------------
mm/huge_memory.c | 40 ----------
3 files changed, 5 insertions(+), 198 deletions(-)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 10e41e8b16349..0115522b102b4 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -441,9 +441,6 @@ static inline bool folio_test_pmd_mappable(struct folio *folio)
return folio_order(folio) >= HPAGE_PMD_ORDER;
}
-struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
- pmd_t *pmd, int flags, struct dev_pagemap **pgmap);
-
vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
extern struct folio *huge_zero_folio;
diff --git a/mm/gup.c b/mm/gup.c
index b539ab0a29fce..914e586137a2f 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -641,31 +641,9 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
return NULL;
pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
-
- if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) &&
- pud_devmap(pud)) {
- /*
- * device mapped pages can only be returned if the caller
- * will manage the page reference count.
- *
- * At least one of FOLL_GET | FOLL_PIN must be set, so
- * assert that here:
- */
- if (!(flags & (FOLL_GET | FOLL_PIN)))
- return ERR_PTR(-EEXIST);
-
- if (flags & FOLL_TOUCH)
- touch_pud(vma, addr, pudp, flags & FOLL_WRITE);
-
- ctx->pgmap = get_dev_pagemap(pfn, ctx->pgmap);
- if (!ctx->pgmap)
- return ERR_PTR(-EFAULT);
- }
-
page = pfn_to_page(pfn);
- if (!pud_devmap(pud) && !pud_write(pud) &&
- gup_must_unshare(vma, flags, page))
+ if (!pud_write(pud) && gup_must_unshare(vma, flags, page))
return ERR_PTR(-EMLINK);
ret = try_grab_folio(page_folio(page), 1, flags);
@@ -864,8 +842,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
page = vm_normal_page(vma, address, pte);
/*
- * We only care about anon pages in can_follow_write_pte() and don't
- * have to worry about pte_devmap() because they are never anon.
+ * We only care about anon pages in can_follow_write_pte().
*/
if ((flags & FOLL_WRITE) &&
!can_follow_write_pte(pte, page, vma, flags)) {
@@ -873,18 +850,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
goto out;
}
- if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
- /*
- * Only return device mapping pages in the FOLL_GET or FOLL_PIN
- * case since they are only valid while holding the pgmap
- * reference.
- */
- *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
- if (*pgmap)
- page = pte_page(pte);
- else
- goto no_page;
- } else if (unlikely(!page)) {
+ if (unlikely(!page)) {
if (flags & FOLL_DUMP) {
/* Avoid special (like zero) pages in core dumps */
page = ERR_PTR(-EFAULT);
@@ -966,14 +932,6 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
return no_page_table(vma, flags, address);
if (!pmd_present(pmdval))
return no_page_table(vma, flags, address);
- if (pmd_devmap(pmdval)) {
- ptl = pmd_lock(mm, pmd);
- page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
- spin_unlock(ptl);
- if (page)
- return page;
- return no_page_table(vma, flags, address);
- }
if (likely(!pmd_leaf(pmdval)))
return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
@@ -2937,7 +2895,7 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
int *nr)
{
struct dev_pagemap *pgmap = NULL;
- int nr_start = *nr, ret = 0;
+ int ret = 0;
pte_t *ptep, *ptem;
ptem = ptep = pte_offset_map(&pmd, addr);
@@ -2961,16 +2919,7 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
if (!pte_access_permitted(pte, flags & FOLL_WRITE))
goto pte_unmap;
- if (pte_devmap(pte)) {
- if (unlikely(flags & FOLL_LONGTERM))
- goto pte_unmap;
-
- pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
- if (unlikely(!pgmap)) {
- gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
- goto pte_unmap;
- }
- } else if (pte_special(pte))
+ if (pte_special(pte))
goto pte_unmap;
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
@@ -3041,91 +2990,6 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
}
#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
-#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static int gup_fast_devmap_leaf(unsigned long pfn, unsigned long addr,
- unsigned long end, unsigned int flags, struct page **pages, int *nr)
-{
- int nr_start = *nr;
- struct dev_pagemap *pgmap = NULL;
-
- do {
- struct folio *folio;
- struct page *page = pfn_to_page(pfn);
-
- pgmap = get_dev_pagemap(pfn, pgmap);
- if (unlikely(!pgmap)) {
- gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
- break;
- }
-
- folio = try_grab_folio_fast(page, 1, flags);
- if (!folio) {
- gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
- break;
- }
- folio_set_referenced(folio);
- pages[*nr] = page;
- (*nr)++;
- pfn++;
- } while (addr += PAGE_SIZE, addr != end);
-
- put_dev_pagemap(pgmap);
- return addr == end;
-}
-
-static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
- unsigned long end, unsigned int flags, struct page **pages,
- int *nr)
-{
- unsigned long fault_pfn;
- int nr_start = *nr;
-
- fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
- if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr))
- return 0;
-
- if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
- gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
- return 0;
- }
- return 1;
-}
-
-static int gup_fast_devmap_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
- unsigned long end, unsigned int flags, struct page **pages,
- int *nr)
-{
- unsigned long fault_pfn;
- int nr_start = *nr;
-
- fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
- if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr))
- return 0;
-
- if (unlikely(pud_val(orig) != pud_val(*pudp))) {
- gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
- return 0;
- }
- return 1;
-}
-#else
-static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
- unsigned long end, unsigned int flags, struct page **pages,
- int *nr)
-{
- BUILD_BUG();
- return 0;
-}
-
-static int gup_fast_devmap_pud_leaf(pud_t pud, pud_t *pudp, unsigned long addr,
- unsigned long end, unsigned int flags, struct page **pages,
- int *nr)
-{
- BUILD_BUG();
- return 0;
-}
-#endif
-
static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags, struct page **pages,
int *nr)
@@ -3140,13 +3004,6 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
if (pmd_special(orig))
return 0;
- if (pmd_devmap(orig)) {
- if (unlikely(flags & FOLL_LONGTERM))
- return 0;
- return gup_fast_devmap_pmd_leaf(orig, pmdp, addr, end, flags,
- pages, nr);
- }
-
page = pmd_page(orig);
refs = record_subpages(page, PMD_SIZE, addr, end, pages + *nr);
@@ -3187,13 +3044,6 @@ static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
if (pud_special(orig))
return 0;
- if (pud_devmap(orig)) {
- if (unlikely(flags & FOLL_LONGTERM))
- return 0;
- return gup_fast_devmap_pud_leaf(orig, pudp, addr, end, flags,
- pages, nr);
- }
-
page = pud_page(orig);
refs = record_subpages(page, PUD_SIZE, addr, end, pages + *nr);
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index d175e93988a92..bc10d50a08fd1 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1575,46 +1575,6 @@ void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
update_mmu_cache_pmd(vma, addr, pmd);
}
-struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
- pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
-{
- unsigned long pfn = pmd_pfn(*pmd);
- struct mm_struct *mm = vma->vm_mm;
- struct page *page;
- int ret;
-
- assert_spin_locked(pmd_lockptr(mm, pmd));
-
- if (flags & FOLL_WRITE && !pmd_write(*pmd))
- return NULL;
-
- if (pmd_present(*pmd) && pmd_devmap(*pmd))
- /* pass */;
- else
- return NULL;
-
- if (flags & FOLL_TOUCH)
- touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);
-
- /*
- * device mapped pages can only be returned if the
- * caller will manage the page reference count.
- */
- if (!(flags & (FOLL_GET | FOLL_PIN)))
- return ERR_PTR(-EEXIST);
-
- pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
- *pgmap = get_dev_pagemap(pfn, *pgmap);
- if (!*pgmap)
- return ERR_PTR(-EFAULT);
- page = pfn_to_page(pfn);
- ret = try_grab_folio(page_folio(page), 1, flags);
- if (ret)
- page = ERR_PTR(ret);
-
- return page;
-}
-
int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 6.12.y 3/4] mm/gup: remove dead pgmap refcounting code
2026-11-12 2:34 ` [PATCH 6.12.y 1/4] mm/gup: remove redundant check for PCI P2PDMA page Sasha Levin
2026-11-12 2:34 ` [PATCH 6.12.y 2/4] mm/gup: remove pXX_devmap usage from get_user_pages() Sasha Levin
@ 2026-11-12 2:34 ` Sasha Levin
2026-11-12 2:34 ` [PATCH 6.12.y 4/4] mm/gup: recheck the PUD under PTL in follow_pud_mask() Sasha Levin
2 siblings, 0 replies; 5+ messages in thread
From: Sasha Levin @ 2026-11-12 2:34 UTC (permalink / raw)
To: stable
Cc: Alistair Popple, David Hildenbrand, Jason Gunthorpe, Dan Williams,
John Hubbard, Oscar Salvador, Peter Xu, Andrew Morton,
Sasha Levin
From: Alistair Popple <apopple@nvidia.com>
[ Upstream commit d3f7922b929a92873e14f7c61a026a6b576e8773 ]
Prior to commit aed877c2b425 ("device/dax: properly refcount device dax
pages when mapping") ZONE_DEVICE pages were not fully reference counted
when mapped into user page tables. Instead GUP would take a reference on
the associated pgmap to ensure the results of pfn_to_page() remained
valid.
This is no longer required and most of the code was removed by commit
fd2825b0760a ("mm/gup: remove pXX_devmap usage from get_user_pages()").
Finish cleaning this up by removing the dead calls to put_dev_pagemap()
and the temporary context struct.
Link: https://lkml.kernel.org/r/20250903225926.34702-1-apopple@nvidia.com
Signed-off-by: Alistair Popple <apopple@nvidia.com>
Acked-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: 79cb51d538ab ("mm/gup: recheck the PUD under PTL in follow_pud_mask()")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/gup.c | 67 ++++++++++++++++++++++----------------------------------
1 file changed, 26 insertions(+), 41 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index 914e586137a2f..b8367869f7928 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -27,11 +27,6 @@
#include "internal.h"
-struct follow_page_context {
- struct dev_pagemap *pgmap;
- unsigned int page_mask;
-};
-
static inline void sanity_check_pinned_pages(struct page **pages,
unsigned long npages)
{
@@ -624,7 +619,7 @@ static struct page *no_page_table(struct vm_area_struct *vma,
#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
static struct page *follow_huge_pud(struct vm_area_struct *vma,
unsigned long addr, pud_t *pudp,
- int flags, struct follow_page_context *ctx)
+ int flags, unsigned long *page_mask)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
@@ -650,7 +645,7 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
if (ret)
page = ERR_PTR(ret);
else
- ctx->page_mask = HPAGE_PUD_NR - 1;
+ *page_mask = HPAGE_PUD_NR - 1;
return page;
}
@@ -696,7 +691,7 @@ static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
static struct page *follow_huge_pmd(struct vm_area_struct *vma,
unsigned long addr, pmd_t *pmd,
unsigned int flags,
- struct follow_page_context *ctx)
+ unsigned long *page_mask)
{
struct mm_struct *mm = vma->vm_mm;
pmd_t pmdval = *pmd;
@@ -733,7 +728,7 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma,
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
- ctx->page_mask = HPAGE_PMD_NR - 1;
+ *page_mask = HPAGE_PMD_NR - 1;
return page;
}
@@ -741,7 +736,7 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma,
#else /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */
static struct page *follow_huge_pud(struct vm_area_struct *vma,
unsigned long addr, pud_t *pudp,
- int flags, struct follow_page_context *ctx)
+ int flags, unsigned long *page_mask)
{
return NULL;
}
@@ -749,7 +744,7 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
static struct page *follow_huge_pmd(struct vm_area_struct *vma,
unsigned long addr, pmd_t *pmd,
unsigned int flags,
- struct follow_page_context *ctx)
+ unsigned long *page_mask)
{
return NULL;
}
@@ -815,8 +810,7 @@ static inline bool can_follow_write_pte(pte_t pte, struct page *page,
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
- unsigned long address, pmd_t *pmd, unsigned int flags,
- struct dev_pagemap **pgmap)
+ unsigned long address, pmd_t *pmd, unsigned int flags)
{
struct mm_struct *mm = vma->vm_mm;
struct folio *folio;
@@ -919,7 +913,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp,
unsigned int flags,
- struct follow_page_context *ctx)
+ unsigned long *page_mask)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
@@ -933,7 +927,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
if (!pmd_present(pmdval))
return no_page_table(vma, flags, address);
if (likely(!pmd_leaf(pmdval)))
- return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
+ return follow_page_pte(vma, address, pmd, flags);
if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags))
return no_page_table(vma, flags, address);
@@ -946,16 +940,16 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
}
if (unlikely(!pmd_leaf(pmdval))) {
spin_unlock(ptl);
- return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
+ return follow_page_pte(vma, address, pmd, flags);
}
if (pmd_trans_huge(pmdval) && (flags & FOLL_SPLIT_PMD)) {
spin_unlock(ptl);
split_huge_pmd(vma, pmd, address);
/* If pmd was left empty, stuff a page table in there quickly */
return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) :
- follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
+ follow_page_pte(vma, address, pmd, flags);
}
- page = follow_huge_pmd(vma, address, pmd, flags, ctx);
+ page = follow_huge_pmd(vma, address, pmd, flags, page_mask);
spin_unlock(ptl);
return page;
}
@@ -963,7 +957,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
static struct page *follow_pud_mask(struct vm_area_struct *vma,
unsigned long address, p4d_t *p4dp,
unsigned int flags,
- struct follow_page_context *ctx)
+ unsigned long *page_mask)
{
pud_t *pudp, pud;
spinlock_t *ptl;
@@ -976,7 +970,7 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
return no_page_table(vma, flags, address);
if (pud_leaf(pud)) {
ptl = pud_lock(mm, pudp);
- page = follow_huge_pud(vma, address, pudp, flags, ctx);
+ page = follow_huge_pud(vma, address, pudp, flags, page_mask);
spin_unlock(ptl);
if (page)
return page;
@@ -985,13 +979,13 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
if (unlikely(pud_bad(pud)))
return no_page_table(vma, flags, address);
- return follow_pmd_mask(vma, address, pudp, flags, ctx);
+ return follow_pmd_mask(vma, address, pudp, flags, page_mask);
}
static struct page *follow_p4d_mask(struct vm_area_struct *vma,
unsigned long address, pgd_t *pgdp,
unsigned int flags,
- struct follow_page_context *ctx)
+ unsigned long *page_mask)
{
p4d_t *p4dp, p4d;
@@ -1002,7 +996,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
if (!p4d_present(p4d) || p4d_bad(p4d))
return no_page_table(vma, flags, address);
- return follow_pud_mask(vma, address, p4dp, flags, ctx);
+ return follow_pud_mask(vma, address, p4dp, flags, page_mask);
}
/**
@@ -1010,20 +1004,16 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
* @vma: vm_area_struct mapping @address
* @address: virtual address to look up
* @flags: flags modifying lookup behaviour
- * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
- * pointer to output page_mask
+ * @page_mask: a pointer to output page_mask
*
* @flags can have FOLL_ flags set, defined in <linux/mm.h>
*
- * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
- * the device's dev_pagemap metadata to avoid repeating expensive lookups.
- *
* When getting an anonymous page and the caller has to trigger unsharing
* of a shared anonymous page first, -EMLINK is returned. The caller should
* trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only
* relevant with FOLL_PIN and !FOLL_WRITE.
*
- * On output, the @ctx->page_mask is set according to the size of the page.
+ * On output, @page_mask is set according to the size of the page.
*
* Return: the mapped (struct page *), %NULL if no mapping exists, or
* an error pointer if there is a mapping to something not represented
@@ -1031,7 +1021,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
*/
static struct page *follow_page_mask(struct vm_area_struct *vma,
unsigned long address, unsigned int flags,
- struct follow_page_context *ctx)
+ unsigned long *page_mask)
{
pgd_t *pgd;
struct mm_struct *mm = vma->vm_mm;
@@ -1039,13 +1029,13 @@ static struct page *follow_page_mask(struct vm_area_struct *vma,
vma_pgtable_walk_begin(vma);
- ctx->page_mask = 0;
+ *page_mask = 0;
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
page = no_page_table(vma, flags, address);
else
- page = follow_p4d_mask(vma, address, pgd, flags, ctx);
+ page = follow_p4d_mask(vma, address, pgd, flags, page_mask);
vma_pgtable_walk_end(vma);
@@ -1389,7 +1379,7 @@ static long __get_user_pages(struct mm_struct *mm,
{
long ret = 0, i = 0;
struct vm_area_struct *vma = NULL;
- struct follow_page_context ctx = { NULL };
+ unsigned long page_mask = 0;
if (!nr_pages)
return 0;
@@ -1427,7 +1417,7 @@ static long __get_user_pages(struct mm_struct *mm,
pages ? &page : NULL);
if (ret)
goto out;
- ctx.page_mask = 0;
+ page_mask = 0;
goto next_page;
}
@@ -1450,7 +1440,7 @@ static long __get_user_pages(struct mm_struct *mm,
}
cond_resched();
- page = follow_page_mask(vma, start, gup_flags, &ctx);
+ page = follow_page_mask(vma, start, gup_flags, &page_mask);
if (!page || PTR_ERR(page) == -EMLINK) {
ret = faultin_page(vma, start, gup_flags,
PTR_ERR(page) == -EMLINK, locked);
@@ -1483,7 +1473,7 @@ static long __get_user_pages(struct mm_struct *mm,
goto out;
}
next_page:
- page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
+ page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
if (page_increm > nr_pages)
page_increm = nr_pages;
@@ -1533,8 +1523,6 @@ static long __get_user_pages(struct mm_struct *mm,
nr_pages -= page_increm;
} while (nr_pages);
out:
- if (ctx.pgmap)
- put_dev_pagemap(ctx.pgmap);
return i ? i : ret;
}
@@ -2894,7 +2882,6 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags, struct page **pages,
int *nr)
{
- struct dev_pagemap *pgmap = NULL;
int ret = 0;
pte_t *ptep, *ptem;
@@ -2966,8 +2953,6 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
ret = 1;
pte_unmap:
- if (pgmap)
- put_dev_pagemap(pgmap);
pte_unmap(ptem);
return ret;
}
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 6.12.y 4/4] mm/gup: recheck the PUD under PTL in follow_pud_mask()
2026-11-12 2:34 ` [PATCH 6.12.y 1/4] mm/gup: remove redundant check for PCI P2PDMA page Sasha Levin
2026-11-12 2:34 ` [PATCH 6.12.y 2/4] mm/gup: remove pXX_devmap usage from get_user_pages() Sasha Levin
2026-11-12 2:34 ` [PATCH 6.12.y 3/4] mm/gup: remove dead pgmap refcounting code Sasha Levin
@ 2026-11-12 2:34 ` Sasha Levin
2 siblings, 0 replies; 5+ messages in thread
From: Sasha Levin @ 2026-11-12 2:34 UTC (permalink / raw)
To: stable
Cc: David Hildenbrand (Arm), Josef Bacik, Muhammad Usama Anjum,
Sasha Levin
From: "David Hildenbrand (Arm)" <david@kernel.org>
[ Upstream commit 79cb51d538abc459f74e046d27797d36b133ce0e ]
follow_pud_mask() reads the PUD without a lock. If it is a leaf, it takes
the lock and calls follow_huge_pud(), which reads the entry again, but
only tests whether it is present.
A leaf can be zapped concurrently and a page fault can fill the empty
entry with a page table.
This currently only applies when device-dax makes use of PUD entries,
and the fault after e.g., MADV_DONTNEED would find THPs to be disabled.
That requires a sequence like
1) Fault in the PUD
2) Disable THP
3) Have GUP race against concurrent MADV_DONTNEED + refault
Which is rather unlikely to happen in practice, which is why we never
received a report about such a problem.
Before commit 1b1676180246 ("mm/gup: handle huge pud for
follow_pud_mask()"), we performed that check in follow_devmap_pud() by
testing for pud_devmap().
Let's fix it by just structuring follow_pud_mask() similar to
follow_pmd_mask(), minus protnone stuff that does not apply and minus
FOLL_SPLIT_PMD handling. That makes it much easier to spot any such
inconsistencies.
Perform the pud_bad() check early in follow_pmd_mask() and just forward
the PUD value. We can drop the pud_present() check in follow_huge_pud(),
just like we have in follow_huge_pmd().
Even though we are performing more changes than strictly necessary, the
fix should be easy to backport.
Fixes: 1b1676180246 ("mm/gup: handle huge pud for follow_pud_mask()")
Cc: stable@vger.kernel.org
Reported-by: Josef Bacik <josef@toxicpanda.com>
Tested-by: Muhammad Usama Anjum <usama.anjum@arm.com>
Link: https://patch.msgid.link/20261031-fixes_core-v3-19-d6437171ecc3@kernel.org
Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
[ sashal: adjusted hunk context for the older pud_write() permission check in place of can_follow_write_pud() ]
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/gup.c | 32 ++++++++++++++++++--------------
1 file changed, 18 insertions(+), 14 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index b8367869f7928..5c7ebf897476a 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -632,9 +632,6 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
if ((flags & FOLL_WRITE) && !pud_write(pud))
return NULL;
- if (!pud_present(pud))
- return NULL;
-
pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
page = pfn_to_page(pfn);
@@ -911,15 +908,17 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
}
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
- unsigned long address, pud_t *pudp,
- unsigned int flags,
- unsigned long *page_mask)
+ unsigned long address, pud_t *pudp, pud_t pud,
+ unsigned int flags, unsigned long *page_mask)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
struct page *page;
struct mm_struct *mm = vma->vm_mm;
+ if (unlikely(pud_bad(pud)))
+ return no_page_table(vma, flags, address);
+
pmd = pmd_offset(pudp, address);
pmdval = pmdp_get_lockless(pmd);
if (pmd_none(pmdval))
@@ -968,18 +967,23 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
pud = pudp_get(pudp);
if (!pud_present(pud))
return no_page_table(vma, flags, address);
- if (pud_leaf(pud)) {
- ptl = pud_lock(mm, pudp);
- page = follow_huge_pud(vma, address, pudp, flags, page_mask);
+ if (!pud_leaf(pud))
+ return follow_pmd_mask(vma, address, pudp, pud, flags, page_mask);
+
+ ptl = pud_lock(mm, pudp);
+ pud = *pudp;
+ if (unlikely(!pud_present(pud))) {
spin_unlock(ptl);
- if (page)
- return page;
return no_page_table(vma, flags, address);
}
- if (unlikely(pud_bad(pud)))
- return no_page_table(vma, flags, address);
+ if (unlikely(!pud_leaf(pud))) {
+ spin_unlock(ptl);
+ return follow_pmd_mask(vma, address, pudp, pud, flags, page_mask);
+ }
- return follow_pmd_mask(vma, address, pudp, flags, page_mask);
+ page = follow_huge_pud(vma, address, pudp, flags, page_mask);
+ spin_unlock(ptl);
+ return page;
}
static struct page *follow_p4d_mask(struct vm_area_struct *vma,
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread