[PATCH v2 5/6] mm/sparse-vmemmap: Fix missing architecture-specific page table sync

Muchun Song
2026-04-15T11:15:04Z

On x86-64, vmemmap_populate() normally calls sync_global_pgds() to
keep the page tables in sync. However, when vmemmap optimization for
compound devmaps is enabled, vmemmap_populate_compound_pages() is called
directly from __populate_section_memmap(), bypassing the architecture-
specific vmemmap_populate() entirely. This skips the sync on x86-64
and can later trigger vmemmap-access faults.

Fix this by moving the vmemmap_can_optimize() dispatch from
__populate_section_memmap() into the generic helpers --
vmemmap_populate_basepages() and vmemmap_populate_hugepages(). This way,
the architecture vmemmap_populate() is always invoked first, ensuring
any arch-specific post-population steps (e.g. sync_global_pgds()) are
executed before returning.

Architectures that override vmemmap_populate() (e.g. powerpc) handle
the optimization dispatch in their own implementation instead.

Fixes: 4917f55b4ef9 ("mm/sparse-vmemmap: improve memory savings for compound devmaps")
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
arch/powerpc/include/asm/book3s/64/radix.h | 6 ------
arch/powerpc/mm/book3s64/radix_pgtable.c | 16 ++++++++++-----
mm/sparse-vmemmap.c | 24 +++++++++++-----------
3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h
index bde07c6f900f..2600defa2dc2 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -357,11 +357,5 @@ int radix__remove_section_mapping(unsigned long start, unsigned long end);
#define vmemmap_can_optimize vmemmap_can_optimize
bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
#endif
-
-#define vmemmap_populate_compound_pages vmemmap_populate_compound_pages
-int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
- unsigned long start,
- unsigned long end, int node,
- struct dev_pagemap *pgmap);
#endif /* __ASSEMBLER__ */
#endif
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index 568500343e5f..21fece355fbb 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -1109,7 +1109,10 @@ static inline pte_t *vmemmap_pte_alloc(pmd_t *pmdp, int node,
return pte_offset_kernel(pmdp, address);
}

-
+static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
+ unsigned long start,
+ unsigned long end, int node,
+ struct dev_pagemap *pgmap);

int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, int node,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
@@ -1122,6 +1125,9 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in
pmd_t *pmd;
pte_t *pte;

+ if (vmemmap_can_optimize(altmap, pgmap))
+ return vmemmap_populate_compound_pages(page_to_pfn((struct page *)start),
+ start, end, node, pgmap);
/*
* If altmap is present, Make sure we align the start vmemmap addr
* to PAGE_SIZE so that we calculate the correct start_pfn in
@@ -1303,10 +1309,10 @@ static pte_t * __meminit vmemmap_compound_tail_page(unsigned long addr,
return pte;
}

-int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
- unsigned long start,
- unsigned long end, int node,
- struct dev_pagemap *pgmap)
+static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
+ unsigned long start,
+ unsigned long end, int node,
+ struct dev_pagemap *pgmap)
{
/*
* we want to map things as base page size mapping so that
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index f5245647afee..7f684ed3479e 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -296,10 +296,16 @@ static int __meminit vmemmap_populate_range(unsigned long start,
return 0;
}

+static int __meminit vmemmap_populate_compound_pages(unsigned long start,
+ unsigned long end, int node,
+ struct dev_pagemap *pgmap);
+
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
+ if (vmemmap_can_optimize(altmap, pgmap))
+ return vmemmap_populate_compound_pages(start, end, node, pgmap);
return vmemmap_populate_range(start, end, node, altmap, -1, 0);
}

@@ -411,6 +417,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
pud_t *pud;
pmd_t *pmd;

+ if (vmemmap_can_optimize(altmap, pgmap))
+ return vmemmap_populate_compound_pages(start, end, node, pgmap);
+
for (addr = start; addr < end; addr = next) {
next = pmd_addr_end(addr, end);

@@ -453,7 +462,6 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
return 0;
}

-#ifndef vmemmap_populate_compound_pages
/*
* For compound pages bigger than section size (e.g. x86 1G compound
* pages with 2M subsection size) fill the rest of sections as tail
@@ -491,14 +499,14 @@ static pte_t * __meminit compound_section_tail_page(unsigned long addr)
return pte;
}

-static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
- unsigned long start,
+static int __meminit vmemmap_populate_compound_pages(unsigned long start,
unsigned long end, int node,
struct dev_pagemap *pgmap)
{
unsigned long size, addr;
pte_t *pte;
int rc;
+ unsigned long start_pfn = page_to_pfn((struct page *)start);

if (reuse_compound_section(start_pfn, pgmap)) {
pte = compound_section_tail_page(start);
@@ -544,26 +552,18 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
return 0;
}

-#endif
-
struct page * __meminit __populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
unsigned long start = (unsigned long) pfn_to_page(pfn);
unsigned long end = start + nr_pages * sizeof(struct page);
- int r;

if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
return NULL;

- if (vmemmap_can_optimize(altmap, pgmap))
- r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap);
- else
- r = vmemmap_populate(start, end, nid, altmap, pgmap);
-
- if (r < 0)
+ if (vmemmap_populate(start, end, nid, altmap, pgmap))
return NULL;

return pfn_to_page(pfn);
--
2.20.1