[PATCH 04/49] mm/sparse: add a @pgmap parameter to arch vmemmap_populate()

Muchun Song
2026-04-05T12:53:51Z

Add the struct dev_pagemap pointer as a parameter to the architecture
specific vmemmap_populate(), vmemmap_populate_hugepages() and
vmemmap_populate_basepages() functions.

Currently, the vmemmap optimization for DAX is handled mostly in an
architecture-agnostic way via vmemmap_populate_compound_pages().
However, this approach skips crucial architecture-specific initialization
steps. For example, the x86 path must call sync_global_pgds() after
populating the vmemmap, which is currently being bypassed.

To fix this, we need to push the awareness of device memory optimization
(via the pgmap) down into the architecture-specific vmemmap_populate()
paths. This will allow each architecture to handle the optimization while
ensuring their specific initialization routines (like page directory
synchronization) are correctly invoked.

This is a preparatory patch only; it changes no behavior. The actual
architecture-specific implementations and fixes will follow.

Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
arch/arm64/mm/mmu.c | 6 +++---
arch/loongarch/mm/init.c | 7 ++++---
arch/powerpc/include/asm/book3s/64/radix.h | 3 ++-
arch/powerpc/mm/book3s64/radix_pgtable.c | 2 +-
arch/powerpc/mm/init_64.c | 4 ++--
arch/riscv/mm/init.c | 4 ++--
arch/s390/mm/vmem.c | 2 +-
arch/sparc/mm/init_64.c | 5 +++--
arch/x86/mm/init_64.c | 8 ++++----
include/linux/mm.h | 8 +++++---
mm/hugetlb_vmemmap.c | 4 ++--
mm/sparse-vmemmap.c | 10 ++++++----
12 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index dc8a8281888c..86162aab5185 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1760,7 +1760,7 @@ int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
}

int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap)
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
/* [start, end] should be within one section */
@@ -1768,9 +1768,9 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,

if (!IS_ENABLED(CONFIG_ARM64_4K_PAGES) ||
(end - start < PAGES_PER_SECTION * sizeof(struct page)))
- return vmemmap_populate_basepages(start, end, node, altmap);
+ return vmemmap_populate_basepages(start, end, node, altmap, pgmap);
else
- return vmemmap_populate_hugepages(start, end, node, altmap);
+ return vmemmap_populate_hugepages(start, end, node, altmap, pgmap);
}

#ifdef CONFIG_MEMORY_HOTPLUG
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index c9c57f08fa2c..d61c2e09caae 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -123,12 +123,13 @@ int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
}

int __meminit vmemmap_populate(unsigned long start, unsigned long end,
- int node, struct vmem_altmap *altmap)
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap)
{
#if CONFIG_PGTABLE_LEVELS == 2
- return vmemmap_populate_basepages(start, end, node, NULL);
+ return vmemmap_populate_basepages(start, end, node, NULL, pgmap);
#else
- return vmemmap_populate_hugepages(start, end, node, NULL);
+ return vmemmap_populate_hugepages(start, end, node, NULL, pgmap);
#endif
}

diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h
index da954e779744..bde07c6f900f 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -321,7 +321,8 @@ extern int __meminit radix__vmemmap_create_mapping(unsigned long start,
unsigned long page_size,
unsigned long phys);
int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end,
- int node, struct vmem_altmap *altmap);
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap);
void __ref radix__vmemmap_free(unsigned long start, unsigned long end,
struct vmem_altmap *altmap);
extern void radix__vmemmap_remove_mapping(unsigned long start,
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index 10aced261cff..568500343e5f 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -1112,7 +1112,7 @@ static inline pte_t *vmemmap_pte_alloc(pmd_t *pmdp, int node,

int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap)
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
unsigned long addr;
unsigned long next;
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index b6f3ae03ca9e..8f4aa5b32186 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -275,12 +275,12 @@ static int __meminit __vmemmap_populate(unsigned long start, unsigned long end,
}

int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap)
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{

#ifdef CONFIG_PPC_BOOK3S_64
if (radix_enabled())
- return radix__vmemmap_populate(start, end, node, altmap);
+ return radix__vmemmap_populate(start, end, node, altmap, pgmap);
#endif

return __vmemmap_populate(start, end, node, altmap);
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 980f693e6b19..277c89661dff 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -1443,7 +1443,7 @@ int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
}

int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap)
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
/*
* Note that SPARSEMEM_VMEMMAP is only selected for rv64 and that we
@@ -1451,7 +1451,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
* memory hotplug, we are not able to update all the page tables with
* the new PMDs.
*/
- return vmemmap_populate_hugepages(start, end, node, altmap);
+ return vmemmap_populate_hugepages(start, end, node, altmap, pgmap);
}
#endif

diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index eeadff45e0e1..a7bf8d3d5601 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -506,7 +506,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
* Add a backed mem_map array to the virtual mem_map array.
*/
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap)
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
int ret;

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 367c269305e5..f870ca330f9e 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2591,9 +2591,10 @@ int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
}

int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
- int node, struct vmem_altmap *altmap)
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap)
{
- return vmemmap_populate_hugepages(vstart, vend, node, NULL);
+ return vmemmap_populate_hugepages(vstart, vend, node, NULL, pgmap);
}
#endif /* CONFIG_SPARSEMEM_VMEMMAP */

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 77b889b71cf3..e18cc81a30b4 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1557,7 +1557,7 @@ int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
}

int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap)
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
int err;

@@ -1565,15 +1565,15 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
VM_BUG_ON(!PAGE_ALIGNED(end));

if (end - start < PAGES_PER_SECTION * sizeof(struct page))
- err = vmemmap_populate_basepages(start, end, node, NULL);
+ err = vmemmap_populate_basepages(start, end, node, NULL, pgmap);
else if (boot_cpu_has(X86_FEATURE_PSE))
- err = vmemmap_populate_hugepages(start, end, node, altmap);
+ err = vmemmap_populate_hugepages(start, end, node, altmap, pgmap);
else if (altmap) {
pr_err_once("%s: no cpu support for altmap allocations\n",
__func__);
err = -ENOMEM;
} else
- err = vmemmap_populate_basepages(start, end, node, NULL);
+ err = vmemmap_populate_basepages(start, end, node, NULL, pgmap);
if (!err)
sync_global_pgds(start, end - 1);
return err;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 0b776907152e..bebc5f892f81 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -4877,11 +4877,13 @@ void vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
int vmemmap_check_pmd(pmd_t *pmd, int node,
unsigned long addr, unsigned long next);
int vmemmap_populate_basepages(unsigned long start, unsigned long end,
- int node, struct vmem_altmap *altmap);
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap);
int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
- int node, struct vmem_altmap *altmap);
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap);
int vmemmap_populate(unsigned long start, unsigned long end, int node,
- struct vmem_altmap *altmap);
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
int vmemmap_populate_hvo(unsigned long start, unsigned long end,
unsigned int order, struct zone *zone,
unsigned long headsize);
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 4a077d231d3a..50b7123f3bdd 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -829,7 +829,7 @@ void __init hugetlb_vmemmap_init_late(int nid)
*/
list_del(&m->list);

- vmemmap_populate(start, end, nid, NULL);
+ vmemmap_populate(start, end, nid, NULL, NULL);
nr_mmap = end - start;
memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));

@@ -845,7 +845,7 @@ void __init hugetlb_vmemmap_init_late(int nid)
if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone,
HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) {
/* Fallback if HVO population fails */
- vmemmap_populate(start, end, nid, NULL);
+ vmemmap_populate(start, end, nid, NULL, NULL);
nr_mmap = end - start;
} else {
m->flags |= HUGE_BOOTMEM_ZONES_VALID;
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 0ef96b1afbcc..387337bba05e 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -297,7 +297,8 @@ static int __meminit vmemmap_populate_range(unsigned long start,
}

int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
- int node, struct vmem_altmap *altmap)
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap)
{
return vmemmap_populate_range(start, end, node, altmap, -1, 0);
}
@@ -400,7 +401,8 @@ int __weak __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
}

int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
- int node, struct vmem_altmap *altmap)
+ int node, struct vmem_altmap *altmap,
+ struct dev_pagemap *pgmap)
{
unsigned long addr;
unsigned long next;
@@ -445,7 +447,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
}
} else if (vmemmap_check_pmd(pmd, node, addr, next))
continue;
- if (vmemmap_populate_basepages(addr, next, node, altmap))
+ if (vmemmap_populate_basepages(addr, next, node, altmap, pgmap))
return -ENOMEM;
}
return 0;
@@ -559,7 +561,7 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
if (vmemmap_can_optimize(altmap, pgmap))
r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap);
else
- r = vmemmap_populate(start, end, nid, altmap);
+ r = vmemmap_populate(start, end, nid, altmap, pgmap);

if (r < 0)
return NULL;
--
2.20.1