* [PATCH v2 10/69] mm/mm_init: Remove set_pageblock_order() call from sparse_init()
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
free_area_init() already sets pageblock_order before sparse_init() runs
for CONFIG_HUGETLB_PAGE_SIZE_VARIABLE, so sparse_init() does not need to
call set_pageblock_order() again.
With that call removed, set_pageblock_order() is only used in mm/mm_init.c.
Make it static.
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v1->v2:
- Move the removal of set_pageblock_order() into this patch
- Update the commit message accordingly
- Add Reviewed-by from Mike Rapoport
---
mm/internal.h | 1 -
mm/mm_init.c | 4 ++--
mm/sparse.c | 3 ---
3 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index 28d179cbc451..6bd9aa37b952 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1436,7 +1436,6 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long,
unsigned long, unsigned long,
unsigned long, unsigned long);
-extern void set_pageblock_order(void);
unsigned long reclaim_pages(struct list_head *folio_list);
unsigned int reclaim_clean_pages_from_list(struct zone *zone,
struct list_head *folio_list);
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 75f98abfed97..6646d4b47796 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -1508,7 +1508,7 @@ static inline void setup_usemap(struct zone *zone) {}
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
-void __init set_pageblock_order(void)
+static void __init set_pageblock_order(void)
{
unsigned int order = PAGE_BLOCK_MAX_ORDER;
@@ -1534,7 +1534,7 @@ void __init set_pageblock_order(void)
* include/linux/pageblock-flags.h for the values of pageblock_order based on
* the kernel config
*/
-void __init set_pageblock_order(void)
+static inline void __init set_pageblock_order(void)
{
}
diff --git a/mm/sparse.c b/mm/sparse.c
index 85557ef387c7..324213d8bdcb 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -343,9 +343,6 @@ void __init sparse_init(void)
pnum_begin = first_present_section_nr();
nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
- /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
- set_pageblock_order();
-
for_each_present_section_nr(pnum_begin + 1, pnum_end) {
int nid = sparse_early_nid(__nr_to_section(pnum_end));
--
2.54.0
^ permalink raw reply related
* [PATCH v2 11/69] mm/sparse: Move sparse_vmemmap_init_nid_late() into sparse_init_nid()
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
sparse_vmemmap_init_nid_late() is still called separately from
mm_core_init_early(), away from the rest of the sparse initialization
path.
Now that sparse_init() runs after zone initialization, call
sparse_vmemmap_init_nid_late() from sparse_init_nid() instead. This
keeps both sparse_vmemmap_init_nid_early() and
sparse_vmemmap_init_nid_late() in the sparse setup path.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
---
v1->v2:
- Add Reviewed-by from Mike Rapoport
---
mm/mm_init.c | 4 ----
mm/sparse.c | 1 +
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 6646d4b47796..165b83c9a9c3 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2686,16 +2686,12 @@ void __init __weak mem_init(void)
void __init mm_core_init_early(void)
{
- int nid;
-
free_area_init();
hugetlb_cma_reserve();
hugetlb_bootmem_alloc();
sparse_init();
- for_each_node_state(nid, N_MEMORY)
- sparse_vmemmap_init_nid_late(nid);
memmap_init();
}
diff --git a/mm/sparse.c b/mm/sparse.c
index 324213d8bdcb..3917a47153d8 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -320,6 +320,7 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
}
}
sparse_usage_fini();
+ sparse_vmemmap_init_nid_late(nid);
}
/*
--
2.54.0
^ permalink raw reply related
* [PATCH v2 12/69] mm/hugetlb_cma: Validate hugetlb CMA range by zone at reserve time
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Hugetlb CMA allocation currently has to cope with CMA areas that span
multiple zones.
Validate the reserved CMA range up front in hugetlb_cma_reserve() so
later hugetlb CMA allocations can assume a zone-consistent area.
Also drop the pfn_valid() check from cma_validate_zones(). mem_section
is not fully initialized at this point, so the check can trigger false
warnings. Keep the sanity check in cma_activate_area() instead.
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v1->v2:
- Update the warning message for zone validation failures
- Add Acked-by from Mike Rapoport
---
mm/cma.c | 3 ++-
mm/hugetlb_cma.c | 6 ++++--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/mm/cma.c b/mm/cma.c
index c7ca567f4c5c..0369f04c7ba5 100644
--- a/mm/cma.c
+++ b/mm/cma.c
@@ -126,7 +126,6 @@ bool cma_validate_zones(struct cma *cma)
* to be in the same zone. Simplify by forcing the entire
* CMA resv range to be in the same zone.
*/
- WARN_ON_ONCE(!pfn_valid(base_pfn));
if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) {
set_bit(CMA_ZONES_INVALID, &cma->flags);
return false;
@@ -165,6 +164,8 @@ static void __init cma_activate_area(struct cma *cma)
bitmap_set(cmr->bitmap, 0, bitmap_count);
}
+ WARN_ON_ONCE(!pfn_valid(cmr->base_pfn));
+
for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count;
pfn += pageblock_nr_pages)
init_cma_reserved_pageblock(pfn_to_page(pfn));
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index 7693ccefd0c6..57a7b3acc758 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -234,9 +234,11 @@ void __init hugetlb_cma_reserve(void)
res = cma_declare_contiguous_multi(size, PAGE_SIZE << order,
HUGETLB_PAGE_ORDER, name,
&hugetlb_cma[nid], nid);
- if (res) {
- pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
+ if (res || !cma_validate_zones(hugetlb_cma[nid])) {
+ pr_warn("hugetlb_cma: %s: err %d, node %d\n",
+ res ? "reservation failed" : "reserved area spans zones",
res, nid);
+ hugetlb_cma[nid] = NULL;
continue;
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 13/69] mm/hugetlb: Refactor early boot gigantic hugepage allocation
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
The early boot gigantic hugepage allocation helpers currently mix
allocation with huge_bootmem_page setup, and leave part of the
initialization flow in architecture code.
Refactor the interface to return the allocated huge page pointer and
move the huge_bootmem_page setup into the generic hugetlb code. This
makes the architecture-specific paths focus only on finding memory,
while the common code handles node placement and early page metadata
setup in one place.
This also lets powerpc benefit from memblock_reserved_mark_noinit(),
which it did not enable before.
In addition, upcoming cross-zone validation for boot-time gigantic
hugetlb reservation is common logic. With this refactoring, that logic
can stay in the generic code instead of being duplicated in
architecture-specific paths.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
arch/powerpc/mm/hugetlbpage.c | 11 ++--
include/linux/hugetlb.h | 8 +--
mm/hugetlb.c | 95 ++++++++++++++---------------------
mm/hugetlb_cma.c | 12 ++---
mm/hugetlb_cma.h | 4 +-
5 files changed, 52 insertions(+), 78 deletions(-)
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 558fafb82b8a..ff8c5ec831bb 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -104,17 +104,14 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p
}
}
-static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate)
+static __init void *pseries_alloc_bootmem_huge_page(struct hstate *hstate)
{
struct huge_bootmem_page *m;
if (nr_gpages == 0)
- return 0;
+ return NULL;
m = phys_to_virt(gpage_freearray[--nr_gpages]);
gpage_freearray[nr_gpages] = 0;
- list_add(&m->list, &huge_boot_pages[0]);
- m->hstate = hstate;
- m->flags = 0;
- return 1;
+ return m;
}
bool __init hugetlb_node_alloc_supported(void)
@@ -124,7 +121,7 @@ bool __init hugetlb_node_alloc_supported(void)
#endif
-int __init alloc_bootmem_huge_page(struct hstate *h, int nid)
+void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
{
#ifdef CONFIG_PPC_BOOK3S_64
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 52a2c30f866c..9a65271d167c 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -720,8 +720,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
unsigned long address, struct folio *folio);
/* arch callback */
-int __init __alloc_bootmem_huge_page(struct hstate *h, int nid);
-int __init alloc_bootmem_huge_page(struct hstate *h, int nid);
+void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid);
+void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid);
bool __init hugetlb_node_alloc_supported(void);
void __init hugetlb_add_hstate(unsigned order);
@@ -1152,9 +1152,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
return NULL;
}
-static inline int __alloc_bootmem_huge_page(struct hstate *h)
+static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid)
{
- return 0;
+ return NULL;
}
static inline struct hstate *hstate_file(struct file *f)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index b4999653a156..e9ba0be2eb17 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3044,79 +3044,58 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact)
{
- struct huge_bootmem_page *m;
- int listnode = nid;
-
if (hugetlb_early_cma(h))
- m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact);
- else {
- if (node_exact)
- m = memblock_alloc_exact_nid_raw(huge_page_size(h),
+ return hugetlb_cma_alloc_bootmem(h, nid, node_exact);
+
+ if (node_exact)
+ return memblock_alloc_exact_nid_raw(huge_page_size(h),
huge_page_size(h), 0,
MEMBLOCK_ALLOC_ACCESSIBLE, nid);
- else {
- m = memblock_alloc_try_nid_raw(huge_page_size(h),
+
+ return memblock_alloc_try_nid_raw(huge_page_size(h),
huge_page_size(h), 0,
MEMBLOCK_ALLOC_ACCESSIBLE, nid);
- /*
- * For pre-HVO to work correctly, pages need to be on
- * the list for the node they were actually allocated
- * from. That node may be different in the case of
- * fallback by memblock_alloc_try_nid_raw. So,
- * extract the actual node first.
- */
- if (m)
- listnode = early_pfn_to_nid(PHYS_PFN(__pa(m)));
- }
-
- if (m) {
- m->flags = 0;
- m->cma = NULL;
- }
- }
-
- if (m) {
- /*
- * Use the beginning of the huge page to store the
- * huge_bootmem_page struct (until gather_bootmem
- * puts them into the mem_map).
- *
- * Put them into a private list first because mem_map
- * is not up yet.
- */
- INIT_LIST_HEAD(&m->list);
- list_add(&m->list, &huge_boot_pages[listnode]);
- m->hstate = h;
- }
-
- return m;
}
-int alloc_bootmem_huge_page(struct hstate *h, int nid)
+void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
-int __alloc_bootmem_huge_page(struct hstate *h, int nid)
+void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid)
{
- struct huge_bootmem_page *m = NULL; /* initialize for clang */
int nr_nodes, node = nid;
/* do node specific alloc */
- if (nid != NUMA_NO_NODE) {
- m = alloc_bootmem(h, node, true);
- if (!m)
- return 0;
- goto found;
- }
+ if (nid != NUMA_NO_NODE)
+ return alloc_bootmem(h, node, true);
/* allocate from next node when distributing huge pages */
for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node,
- &hugetlb_bootmem_nodes) {
- m = alloc_bootmem(h, node, false);
- if (!m)
- return 0;
- goto found;
- }
+ &hugetlb_bootmem_nodes)
+ return alloc_bootmem(h, node, false);
-found:
+ return NULL;
+}
+
+static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
+{
+ struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid);
+
+ if (!m)
+ return false;
+
+ nid = early_pfn_to_nid(PHYS_PFN(__pa(m)));
+ /*
+ * Use the beginning of the huge page to store the huge_bootmem_page
+ * struct (until gather_bootmem puts them into the mem_map).
+ *
+ * Put them into a private list first because mem_map is not up yet.
+ */
+ INIT_LIST_HEAD(&m->list);
+ list_add(&m->list, &huge_boot_pages[nid]);
+ m->hstate = h;
+ if (!hugetlb_early_cma(h)) {
+ m->cma = NULL;
+ m->flags = 0;
+ }
/*
* Only initialize the head struct page in memmap_init_reserved_pages,
@@ -3128,7 +3107,7 @@ int __alloc_bootmem_huge_page(struct hstate *h, int nid)
memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
huge_page_size(h) - PAGE_SIZE);
- return 1;
+ return true;
}
/* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index 57a7b3acc758..6b5c2aec4449 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -57,13 +57,13 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
}
struct huge_bootmem_page * __init
-hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact)
+hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact)
{
struct cma *cma;
struct huge_bootmem_page *m;
- int node = *nid;
+ int node;
- cma = hugetlb_cma[*nid];
+ cma = hugetlb_cma[nid];
m = cma_reserve_early(cma, huge_page_size(h));
if (!m) {
if (node_exact)
@@ -71,13 +71,11 @@ hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact)
for_each_node_mask(node, hugetlb_bootmem_nodes) {
cma = hugetlb_cma[node];
- if (!cma || node == *nid)
+ if (!cma || node == nid)
continue;
m = cma_reserve_early(cma, huge_page_size(h));
- if (m) {
- *nid = node;
+ if (m)
break;
- }
}
}
diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h
index c619c394b1ae..057852c792bd 100644
--- a/mm/hugetlb_cma.h
+++ b/mm/hugetlb_cma.h
@@ -6,7 +6,7 @@
void hugetlb_cma_free_frozen_folio(struct folio *folio);
struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
int nid, nodemask_t *nodemask);
-struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid,
+struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid,
bool node_exact);
bool hugetlb_cma_exclusive_alloc(void);
unsigned long hugetlb_cma_total_size(void);
@@ -24,7 +24,7 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order,
}
static inline
-struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid,
+struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid,
bool node_exact)
{
return NULL;
--
2.54.0
^ permalink raw reply related
* [PATCH v2 14/69] mm/hugetlb: Free cross-zone bootmem gigantic pages after allocation
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Now that hugetlb reservation runs after zone initialization, bootmem
gigantic page allocation can detect pages that span multiple zones.
Keep those cross-zone pages separate during allocation and free them
after allocation completes, so later hugetlb initialization only sees
zone-valid gigantic pages.
This chooses to free cross-zone gigantic pages directly instead of
retrying allocation. In practice, such cross-zone cases are expected to
be very rare, so adding retry logic does not seem justified at this
point. Keeping the handling simple also preserves the previous behavior.
If similar real-world reports show up later, retry support can be
reconsidered then.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/hugetlb.c | 75 ++++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 64 insertions(+), 11 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e9ba0be2eb17..d5d324f69d7a 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3077,12 +3077,15 @@ void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid)
static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
{
+ unsigned long pfn;
+ unsigned int nid_request = nid;
struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid);
if (!m)
return false;
- nid = early_pfn_to_nid(PHYS_PFN(__pa(m)));
+ pfn = PHYS_PFN(__pa(m));
+ nid = early_pfn_to_nid(pfn);
/*
* Use the beginning of the huge page to store the huge_bootmem_page
* struct (until gather_bootmem puts them into the mem_map).
@@ -3090,22 +3093,38 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
* Put them into a private list first because mem_map is not up yet.
*/
INIT_LIST_HEAD(&m->list);
- list_add(&m->list, &huge_boot_pages[nid]);
m->hstate = h;
if (!hugetlb_early_cma(h)) {
m->cma = NULL;
m->flags = 0;
}
- /*
- * Only initialize the head struct page in memmap_init_reserved_pages,
- * rest of the struct pages will be initialized by the HugeTLB
- * subsystem itself.
- * The head struct page is used to get folio information by the HugeTLB
- * subsystem like zone id and node id.
- */
- memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
- huge_page_size(h) - PAGE_SIZE);
+ /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */
+ if (!hugetlb_early_cma(h) &&
+ pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) {
+ /*
+ * If the allocated page is on a different node than requested
+ * (e.g. on PowerPC LPARs), put it on the requested node's list.
+ * Otherwise, the cross-zone page will be stranded and never
+ * freed, as the cleanup code only operates on the requested node.
+ */
+ if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request))
+ list_add(&m->list, &huge_boot_pages[nid_request]);
+ else
+ list_add(&m->list, &huge_boot_pages[nid]);
+ } else {
+ list_add_tail(&m->list, &huge_boot_pages[nid]);
+ m->flags |= HUGE_BOOTMEM_ZONES_VALID;
+ /*
+ * Only initialize the head struct page in memmap_init_reserved_pages,
+ * rest of the struct pages will be initialized by the HugeTLB
+ * subsystem itself.
+ * The head struct page is used to get folio information by the HugeTLB
+ * subsystem like zone id and node id.
+ */
+ memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
+ huge_page_size(h) - PAGE_SIZE);
+ }
return true;
}
@@ -3384,6 +3403,34 @@ void __init hugetlb_struct_page_init(void)
padata_do_multithreaded(&job);
}
+static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid)
+{
+ unsigned long freed = 0;
+ struct huge_bootmem_page *m, *tmp;
+
+ if (!hstate_is_gigantic(h))
+ return freed;
+
+ list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) {
+ if (m->flags & HUGE_BOOTMEM_ZONES_VALID)
+ break;
+
+ list_del(&m->list);
+ memblock_free(m, huge_page_size(h));
+ freed++;
+ }
+
+ if (freed) {
+ char buf[32];
+
+ string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf));
+ pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n",
+ freed, buf, nid);
+ }
+
+ return freed;
+}
+
static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
{
unsigned long i;
@@ -3414,6 +3461,8 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
cond_resched();
}
+ i -= hugetlb_free_cross_zone_pages(h, nid);
+
if (!list_empty(&folio_list))
prep_and_add_allocated_folios(h, &folio_list);
@@ -3487,6 +3536,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l
static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
{
+ int nid;
unsigned long i;
for (i = 0; i < h->max_huge_pages; ++i) {
@@ -3495,6 +3545,9 @@ static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
cond_resched();
}
+ for_each_node(nid)
+ i -= hugetlb_free_cross_zone_pages(h, nid);
+
return i;
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 15/69] mm/hugetlb_vmemmap: Move bootmem HVO setup to early init
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Bootmem HugeTLB pages currently defer HVO setup to
hugetlb_vmemmap_init_late(), because the optimization needs zone
information.
Now that zone initialization is available earlier, the bootmem HVO setup
can be done directly from hugetlb_vmemmap_init_early(). This lets
gigantic HugeTLB pages apply HVO as soon as they are allocated.
Bootmem gigantic pages that span multiple zones are now filtered out
when they are allocated, so the remaining bootmem gigantic pages seen by
later hugetlb initialization are already zone-valid. As a result,
hugetlb_vmemmap_init_late() no longer needs to handle bootmem HVO setup.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/hugetlb_vmemmap.c | 67 +++++++++-----------------------------------
1 file changed, 13 insertions(+), 54 deletions(-)
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 4f58cd940f61..e2251bc47444 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -745,6 +745,8 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
return true;
}
+static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn);
+
/*
* Initialize memmap section for a gigantic page, HVO-style.
*/
@@ -752,6 +754,7 @@ void __init hugetlb_vmemmap_init_early(int nid)
{
unsigned long psize, paddr, section_size;
unsigned long ns, i, pnum, pfn, nr_pages;
+ unsigned long start, end;
struct huge_bootmem_page *m = NULL;
void *map;
@@ -761,6 +764,8 @@ void __init hugetlb_vmemmap_init_early(int nid)
section_size = (1UL << PA_SECTION_SHIFT);
list_for_each_entry(m, &huge_boot_pages[nid], list) {
+ struct zone *zone;
+
if (!vmemmap_should_optimize_bootmem_page(m))
continue;
@@ -769,6 +774,14 @@ void __init hugetlb_vmemmap_init_early(int nid)
paddr = virt_to_phys(m);
pfn = PHYS_PFN(paddr);
map = pfn_to_page(pfn);
+ start = (unsigned long)map;
+ end = start + hugetlb_vmemmap_size(m->hstate);
+ zone = pfn_to_zone(nid, pfn);
+
+ if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
+ zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
+ panic("Failed to allocate memmap for HugeTLB page\n");
+ memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
pnum = pfn_to_section_nr(pfn);
ns = psize / section_size;
@@ -800,60 +813,6 @@ static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
void __init hugetlb_vmemmap_init_late(int nid)
{
- struct huge_bootmem_page *m, *tm;
- unsigned long phys, nr_pages, start, end;
- unsigned long pfn, nr_mmap;
- struct zone *zone = NULL;
- struct hstate *h;
- void *map;
-
- if (!READ_ONCE(vmemmap_optimize_enabled))
- return;
-
- list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
- if (!(m->flags & HUGE_BOOTMEM_HVO))
- continue;
-
- phys = virt_to_phys(m);
- h = m->hstate;
- pfn = PHYS_PFN(phys);
- nr_pages = pages_per_huge_page(h);
- map = pfn_to_page(pfn);
- start = (unsigned long)map;
- end = start + nr_pages * sizeof(struct page);
-
- if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
- /*
- * Oops, the hugetlb page spans multiple zones.
- * Remove it from the list, and populate it normally.
- */
- list_del(&m->list);
-
- vmemmap_populate(start, end, nid, NULL);
- nr_mmap = end - start;
- memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
-
- memblock_phys_free(phys, huge_page_size(h));
- continue;
- }
-
- if (!zone || !zone_spans_pfn(zone, pfn))
- zone = pfn_to_zone(nid, pfn);
- if (WARN_ON_ONCE(!zone))
- continue;
-
- if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone,
- HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) {
- /* Fallback if HVO population fails */
- vmemmap_populate(start, end, nid, NULL);
- nr_mmap = end - start;
- } else {
- m->flags |= HUGE_BOOTMEM_ZONES_VALID;
- nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE;
- }
-
- memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
- }
}
#endif
--
2.54.0
^ permalink raw reply related
* [PATCH v2 16/69] mm/hugetlb: Remove obsolete bootmem cross-zone checks
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Bootmem gigantic HugeTLB pages used to be validated again during
gather_bootmem_prealloc_node() and any cross-zone pages were discarded
there.
That validation is no longer needed. Cross-zone bootmem gigantic pages
are now detected during allocation and freed before they reach the later
bootmem gathering path, so the remaining pages are already zone-valid.
Remove the obsolete cross-zone validation, invalid-page freeing, and the
associated discarded-page accounting.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/hugetlb.h | 2 --
mm/hugetlb.c | 70 -----------------------------------------
2 files changed, 72 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 9a65271d167c..ece4e6a4a4c6 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -701,8 +701,6 @@ struct huge_bootmem_page {
#define HUGE_BOOTMEM_ZONES_VALID 0x0002
#define HUGE_BOOTMEM_CMA 0x0004
-bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m);
-
int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list);
int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn);
void wait_for_freed_hugetlb_folios(void);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index d5d324f69d7a..dcf8e09ec6be 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -58,7 +58,6 @@ struct hstate hstates[HUGE_MAX_HSTATE];
__initdata nodemask_t hugetlb_bootmem_nodes;
__initdata struct list_head huge_boot_pages[MAX_NUMNODES];
-static unsigned long hstate_boot_nrinvalid[HUGE_MAX_HSTATE] __initdata;
/*
* Due to ordering constraints across the init code for various
@@ -3238,57 +3237,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h,
}
}
-bool __init hugetlb_bootmem_page_zones_valid(int nid,
- struct huge_bootmem_page *m)
-{
- unsigned long start_pfn;
- bool valid;
-
- if (m->flags & HUGE_BOOTMEM_ZONES_VALID) {
- /*
- * Already validated, skip check.
- */
- return true;
- }
-
- if (hugetlb_bootmem_page_earlycma(m)) {
- valid = cma_validate_zones(m->cma);
- goto out;
- }
-
- start_pfn = virt_to_phys(m) >> PAGE_SHIFT;
-
- valid = !pfn_range_intersects_zones(nid, start_pfn,
- pages_per_huge_page(m->hstate));
-out:
- if (!valid)
- hstate_boot_nrinvalid[hstate_index(m->hstate)]++;
-
- return valid;
-}
-
-/*
- * Free a bootmem page that was found to be invalid (intersecting with
- * multiple zones).
- *
- * Since it intersects with multiple zones, we can't just do a free
- * operation on all pages at once, but instead have to walk all
- * pages, freeing them one by one.
- */
-static void __init hugetlb_bootmem_free_invalid_page(int nid, struct page *page,
- struct hstate *h)
-{
- unsigned long npages = pages_per_huge_page(h);
- unsigned long pfn;
-
- while (npages--) {
- pfn = page_to_pfn(page);
- __init_page_from_nid(pfn, nid);
- free_reserved_page(page);
- page++;
- }
-}
-
/*
* Put bootmem huge pages into the standard lists after mem_map is up.
* Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
@@ -3304,17 +3252,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
struct folio *folio = (void *)page;
h = m->hstate;
- if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
- /*
- * Can't use this page. Initialize the
- * page structures if that hasn't already
- * been done, and give them to the page
- * allocator.
- */
- hugetlb_bootmem_free_invalid_page(nid, page, h);
- continue;
- }
-
/*
* It is possible to have multiple huge page sizes (hstates)
* in this list. If so, process each size separately.
@@ -3703,20 +3640,13 @@ static void __init hugetlb_init_hstates(void)
static void __init report_hugepages(void)
{
struct hstate *h;
- unsigned long nrinvalid;
for_each_hstate(h) {
char buf[32];
- nrinvalid = hstate_boot_nrinvalid[hstate_index(h)];
- h->max_huge_pages -= nrinvalid;
-
string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
buf, h->nr_huge_pages);
- if (nrinvalid)
- pr_info("HugeTLB: %s page size: %lu invalid page%s discarded\n",
- buf, nrinvalid, str_plural(nrinvalid));
pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 17/69] mm/sparse-vmemmap: Remove sparse_vmemmap_init_nid_late()
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
hugetlb_vmemmap_init_late() no longer has any users, so the remaining
late-init path in sparse_vmemmap_init_nid_late() is dead code.
Remove sparse_vmemmap_init_nid_late() and its declarations.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 7 -------
mm/hugetlb_vmemmap.c | 4 ----
mm/hugetlb_vmemmap.h | 5 -----
mm/sparse-vmemmap.c | 11 -----------
mm/sparse.c | 1 -
5 files changed, 28 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9adb2ad21da5..362e16497533 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2167,8 +2167,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section)
}
void sparse_vmemmap_init_nid_early(int nid);
-void sparse_vmemmap_init_nid_late(int nid);
-
#else
static inline int preinited_vmemmap_section(const struct mem_section *section)
{
@@ -2177,10 +2175,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section)
static inline void sparse_vmemmap_init_nid_early(int nid)
{
}
-
-static inline void sparse_vmemmap_init_nid_late(int nid)
-{
-}
#endif
static inline int online_section_nr(unsigned long nr)
@@ -2385,7 +2379,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
#else
#define sparse_vmemmap_init_nid_early(_nid) do {} while (0)
-#define sparse_vmemmap_init_nid_late(_nid) do {} while (0)
#define pfn_in_present_section pfn_valid
#endif /* CONFIG_SPARSEMEM */
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index e2251bc47444..952216a49bcb 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -810,10 +810,6 @@ static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
return NULL;
}
-
-void __init hugetlb_vmemmap_init_late(int nid)
-{
-}
#endif
static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 18b490825215..7ac49c52457d 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -29,7 +29,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
void hugetlb_vmemmap_init_early(int nid);
-void hugetlb_vmemmap_init_late(int nid);
#endif
@@ -81,10 +80,6 @@ static inline void hugetlb_vmemmap_init_early(int nid)
{
}
-static inline void hugetlb_vmemmap_init_late(int nid)
-{
-}
-
static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
return 0;
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index fcf0ce5212f1..17d45dac4324 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -574,17 +574,6 @@ void __init sparse_vmemmap_init_nid_early(int nid)
{
hugetlb_vmemmap_init_early(nid);
}
-
-/*
- * This is called just before the initialization of page structures
- * through memmap_init. Zones are now initialized, so any work that
- * needs to be done that needs zone information can be done from
- * here.
- */
-void __init sparse_vmemmap_init_nid_late(int nid)
-{
- hugetlb_vmemmap_init_late(nid);
-}
#endif
static void subsection_mask_set(unsigned long *map, unsigned long pfn,
diff --git a/mm/sparse.c b/mm/sparse.c
index 3917a47153d8..324213d8bdcb 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -320,7 +320,6 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
}
}
sparse_usage_fini();
- sparse_vmemmap_init_nid_late(nid);
}
/*
--
2.54.0
^ permalink raw reply related
* [PATCH v2 18/69] mm/hugetlb: Remove unused bootmem cma field
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
struct huge_bootmem_page no longer needs to keep the CMA pointer. The
bootmem path only needs to remember whether a huge page came from CMA,
which is already encoded in the flags field.
Set HUGE_BOOTMEM_CMA when the page is allocated and drop the unused cma
field together with the redundant assignments.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/hugetlb.h | 1 -
mm/hugetlb.c | 5 +----
mm/hugetlb_cma.c | 27 ++++++++++-----------------
3 files changed, 11 insertions(+), 22 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index ece4e6a4a4c6..fd901bb3630c 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -694,7 +694,6 @@ struct huge_bootmem_page {
struct list_head list;
struct hstate *hstate;
unsigned long flags;
- struct cma *cma;
};
#define HUGE_BOOTMEM_HVO 0x0001
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index dcf8e09ec6be..1f0a0e31d624 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3093,10 +3093,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
*/
INIT_LIST_HEAD(&m->list);
m->hstate = h;
- if (!hugetlb_early_cma(h)) {
- m->cma = NULL;
- m->flags = 0;
- }
+ m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0;
/* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */
if (!hugetlb_early_cma(h) &&
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index 6b5c2aec4449..fbe5ed7ffaa7 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -65,26 +65,19 @@ hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact)
cma = hugetlb_cma[nid];
m = cma_reserve_early(cma, huge_page_size(h));
- if (!m) {
- if (node_exact)
- return NULL;
+ if (m || node_exact)
+ return m;
- for_each_node_mask(node, hugetlb_bootmem_nodes) {
- cma = hugetlb_cma[node];
- if (!cma || node == nid)
- continue;
- m = cma_reserve_early(cma, huge_page_size(h));
- if (m)
- break;
- }
- }
-
- if (m) {
- m->flags = HUGE_BOOTMEM_CMA;
- m->cma = cma;
+ for_each_node_mask(node, hugetlb_bootmem_nodes) {
+ cma = hugetlb_cma[node];
+ if (!cma || node == nid)
+ continue;
+ m = cma_reserve_early(cma, huge_page_size(h));
+ if (m)
+ return m;
}
- return m;
+ return NULL;
}
static int __init cmdline_parse_hugetlb_cma(char *p)
--
2.54.0
^ permalink raw reply related
* [PATCH v2 19/69] mm/mm_init: Make __init_page_from_nid() static
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
__init_page_from_nid() no longer has external users and is only used
locally in mm/mm_init.c under CONFIG_DEFERRED_STRUCT_PAGE_INIT.
Make it static and keep it inside that block.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/internal.h | 1 -
mm/mm_init.c | 4 ++--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index 6bd9aa37b952..4a5053368078 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1754,7 +1754,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte
void __meminit __init_single_page(struct page *page, unsigned long pfn,
unsigned long zone, int nid);
-void __meminit __init_page_from_nid(unsigned long pfn, int nid);
/* shrinker related functions */
unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 165b83c9a9c3..c64e5d63c4ae 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -688,10 +688,11 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
}
#endif
+#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
/*
* Initialize a reserved page unconditionally, finding its zone first.
*/
-void __meminit __init_page_from_nid(unsigned long pfn, int nid)
+static void __meminit __init_page_from_nid(unsigned long pfn, int nid)
{
pg_data_t *pgdat;
int zid;
@@ -713,7 +714,6 @@ void __meminit __init_page_from_nid(unsigned long pfn, int nid)
}
}
-#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
{
pgdat->first_deferred_pfn = ULONG_MAX;
--
2.54.0
^ permalink raw reply related
* [PATCH v2 20/69] mm/sparse-vmemmap: Drop VMEMMAP_POPULATE_PAGEREF
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
VMEMMAP_POPULATE_PAGEREF is only needed once the slab allocator is
available, so it does not need to be passed through the vmemmap
population call chain.
Drop the flag and test slab_is_available() directly in
vmemmap_pte_populate() instead.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/sparse-vmemmap.c | 40 ++++++++++++++--------------------------
1 file changed, 14 insertions(+), 26 deletions(-)
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 17d45dac4324..d7e9fb47f7ee 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -33,13 +33,6 @@
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"
-
-/*
- * Flags for vmemmap_populate_range and friends.
- */
-/* Get a ref on the head page struct page, for ZONE_DEVICE compound pages */
-#define VMEMMAP_POPULATE_PAGEREF 0x0001
-
#include "internal.h"
/*
@@ -147,8 +140,8 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
}
static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
- struct vmem_altmap *altmap,
- unsigned long ptpfn, unsigned long flags)
+ struct vmem_altmap *altmap,
+ unsigned long ptpfn)
{
pte_t *pte = pte_offset_kernel(pmd, addr);
if (pte_none(ptep_get(pte))) {
@@ -170,7 +163,7 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in
* and through vmemmap_populate_compound_pages() when
* slab is available.
*/
- if (flags & VMEMMAP_POPULATE_PAGEREF)
+ if (slab_is_available())
get_page(pfn_to_page(ptpfn));
}
entry = pfn_pte(ptpfn, PAGE_KERNEL);
@@ -243,8 +236,7 @@ static pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node,
struct vmem_altmap *altmap,
- unsigned long ptpfn,
- unsigned long flags)
+ unsigned long ptpfn)
{
pgd_t *pgd;
p4d_t *p4d;
@@ -264,7 +256,7 @@ static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node,
pmd = vmemmap_pmd_populate(pud, addr, node);
if (!pmd)
return NULL;
- pte = vmemmap_pte_populate(pmd, addr, node, altmap, ptpfn, flags);
+ pte = vmemmap_pte_populate(pmd, addr, node, altmap, ptpfn);
if (!pte)
return NULL;
vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
@@ -275,15 +267,14 @@ static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node,
static int __meminit vmemmap_populate_range(unsigned long start,
unsigned long end, int node,
struct vmem_altmap *altmap,
- unsigned long ptpfn,
- unsigned long flags)
+ unsigned long ptpfn)
{
unsigned long addr = start;
pte_t *pte;
for (; addr < end; addr += PAGE_SIZE) {
pte = vmemmap_populate_address(addr, node, altmap,
- ptpfn, flags);
+ ptpfn);
if (!pte)
return -ENOMEM;
}
@@ -294,7 +285,7 @@ static int __meminit vmemmap_populate_range(unsigned long start,
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap)
{
- return vmemmap_populate_range(start, end, node, altmap, -1, 0);
+ return vmemmap_populate_range(start, end, node, altmap, -1);
}
/*
@@ -370,7 +361,7 @@ int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end,
return -ENOMEM;
for (maddr = addr; maddr < addr + headsize; maddr += PAGE_SIZE) {
- pte = vmemmap_populate_address(maddr, node, NULL, -1, 0);
+ pte = vmemmap_populate_address(maddr, node, NULL, -1);
if (!pte)
return -ENOMEM;
}
@@ -378,8 +369,7 @@ int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end,
/*
* Reuse the last page struct page mapped above for the rest.
*/
- return vmemmap_populate_range(maddr, end, node, NULL,
- page_to_pfn(tail), 0);
+ return vmemmap_populate_range(maddr, end, node, NULL, page_to_pfn(tail));
}
#endif
@@ -503,8 +493,7 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
* with just tail struct pages.
*/
return vmemmap_populate_range(start, end, node, NULL,
- pte_pfn(ptep_get(pte)),
- VMEMMAP_POPULATE_PAGEREF);
+ pte_pfn(ptep_get(pte)));
}
size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page));
@@ -512,13 +501,13 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
unsigned long next, last = addr + size;
/* Populate the head page vmemmap page */
- pte = vmemmap_populate_address(addr, node, NULL, -1, 0);
+ pte = vmemmap_populate_address(addr, node, NULL, -1);
if (!pte)
return -ENOMEM;
/* Populate the tail pages vmemmap page */
next = addr + PAGE_SIZE;
- pte = vmemmap_populate_address(next, node, NULL, -1, 0);
+ pte = vmemmap_populate_address(next, node, NULL, -1);
if (!pte)
return -ENOMEM;
@@ -528,8 +517,7 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
*/
next += PAGE_SIZE;
rc = vmemmap_populate_range(next, last, node, NULL,
- pte_pfn(ptep_get(pte)),
- VMEMMAP_POPULATE_PAGEREF);
+ pte_pfn(ptep_get(pte)));
if (rc)
return -ENOMEM;
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 21/69] mm: Rename vmemmap optimization macros around folio semantics
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
The existing vmemmap optimization macros are named in terms of tail
pages, but they actually describe which folio sizes can use the
optimization and how much vmemmap backing an optimized folio keeps.
Rename them to reflect that meaning directly. This makes the names work
for both HugeTLB and other folio-based users such as DAX.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 18 ++++++++++--------
mm/hugetlb.c | 4 ++--
mm/hugetlb_vmemmap.c | 2 +-
mm/sparse-vmemmap.c | 4 ++--
4 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 362e16497533..40b1cea98b82 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -107,13 +107,15 @@
is_power_of_2(sizeof(struct page)) ? \
MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
-/*
- * vmemmap optimization (like HVO) is only possible for page orders that fill
- * two or more pages with struct pages.
- */
-#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page)))
-#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1)
-#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0)
+/* The number of vmemmap pages required by a vmemmap-optimized folio. */
+#define OPTIMIZED_FOLIO_VMEMMAP_PAGES 1
+#define OPTIMIZED_FOLIO_VMEMMAP_SIZE (OPTIMIZED_FOLIO_VMEMMAP_PAGES * PAGE_SIZE)
+#define OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES (OPTIMIZED_FOLIO_VMEMMAP_SIZE / sizeof(struct page))
+#define OPTIMIZABLE_FOLIO_MIN_ORDER (ilog2(OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES) + 1)
+
+#define __NR_OPTIMIZABLE_FOLIO_ORDERS (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1)
+#define NR_OPTIMIZABLE_FOLIO_ORDERS \
+ (__NR_OPTIMIZABLE_FOLIO_ORDERS > 0 ? __NR_OPTIMIZABLE_FOLIO_ORDERS : 0)
enum migratetype {
MIGRATE_UNMOVABLE,
@@ -1146,7 +1148,7 @@ struct zone {
atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
- struct page *vmemmap_tails[NR_VMEMMAP_TAILS];
+ struct page *vmemmap_tails[NR_OPTIMIZABLE_FOLIO_ORDERS];
#endif
} ____cacheline_internodealigned_in_smp;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 1f0a0e31d624..53448b05ca11 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3318,7 +3318,7 @@ void __init hugetlb_struct_page_init(void)
struct zone *zone;
for_each_zone(zone) {
- for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
+ for (int i = 0; i < NR_OPTIMIZABLE_FOLIO_ORDERS; i++) {
struct page *tail, *p;
unsigned int order;
@@ -3326,7 +3326,7 @@ void __init hugetlb_struct_page_init(void)
if (!tail)
continue;
- order = i + VMEMMAP_TAIL_MIN_ORDER;
+ order = i + OPTIMIZABLE_FOLIO_MIN_ORDER;
p = page_to_virt(tail);
for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
init_compound_tail(p + j, NULL, order, zone);
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 952216a49bcb..e9906d32a64c 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -495,7 +495,7 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *
static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
{
- const unsigned int idx = order - VMEMMAP_TAIL_MIN_ORDER;
+ const unsigned int idx = order - OPTIMIZABLE_FOLIO_MIN_ORDER;
struct page *tail, *p;
int node = zone_to_nid(zone);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index d7e9fb47f7ee..39529245d790 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -318,12 +318,12 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
unsigned int idx;
int node = zone_to_nid(zone);
- if (WARN_ON_ONCE(order < VMEMMAP_TAIL_MIN_ORDER))
+ if (WARN_ON_ONCE(order < OPTIMIZABLE_FOLIO_MIN_ORDER))
return NULL;
if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER))
return NULL;
- idx = order - VMEMMAP_TAIL_MIN_ORDER;
+ idx = order - OPTIMIZABLE_FOLIO_MIN_ORDER;
tail = zone->vmemmap_tails[idx];
if (tail)
return tail;
--
2.54.0
^ permalink raw reply related
* [PATCH v2 22/69] mm/sparse: Drop power-of-2 size requirement for struct mem_section
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
struct mem_section is currently forced to a power-of-2 size so the
section-to-root lookup can use a mask instead of a modulo.
That requirement adds configuration-dependent padding, especially with
CONFIG_PAGE_EXTENSION, just to preserve the lookup scheme.
Drop the constraint and use a plain modulo for the lookup instead. The
divisor is constant, so the generated code remains cheap while avoiding
the extra padding. It also removes an unnecessary layout constraint
from the type.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 8 +-------
mm/sparse.c | 2 --
scripts/gdb/linux/mm.py | 6 ++----
3 files changed, 3 insertions(+), 13 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 40b1cea98b82..ae0271eaec05 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2027,12 +2027,7 @@ struct mem_section {
* section. (see page_ext.h about this.)
*/
struct page_ext *page_ext;
- unsigned long pad;
#endif
- /*
- * WARNING: mem_section must be a power-of-2 in size for the
- * calculation and use of SECTION_ROOT_MASK to make sense.
- */
};
#ifdef CONFIG_SPARSEMEM_EXTREME
@@ -2043,7 +2038,6 @@ struct mem_section {
#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
#define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
-#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
#ifdef CONFIG_SPARSEMEM_EXTREME
extern struct mem_section **mem_section;
@@ -2067,7 +2061,7 @@ static inline struct mem_section *__nr_to_section(unsigned long nr)
if (!mem_section || !mem_section[root])
return NULL;
#endif
- return &mem_section[root][nr & SECTION_ROOT_MASK];
+ return &mem_section[root][nr % SECTIONS_PER_ROOT];
}
extern size_t mem_section_usage_size(void);
diff --git a/mm/sparse.c b/mm/sparse.c
index 324213d8bdcb..9457a4d6a6fc 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -331,8 +331,6 @@ void __init sparse_init(void)
unsigned long pnum_end, pnum_begin, map_count = 1;
int nid_begin;
- /* see include/linux/mmzone.h 'struct mem_section' definition */
- BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
memblocks_present();
if (compound_info_has_mask()) {
diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py
index dffadccbb01d..da4e8e9655a6 100644
--- a/scripts/gdb/linux/mm.py
+++ b/scripts/gdb/linux/mm.py
@@ -70,7 +70,6 @@ class x86_page_ops():
self.SECTIONS_PER_ROOT = 1
self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT)
- self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1
try:
self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT'))
@@ -100,7 +99,7 @@ class x86_page_ops():
def __nr_to_section(self, nr):
root = self.SECTION_NR_TO_ROOT(nr)
mem_section = gdb.parse_and_eval("mem_section")
- return mem_section[root][nr & self.SECTION_ROOT_MASK]
+ return mem_section[root][nr % self.SECTIONS_PER_ROOT]
def pfn_to_section_nr(self, pfn):
return pfn >> self.PFN_SECTION_SHIFT
@@ -249,7 +248,6 @@ class aarch64_page_ops():
self.SECTIONS_PER_ROOT = 1
self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT)
- self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1
self.SUBSECTION_SHIFT = 21
self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT
self.PFN_SUBSECTION_SHIFT = self.SUBSECTION_SHIFT - self.PAGE_SHIFT
@@ -304,7 +302,7 @@ class aarch64_page_ops():
def __nr_to_section(self, nr):
root = self.SECTION_NR_TO_ROOT(nr)
mem_section = gdb.parse_and_eval("mem_section")
- return mem_section[root][nr & self.SECTION_ROOT_MASK]
+ return mem_section[root][nr % self.SECTIONS_PER_ROOT]
def pfn_to_section_nr(self, pfn):
return pfn >> self.PFN_SECTION_SHIFT
--
2.54.0
^ permalink raw reply related
* [PATCH v2 23/69] mm/sparse-vmemmap: track compound page order in struct mem_section
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
HugeTLB and DAX both rely on vmemmap optimization, but sparsemem does
not record what compound page order a section is populated with.
As a result, code that needs this information has to open-code
separate handling across users of vmemmap optimization. It also
prevents other memory management code, such as struct page
initialization, from skipping initialization of shared vmemmap pages
when needed.
Track the compound page order in struct mem_section and provide small
helpers to access it. A compound page larger than a section naturally
carries the same order across all covered sections.
This is a preparatory change for consolidating vmemmap optimization
handling and for letting later code make initialization decisions
based on the section's compound page order.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index ae0271eaec05..6f112e6f42bb 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2028,6 +2028,14 @@ struct mem_section {
*/
struct page_ext *page_ext;
#endif
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+ /*
+ * The order of compound pages in this section. Typically, the section
+ * holds compound pages of this order; a larger compound page will span
+ * multiple sections.
+ */
+ unsigned int order;
+#endif
};
#ifdef CONFIG_SPARSEMEM_EXTREME
@@ -2224,6 +2232,17 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long
*pfn = (*pfn & PAGE_SECTION_MASK) + (bit * PAGES_PER_SUBSECTION);
return true;
}
+
+static inline void section_set_order(struct mem_section *section, unsigned int order)
+{
+ VM_WARN_ON(section->order && order && section->order != order);
+ section->order = order;
+}
+
+static inline unsigned int section_order(const struct mem_section *section)
+{
+ return section->order;
+}
#else
static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
{
@@ -2234,6 +2253,15 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long
{
return true;
}
+
+static inline void section_set_order(struct mem_section *section, unsigned int order)
+{
+}
+
+static inline unsigned int section_order(const struct mem_section *section)
+{
+ return 0;
+}
#endif
void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
--
2.54.0
^ permalink raw reply related
* [PATCH v2 24/69] mm/mm_init: Skip initializing shared vmemmap tail pages
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
memmap_init_range() initializes every struct page in the target range.
For compound pages with vmemmap optimization, the tail struct pages are
backed by a shared vmemmap page.
Initializing those tail struct pages would overwrite the shared
vmemmap page contents, so users such as HugeTLB have to open-code
follow-up handling to restore the metadata afterwards.
Use the section's compound page order to detect struct pages that fall
into the shared tail vmemmap range and skip their initialization in
memmap_init_range(). Still initialize the pageblock migratetypes for
the skipped range so the surrounding setup remains intact.
This is a preparatory change for consolidating handling across users of
vmemmap optimization, and it also avoids redundant initialization of
shared tail vmemmap pages during early boot.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 9 +++++++++
mm/internal.h | 16 ++++++++++++++++
mm/mm_init.c | 19 +++++++++++++------
3 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 6f112e6f42bb..5fc968bac1f7 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2264,6 +2264,11 @@ static inline unsigned int section_order(const struct mem_section *section)
}
#endif
+static inline unsigned int pfn_to_section_order(unsigned long pfn)
+{
+ return section_order(__pfn_to_section(pfn));
+}
+
void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
unsigned long flags);
@@ -2404,6 +2409,10 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
#else
#define sparse_vmemmap_init_nid_early(_nid) do {} while (0)
#define pfn_in_present_section pfn_valid
+static inline unsigned int pfn_to_section_order(unsigned long pfn)
+{
+ return 0;
+}
#endif /* CONFIG_SPARSEMEM */
/*
diff --git a/mm/internal.h b/mm/internal.h
index 4a5053368078..1f1c07eb70e2 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1004,10 +1004,26 @@ static inline void sparse_init(void) {}
*/
#ifdef CONFIG_SPARSEMEM_VMEMMAP
void sparse_init_subsection_map(void);
+
+static inline bool vmemmap_page_optimizable(const struct page *page)
+{
+ unsigned long pfn = page_to_pfn(page);
+ unsigned long nr_pages = 1UL << pfn_to_section_order(pfn);
+
+ if (!is_power_of_2(sizeof(struct page)))
+ return false;
+
+ return (pfn & (nr_pages - 1)) >= OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES;
+}
#else
static inline void sparse_init_subsection_map(void)
{
}
+
+static inline bool vmemmap_page_optimizable(const struct page *page)
+{
+ return false;
+}
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
diff --git a/mm/mm_init.c b/mm/mm_init.c
index c64e5d63c4ae..3aaee1cf7bf0 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -674,19 +674,17 @@ static inline void fixup_hashdist(void)
static inline void fixup_hashdist(void) {}
#endif /* CONFIG_NUMA */
-#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
- unsigned long nr_pages, int migratetype, bool atomic)
+ unsigned long nr_pages, int migratetype, bool isolate, bool atomic)
{
const unsigned long end = pfn + nr_pages;
for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) {
- init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false);
+ init_pageblock_migratetype(pfn_to_page(pfn), migratetype, isolate);
if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION))
cond_resched();
}
}
-#endif
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
/*
@@ -916,6 +914,15 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
}
page = pfn_to_page(pfn);
+ if (vmemmap_page_optimizable(page)) {
+ unsigned long start = pfn;
+
+ pfn = min(ALIGN(start, 1UL << pfn_to_section_order(pfn)), end_pfn);
+ pageblock_migratetype_init_range(start, pfn - start, migratetype,
+ isolate_pageblock, false);
+ continue;
+ }
+
__init_single_page(page, pfn, zone, nid);
if (context == MEMINIT_HOTPLUG) {
#ifdef CONFIG_ZONE_DEVICE
@@ -1142,7 +1149,7 @@ void __ref memmap_init_zone_device(struct zone *zone,
compound_nr_pages(pfn, altmap, pgmap));
}
- pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false);
+ pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false, false);
pr_debug("%s initialised %lu pages in %ums\n", __func__,
nr_pages, jiffies_to_msecs(jiffies - start));
@@ -1982,7 +1989,7 @@ static void __init deferred_free_pages(unsigned long pfn,
if (!nr_pages)
return;
- pageblock_migratetype_init_range(pfn, nr_pages, mt, true);
+ pageblock_migratetype_init_range(pfn, nr_pages, mt, false, true);
page = pfn_to_page(pfn);
--
2.54.0
^ permalink raw reply related
* [PATCH v2 25/69] mm/sparse-vmemmap: Initialize shared tail vmemmap pages on allocation
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
The shared tail vmemmap page allocated in vmemmap_get_tail() used to be
left uninitialized, because memmap_init_range() would later overwrite
it. That forced users such as HugeTLB to defer the initialization to
their own setup paths.
Now that memmap_init_range() skips shared tail vmemmap pages, initialize
them immediately in vmemmap_get_tail() with init_compound_tail()
instead.
This moves the initialization to the point where the shared tail page is
allocated and avoids relying on deferred handling in individual users.
The remaining deferred initialization in HugeTLB will be removed once it
switches to the section compound page order mechanism.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/sparse-vmemmap.c | 11 ++---------
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 39529245d790..60d5330a8399 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -328,18 +328,11 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
if (tail)
return tail;
- /*
- * Only allocate the page, but do not initialize it.
- *
- * Any initialization done here will be overwritten by memmap_init().
- *
- * hugetlb_vmemmap_init() will take care of initialization after
- * memmap_init().
- */
-
p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
if (!p)
return NULL;
+ for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++)
+ init_compound_tail(p + i, NULL, order, zone);
tail = virt_to_page(p);
zone->vmemmap_tails[idx] = tail;
--
2.54.0
^ permalink raw reply related
* [PATCH v2 26/69] mm/sparse-vmemmap: Support section-based vmemmap accounting
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Teach section_nr_vmemmap_pages() to account for section-based vmemmap
optimization, so the helper can report the vmemmap page usage for a
memory section with or without shared tail vmemmap pages.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 8 ++++++++
mm/sparse-vmemmap.c | 13 +++++++++----
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 5fc968bac1f7..0974205abd3d 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2269,6 +2269,14 @@ static inline unsigned int pfn_to_section_order(unsigned long pfn)
return section_order(__pfn_to_section(pfn));
}
+static inline bool section_vmemmap_optimizable(const struct mem_section *section)
+{
+ if (!is_power_of_2(sizeof(struct page)))
+ return false;
+
+ return section_order(section) >= OPTIMIZABLE_FOLIO_MIN_ORDER;
+}
+
void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
unsigned long flags);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 60d5330a8399..94964363d95c 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -629,24 +629,29 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
- const unsigned int order = pgmap ? pgmap->vmemmap_shift : 0;
+ const struct mem_section *ms = __pfn_to_section(pfn);
+ const unsigned int order = pgmap ? pgmap->vmemmap_shift : section_order(ms);
const unsigned long pages_per_compound = 1UL << order;
+ unsigned int vmemmap_pages = OPTIMIZED_FOLIO_VMEMMAP_PAGES;
VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION);
- if (!vmemmap_can_optimize(altmap, pgmap))
+ if (vmemmap_can_optimize(altmap, pgmap))
+ vmemmap_pages = VMEMMAP_RESERVE_NR;
+
+ if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms))
return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
if (order < PFN_SECTION_SHIFT) {
VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
- return VMEMMAP_RESERVE_NR * nr_pages / pages_per_compound;
+ return vmemmap_pages * nr_pages / pages_per_compound;
}
VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
if (IS_ALIGNED(pfn, pages_per_compound))
- return VMEMMAP_RESERVE_NR;
+ return vmemmap_pages;
return 0;
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 27/69] mm/sparse-vmemmap: Support section-based vmemmap optimization
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Teach sparse-vmemmap population code to use the compound page order
when deciding whether a vmemmap page can be optimized.
With this information, the common sparse-vmemmap population path can
allocate or reuse shared tail vmemmap pages directly instead of relying
on HugeTLB/DAX-specific handling.
This centralizes vmemmap optimization logic in the sparse-vmemmap code,
based on section metadata, and prepares for sharing the same mechanism
across different users of vmemmap optimization, including HugeTLB and
DAX.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 2 +-
mm/internal.h | 3 ++
mm/sparse-vmemmap.c | 89 +++++++++++++++++++++++++-----------------
mm/sparse.c | 34 +++++++++++++++-
4 files changed, 89 insertions(+), 39 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 0974205abd3d..bf4c40818b63 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1147,7 +1147,7 @@ struct zone {
/* Zone statistics */
atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
struct page *vmemmap_tails[NR_OPTIMIZABLE_FOLIO_ORDERS];
#endif
} ____cacheline_internodealigned_in_smp;
diff --git a/mm/internal.h b/mm/internal.h
index 1f1c07eb70e2..2defdef1aedf 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -995,6 +995,9 @@ static inline void __section_mark_present(struct mem_section *ms,
ms->section_mem_map |= SECTION_MARKED_PRESENT;
}
+
+int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
#else
static inline void sparse_init(void) {}
#endif /* CONFIG_SPARSEMEM */
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 94964363d95c..69ae40692e41 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -139,17 +139,49 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
start, end - 1);
}
+static struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid)
+{
+ pg_data_t *pgdat = NODE_DATA(nid);
+
+ for (enum zone_type zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
+ struct zone *zone = &pgdat->node_zones[zone_type];
+
+ if (zone_spans_pfn(zone, pfn))
+ return zone;
+ }
+
+ return NULL;
+}
+
+static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone);
+
static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
struct vmem_altmap *altmap,
unsigned long ptpfn)
{
pte_t *pte = pte_offset_kernel(pmd, addr);
+
if (pte_none(ptep_get(pte))) {
pte_t entry;
- void *p;
+
+ if (vmemmap_page_optimizable((struct page *)addr) &&
+ ptpfn == (unsigned long)-1) {
+ struct page *page;
+ unsigned long pfn = page_to_pfn((struct page *)addr);
+ const struct mem_section *ms = __pfn_to_section(pfn);
+ struct zone *zone = pfn_to_zone(pfn, node);
+
+ if (WARN_ON_ONCE(!zone))
+ return NULL;
+ page = vmemmap_get_tail(section_order(ms), zone);
+ if (!page)
+ return NULL;
+ ptpfn = page_to_pfn(page);
+ }
if (ptpfn == (unsigned long)-1) {
- p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
+ void *p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
+
if (!p)
return NULL;
ptpfn = PHYS_PFN(__pa(p));
@@ -168,7 +200,8 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in
}
entry = pfn_pte(ptpfn, PAGE_KERNEL);
set_pte_at(&init_mm, addr, pte, entry);
- }
+ } else if (WARN_ON_ONCE(vmemmap_page_optimizable((struct page *)addr)))
+ return NULL;
return pte;
}
@@ -311,7 +344,6 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end,
}
}
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
{
struct page *p, *tail;
@@ -340,6 +372,7 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
return tail;
}
+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end,
unsigned int order, struct zone *zone,
unsigned long headsize)
@@ -388,6 +421,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
pmd_t *pmd;
for (addr = start; addr < end; addr = next) {
+ unsigned long pfn = page_to_pfn((struct page *)addr);
+ struct mem_section *ms = __pfn_to_section(pfn);
+
next = pmd_addr_end(addr, end);
pgd = vmemmap_pgd_populate(addr, node);
@@ -403,7 +439,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
return -ENOMEM;
pmd = pmd_offset(pud, addr);
- if (pmd_none(pmdp_get(pmd))) {
+ if (pmd_none(pmdp_get(pmd)) && !section_vmemmap_optimizable(ms)) {
void *p;
p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
@@ -421,8 +457,19 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
*/
return -ENOMEM;
}
- } else if (vmemmap_check_pmd(pmd, node, addr, next))
+ } else if (vmemmap_check_pmd(pmd, node, addr, next)) {
+ const struct mem_section *start_ms;
+ unsigned long align = max(1UL << section_order(ms), PAGES_PER_SECTION);
+
+ /* HVO-covered sections must not use PMD mappings. */
+ start_ms = __pfn_to_section(ALIGN_DOWN(pfn, align));
+ if (!IS_ALIGNED(pfn, align) && section_vmemmap_optimizable(start_ms))
+ return -ENOTSUPP;
+
+ /* PMD mappings end HVO coverage for this section. */
+ section_set_order(ms, 0);
continue;
+ }
if (vmemmap_populate_basepages(addr, next, node, altmap))
return -ENOMEM;
}
@@ -626,36 +673,6 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
}
}
-static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
- struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
-{
- const struct mem_section *ms = __pfn_to_section(pfn);
- const unsigned int order = pgmap ? pgmap->vmemmap_shift : section_order(ms);
- const unsigned long pages_per_compound = 1UL << order;
- unsigned int vmemmap_pages = OPTIMIZED_FOLIO_VMEMMAP_PAGES;
-
- VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
- VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION);
-
- if (vmemmap_can_optimize(altmap, pgmap))
- vmemmap_pages = VMEMMAP_RESERVE_NR;
-
- if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms))
- return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
-
- if (order < PFN_SECTION_SHIFT) {
- VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
- return vmemmap_pages * nr_pages / pages_per_compound;
- }
-
- VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
-
- if (IS_ALIGNED(pfn, pages_per_compound))
- return vmemmap_pages;
-
- return 0;
-}
-
static struct page * __meminit populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
diff --git a/mm/sparse.c b/mm/sparse.c
index 9457a4d6a6fc..3e96478a63e0 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -284,6 +284,36 @@ static void __init sparse_usage_fini(void)
sparse_usagebuf = sparse_usagebuf_end = NULL;
}
+int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
+ struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
+{
+ const struct mem_section *ms = __pfn_to_section(pfn);
+ const unsigned int order = pgmap ? pgmap->vmemmap_shift : section_order(ms);
+ const unsigned long pages_per_compound = 1UL << order;
+ unsigned int vmemmap_pages = OPTIMIZED_FOLIO_VMEMMAP_PAGES;
+
+ VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
+ VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION);
+
+ if (vmemmap_can_optimize(altmap, pgmap))
+ vmemmap_pages = VMEMMAP_RESERVE_NR;
+
+ if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms))
+ return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
+
+ if (order < PFN_SECTION_SHIFT) {
+ VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
+ return vmemmap_pages * nr_pages / pages_per_compound;
+ }
+
+ VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
+
+ if (IS_ALIGNED(pfn, pages_per_compound))
+ return vmemmap_pages;
+
+ return 0;
+}
+
/*
* Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
* And number of present sections in this node is map_count.
@@ -314,8 +344,8 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
nid, NULL, NULL);
if (!map)
panic("Failed to allocate memmap for section %lu\n", pnum);
- memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
- PAGE_SIZE));
+ memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
+ NULL, NULL));
sparse_init_early_section(nid, map, pnum, 0);
}
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 28/69] mm/hugetlb: Use generic vmemmap optimization macros
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Vmemmap optimization is no longer hugetlb-specific, so the remaining
hugetlb-local reserve macros are redundant.
Replace them with the generic definitions to remove duplication and keep
the hugetlb vmemmap code aligned with the common optimization macros.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/hugetlb.c | 4 ++--
mm/hugetlb_vmemmap.c | 14 +++++++-------
mm/hugetlb_vmemmap.h | 9 +--------
3 files changed, 10 insertions(+), 17 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 53448b05ca11..8debe5c5abce 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3222,7 +3222,7 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h,
* be no contention.
*/
hugetlb_folio_init_tail_vmemmap(folio, h,
- HUGETLB_VMEMMAP_RESERVE_PAGES,
+ OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES,
pages_per_huge_page(h));
}
hugetlb_bootmem_init_migratetype(folio, h);
@@ -3261,7 +3261,7 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
WARN_ON(folio_ref_count(folio) != 1);
hugetlb_folio_init_vmemmap(folio, h,
- HUGETLB_VMEMMAP_RESERVE_PAGES);
+ OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES);
init_new_hugetlb_folio(folio);
if (hugetlb_bootmem_page_prehvo(m))
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index e9906d32a64c..4367118f8f57 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -407,7 +407,7 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
- vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE;
+ vmemmap_start += OPTIMIZED_FOLIO_VMEMMAP_SIZE;
/*
* The pages which the vmemmap virtual address range [@vmemmap_start,
@@ -637,10 +637,10 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
spfn = (unsigned long)&folio->page;
epfn = spfn + hugetlb_vmemmap_size(h);
vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
- HUGETLB_VMEMMAP_RESERVE_SIZE);
+ OPTIMIZED_FOLIO_VMEMMAP_SIZE);
register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)),
&folio->page,
- HUGETLB_VMEMMAP_RESERVE_PAGES);
+ OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES);
continue;
}
@@ -779,9 +779,9 @@ void __init hugetlb_vmemmap_init_early(int nid)
zone = pfn_to_zone(nid, pfn);
if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
- zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
+ zone, OPTIMIZED_FOLIO_VMEMMAP_SIZE))
panic("Failed to allocate memmap for HugeTLB page\n");
- memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
+ memmap_boot_pages_add(OPTIMIZED_FOLIO_VMEMMAP_PAGES);
pnum = pfn_to_section_nr(pfn);
ns = psize / section_size;
@@ -826,8 +826,8 @@ static int __init hugetlb_vmemmap_init(void)
{
const struct hstate *h;
- /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */
- BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES);
+ /* OPTIMIZED_FOLIO_VMEMMAP_SIZE should cover all used struct pages */
+ BUILD_BUG_ON(__NR_USED_SUBPAGE > OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES);
for_each_hstate(h) {
if (hugetlb_vmemmap_optimizable(h)) {
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 7ac49c52457d..66e11893d076 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -12,13 +12,6 @@
#include <linux/io.h>
#include <linux/memblock.h>
-/*
- * Reserve one vmemmap page, all vmemmap addresses are mapped to it. See
- * Documentation/mm/vmemmap_dedup.rst.
- */
-#define HUGETLB_VMEMMAP_RESERVE_SIZE PAGE_SIZE
-#define HUGETLB_VMEMMAP_RESERVE_PAGES (HUGETLB_VMEMMAP_RESERVE_SIZE / sizeof(struct page))
-
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio);
long hugetlb_vmemmap_restore_folios(const struct hstate *h,
@@ -43,7 +36,7 @@ static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
*/
static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
- int size = hugetlb_vmemmap_size(h) - HUGETLB_VMEMMAP_RESERVE_SIZE;
+ int size = hugetlb_vmemmap_size(h) - OPTIMIZED_FOLIO_VMEMMAP_SIZE;
if (!is_power_of_2(sizeof(struct page)))
return 0;
--
2.54.0
^ permalink raw reply related
* [PATCH v2 29/69] mm/sparse: Mark memblocks present earlier
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
Later patches need struct mem_section entries to be available before
HugeTLB bootmem allocation starts, so the section metadata can be set up
at that stage.
Move the memblock-based section present marking out of sparse_init() and
call it earlier from mm_core_init_early(). Rename the helper to
sparse_memblocks_present() while doing so.
This prepares sparsemem section metadata before the early HugeTLB setup
path.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/internal.h | 2 ++
mm/mm_init.c | 1 +
mm/sparse.c | 4 +---
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index 2defdef1aedf..bf30617c78d8 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -962,6 +962,7 @@ void memmap_init_range(unsigned long, int, unsigned long, unsigned long,
* mm/sparse.c
*/
#ifdef CONFIG_SPARSEMEM
+void sparse_memblocks_present(void);
void sparse_init(void);
int sparse_index_init(unsigned long section_nr, int nid);
@@ -999,6 +1000,7 @@ static inline void __section_mark_present(struct mem_section *ms,
int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
#else
+static inline void sparse_memblocks_present(void) {}
static inline void sparse_init(void) {}
#endif /* CONFIG_SPARSEMEM */
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 3aaee1cf7bf0..6723c604eefd 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2693,6 +2693,7 @@ void __init __weak mem_init(void)
void __init mm_core_init_early(void)
{
+ sparse_memblocks_present();
free_area_init();
hugetlb_cma_reserve();
diff --git a/mm/sparse.c b/mm/sparse.c
index 3e96478a63e0..33e89bf1ec0c 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -195,7 +195,7 @@ static void __init memory_present(int nid, unsigned long start, unsigned long en
* This is a convenience function that is useful to mark all of the systems
* memory as present during initialization.
*/
-static void __init memblocks_present(void)
+void __init sparse_memblocks_present(void)
{
unsigned long start, end;
int i, nid;
@@ -361,8 +361,6 @@ void __init sparse_init(void)
unsigned long pnum_end, pnum_begin, map_count = 1;
int nid_begin;
- memblocks_present();
-
if (compound_info_has_mask()) {
VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),
MAX_FOLIO_VMEMMAP_ALIGN));
--
2.54.0
^ permalink raw reply related
* [PATCH v2 30/69] mm/hugetlb: Switch HugeTLB to section-based vmemmap optimization
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
HugeTLB bootmem vmemmap optimization still carries its own early setup
path, including pre-populating optimized mappings before the generic
sparse-vmemmap code runs.
Now that section metadata records the compound page order, HugeTLB only
needs to mark the bootmem huge page range with that order. The generic
sparse-vmemmap population path can then allocate and map the shared tail
vmemmap pages without any HugeTLB-specific early population code.
Do that by setting the section order when a bootmem huge page is
allocated and dropping the dedicated pre-HVO helpers and related
special-casing.
This removes duplicate early setup logic and switches HugeTLB to the
section-based vmemmap optimization path.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/hugetlb.h | 1 -
include/linux/mm.h | 3 -
include/linux/mmzone.h | 17 ++++++
mm/bootmem_info.c | 5 +-
mm/hugetlb.c | 26 ++-------
mm/hugetlb_vmemmap.c | 124 ++++++----------------------------------
mm/hugetlb_vmemmap.h | 13 ++---
mm/sparse-vmemmap.c | 29 ----------
8 files changed, 45 insertions(+), 173 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index fd901bb3630c..dce8969961ea 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -171,7 +171,6 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio);
extern int movable_gigantic_pages __read_mostly;
extern int sysctl_hugetlb_shm_group __read_mostly;
-extern struct list_head huge_boot_pages[MAX_NUMNODES];
void hugetlb_struct_page_init(void);
void hugetlb_bootmem_alloc(void);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 31e27ff6a35f..f39f6fca6551 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -4864,9 +4864,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap);
int vmemmap_populate(unsigned long start, unsigned long end, int node,
struct vmem_altmap *altmap);
-int vmemmap_populate_hvo(unsigned long start, unsigned long end,
- unsigned int order, struct zone *zone,
- unsigned long headsize);
void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
unsigned long headsize);
void vmemmap_populate_print_last(void);
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index bf4c40818b63..d6a5dd042c25 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2264,6 +2264,18 @@ static inline unsigned int section_order(const struct mem_section *section)
}
#endif
+static inline void section_set_order_range(unsigned long pfn, unsigned long nr_pages,
+ unsigned int order)
+{
+ unsigned long section_nr = pfn_to_section_nr(pfn);
+
+ if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))
+ return;
+
+ for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++)
+ section_set_order(__nr_to_section(section_nr + i), order);
+}
+
static inline unsigned int pfn_to_section_order(unsigned long pfn)
{
return section_order(__pfn_to_section(pfn));
@@ -2417,6 +2429,11 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
#else
#define sparse_vmemmap_init_nid_early(_nid) do {} while (0)
#define pfn_in_present_section pfn_valid
+static inline void section_set_order_range(unsigned long pfn, unsigned long nr_pages,
+ unsigned int order)
+{
+}
+
static inline unsigned int pfn_to_section_order(unsigned long pfn)
{
return 0;
diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c
index 3d7675a3ae04..24f45d86ffb3 100644
--- a/mm/bootmem_info.c
+++ b/mm/bootmem_info.c
@@ -51,9 +51,8 @@ static void __init register_page_bootmem_info_section(unsigned long start_pfn)
section_nr = pfn_to_section_nr(start_pfn);
ms = __nr_to_section(section_nr);
- if (!preinited_vmemmap_section(ms))
- register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn),
- PAGES_PER_SECTION);
+ register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn),
+ PAGES_PER_SECTION);
usage = ms->usage;
page = virt_to_page(usage);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 8debe5c5abce..080f130017e3 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -57,7 +57,7 @@ unsigned int default_hstate_idx;
struct hstate hstates[HUGE_MAX_HSTATE];
__initdata nodemask_t hugetlb_bootmem_nodes;
-__initdata struct list_head huge_boot_pages[MAX_NUMNODES];
+static __initdata struct list_head huge_boot_pages[MAX_NUMNODES];
/*
* Due to ordering constraints across the init code for various
@@ -3111,6 +3111,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
} else {
list_add_tail(&m->list, &huge_boot_pages[nid]);
m->flags |= HUGE_BOOTMEM_ZONES_VALID;
+ hugetlb_vmemmap_optimize_bootmem_page(m);
/*
* Only initialize the head struct page in memmap_init_reserved_pages,
* rest of the struct pages will be initialized by the HugeTLB
@@ -3264,13 +3265,15 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES);
init_new_hugetlb_folio(folio);
- if (hugetlb_bootmem_page_prehvo(m))
+ if (hugetlb_bootmem_page_prehvo(m)) {
/*
* If pre-HVO was done, just set the
* flag, the HVO code will then skip
* this folio.
*/
folio_set_hugetlb_vmemmap_optimized(folio);
+ section_set_order_range(folio_pfn(folio), folio_nr_pages(folio), 0);
+ }
if (hugetlb_bootmem_page_earlycma(m))
folio_set_hugetlb_cma(folio);
@@ -3314,25 +3317,6 @@ void __init hugetlb_struct_page_init(void)
.max_threads = num_node_state(N_MEMORY),
.numa_aware = true,
};
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
- struct zone *zone;
-
- for_each_zone(zone) {
- for (int i = 0; i < NR_OPTIMIZABLE_FOLIO_ORDERS; i++) {
- struct page *tail, *p;
- unsigned int order;
-
- tail = zone->vmemmap_tails[i];
- if (!tail)
- continue;
-
- order = i + OPTIMIZABLE_FOLIO_MIN_ORDER;
- p = page_to_virt(tail);
- for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
- init_compound_tail(p + j, NULL, order, zone);
- }
- }
-#endif
padata_do_multithreaded(&job);
}
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 4367118f8f57..730190390ba9 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -16,6 +16,7 @@
#include <linux/mmdebug.h>
#include <linux/pagewalk.h>
#include <linux/pgalloc.h>
+#include <linux/io.h>
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"
@@ -478,12 +479,8 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
return ret;
}
-/* Return true iff a HugeTLB whose vmemmap should and can be optimized. */
-static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *folio)
+static inline bool vmemmap_should_optimize(const struct hstate *h)
{
- if (folio_test_hugetlb_vmemmap_optimized(folio))
- return false;
-
if (!READ_ONCE(vmemmap_optimize_enabled))
return false;
@@ -493,6 +490,15 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *
return true;
}
+/* Return true iff a HugeTLB whose vmemmap should and can be optimized. */
+static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *folio)
+{
+ if (folio_test_hugetlb_vmemmap_optimized(folio))
+ return false;
+
+ return vmemmap_should_optimize(h);
+}
+
static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
{
const unsigned int idx = order - OPTIMIZABLE_FOLIO_MIN_ORDER;
@@ -638,9 +644,6 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
epfn = spfn + hugetlb_vmemmap_size(h);
vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
OPTIMIZED_FOLIO_VMEMMAP_SIZE);
- register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)),
- &folio->page,
- OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES);
continue;
}
@@ -706,111 +709,18 @@ void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head
__hugetlb_vmemmap_optimize_folios(h, folio_list, true);
}
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-
-/* Return true of a bootmem allocated HugeTLB page should be pre-HVO-ed */
-static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
-{
- unsigned long section_size, psize, pmd_vmemmap_size;
- phys_addr_t paddr;
-
- if (!READ_ONCE(vmemmap_optimize_enabled))
- return false;
-
- if (!hugetlb_vmemmap_optimizable(m->hstate))
- return false;
-
- psize = huge_page_size(m->hstate);
- paddr = virt_to_phys(m);
-
- /*
- * Pre-HVO only works if the bootmem huge page
- * is aligned to the section size.
- */
- section_size = (1UL << PA_SECTION_SHIFT);
- if (!IS_ALIGNED(paddr, section_size) ||
- !IS_ALIGNED(psize, section_size))
- return false;
-
- /*
- * The pre-HVO code does not deal with splitting PMDS,
- * so the bootmem page must be aligned to the number
- * of base pages that can be mapped with one vmemmap PMD.
- */
- pmd_vmemmap_size = (PMD_SIZE / (sizeof(struct page))) << PAGE_SHIFT;
- if (!IS_ALIGNED(paddr, pmd_vmemmap_size) ||
- !IS_ALIGNED(psize, pmd_vmemmap_size))
- return false;
-
- return true;
-}
-
-static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn);
-
-/*
- * Initialize memmap section for a gigantic page, HVO-style.
- */
-void __init hugetlb_vmemmap_init_early(int nid)
+void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m)
{
- unsigned long psize, paddr, section_size;
- unsigned long ns, i, pnum, pfn, nr_pages;
- unsigned long start, end;
- struct huge_bootmem_page *m = NULL;
- void *map;
+ struct hstate *h = m->hstate;
+ unsigned long pfn = PHYS_PFN(__pa(m));
- if (!READ_ONCE(vmemmap_optimize_enabled))
+ if (!vmemmap_should_optimize(h))
return;
- section_size = (1UL << PA_SECTION_SHIFT);
-
- list_for_each_entry(m, &huge_boot_pages[nid], list) {
- struct zone *zone;
-
- if (!vmemmap_should_optimize_bootmem_page(m))
- continue;
-
- nr_pages = pages_per_huge_page(m->hstate);
- psize = nr_pages << PAGE_SHIFT;
- paddr = virt_to_phys(m);
- pfn = PHYS_PFN(paddr);
- map = pfn_to_page(pfn);
- start = (unsigned long)map;
- end = start + hugetlb_vmemmap_size(m->hstate);
- zone = pfn_to_zone(nid, pfn);
-
- if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
- zone, OPTIMIZED_FOLIO_VMEMMAP_SIZE))
- panic("Failed to allocate memmap for HugeTLB page\n");
- memmap_boot_pages_add(OPTIMIZED_FOLIO_VMEMMAP_PAGES);
-
- pnum = pfn_to_section_nr(pfn);
- ns = psize / section_size;
-
- for (i = 0; i < ns; i++) {
- sparse_init_early_section(nid, map, pnum,
- SECTION_IS_VMEMMAP_PREINIT);
- map += section_map_size();
- pnum++;
- }
-
+ section_set_order_range(pfn, pages_per_huge_page(h), huge_page_order(h));
+ if (section_vmemmap_optimizable(__pfn_to_section(pfn)))
m->flags |= HUGE_BOOTMEM_HVO;
- }
-}
-
-static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
-{
- struct zone *zone;
- enum zone_type zone_type;
-
- for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
- zone = &NODE_DATA(nid)->node_zones[zone_type];
- if (zone_spans_pfn(zone, pfn))
- return zone;
- }
-
- return NULL;
}
-#endif
static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
{
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 66e11893d076..0d8c88997066 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -9,8 +9,6 @@
#ifndef _LINUX_HUGETLB_VMEMMAP_H
#define _LINUX_HUGETLB_VMEMMAP_H
#include <linux/hugetlb.h>
-#include <linux/io.h>
-#include <linux/memblock.h>
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio);
@@ -20,10 +18,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio);
void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);
void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-void hugetlb_vmemmap_init_early(int nid);
-#endif
-
+void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m);
static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
{
@@ -69,13 +64,13 @@ static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h,
{
}
-static inline void hugetlb_vmemmap_init_early(int nid)
+static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
+ return 0;
}
-static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
+static inline void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m)
{
- return 0;
}
#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 69ae40692e41..b86634903fc0 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -32,7 +32,6 @@
#include <asm/dma.h>
#include <asm/tlbflush.h>
-#include "hugetlb_vmemmap.h"
#include "internal.h"
/*
@@ -372,33 +371,6 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
return tail;
}
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
-int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end,
- unsigned int order, struct zone *zone,
- unsigned long headsize)
-{
- unsigned long maddr;
- struct page *tail;
- pte_t *pte;
- int node = zone_to_nid(zone);
-
- tail = vmemmap_get_tail(order, zone);
- if (!tail)
- return -ENOMEM;
-
- for (maddr = addr; maddr < addr + headsize; maddr += PAGE_SIZE) {
- pte = vmemmap_populate_address(maddr, node, NULL, -1);
- if (!pte)
- return -ENOMEM;
- }
-
- /*
- * Reuse the last page struct page mapped above for the rest.
- */
- return vmemmap_populate_range(maddr, end, node, NULL, page_to_pfn(tail));
-}
-#endif
-
void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
unsigned long addr, unsigned long next)
{
@@ -600,7 +572,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
*/
void __init sparse_vmemmap_init_nid_early(int nid)
{
- hugetlb_vmemmap_init_early(nid);
}
#endif
--
2.54.0
^ permalink raw reply related
* [PATCH v2 31/69] mm/sparse: Remove section_map_size()
From: Muchun Song @ 2026-05-13 13:04 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
section_map_size() no longer provides any shared logic.
After the sparse-vmemmap changes, its only remaining user is the
!CONFIG_SPARSEMEM_VMEMMAP path in __populate_section_memmap(), which can
compute the size inline with PAGE_ALIGN(sizeof(struct page) *
PAGES_PER_SECTION).
Remove section_map_size() and inline the remaining calculation.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mm.h | 1 -
mm/sparse.c | 15 ++-------------
2 files changed, 2 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index f39f6fca6551..fef39be8acd2 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -4845,7 +4845,6 @@ static inline void print_vma_addr(char *prefix, unsigned long rip)
}
#endif
-unsigned long section_map_size(void);
struct page * __populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap);
diff --git a/mm/sparse.c b/mm/sparse.c
index 33e89bf1ec0c..47349f6f463f 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -222,23 +222,12 @@ size_t mem_section_usage_size(void)
return sizeof(struct mem_section_usage) + usemap_size();
}
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-unsigned long __init section_map_size(void)
-{
- return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
-}
-
-#else
-unsigned long __init section_map_size(void)
-{
- return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
-}
-
+#ifndef CONFIG_SPARSEMEM_VMEMMAP
struct page __init *__populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
- unsigned long size = section_map_size();
+ unsigned long size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false);
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 32/69] mm/mm_init: Factor out pfn_to_zone() as a shared helper
From: Muchun Song @ 2026-05-13 13:05 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
pfn_to_zone() in sparse-vmemmap.c duplicates the zone lookup logic in
__init_page_from_nid().
Move it to mm_init.c, declare it in mm/internal.h, and reuse it from
__init_page_from_nid() instead of open-coding the zone walk there.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
mm/internal.h | 1 +
mm/mm_init.c | 28 ++++++++++++++++------------
mm/sparse-vmemmap.c | 14 --------------
3 files changed, 17 insertions(+), 26 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index bf30617c78d8..18276cd15622 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1354,6 +1354,7 @@ static inline bool deferred_pages_enabled(void)
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
+struct zone *pfn_to_zone(unsigned long pfn, int nid);
void init_deferred_page(unsigned long pfn, int nid);
enum mminit_level {
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 6723c604eefd..35c99e5c215c 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -686,25 +686,29 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
}
}
+struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid)
+{
+ pg_data_t *pgdat = NODE_DATA(nid);
+
+ for (enum zone_type zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
+ struct zone *zone = &pgdat->node_zones[zone_type];
+
+ if (zone_spans_pfn(zone, pfn))
+ return zone;
+ }
+
+ return NULL;
+}
+
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
/*
* Initialize a reserved page unconditionally, finding its zone first.
*/
static void __meminit __init_page_from_nid(unsigned long pfn, int nid)
{
- pg_data_t *pgdat;
- int zid;
-
- pgdat = NODE_DATA(nid);
-
- for (zid = 0; zid < MAX_NR_ZONES; zid++) {
- struct zone *zone = &pgdat->node_zones[zid];
-
- if (zone_spans_pfn(zone, pfn))
- break;
- }
- __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
+ struct zone *zone = pfn_to_zone(pfn, nid);
+ __init_single_page(pfn_to_page(pfn), pfn, zone_idx(zone), nid);
if (pageblock_aligned(pfn)) {
enum migratetype mt =
kho_scratch_migratetype(pfn, MIGRATE_MOVABLE);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index b86634903fc0..f1c3b2d0f23c 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -138,20 +138,6 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
start, end - 1);
}
-static struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid)
-{
- pg_data_t *pgdat = NODE_DATA(nid);
-
- for (enum zone_type zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
- struct zone *zone = &pgdat->node_zones[zone_type];
-
- if (zone_spans_pfn(zone, pfn))
- return zone;
- }
-
- return NULL;
-}
-
static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone);
static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
--
2.54.0
^ permalink raw reply related
* [PATCH v2 33/69] mm/sparse: Remove SPARSEMEM_VMEMMAP_PREINIT
From: Muchun Song @ 2026-05-13 13:05 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
SPARSEMEM_VMEMMAP_PREINIT was only there to support HugeTLB's early
vmemmap optimization setup.
Now that HugeTLB bootmem vmemmap optimization uses the common
section-based sparse-vmemmap path, sparsemem no longer needs a separate
pre-initialization mechanism.
Remove the Kconfig symbols, section flag, and empty sparse-vmemmap early
hook, and always initialize present sections through the normal sparse
setup path.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
arch/x86/Kconfig | 1 -
fs/Kconfig | 1 -
include/linux/mmzone.h | 25 -------------------------
mm/Kconfig | 5 -----
mm/sparse-vmemmap.c | 13 -------------
mm/sparse.c | 23 ++++++++---------------
6 files changed, 8 insertions(+), 60 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index f24810015234..ed2aa0e4c472 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -148,7 +148,6 @@ config X86
select ARCH_WANT_LD_ORPHAN_WARN
select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if X86_64
select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64
- select ARCH_WANT_HUGETLB_VMEMMAP_PREINIT if X86_64
select ARCH_WANTS_THP_SWAP if X86_64
select ARCH_HAS_PARANOID_L1D_FLUSH
select ARCH_WANT_IRQS_OFF_ACTIVATE_MM
diff --git a/fs/Kconfig b/fs/Kconfig
index cf6ae64776e6..ccb9dd480523 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -278,7 +278,6 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP
def_bool HUGETLB_PAGE
depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
depends on SPARSEMEM_VMEMMAP
- select SPARSEMEM_VMEMMAP_PREINIT if ARCH_WANT_HUGETLB_VMEMMAP_PREINIT
config HUGETLB_PMD_PAGE_TABLE_SHARING
def_bool HUGETLB_PAGE
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index d6a5dd042c25..b9baef8cca91 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2092,9 +2092,6 @@ enum {
SECTION_IS_EARLY_BIT,
#ifdef CONFIG_ZONE_DEVICE
SECTION_TAINT_ZONE_DEVICE_BIT,
-#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
- SECTION_IS_VMEMMAP_PREINIT_BIT,
#endif
SECTION_MAP_LAST_BIT,
};
@@ -2106,9 +2103,6 @@ enum {
#ifdef CONFIG_ZONE_DEVICE
#define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT)
#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-#define SECTION_IS_VMEMMAP_PREINIT BIT(SECTION_IS_VMEMMAP_PREINIT_BIT)
-#endif
#define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1))
#define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT
@@ -2163,24 +2157,6 @@ static inline int online_device_section(const struct mem_section *section)
}
#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-static inline int preinited_vmemmap_section(const struct mem_section *section)
-{
- return (section &&
- (section->section_mem_map & SECTION_IS_VMEMMAP_PREINIT));
-}
-
-void sparse_vmemmap_init_nid_early(int nid);
-#else
-static inline int preinited_vmemmap_section(const struct mem_section *section)
-{
- return 0;
-}
-static inline void sparse_vmemmap_init_nid_early(int nid)
-{
-}
-#endif
-
static inline int online_section_nr(unsigned long nr)
{
return online_section(__nr_to_section(nr));
@@ -2427,7 +2403,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
#endif
#else
-#define sparse_vmemmap_init_nid_early(_nid) do {} while (0)
#define pfn_in_present_section pfn_valid
static inline void section_set_order_range(unsigned long pfn, unsigned long nr_pages,
unsigned int order)
diff --git a/mm/Kconfig b/mm/Kconfig
index bb0202cf8b15..c26d2d2050d5 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -410,8 +410,6 @@ config SPARSEMEM_VMEMMAP
pfn_to_page and page_to_pfn operations. This is the most
efficient option when sufficient kernel resources are available.
-config SPARSEMEM_VMEMMAP_PREINIT
- bool
#
# Select this config option from the architecture Kconfig, if it is preferred
# to enable the feature of HugeTLB/dev_dax vmemmap optimization.
@@ -422,9 +420,6 @@ config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
bool
-config ARCH_WANT_HUGETLB_VMEMMAP_PREINIT
- bool
-
config HAVE_MEMBLOCK_PHYS_MAP
bool
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index f1c3b2d0f23c..dde4486195ad 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -548,19 +548,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
return pfn_to_page(pfn);
}
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-/*
- * This is called just before initializing sections for a NUMA node.
- * Any special initialization that needs to be done before the
- * generic initialization can be done from here. Sections that
- * are initialized in hooks called from here will be skipped by
- * the generic initialization.
- */
-void __init sparse_vmemmap_init_nid_early(int nid)
-{
-}
-#endif
-
static void subsection_mask_set(unsigned long *map, unsigned long pfn,
unsigned long nr_pages)
{
diff --git a/mm/sparse.c b/mm/sparse.c
index 47349f6f463f..eab37504819d 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -316,27 +316,20 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
if (sparse_usage_init(nid, map_count))
panic("Failed to allocate usemap for node %d\n", nid);
- sparse_vmemmap_init_nid_early(nid);
-
for_each_present_section_nr(pnum_begin, pnum) {
- struct mem_section *ms;
unsigned long pfn = section_nr_to_pfn(pnum);
+ struct page *map;
if (pnum >= pnum_end)
break;
- ms = __nr_to_section(pnum);
- if (!preinited_vmemmap_section(ms)) {
- struct page *map;
-
- map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
- nid, NULL, NULL);
- if (!map)
- panic("Failed to allocate memmap for section %lu\n", pnum);
- memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
- NULL, NULL));
- sparse_init_early_section(nid, map, pnum, 0);
- }
+ map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
+ nid, NULL, NULL);
+ if (!map)
+ panic("Failed to allocate memmap for section %lu\n", pnum);
+ memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
+ NULL, NULL));
+ sparse_init_early_section(nid, map, pnum, 0);
}
sparse_usage_fini();
}
--
2.54.0
^ permalink raw reply related
* [PATCH v2 34/69] mm/sparse: Inline usemap allocation into sparse_init_nid()
From: Muchun Song @ 2026-05-13 13:05 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Muchun Song, Oscar Salvador,
Michael Ellerman, Madhavan Srinivasan
Cc: Lorenzo Stoakes, Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Nicholas Piggin,
Christophe Leroy, Ackerley Tng, Frank van der Linden,
aneesh.kumar, joao.m.martins, linux-mm, linuxppc-dev,
linux-kernel, Muchun Song
In-Reply-To: <20260513130542.35604-1-songmuchun@bytedance.com>
After removing SPARSEMEM_VMEMMAP_PREINIT, sparse_init_nid() no longer
needs the transient sparse_usagebuf state and its helper wrappers.
Allocate the usemap buffer directly in sparse_init_nid(), pass it to
sparse_init_one_section(), and drop sparse_usage_init(),
sparse_usage_fini(), and sparse_init_early_section().
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
include/linux/mmzone.h | 3 ---
mm/sparse.c | 46 +++++++-----------------------------------
2 files changed, 7 insertions(+), 42 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index b9baef8cca91..a60fd5785fa5 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2265,9 +2265,6 @@ static inline bool section_vmemmap_optimizable(const struct mem_section *section
return section_order(section) >= OPTIMIZABLE_FOLIO_MIN_ORDER;
}
-void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
- unsigned long flags);
-
#ifndef CONFIG_HAVE_ARCH_PFN_VALID
/**
* pfn_valid - check if there is a valid memory map entry for a PFN
diff --git a/mm/sparse.c b/mm/sparse.c
index eab37504819d..54c38ea08190 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -237,42 +237,6 @@ void __weak __meminit vmemmap_populate_print_last(void)
{
}
-static void *sparse_usagebuf __meminitdata;
-static void *sparse_usagebuf_end __meminitdata;
-
-/*
- * Helper function that is used for generic section initialization, and
- * can also be used by any hooks added above.
- */
-void __init sparse_init_early_section(int nid, struct page *map,
- unsigned long pnum, unsigned long flags)
-{
- BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);
- sparse_init_one_section(__nr_to_section(pnum), pnum, map,
- sparse_usagebuf, SECTION_IS_EARLY | flags);
- sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();
-}
-
-static int __init sparse_usage_init(int nid, unsigned long map_count)
-{
- unsigned long size;
-
- size = mem_section_usage_size() * map_count;
- sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid);
- if (!sparse_usagebuf) {
- sparse_usagebuf_end = NULL;
- return -ENOMEM;
- }
-
- sparse_usagebuf_end = sparse_usagebuf + size;
- return 0;
-}
-
-static void __init sparse_usage_fini(void)
-{
- sparse_usagebuf = sparse_usagebuf_end = NULL;
-}
-
int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
@@ -312,8 +276,11 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
unsigned long map_count)
{
unsigned long pnum;
+ struct mem_section_usage *usage;
- if (sparse_usage_init(nid, map_count))
+ usage = memblock_alloc_node(map_count * mem_section_usage_size(),
+ SMP_CACHE_BYTES, nid);
+ if (!usage)
panic("Failed to allocate usemap for node %d\n", nid);
for_each_present_section_nr(pnum_begin, pnum) {
@@ -329,9 +296,10 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
panic("Failed to allocate memmap for section %lu\n", pnum);
memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
NULL, NULL));
- sparse_init_early_section(nid, map, pnum, 0);
+ sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage,
+ SECTION_IS_EARLY);
+ usage = (void *)usage + mem_section_usage_size();
}
- sparse_usage_fini();
}
/*
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox