From: Yang Shi <yang@os.amperecomputing.com>
To: cl@gentwo.org, dennis@kernel.org, tj@kernel.org,
urezki@gmail.com, catalin.marinas@arm.com, will@kernel.org,
ryan.roberts@arm.com, david@kernel.org,
akpm@linux-foundation.org, hca@linux.ibm.com, gor@linux.ibm.com,
agordeev@linux.ibm.com
Cc: yang@os.amperecomputing.com, linux-mm@kvack.org,
linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org
Subject: [PATCH 07/16] arm64: mm: sync percpu page table for memory hotplug/unplug
Date: Wed, 15 Jul 2026 11:04:09 -0700 [thread overview]
Message-ID: <20260715180455.515692-8-yang@os.amperecomputing.com> (raw)
In-Reply-To: <20260715180455.515692-1-yang@os.amperecomputing.com>
Need to sync kernel percpu page table for memory hotplug/unplug,
including both linear mapping and vmemmap.
Signed-off-by: Yang Shi <yang@os.amperecomputing.com>
---
arch/arm64/mm/mmu.c | 85 ++++++++++++++++++++++++++++++---------------
1 file changed, 57 insertions(+), 28 deletions(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 5a235491f268..e1290ccaa5be 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -214,7 +214,7 @@ static void init_clear_pgtable(void *table)
}
static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
- phys_addr_t phys, pgprot_t prot)
+ phys_addr_t phys, pgprot_t prot, pgtbl_mod_mask *mask)
{
do {
pte_t old_pte = __ptep_get(ptep);
@@ -224,6 +224,7 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
* are deferred to the end of alloc_init_cont_pte().
*/
__set_pte_nosync(ptep, pfn_pte(__phys_to_pfn(phys), prot));
+ *mask |= PGTBL_PTE_MODIFIED;
/*
* After the PTE entry has been populated once, we
@@ -251,7 +252,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
unsigned long next;
pmd_t pmd = READ_ONCE(*pmdp);
@@ -272,6 +273,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
init_clear_pgtable(ptep);
ptep += pte_index(addr);
__pmd_populate(pmdp, pte_phys, pmdval);
+ *mask |= PGTBL_PMD_MODIFIED;
} else {
BUG_ON(pmd_bad(pmd));
ptep = pte_set_fixmap_offset(pmdp, addr);
@@ -288,7 +290,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
!pte_range_has_valid_noncont(ptep))
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
- init_pte(ptep, addr, next, phys, __prot);
+ init_pte(ptep, addr, next, phys, __prot, mask);
ptep += pte_index(next) - pte_index(addr);
phys += next - addr;
@@ -306,7 +308,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
- phys_addr_t (*pgtable_alloc)(enum pgtable_level), int flags)
+ phys_addr_t (*pgtable_alloc)(enum pgtable_level), int flags,
+ pgtbl_mod_mask *mask)
{
unsigned long next;
@@ -319,7 +322,10 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
if (((addr | next | phys) & ~PMD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0 &&
!pmd_table(old_pmd)) {
- WARN_ON(!pmd_set_huge(pmdp, phys, prot));
+ if (pmd_set_huge(pmdp, phys, prot))
+ *mask |= PGTBL_PMD_MODIFIED;
+ else
+ WARN_ON(1);
/*
* After the PMD entry has been populated once, we
@@ -331,7 +337,7 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
int ret;
ret = alloc_init_cont_pte(pmdp, addr, next, phys, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
if (ret)
return ret;
@@ -359,7 +365,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
unsigned long next;
@@ -384,6 +390,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
init_clear_pgtable(pmdp);
pmdp += pmd_index(addr);
__pud_populate(pudp, pmd_phys, pudval);
+ *mask |= PGTBL_PUD_MODIFIED;
} else {
BUG_ON(pud_bad(pud));
pmdp = pmd_set_fixmap_offset(pudp, addr);
@@ -400,7 +407,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
!pmd_range_has_valid_noncont(pmdp))
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
- ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags);
+ ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags,
+ mask);
if (ret)
goto out;
@@ -417,7 +425,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret = 0;
unsigned long next;
@@ -438,6 +446,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
init_clear_pgtable(pudp);
pudp += pud_index(addr);
__p4d_populate(p4dp, pud_phys, p4dval);
+ *mask |= PGTBL_P4D_MODIFIED;
} else {
BUG_ON(p4d_bad(p4d));
pudp = pud_set_fixmap_offset(p4dp, addr);
@@ -455,7 +464,10 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
((addr | next | phys) & ~PUD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0 &&
!pud_table(old_pud)) {
- WARN_ON(!pud_set_huge(pudp, phys, prot));
+ if (pud_set_huge(pudp, phys, prot))
+ *mask |= PGTBL_PUD_MODIFIED;
+ else
+ WARN_ON(1);
/*
* After the PUD entry has been populated once, we
@@ -465,7 +477,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
READ_ONCE(pud_val(*pudp))));
} else {
ret = alloc_init_cont_pmd(pudp, addr, next, phys, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
if (ret)
goto out;
@@ -484,7 +496,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
unsigned long next;
@@ -505,6 +517,7 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
init_clear_pgtable(p4dp);
p4dp += p4d_index(addr);
__pgd_populate(pgdp, p4d_phys, pgdval);
+ *mask |= PGTBL_PGD_MODIFIED;
} else {
BUG_ON(pgd_bad(pgd));
p4dp = p4d_set_fixmap_offset(pgdp, addr);
@@ -516,7 +529,7 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
next = p4d_addr_end(addr, end);
ret = alloc_init_pud(p4dp, addr, next, phys, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
if (ret)
goto out;
@@ -536,7 +549,7 @@ static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
unsigned long addr, end, next;
@@ -556,7 +569,7 @@ static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
do {
next = pgd_addr_end(addr, end);
ret = alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc,
- flags);
+ flags, mask);
if (ret)
return ret;
phys += next - addr;
@@ -569,13 +582,13 @@ static int __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
mutex_lock(&fixmap_lock);
ret = __create_pgd_mapping_locked(pgdir, phys, virt, size, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
mutex_unlock(&fixmap_lock);
return ret;
@@ -588,9 +601,10 @@ static void early_create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
int flags)
{
int ret;
+ pgtbl_mod_mask mask = 0;
ret = __create_pgd_mapping(pgdir, phys, virt, size, prot, pgtable_alloc,
- flags);
+ flags, &mask);
if (ret)
panic("Failed to create page tables\n");
}
@@ -1332,6 +1346,7 @@ static int __init __kpti_install_ng_mappings(void *__unused)
if (!cpu) {
int ret;
+ pgtbl_mod_mask mask = 0;
alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);
@@ -1355,7 +1370,7 @@ static int __init __kpti_install_ng_mappings(void *__unused)
//
ret = __create_pgd_mapping_locked(kpti_ng_temp_pgd, __pa(alloc),
KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
- kpti_ng_pgd_alloc, 0);
+ kpti_ng_pgd_alloc, 0, &mask);
if (ret)
panic("Failed to create page tables\n");
}
@@ -1661,7 +1676,7 @@ static void unmap_hotplug_range(unsigned long addr, unsigned long end,
static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
pte_t *ptep, pte;
unsigned long i, start = addr;
@@ -1677,6 +1692,7 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
WARN_ON(!pte_none(pte));
} while (addr += PAGE_SIZE, addr < end);
+ *mask |= PGTBL_PTE_MODIFIED;
if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK))
return;
@@ -1692,13 +1708,14 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
}
pmd_clear(pmdp);
+ *mask |= PGTBL_PMD_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(ptep));
}
static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
pmd_t *pmdp, pmd;
unsigned long i, next, start = addr;
@@ -1711,7 +1728,7 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
continue;
WARN_ON(!pmd_present(pmd) || !pmd_table(pmd));
- free_empty_pte_table(pmdp, addr, next, floor, ceiling);
+ free_empty_pte_table(pmdp, addr, next, floor, ceiling, mask);
} while (addr = next, addr < end);
if (CONFIG_PGTABLE_LEVELS <= 2)
@@ -1732,13 +1749,14 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
}
pud_clear(pudp);
+ *mask |= PGTBL_PUD_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(pmdp));
}
static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
pud_t *pudp, pud;
unsigned long i, next, start = addr;
@@ -1751,7 +1769,7 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
continue;
WARN_ON(!pud_present(pud) || !pud_table(pud));
- free_empty_pmd_table(pudp, addr, next, floor, ceiling);
+ free_empty_pmd_table(pudp, addr, next, floor, ceiling, mask);
} while (addr = next, addr < end);
if (!pgtable_l4_enabled())
@@ -1772,13 +1790,14 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
}
p4d_clear(p4dp);
+ *mask |= PGTBL_P4D_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(pudp));
}
static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
p4d_t *p4dp, p4d;
unsigned long i, next, start = addr;
@@ -1791,7 +1810,7 @@ static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
continue;
WARN_ON(!p4d_present(p4d));
- free_empty_pud_table(p4dp, addr, next, floor, ceiling);
+ free_empty_pud_table(p4dp, addr, next, floor, ceiling, mask);
} while (addr = next, addr < end);
if (!pgtable_l5_enabled())
@@ -1812,6 +1831,7 @@ static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
}
pgd_clear(pgdp);
+ *mask |= PGTBL_PGD_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(p4dp));
}
@@ -1821,6 +1841,8 @@ static void free_empty_tables(unsigned long addr, unsigned long end,
{
unsigned long next;
pgd_t *pgdp, pgd;
+ pgtbl_mod_mask mask = 0;
+ unsigned long start = addr;
do {
next = pgd_addr_end(addr, end);
@@ -1830,8 +1852,11 @@ static void free_empty_tables(unsigned long addr, unsigned long end,
continue;
WARN_ON(!pgd_present(pgd));
- free_empty_p4d_table(pgdp, addr, next, floor, ceiling);
+ free_empty_p4d_table(pgdp, addr, next, floor, ceiling, &mask);
} while (addr = next, addr < end);
+
+ if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
+ arch_sync_kernel_mappings(start, end);
}
#endif
@@ -2039,6 +2064,7 @@ int arch_add_memory(int nid, u64 start, u64 size,
struct mhp_params *params)
{
int ret, flags = NO_EXEC_MAPPINGS;
+ pgtbl_mod_mask mask = 0;
VM_BUG_ON(!mhp_range_allowed(start, size, true));
@@ -2047,10 +2073,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
ret = __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
size, params->pgprot, pgd_pgtable_alloc_init_mm,
- flags);
+ flags, &mask);
if (ret)
goto err;
+ if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
+ arch_sync_kernel_mappings(start, start + size);
+
memblock_clear_nomap(start, size);
ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
--
2.47.0
next prev parent reply other threads:[~2026-07-15 18:06 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-07-15 18:04 [RFC v2 PATCH 0/16] Optimize this_cpu_*() ops for non-x86 (ARM64 for this series) Yang Shi
2026-07-15 18:04 ` [PATCH 01/16] drivers: arch_numa: move percpu set up code to arch Yang Shi
2026-07-15 18:04 ` [PATCH 02/16] arm64: kconfig: make percpu related configs not depend on NUMA Yang Shi
2026-07-15 18:04 ` [PATCH 03/16] mm: pgalloc: introduce {pud|pmd}_populate_sync() Yang Shi
2026-07-15 18:04 ` [PATCH 04/16] vmalloc: pass in pgd pointer for vmap{__vunmap}_range_noflush() Yang Shi
2026-07-15 18:04 ` [PATCH 05/16] arm64: mm: enable percpu kernel page table Yang Shi
2026-07-15 18:04 ` [PATCH 06/16] arm64: mm: defined {pud|pmd}_populate_sync() Yang Shi
2026-07-15 18:04 ` Yang Shi [this message]
2026-07-15 18:04 ` [PATCH 08/16] arm64: kasan: sync up kasan shadow area page table Yang Shi
2026-07-15 18:04 ` [PATCH 09/16] arm64: mm: define percpu virtual space area Yang Shi
2026-07-15 18:04 ` [PATCH 10/16] mm: percpu: prepare to use dedicated percpu area Yang Shi
2026-07-15 18:04 ` [PATCH 11/16] arm64: mm: map local percpu first chunk Yang Shi
2026-07-15 18:04 ` [PATCH 12/16] mm: percpu: set up first chunk and reserve chunk Yang Shi
2026-07-15 18:04 ` [PATCH 13/16] arm64: mm: introduce __per_cpu_local_off Yang Shi
2026-07-15 18:04 ` [PATCH 14/16] mm: percpu: allocate and free local percpu vm area Yang Shi
2026-07-15 18:04 ` [PATCH 15/16] arm64: kconfig: select HAVE_LOCAL_PER_CPU_MAP Yang Shi
2026-07-15 18:04 ` [PATCH 16/16] arm64: percpu: use local percpu for this_cpu_*() APIs Yang Shi
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260715180455.515692-8-yang@os.amperecomputing.com \
--to=yang@os.amperecomputing.com \
--cc=agordeev@linux.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=catalin.marinas@arm.com \
--cc=cl@gentwo.org \
--cc=david@kernel.org \
--cc=dennis@kernel.org \
--cc=gor@linux.ibm.com \
--cc=hca@linux.ibm.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ryan.roberts@arm.com \
--cc=tj@kernel.org \
--cc=urezki@gmail.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.