[PATCH v2] kasan: fix runtime shadow page table synchronization

[PATCH v2] kasan: fix runtime shadow page table synchronization

[Muchun Song]

kasan_populate_early_shadow() is also used after slab becomes available
to install shadow mappings for hot-added memory. When this path allocates
a new page table at a level that requires synchronization, the new entry
is initially published only in init_mm.

On architectures that maintain separate kernel page tables, runtime
changes to the kernel page tables must be propagated to existing process
page tables. Otherwise, the KASAN shadow mapping can remain unreachable
and fault when the hot-added memory is accessed.

Use the existing page-table allocation tracking helpers to collect the
modified levels, then synchronize the affected kernel mappings once after
the walk.

Once synchronized, an entry remains in other process page tables. Clearing
only init_mm and freeing its target table during teardown would leave stale
pointers behind. Retain page tables referenced by synchronized entries.

Fixes: 0207df4fa1a8 ("kernel/memremap, kasan: make ZONE_DEVICE with work with KASAN")
Cc: stable@vger.kernel.org
Assisted-by: LLM
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Use the existing page-table tracking helpers and synchronize once after
  shadow population (suggested by Andrey Ryabinin)
- Fix aligned shadow population to reuse existing page tables instead of
  replacing them (suggested by Andrey Ryabinin)
---
 mm/kasan/init.c | 125 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 90 insertions(+), 35 deletions(-)

diff --git a/mm/kasan/init.c b/mm/kasan/init.c
index 66a883887987..f94f461d864c 100644
--- a/mm/kasan/init.c
+++ b/mm/kasan/init.c
@@ -17,6 +17,7 @@
 
 #include <asm/page.h>
 
+#include "../pgalloc-track.h"
 #include "kasan.h"
 
 /*
@@ -107,7 +108,7 @@ static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr,
 }
 
 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
-				unsigned long end)
+				unsigned long end, pgtbl_mod_mask *mask)
 {
 	pmd_t *pmd = pmd_offset(pud, addr);
 	unsigned long next;
@@ -115,25 +116,26 @@ static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
 	do {
 		next = pmd_addr_end(addr, end);
 
-		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
+		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE &&
+		    pmd_none(*pmd)) {
 			pmd_populate_kernel(&init_mm, pmd,
 					lm_alias(kasan_early_shadow_pte));
+			*mask |= PGTBL_PMD_MODIFIED;
 			continue;
 		}
 
 		if (pmd_none(*pmd)) {
-			pte_t *p;
+			if (slab_is_available()) {
+				if (!pte_alloc_kernel_track(pmd, addr, mask))
+					return -ENOMEM;
+			} else {
+				pte_t *p;
 
-			if (slab_is_available())
-				p = pte_alloc_one_kernel(&init_mm);
-			else {
 				p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
 				kernel_pte_init(p);
+				pmd_populate_kernel(&init_mm, pmd, p);
+				*mask |= PGTBL_PMD_MODIFIED;
 			}
-			if (!p)
-				return -ENOMEM;
-
-			pmd_populate_kernel(&init_mm, pmd, p);
 		}
 		zero_pte_populate(pmd, addr, next);
 	} while (pmd++, addr = next, addr != end);
@@ -142,80 +144,93 @@ static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
 }
 
 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr,
-				unsigned long end)
+				unsigned long end, pgtbl_mod_mask *mask)
 {
 	pud_t *pud = pud_offset(p4d, addr);
 	unsigned long next;
+	int ret;
 
 	do {
 		next = pud_addr_end(addr, end);
-		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
+		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE &&
+		    pud_none(*pud)) {
 			pmd_t *pmd;
 
 			pud_populate(&init_mm, pud,
 					lm_alias(kasan_early_shadow_pmd));
+			*mask |= PGTBL_PUD_MODIFIED;
 			pmd = pmd_offset(pud, addr);
 			pmd_populate_kernel(&init_mm, pmd,
 					lm_alias(kasan_early_shadow_pte));
+			*mask |= PGTBL_PMD_MODIFIED;
 			continue;
 		}
 
 		if (pud_none(*pud)) {
-			pmd_t *p;
-
 			if (slab_is_available()) {
-				p = pmd_alloc(&init_mm, pud, addr);
-				if (!p)
+				if (!pmd_alloc_track(&init_mm, pud, addr, mask))
 					return -ENOMEM;
 			} else {
+				pmd_t *p;
+
 				p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
 				pmd_init(p);
 				pud_populate(&init_mm, pud, p);
+				*mask |= PGTBL_PUD_MODIFIED;
 			}
 		}
-		zero_pmd_populate(pud, addr, next);
+		ret = zero_pmd_populate(pud, addr, next, mask);
+		if (ret)
+			return ret;
 	} while (pud++, addr = next, addr != end);
 
 	return 0;
 }
 
 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,
-				unsigned long end)
+				unsigned long end, pgtbl_mod_mask *mask)
 {
 	p4d_t *p4d = p4d_offset(pgd, addr);
 	unsigned long next;
+	int ret;
 
 	do {
 		next = p4d_addr_end(addr, end);
-		if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
+		if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE &&
+		    p4d_none(*p4d)) {
 			pud_t *pud;
 			pmd_t *pmd;
 
-			p4d_populate_kernel(addr, p4d,
+			p4d_populate(&init_mm, p4d,
 					lm_alias(kasan_early_shadow_pud));
+			*mask |= PGTBL_P4D_MODIFIED;
 			pud = pud_offset(p4d, addr);
 			pud_populate(&init_mm, pud,
 					lm_alias(kasan_early_shadow_pmd));
+			*mask |= PGTBL_PUD_MODIFIED;
 			pmd = pmd_offset(pud, addr);
 			pmd_populate_kernel(&init_mm, pmd,
 					lm_alias(kasan_early_shadow_pte));
+			*mask |= PGTBL_PMD_MODIFIED;
 			continue;
 		}
 
 		if (p4d_none(*p4d)) {
-			pud_t *p;
-
 			if (slab_is_available()) {
-				p = pud_alloc(&init_mm, p4d, addr);
-				if (!p)
+				if (!pud_alloc_track(&init_mm, p4d, addr, mask))
 					return -ENOMEM;
 			} else {
+				pud_t *p;
+
 				p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
 				pud_init(p);
-				p4d_populate_kernel(addr, p4d, p);
+				p4d_populate(&init_mm, p4d, p);
+				*mask |= PGTBL_P4D_MODIFIED;
 			}
 		}
-		zero_pud_populate(p4d, addr, next);
+		ret = zero_pud_populate(p4d, addr, next, mask);
+		if (ret)
+			return ret;
 	} while (p4d++, addr = next, addr != end);
 
 	return 0;
@@ -230,15 +245,19 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,
 int __ref kasan_populate_early_shadow(const void *shadow_start,
 					const void *shadow_end)
 {
-	unsigned long addr = (unsigned long)shadow_start;
+	unsigned long start = (unsigned long)shadow_start;
+	unsigned long addr = start;
 	unsigned long end = (unsigned long)shadow_end;
 	pgd_t *pgd = pgd_offset_k(addr);
 	unsigned long next;
+	pgtbl_mod_mask mask = 0;
+	int ret = 0;
 
 	do {
 		next = pgd_addr_end(addr, end);
 
-		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
+		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE &&
+		    pgd_none(*pgd)) {
 			p4d_t *p4d;
 			pud_t *pud;
 			pmd_t *pmd;
@@ -251,34 +270,46 @@ int __ref kasan_populate_early_shadow(const void *shadow_start,
 			 * puds,pmds, so pgd_populate(), pud_populate()
 			 * is noops.
 			 */
-			pgd_populate_kernel(addr, pgd,
+			pgd_populate(&init_mm, pgd,
 					lm_alias(kasan_early_shadow_p4d));
+			mask |= PGTBL_PGD_MODIFIED;
 			p4d = p4d_offset(pgd, addr);
-			p4d_populate_kernel(addr, p4d,
+			p4d_populate(&init_mm, p4d,
 					lm_alias(kasan_early_shadow_pud));
+			mask |= PGTBL_P4D_MODIFIED;
 			pud = pud_offset(p4d, addr);
 			pud_populate(&init_mm, pud,
 					lm_alias(kasan_early_shadow_pmd));
+			mask |= PGTBL_PUD_MODIFIED;
 			pmd = pmd_offset(pud, addr);
 			pmd_populate_kernel(&init_mm, pmd,
 					lm_alias(kasan_early_shadow_pte));
+			mask |= PGTBL_PMD_MODIFIED;
 			continue;
 		}
 
 		if (pgd_none(*pgd)) {
 
 			if (slab_is_available()) {
-				if (!p4d_alloc(&init_mm, pgd, addr))
-					return -ENOMEM;
+				if (!p4d_alloc_track(&init_mm, pgd, addr, &mask)) {
+					ret = -ENOMEM;
+					break;
+				}
 			} else {
-				pgd_populate_kernel(addr, pgd,
+				pgd_populate(&init_mm, pgd,
 					early_alloc(PAGE_SIZE, NUMA_NO_NODE));
+				mask |= PGTBL_PGD_MODIFIED;
 			}
 		}
-		zero_p4d_populate(pgd, addr, next);
+		ret = zero_p4d_populate(pgd, addr, next, &mask);
+		if (ret)
+			break;
 	} while (pgd++, addr = next, addr != end);
 
-	return 0;
+	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
+		arch_sync_kernel_mappings(start, end);
+
+	return ret;
 }
 
 static void kasan_free_pte(pte_t *pte_start, pmd_t *pmd)
@@ -286,6 +317,9 @@ static void kasan_free_pte(pte_t *pte_start, pmd_t *pmd)
 	pte_t *pte;
 	int i;
 
+	if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PMD_MODIFIED)
+		return;
+
 	for (i = 0; i < PTRS_PER_PTE; i++) {
 		pte = pte_start + i;
 		if (!pte_none(ptep_get(pte)))
@@ -301,6 +335,9 @@ static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud)
 	pmd_t *pmd;
 	int i;
 
+	if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PUD_MODIFIED)
+		return;
+
 	for (i = 0; i < PTRS_PER_PMD; i++) {
 		pmd = pmd_start + i;
 		if (!pmd_none(*pmd))
@@ -316,6 +353,9 @@ static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d)
 	pud_t *pud;
 	int i;
 
+	if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_P4D_MODIFIED)
+		return;
+
 	for (i = 0; i < PTRS_PER_PUD; i++) {
 		pud = pud_start + i;
 		if (!pud_none(*pud))
@@ -331,6 +371,9 @@ static void kasan_free_p4d(p4d_t *p4d_start, pgd_t *pgd)
 	p4d_t *p4d;
 	int i;
 
+	if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PGD_MODIFIED)
+		return;
+
 	for (i = 0; i < PTRS_PER_P4D; i++) {
 		p4d = p4d_start + i;
 		if (!p4d_none(*p4d))
@@ -377,6 +420,9 @@ static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr,
 			continue;
 
 		if (kasan_pte_table(*pmd)) {
+			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PMD_MODIFIED)
+				continue;
+
 			if (IS_ALIGNED(addr, PMD_SIZE) &&
 			    IS_ALIGNED(next, PMD_SIZE)) {
 				pmd_clear(pmd);
@@ -403,6 +449,9 @@ static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,
 			continue;
 
 		if (kasan_pmd_table(*pud)) {
+			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PUD_MODIFIED)
+				continue;
+
 			if (IS_ALIGNED(addr, PUD_SIZE) &&
 			    IS_ALIGNED(next, PUD_SIZE)) {
 				pud_clear(pud);
@@ -430,6 +479,9 @@ static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr,
 			continue;
 
 		if (kasan_pud_table(*p4d)) {
+			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_P4D_MODIFIED)
+				continue;
+
 			if (IS_ALIGNED(addr, P4D_SIZE) &&
 			    IS_ALIGNED(next, P4D_SIZE)) {
 				p4d_clear(p4d);
@@ -464,6 +516,9 @@ void kasan_remove_zero_shadow(void *start, unsigned long size)
 			continue;
 
 		if (kasan_p4d_table(*pgd)) {
+			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PGD_MODIFIED)
+				continue;
+
 			if (IS_ALIGNED(addr, PGDIR_SIZE) &&
 			    IS_ALIGNED(next, PGDIR_SIZE)) {
 				pgd_clear(pgd);

base-commit: 63562804f25cb5b6f5e4f5e3a9f0d1c3c6868596
-- 
2.54.0

Re: [PATCH v2] kasan: fix runtime shadow page table synchronization

[Jose A. Perez de Azpillaga]

On 10/18/26 6:35 AM, Muchun Song wrote:
> -		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
> +		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE &&
> +		    pmd_none(*pmd)) {
>  			pmd_populate_kernel(&init_mm, pmd,
>  					lm_alias(kasan_early_shadow_pte));

what if the entry already points at the early shadow table? this walks
down into it then, which loongarch does on every boot.

[snip]

> -		zero_pmd_populate(pud, addr, next);
> +		ret = zero_pmd_populate(pud, addr, next, mask);
> +		if (ret)
> +			return ret;

isn't this a separate fix? it's not in the changelog.

[snip]

>  		if (kasan_pud_table(*p4d)) {
> +			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_P4D_MODIFIED)
> +				continue;
> +

doesn't this leave the range on the zero shadow, which memory onlined
there later then keeps? would always allocating a table at the synced
level once slab is up avoid that?

-- 
cheers,
jose a. p-a

Re: [PATCH v2] kasan: fix runtime shadow page table synchronization

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [High] kasan: syncing static early shadow tables breaks kasan_remove_zero_shadow()
- [High] kasan: missing kasan_p*d_table() checks in kasan_populate_early_shadow()
--

commit 84e267a96cb1eab3b9fd041e188f62b5f7899dd0
Author: Muchun Song <muchun.song@linux.dev>

kasan: fix runtime shadow page table synchronization

This commit tracks page table modifications during KASAN zero-shadow
population and calls arch_sync_kernel_mappings() once after the walk. It also
avoids replacing already-populated page table entries during aligned shadow
population and retains page tables referenced by synchronized entries during
teardown.

> diff --git a/mm/kasan/init.c b/mm/kasan/init.c
> index 66a8838879876..f94f461d864ca 100644
> --- a/mm/kasan/init.c
> +++ b/mm/kasan/init.c

[ ... ]

> @@ -115,25 +116,26 @@ static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
>  	do {
>  		next = pmd_addr_end(addr, end);
>  
> -		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
> +		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE &&
> +		    pmd_none(*pmd)) {
>  			pmd_populate_kernel(&init_mm, pmd,
>  					lm_alias(kasan_early_shadow_pte));
> +			*mask |= PGTBL_PMD_MODIFIED;
>  			continue;
>  		}

[Severity: High]
What happens in zero_pmd_populate(), zero_pud_populate(), zero_p4d_populate(),
and kasan_populate_early_shadow() when the page table entry is not none
because it already points to the shared static early shadow table
(kasan_pte_table(*pmd), kasan_pmd_table(*pud), kasan_pud_table(*p4d), or
kasan_p4d_table(*pgd))?

For example, in arch/loongarch/mm/kasan_init.c, kasan_init() pre-populates
swapper_pg_dir across the entire shadow range with kasan_early_shadow_* before
calling kasan_populate_early_shadow():

arch/loongarch/mm/kasan_init.c:kasan_init() {
	/* Maps everything to a single page of zeroes */
	kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, true);

	kasan_populate_early_shadow(mem_to_shadow((void *)VMALLOC_START),
					mem_to_shadow((void *)KFENCE_AREA_END));
}

Because *p4d, *pud, and *pmd point to kasan_early_shadow_pud,
kasan_early_shadow_pmd, and kasan_early_shadow_pte rather than
invalid_pud_table, invalid_pmd_table, and invalid_pte_table, p4d_none(*p4d),
pud_none(*pud), and pmd_none(*pmd) all evaluate to false.

Instead of skipping aligned PGDIR_SIZE, P4D_SIZE, PUD_SIZE, and PMD_SIZE
blocks, kasan_populate_early_shadow() descends into zero_pte_populate() for
every single PMD across the multi-terabyte VMALLOC_START..KFENCE_AREA_END
shadow range, executing billions of redundant set_pte_at() calls on the same
shared kasan_early_shadow_pte array (and on 16KB-page LoongArch configs where
mem_to_shadow(VMALLOC_START) is not PMD-aligned, prematurely write-protecting
kasan_early_shadow_pte entries while kasan_pg_dir is still active).

The same descent into the shared static kasan_early_shadow_* tables happens at
runtime if kasan_add_zero_shadow() -> kasan_populate_early_shadow() is called
again on a range where kasan_remove_zero_shadow() previously retained a
synchronized kasan_early_shadow_* entry.

Should these aligned fast paths also check whether the entry already points to
kasan_early_shadow_{p4d,pud,pmd,pte} (and skip descending without needing to
re-populate or set the modified mask)?

[ ... ]

> @@ -377,6 +420,9 @@ static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr,
>  			continue;
>  
>  		if (kasan_pte_table(*pmd)) {
> +			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PMD_MODIFIED)
> +				continue;
> +
>  			if (IS_ALIGNED(addr, PMD_SIZE) &&
>  			    IS_ALIGNED(next, PMD_SIZE)) {
>  				pmd_clear(pmd);
> @@ -403,6 +449,9 @@ static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,
>  			continue;
>  
>  		if (kasan_pmd_table(*pud)) {
> +			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PUD_MODIFIED)
> +				continue;
> +
>  			if (IS_ALIGNED(addr, PUD_SIZE) &&
>  			    IS_ALIGNED(next, PUD_SIZE)) {
>  				pud_clear(pud);
> @@ -430,6 +479,9 @@ static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr,
>  			continue;
>  
>  		if (kasan_pud_table(*p4d)) {
> +			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_P4D_MODIFIED)
> +				continue;
> +
>  			if (IS_ALIGNED(addr, P4D_SIZE) &&
>  			    IS_ALIGNED(next, P4D_SIZE)) {
>  				p4d_clear(p4d);
> @@ -464,6 +516,9 @@ void kasan_remove_zero_shadow(void *start, unsigned long size)
>  			continue;
>  
>  		if (kasan_p4d_table(*pgd)) {
> +			if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PGD_MODIFIED)
> +				continue;
> +
>  			if (IS_ALIGNED(addr, PGDIR_SIZE) &&
>  			    IS_ALIGNED(next, PGDIR_SIZE)) {
>  				pgd_clear(pgd);

[Severity: High]
Can skipping synchronized kasan_early_shadow_{p4d,pud,pmd,pte} entries here
prevent kasan_remove_zero_shadow() from unmapping the zero shadow at all?

When kasan_add_zero_shadow() -> kasan_populate_early_shadow() runs at runtime
(with slab_is_available() == true) for a range covering an aligned PGDIR_SIZE,
P4D_SIZE, PUD_SIZE, or PMD_SIZE boundary at a level included in
ARCH_PAGE_TABLE_SYNC_MASK (for example, PGTBL_P4D_MODIFIED on 4-level x86_64,
PGTBL_PGD_MODIFIED on 5-level x86_64, or PGTBL_PMD_MODIFIED on 32-bit x86/ARM),
it installs the shared static kasan_early_shadow_{p4d,pud,pmd,pte} table
directly into the synchronized entry.

When kasan_remove_zero_shadow() is later called to remove that shadow mapping,
hitting this continue leaves the entry pointing to kasan_early_shadow_*.
Unlike a dynamically allocated page table (where kasan_remove_*_table()
descends and clears all child entries before kasan_free_*() retains only the
empty top-level table), kasan_early_shadow_* is a shared static table whose
entries cannot be cleared and remain permanently mapped to the read-only
kasan_early_shadow_page.

If normal System RAM is subsequently hot-added and onlined in that address
range, kasan_mem_notifier() in mm/kasan/shadow.c checks shadow_mapped():

mm/kasan/shadow.c:kasan_mem_notifier() {
	...
	case MEM_GOING_ONLINE: {
		...
		if (shadow_mapped(shadow_start))
			return NOTIFY_OK;
}

Because the read-only kasan_early_shadow_page PTE is still present,
shadow_mapped() returns true and kasan_mem_notifier() skips allocating
writable shadow memory, causing subsequent KASAN shadow writes to fault on the
read-only zero shadow page. In addition, if a smaller or unaligned range
within that region is later passed to kasan_add_zero_shadow(),
kasan_populate_early_shadow() sees !p*d_none(*p*d) and descends into and
modifies the shared global kasan_early_shadow_* tables.

Should kasan_populate_early_shadow() avoid installing the static
kasan_early_shadow_{p4d,pud,pmd,pte} tables at page table levels that are in
ARCH_PAGE_TABLE_SYNC_MASK when slab_is_available() is true, and instead
allocate a normal page table at that synchronized level so its entries can be
cleared during kasan_remove_zero_shadow()?

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261018043501.3257632-1-songmuchun@bytedance.com?part=1