[PATCH v2 0/5] fixes for data/bss linear alias unmap series

[PATCH v2 0/5] fixes for data/bss linear alias unmap series

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Fixes for the data/bss linear alias unmap series:

- Fix KASAN related issue reported by Mark, by moving all KASAN page
  tables out of BSS [on arm64], not just the ones defined under
  arch/arm64

- Fix two issues spotted by Sashiko

- Fix the CKI reported WARN() splat on BBML2_NOABORT systems that try to
  split block mappings too early.

Changes since v1:

- Split .bss..pgtbl changes off into separate patch
- Take Catalin's suggestion on how to address MTE issue with the
  read-only zero page
- Improved commit logs

Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Mark Brown <broonie@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>

Ard Biesheuvel (5):
  arm64: Rename page table BSS section to .bss..pgtbl
  kasan: Move generic KASAN page tables out of BSS too
  arm64: Avoid double evaluation of __ptep_get()
  KVM: arm64: Omit tag sync on stage-2 mappings of the zero page
  arm64: mm: Defer remap of linear alias of data/bss

 arch/arm64/include/asm/linkage.h |  2 ++
 arch/arm64/include/asm/mmu.h     |  2 --
 arch/arm64/include/asm/pgtable.h |  4 ----
 arch/arm64/kernel/vmlinux.lds.S  |  8 ++++----
 arch/arm64/kvm/mmu.c             |  5 +++++
 arch/arm64/mm/fixmap.c           |  6 +++---
 arch/arm64/mm/kasan_init.c       |  2 +-
 arch/arm64/mm/mmu.c              | 20 +++++++++++++-------
 include/linux/linkage.h          |  4 ++++
 mm/kasan/init.c                  | 10 +++++-----
 10 files changed, 37 insertions(+), 26 deletions(-)


base-commit: 63e0b6a5b6934d6a919d1c65ea185303200a1874
-- 
2.54.0.1032.g2f8565e1d1-goog

[PATCH v2 1/5] arm64: Rename page table BSS section to .bss..pgtbl

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will
notice the leading ".bss" and mark it as NOBITS by default (rather than
PROGBITS, which would take up space in Image binary, forcing all of the
preceding BSS to be emitted into the image as well). This supersedes the
NOLOAD linker directive, which achieves the same thing, and can be
therefore be dropped.

Also, rename .pgdir to .pgtbl to be more generic, as page tables of
various levels will reside here.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/include/asm/linkage.h | 2 ++
 arch/arm64/include/asm/mmu.h     | 2 --
 arch/arm64/kernel/vmlinux.lds.S  | 8 ++++----
 arch/arm64/mm/fixmap.c           | 6 +++---
 arch/arm64/mm/kasan_init.c       | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h
index 40bd17add539..8637f667667c 100644
--- a/arch/arm64/include/asm/linkage.h
+++ b/arch/arm64/include/asm/linkage.h
@@ -43,4 +43,6 @@
 	SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)	\
 	bti c ;
 
+#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE)
+
 #endif
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index fb95754f2876..5e1211c540ab 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -13,8 +13,6 @@
 
 #ifndef __ASSEMBLER__
 
-#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE)
-
 #include <linux/refcount.h>
 #include <asm/cpufeature.h>
 
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 2b0ebfb30c63..d3ed59abab38 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -352,11 +352,11 @@ SECTIONS
 	BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE)
 	__pi___bss_start = __bss_start;
 
-	/* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */
-	.pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) {
-		*(.pgdir.bss)
+	/* page table BSS starts here - preceding data/BSS is omitted from the linear map */
+	.pgtbl : ALIGN(PAGE_SIZE) {
+		*(.bss..pgtbl)
 	}
-	ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS")
+	ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS")
 
 	. = ALIGN(PAGE_SIZE);
 	__pi_init_pg_dir = .;
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index 1a3bbd67dd76..f66a0016dd02 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1);
 
 #define BM_PTE_TABLE_IDX(addr)	__BM_TABLE_IDX(addr, PMD_SHIFT)
 
-static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss;
-static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused;
-static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused;
+static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl;
+static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused;
+static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused;
 
 static inline pte_t *fixmap_pte(unsigned long addr)
 {
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index dbf22cae82ee..3fcad956fdf7 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void)
 		 * shadow pud_t[]/p4d_t[], which could end up getting corrupted
 		 * when the linear region is mapped.
 		 */
-		static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss;
+		static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl;
 		pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START);
 
 		set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE));
-- 
2.54.0.1032.g2f8565e1d1-goog

Re: [PATCH v2 1/5] arm64: Rename page table BSS section to .bss..pgtbl

[Mark Brown]

[-- Attachment #1: Type: text/plain, Size: 863 bytes --]

On Thu, Jun 04, 2026 at 05:11:53PM +0200, Ard Biesheuvel wrote:
> From: Ard Biesheuvel <ardb@kernel.org>
> 
> Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will
> notice the leading ".bss" and mark it as NOBITS by default (rather than
> PROGBITS, which would take up space in Image binary, forcing all of the
> preceding BSS to be emitted into the image as well). This supersedes the
> NOLOAD linker directive, which achieves the same thing, and can be
> therefore be dropped.
> 
> Also, rename .pgdir to .pgtbl to be more generic, as page tables of
> various levels will reside here.

This addresses the boot failure, I'm still seeing KUnit failures due to
an unrelated bug that was fixed post -rc1 but we get into actually
trying to run KUnit tests which was the relevant thing:

Tested-by: Mark Brown <broonie@kernel.org>

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

[PATCH v2 2/5] kasan: Move generic KASAN page tables out of BSS too

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Make sure that all KASAN page tables are emitted into the .pgtbl section
(provided that the arch has one - otherwise, fall back to page aligned
BSS)

This is needed because BSS itself is no longer accessible via the linear
map on arm64.

Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: kasan-dev@googlegroups.com
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 include/linux/linkage.h |  4 ++++
 mm/kasan/init.c         | 10 +++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/include/linux/linkage.h b/include/linux/linkage.h
index b11660b706c5..53fe1f48fd28 100644
--- a/include/linux/linkage.h
+++ b/include/linux/linkage.h
@@ -39,6 +39,10 @@
 #define __page_aligned_data	__section(".data..page_aligned") __aligned(PAGE_SIZE)
 #define __page_aligned_bss	__section(".bss..page_aligned") __aligned(PAGE_SIZE)
 
+#ifndef __bss_pgtbl
+#define __bss_pgtbl		__page_aligned_bss
+#endif
+
 /*
  * For assembly routines.
  *
diff --git a/mm/kasan/init.c b/mm/kasan/init.c
index 9c880f607c6a..66a883887987 100644
--- a/mm/kasan/init.c
+++ b/mm/kasan/init.c
@@ -26,10 +26,10 @@
  *   - Latter it reused it as zero shadow to cover large ranges of memory
  *     that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
  */
-unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss;
+unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl;
 
 #if CONFIG_PGTABLE_LEVELS > 4
-p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss;
+p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl;
 static inline bool kasan_p4d_table(pgd_t pgd)
 {
 	return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d));
@@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd)
 }
 #endif
 #if CONFIG_PGTABLE_LEVELS > 3
-pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss;
+pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl;
 static inline bool kasan_pud_table(p4d_t p4d)
 {
 	return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud));
@@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d)
 }
 #endif
 #if CONFIG_PGTABLE_LEVELS > 2
-pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss;
+pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl;
 static inline bool kasan_pmd_table(pud_t pud)
 {
 	return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd));
@@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud)
 }
 #endif
 pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS]
-	__page_aligned_bss;
+	__bss_pgtbl;
 
 static inline bool kasan_pte_table(pmd_t pmd)
 {
-- 
2.54.0.1032.g2f8565e1d1-goog

[PATCH v2 3/5] arm64: Avoid double evaluation of __ptep_get()

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Sashiko warns that the new pte_valid_noncont() macro is used in a manner
where the argument (which performs a READ_ONCE() of the descriptor) is
evaluated twice.

Drop the macro that we just added, and move the check into the newly
added users.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/include/asm/pgtable.h |  4 ----
 arch/arm64/mm/mmu.c              | 14 ++++++++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 491ba0a6492d..c9e4e00a9af2 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -181,10 +181,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
  * Returns true if the pte is valid and has the contiguous bit set.
  */
 #define pte_valid_cont(pte)	(pte_valid(pte) && pte_cont(pte))
-/*
- * Returns true if the pte is valid and has the contiguous bit cleared.
- */
-#define pte_valid_noncont(pte)	(pte_valid(pte) && !pte_cont(pte))
 /*
  * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
  * so that we don't erroneously return false for pages that have been
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index be51f6cac86f..d68e691c093a 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -186,9 +186,12 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
 
 static bool pte_range_has_valid_noncont(pte_t *ptep)
 {
-	for (int i = 0; i < CONT_PTES; i++)
-		if (pte_valid_noncont(__ptep_get(&ptep[i])))
+	for (int i = 0; i < CONT_PTES; i++) {
+		pte_t pte = __ptep_get(&ptep[i]);
+
+		if (pte_valid(pte) && !pte_cont(pte))
 			return true;
+	}
 	return false;
 }
 
@@ -291,9 +294,12 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
 
 static bool pmd_range_has_valid_noncont(pmd_t *pmdp)
 {
-	for (int i = 0; i < CONT_PMDS; i++)
-		if (pte_valid_noncont(pmd_pte(READ_ONCE(pmdp[i]))))
+	for (int i = 0; i < CONT_PMDS; i++) {
+		pte_t pte = pmd_pte(READ_ONCE(pmdp[i]));
+
+		if (pte_valid(pte) && !pte_cont(pte))
 			return true;
+	}
 	return false;
 }
 
-- 
2.54.0.1032.g2f8565e1d1-goog

[PATCH v2 4/5] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Commit

   f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")

removed the PG_mte_tagged flag from the zero page, but missed a KVM code
path that may set this flag on the zero page when it is used in a
stage-2 CoW mapping of anonymous memory.

So disregard the zero page explicitly in sanitise_mte_tags().

Fixes: f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")
Cc: <stable@vger.kernel.org> # 5.10.x
Suggested-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/kvm/mmu.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index d089c107d9b7..445d6cf035c9 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
 	if (!kvm_has_mte(kvm))
 		return;
 
+	if (is_zero_pfn(pfn)) {
+		WARN_ON_ONCE(nr_pages != 1);
+		return;
+	}
+
 	if (folio_test_hugetlb(folio)) {
 		/* Hugetlb has MTE flags set on head page only */
 		if (folio_try_hugetlb_mte_tagging(folio)) {
-- 
2.54.0.1032.g2f8565e1d1-goog

Re: [PATCH v2 4/5] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page

[Catalin Marinas]

On Thu, Jun 04, 2026 at 05:11:56PM +0200, Ard Biesheuvel wrote:
> From: Ard Biesheuvel <ardb@kernel.org>
> 
> Commit
> 
>    f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")
> 
> removed the PG_mte_tagged flag from the zero page, but missed a KVM code
> path that may set this flag on the zero page when it is used in a
> stage-2 CoW mapping of anonymous memory.
> 
> So disregard the zero page explicitly in sanitise_mte_tags().
> 
> Fixes: f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")
> Cc: <stable@vger.kernel.org> # 5.10.x
> Suggested-by: Catalin Marinas <catalin.marinas@arm.com>
> Signed-off-by: Ard Biesheuvel <ardb@kernel.org>

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>

[PATCH v2 5/5] arm64: mm: Defer remap of linear alias of data/bss

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Marking the linear alias of data/bss invalid involves calling
set_memory_valid(), which calls split_kernel_leaf_mapping() under the
hood.

On BBML2_NOABORT capable systems, this may result in the need to
allocate page tables at a time when the generic memory allocation APIs
are not yet available, resulting in a splat like

   WARNING: arch/arm64/mm/mmu.c:821 at split_kernel_leaf_mapping+0x15c/0x170, CPU#0: swapper/0
   Modules linked in:
   CPU: 0 UID: 0 PID: 0 Comm: swapper Not tainted 7.1.0-rc6 #1 PREEMPT(undef)
   pstate: a04000c9 (NzCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
   pc : split_kernel_leaf_mapping+0x15c/0x170
   lr : update_range_prot+0x40/0x128
   sp : ffffc99ad3863c80
   ...
   Call trace:
    split_kernel_leaf_mapping+0x15c/0x170 (P)
    update_range_prot+0x40/0x128
    set_memory_valid+0x94/0xe0
    mark_linear_data_alias_valid+0x54/0x68
    map_mem+0x1fc/0x240
    paging_init+0x48/0x210
    setup_arch+0x274/0x338
    start_kernel+0x98/0x538
    __primary_switched+0x88/0x98

as reported by CKI automated testing.

So defer the boot-time call to mark_linear_data_alias_valid() to a later
time when page allocations can be made normally.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/mm/mmu.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index d68e691c093a..3134f1c1097c 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1095,6 +1095,9 @@ void __init mark_linear_text_alias_ro(void)
 			    (unsigned long)__init_begin - (unsigned long)_text,
 			    PAGE_KERNEL_RO);
 
+	/* Map the kernel data/bss as invalid in the linear map */
+	mark_linear_data_alias_valid(false);
+
 	/*
 	 * Register a PM notifier to remap the linear alias of data/bss as
 	 * valid read-only before hibernation. This is needed because the
@@ -1237,9 +1240,6 @@ static void __init map_mem(void)
 		__map_memblock(start, end, pgprot_tagged(PAGE_KERNEL),
 			       flags);
 	}
-
-	/* Map the kernel data/bss as invalid in the linear map */
-	mark_linear_data_alias_valid(false);
 }
 
 void mark_rodata_ro(void)
-- 
2.54.0.1032.g2f8565e1d1-goog