[RFC PATCH v2 0/4] arm64: mm: Map fixmap page tables read-only

[RFC PATCH v2 0/4] arm64: mm: Map fixmap page tables read-only

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

This v2 now covers intermediate level page tables as well as the PTE
level page table for the fixmap. The latter is a special case, as it

a) is only accessed via the kernel image's mapping, and never via the
   linear map (except for ptdump etc)

b) must be accessible via a read-write mapping, as all manipulation of
   read-only page table descriptors relies on the fixmap itself

and so it is treated separately. The intermediate page tables may be
shared with other mappings in the upper kernel/vmalloc region, so they
must be updatable using the ordinary APIs.

Build tested and boot tested on a Lenovo Yoga C630 using 16k pages.

v1: https://lore.kernel.org/all/20260805104042.1107678-2-ardb+git@google.com/

Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Liz Prucka <lizprucka@google.com>
Cc: Seth Jenkins <sethjenkins@google.com>
Cc: Kees Cook <kees@kernel.org>
Cc: Jann Horn <jannh@google.com>
Cc: linux-hardening@vger.kernel.org

Ard Biesheuvel (4):
  arm64: mm: Map fixmap PTE tables r/o in the linear map
  arm64: mm: Use fault handler to permit swapper_pg_dir updates
  arm64: mm: Create r/o page table region that permits updates
  arm64: mm: Move fixmap intermediate page tables into .rodata

 arch/arm64/include/asm/fixmap.h  |  4 ++
 arch/arm64/include/asm/linkage.h |  1 +
 arch/arm64/include/asm/pgtable.h | 55 ++++++++------------
 arch/arm64/kernel/vmlinux.lds.S  |  6 +++
 arch/arm64/mm/fixmap.c           | 12 +++--
 arch/arm64/mm/mmu.c              | 48 +++++++++++++----
 6 files changed, 78 insertions(+), 48 deletions(-)

-- 
2.55.0.887.g758fc8c411-goog

[RFC PATCH v2 1/4] arm64: mm: Map fixmap PTE tables r/o in the linear map

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Without physical KASLR, the fixmap page tables will appear at an a
priori known offset in the physical address space, and due to the lack
of randomization, the linear map carries a writeable alias of the fixmap
PTE pages, which appears at an offset in the kernel VA space that is
also predictable.

Given that the placement of the fixmap area is never randomized either,
a single store to this linear alias region is sufficient to map any
physical page with any permissions at a known offset in the kernel VA
space, including on top of the PTI trampoline.

Avoid this, by remapping the fixmap PTE pages read-only in the linear
map. This is possible because all updates to bm_pte[] occur via the
mapping of the kernel image in the vmap area. A read-only mapping is
still needed for things like ptdump that walk the page tables.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/include/asm/fixmap.h | 3 +++
 arch/arm64/mm/fixmap.c          | 8 +++++---
 arch/arm64/mm/mmu.c             | 8 ++++++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 170c3502d723..9191125738e9 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -112,6 +112,9 @@ enum fixed_addresses {
 
 void __init early_fixmap_init(void);
 
+extern pte_t fixmap_bm_pte[][PTRS_PER_PTE];
+extern const size_t fixmap_bm_pte_size;
+
 #define __early_set_fixmap __set_fixmap
 
 extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot);
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index f66a0016dd02..3a8cf6de6a7d 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -31,13 +31,15 @@ static_assert(NR_BM_PMD_TABLES == 1);
 
 #define BM_PTE_TABLE_IDX(addr)	__BM_TABLE_IDX(addr, PMD_SHIFT)
 
-static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl;
+pte_t fixmap_bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl;
 static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused;
 static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused;
 
+const size_t fixmap_bm_pte_size = sizeof(fixmap_bm_pte);
+
 static inline pte_t *fixmap_pte(unsigned long addr)
 {
-	return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)];
+	return &fixmap_bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)];
 }
 
 static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
@@ -46,7 +48,7 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
 	pte_t *ptep;
 
 	if (pmd_none(pmd)) {
-		ptep = bm_pte[BM_PTE_TABLE_IDX(addr)];
+		ptep = fixmap_bm_pte[BM_PTE_TABLE_IDX(addr)];
 		__pmd_populate(pmdp, __pa_symbol(ptep),
 			       PMD_TYPE_TABLE | PMD_TABLE_AF);
 	}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 79d90226fd5d..9c1aa838e9d5 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1184,6 +1184,7 @@ static void __init map_mem(void)
 	phys_addr_t init_begin = __pa_symbol(__init_begin);
 	phys_addr_t init_end = __pa_symbol(__init_end);
 	phys_addr_t kernel_end = __pa_symbol(__bss_stop);
+	phys_addr_t fixmap_pte_base = __pa_symbol(fixmap_bm_pte);
 	phys_addr_t start, end;
 	int flags = NO_EXEC_MAPPINGS;
 	u64 i;
@@ -1225,6 +1226,9 @@ static void __init map_mem(void)
 	__map_memblock(init_end, kernel_end, pgprot_tagged(PAGE_KERNEL),
 		       flags);
 
+	__map_memblock(fixmap_pte_base, fixmap_pte_base + fixmap_bm_pte_size,
+		       pgprot_tagged(PAGE_KERNEL), flags);
+
 	/* map all the memory banks */
 	for_each_mem_range(i, &start, &end) {
 		/*
@@ -1268,6 +1272,10 @@ void mark_rodata_ro(void)
 			    (unsigned long)_stext - (unsigned long)_text,
 			    PAGE_KERNEL_RO);
 
+	update_mapping_prot(__pa_symbol(fixmap_bm_pte),
+			    (unsigned long)lm_alias(fixmap_bm_pte),
+			    fixmap_bm_pte_size, PAGE_KERNEL_RO);
+
 	/* Map the kernel data/bss as invalid in the linear map */
 	mark_linear_data_alias_valid(false);
 }
-- 
2.55.0.887.g758fc8c411-goog

[RFC PATCH v2 2/4] arm64: mm: Use fault handler to permit swapper_pg_dir updates

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Instead of checking the destination of a page table descriptor update
against the virtual address of swapper_pg_dir in the kernel image, and
taking a dedicated path to perform such updates via a fixmap alias,
rely on a fault handler to do so if the plain store faulted.

This is more efficient for the general case, and makes it more
straight-forward to extend this logic to other page tables that may have
been allocated statically in the .rodata region of the kernel image.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/include/asm/fixmap.h  |  1 +
 arch/arm64/include/asm/pgtable.h | 49 +++++++++-----------
 arch/arm64/mm/mmu.c              | 18 +++----
 3 files changed, 32 insertions(+), 36 deletions(-)

diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 9191125738e9..8db791dcb4cb 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -99,6 +99,7 @@ enum fixed_addresses {
 	FIX_PUD,
 	FIX_P4D,
 	FIX_PGD,
+	FIX_PTVAL,
 
 	__end_of_fixed_addresses
 };
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 6000905a2e86..799cd9d52c28 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -35,6 +35,7 @@
 #include <asm/cmpxchg.h>
 #include <asm/fixmap.h>
 #include <asm/por.h>
+#include <asm/uaccess.h>
 #include <linux/mmdebug.h>
 #include <linux/mm_types.h>
 #include <linux/sched.h>
@@ -814,7 +815,7 @@ extern pgd_t idmap_pg_dir[];
 extern pgd_t tramp_pg_dir[];
 extern pgd_t reserved_pg_dir[];
 
-extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
+void try_set_readonly_ptval(ptval_t *ptvalp, ptval_t ptval);
 
 static inline bool in_swapper_pgdir(void *addr)
 {
@@ -824,17 +825,14 @@ static inline bool in_swapper_pgdir(void *addr)
 
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
-#ifdef __PAGETABLE_PMD_FOLDED
-	if (in_swapper_pgdir(pmdp)) {
-		set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
-		return;
-	}
-#endif /* __PAGETABLE_PMD_FOLDED */
-
-	WRITE_ONCE(*pmdp, pmd);
+	__put_kernel_nofault(pmdp, &pmd, pmd_t, fault);
 
 	if (pmd_valid(pmd))
 		queue_pte_barriers();
+	return;
+
+fault:
+	try_set_readonly_ptval(&pmd_val(*pmdp), pmd_val(pmd));
 }
 
 static inline void pmd_clear(pmd_t *pmdp)
@@ -890,15 +888,14 @@ static inline bool pgtable_l4_enabled(void);
 
 static inline void set_pud(pud_t *pudp, pud_t pud)
 {
-	if (!pgtable_l4_enabled() && in_swapper_pgdir(pudp)) {
-		set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
-		return;
-	}
-
-	WRITE_ONCE(*pudp, pud);
+	__put_kernel_nofault(pudp, &pud, pud_t, fault);
 
 	if (pud_valid(pud))
 		queue_pte_barriers();
+	return;
+
+fault:
+	try_set_readonly_ptval(&pud_val(*pudp), pud_val(pud));
 }
 
 static inline void pud_clear(pud_t *pudp)
@@ -971,13 +968,12 @@ static inline bool mm_pud_folded(const struct mm_struct *mm)
 
 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
 {
-	if (in_swapper_pgdir(p4dp)) {
-		set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
-		return;
-	}
-
-	WRITE_ONCE(*p4dp, p4d);
+	__put_kernel_nofault(p4dp, &p4d, p4d_t, fault);
 	queue_pte_barriers();
+	return;
+
+fault:
+	try_set_readonly_ptval(&p4d_val(*p4dp), p4d_val(p4d));
 }
 
 static inline void p4d_clear(p4d_t *p4dp)
@@ -1099,13 +1095,12 @@ static inline bool mm_p4d_folded(const struct mm_struct *mm)
 
 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 {
-	if (in_swapper_pgdir(pgdp)) {
-		set_swapper_pgd(pgdp, __pgd(pgd_val(pgd)));
-		return;
-	}
-
-	WRITE_ONCE(*pgdp, pgd);
+	__put_kernel_nofault(pgdp, &pgd, pgd_t, fault);
 	queue_pte_barriers();
+	return;
+
+fault:
+	try_set_readonly_ptval(&pgd_val(*pgdp), pgd_val(pgd));
 }
 
 static inline void pgd_clear(pgd_t *pgdp)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9c1aa838e9d5..b97e4bf99ca5 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -63,34 +63,34 @@ static bool rodata_is_rw __ro_after_init = true;
  */
 long __section(".mmuoff.data.write") __early_cpu_boot_status;
 
-static DEFINE_SPINLOCK(swapper_pgdir_lock);
 static DEFINE_MUTEX(fixmap_lock);
 
-void noinstr set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+void noinstr try_set_readonly_ptval(ptval_t *ptvalp, ptval_t ptval)
 {
-	pgd_t *fixmap_pgdp;
+	static DEFINE_SPINLOCK(lock);
+
+	BUG_ON(!in_swapper_pgdir(ptvalp));
 
 	/*
 	 * Don't bother with the fixmap if swapper_pg_dir is still mapped
 	 * writable in the kernel mapping.
 	 */
 	if (rodata_is_rw) {
-		WRITE_ONCE(*pgdp, pgd);
+		WRITE_ONCE(*ptvalp, ptval);
 		dsb(ishst);
 		isb();
 		return;
 	}
 
-	spin_lock(&swapper_pgdir_lock);
-	fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp));
-	WRITE_ONCE(*fixmap_pgdp, pgd);
+	guard(spinlock)(&lock);
+	ptvalp = (ptval_t *)set_fixmap_offset(FIX_PTVAL, __pa_symbol(ptvalp));
+	WRITE_ONCE(*ptvalp, ptval);
 	/*
 	 * We need dsb(ishst) here to ensure the page-table-walker sees
 	 * our new entry before set_p?d() returns. The fixmap's
 	 * flush_tlb_kernel_range() via clear_fixmap() does this for us.
 	 */
-	pgd_clear_fixmap();
-	spin_unlock(&swapper_pgdir_lock);
+	clear_fixmap(FIX_PTVAL);
 }
 
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
-- 
2.55.0.887.g758fc8c411-goog

[RFC PATCH v2 3/4] arm64: mm: Create r/o page table region that permits updates

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

Generalize the handling of swapper_pg_dir, and apply the fixmap fallback
for page table updates to all page tables in the .rodata..pgtbl section.

Currently, this holds only swapper_pg_dir, but this will be expanded in
subsequent patches.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/include/asm/pgtable.h |  6 -----
 arch/arm64/kernel/vmlinux.lds.S  |  6 +++++
 arch/arm64/mm/mmu.c              | 26 +++++++++++++++++---
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 799cd9d52c28..de0a07dae83a 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -817,12 +817,6 @@ extern pgd_t reserved_pg_dir[];
 
 void try_set_readonly_ptval(ptval_t *ptvalp, ptval_t ptval);
 
-static inline bool in_swapper_pgdir(void *addr)
-{
-	return ((unsigned long)addr & PAGE_MASK) ==
-	        ((unsigned long)swapper_pg_dir & PAGE_MASK);
-}
-
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
 	__put_kernel_nofault(pmdp, &pmd, pmd_t, fault);
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index af1d72020976..2cda0b7c41d4 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -243,9 +243,15 @@ SECTIONS
 	reserved_pg_dir = .;
 	. += PAGE_SIZE;
 
+	__rodata_pgtbl_start = .;
 	swapper_pg_dir = .;
 	. += PAGE_SIZE;
 
+	.pgtbl.ro_after_init : ALIGN(PAGE_SIZE) {
+		*(.pgtbl.ro_after_init)
+	}
+	__rodata_pgtbl_end = .;
+
 	. = ALIGN(SEGMENT_ALIGN);
 	__init_begin = .;
 	__inittext_begin = .;
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index b97e4bf99ca5..dad482d66f17 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -65,17 +65,27 @@ long __section(".mmuoff.data.write") __early_cpu_boot_status;
 
 static DEFINE_MUTEX(fixmap_lock);
 
+static struct range kimg_ropgtbl_range __ro_after_init;
+
 void noinstr try_set_readonly_ptval(ptval_t *ptvalp, ptval_t ptval)
 {
 	static DEFINE_SPINLOCK(lock);
+	bool is_lm = __is_lm_address(ptvalp);
+	u64 pa = is_lm ? __pa(ptvalp) : __pa_symbol(ptvalp);
 
-	BUG_ON(!in_swapper_pgdir(ptvalp));
+	if (!range_contains(&kimg_ropgtbl_range,
+			    &DEFINE_RANGE(pa, pa + sizeof(ptval_t))))
+		BUG();
 
 	/*
-	 * Don't bother with the fixmap if swapper_pg_dir is still mapped
+	 * Don't bother with the fixmap if .rodata is still mapped
 	 * writable in the kernel mapping.
 	 */
 	if (rodata_is_rw) {
+		/* no fault should have occurred for a kimg address */
+		BUG_ON(!is_lm);
+
+		ptvalp = (ptval_t *)__phys_to_kimg(pa);
 		WRITE_ONCE(*ptvalp, ptval);
 		dsb(ishst);
 		isb();
@@ -83,7 +93,7 @@ void noinstr try_set_readonly_ptval(ptval_t *ptvalp, ptval_t ptval)
 	}
 
 	guard(spinlock)(&lock);
-	ptvalp = (ptval_t *)set_fixmap_offset(FIX_PTVAL, __pa_symbol(ptvalp));
+	ptvalp = (ptval_t *)set_fixmap_offset(FIX_PTVAL, pa);
 	WRITE_ONCE(*ptvalp, ptval);
 	/*
 	 * We need dsb(ishst) here to ensure the page-table-walker sees
@@ -1177,6 +1187,14 @@ static inline void arm64_kfence_map_pool(void) { }
 
 #endif /* CONFIG_KFENCE */
 
+static void __init record_ropgtbl_phys_range(void)
+{
+	extern const char __rodata_pgtbl_start[], __rodata_pgtbl_end[];
+
+	kimg_ropgtbl_range = DEFINE_RANGE(__pa_symbol(__rodata_pgtbl_start),
+					  __pa_symbol(__rodata_pgtbl_end));
+}
+
 static void __init map_mem(void)
 {
 	static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
@@ -1189,6 +1207,8 @@ static void __init map_mem(void)
 	int flags = NO_EXEC_MAPPINGS;
 	u64 i;
 
+	record_ropgtbl_phys_range();
+
 	/*
 	 * Setting hierarchical PXNTable attributes on table entries covering
 	 * the linear region is only possible if it is guaranteed that no table
-- 
2.55.0.887.g758fc8c411-goog

[RFC PATCH v2 4/4] arm64: mm: Move fixmap intermediate page tables into .rodata

[Ard Biesheuvel]

From: Ard Biesheuvel <ardb@kernel.org>

The fixmap intermediate page tables are allocated statically, are
installed into the kernel's page table hierarchy early during boot, and
control a slice of the kernel's virtual address space that is not
subject to KASLR randomization.

Combined with the lack of randomization of the linear map, and the
tendency of some Android bootloaders to place the kernel image at the
base of DRAM in the physical space, the placement of these page tables
produces a vulnerability that is comparatively easy to exploit.

Avoid this, by moving these intermediate page tables into .rodata, so
that they cannot be manipulated directly via the linear map.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/include/asm/linkage.h | 1 +
 arch/arm64/mm/fixmap.c           | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h
index d1f7a16729d2..00963e11ebf0 100644
--- a/arch/arm64/include/asm/linkage.h
+++ b/arch/arm64/include/asm/linkage.h
@@ -45,6 +45,7 @@
 
 #define _THIS_IP_ ({ unsigned long __ip; asm volatile("adr %0, ." : "=r" (__ip)); __ip; })
 
+#define __rodata_pgtbl __section(".pgtbl.ro_after_init") __aligned(PAGE_SIZE)
 #define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE)
 
 #endif
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index 3a8cf6de6a7d..ab0f9ba7b712 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -32,8 +32,8 @@ static_assert(NR_BM_PMD_TABLES == 1);
 #define BM_PTE_TABLE_IDX(addr)	__BM_TABLE_IDX(addr, PMD_SHIFT)
 
 pte_t fixmap_bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl;
-static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused;
-static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused;
+static pmd_t bm_pmd[PTRS_PER_PMD] __rodata_pgtbl;
+static pud_t bm_pud[PTRS_PER_PUD] __rodata_pgtbl;
 
 const size_t fixmap_bm_pte_size = sizeof(fixmap_bm_pte);
 
-- 
2.55.0.887.g758fc8c411-goog