[PATCH 2/2] riscv: Prepare late KFENCE pools with PTE mappings

[Jackie Liu <liu.yun@linux.dev>]

From: Jackie Liu <liuyun01@kylinos.cn>

RISC-V preallocates a PTE-mapped KFENCE pool, but pools allocated after
boot may use huge mappings. arch_kfence_init_pool() leaves those mappings
unchanged, while kfence_protect_page() requires PTEs.

Use set_memory_4k() to prepare each pool. Add a force_pte mode to the
linear-map splitter so fully covered huge mappings are split as well.
Preserve permissions and flush the TLB even if splitting fails partway.

Keep the early PTE mapping to avoid boot-time page-table allocations.
Release that pool when boot parameters disable sampling, so late enabling
initializes a fresh pool. If reservation fails, continue booting with
sampling disabled to avoid an unprepared fallback pool, while allowing
a later retry.

Link: https://lore.kernel.org/all/sashiko-outbox-190189@kernel.org/
Fixes: 25abe0db9243 ("riscv: Fix kfence now that the linear mapping can be backed by PUD/P4D/PGD")
Signed-off-by: Jackie Liu <liuyun01@kylinos.cn>
---
 arch/riscv/include/asm/kfence.h     |  4 ++-
 arch/riscv/include/asm/set_memory.h |  3 ++
 arch/riscv/mm/init.c                | 23 +++++++++---
 arch/riscv/mm/pageattr.c            | 56 ++++++++++++++++++++---------
 4 files changed, 65 insertions(+), 21 deletions(-)

diff --git a/arch/riscv/include/asm/kfence.h b/arch/riscv/include/asm/kfence.h
index 29cb3a6ee113d..d7bc84cc5107a 100644
--- a/arch/riscv/include/asm/kfence.h
+++ b/arch/riscv/include/asm/kfence.h
@@ -8,10 +8,12 @@
 #include <asm-generic/pgalloc.h>
 #include <asm/cacheflush.h>
 #include <asm/pgtable.h>
+#include <asm/set_memory.h>
 
 static inline bool arch_kfence_init_pool(void)
 {
-	return true;
+	return !set_memory_4k((unsigned long)__kfence_pool,
+			      KFENCE_POOL_SIZE / PAGE_SIZE);
 }
 
 static inline bool kfence_protect_page(unsigned long addr, bool protect)
diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h
index e9f9960c19477..dfd41466265f3 100644
--- a/arch/riscv/include/asm/set_memory.h
+++ b/arch/riscv/include/asm/set_memory.h
@@ -11,6 +11,9 @@
  * Functions to change memory attributes.
  */
 #ifdef CONFIG_MMU
+#ifdef CONFIG_64BIT
+int set_memory_4k(unsigned long addr, int numpages);
+#endif
 int set_memory_ro(unsigned long addr, int numpages);
 int set_memory_rw(unsigned long addr, int numpages);
 int set_memory_x(unsigned long addr, int numpages);
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index b50de11605c6c..aee6fc4d6c5df 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -173,6 +173,16 @@ void __init arch_mm_preinit(void)
 	BUG_ON(!mem_map);
 #endif /* CONFIG_FLATMEM */
 
+#ifdef CONFIG_KFENCE
+	/* Boot parameters have been parsed, and memblock can still free the pool. */
+	if (!__kfence_pool) {
+		kfence_sample_interval = 0;
+	} else if (!kfence_sample_interval) {
+		memblock_free(__kfence_pool, KFENCE_POOL_SIZE);
+		__kfence_pool = NULL;
+	}
+#endif
+
 	if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb &&
 	    dma_cache_alignment != 1) {
 		/*
@@ -1230,10 +1240,12 @@ static void __init create_linear_mapping_page_table(void)
 	 *  for this region.
 	 */
 	kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
-	BUG_ON(!kfence_pool);
-
-	memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
-	__kfence_pool = __va(kfence_pool);
+	if (kfence_pool) {
+		memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
+		__kfence_pool = __va(kfence_pool);
+	} else {
+		pr_err("failed to allocate kfence pool\n");
+	}
 #endif
 
 	/* Map all memory banks in the linear mapping */
@@ -1254,6 +1266,9 @@ static void __init create_linear_mapping_page_table(void)
 #endif
 
 #ifdef CONFIG_KFENCE
+	if (!kfence_pool)
+		return;
+
 	create_linear_mapping_range(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, PAGE_SIZE, NULL);
 
 	memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c
index 32834a09a8097..35990e7f9ebe5 100644
--- a/arch/riscv/mm/pageattr.c
+++ b/arch/riscv/mm/pageattr.c
@@ -94,7 +94,8 @@ static const struct mm_walk_ops pageattr_ops = {
 
 #ifdef CONFIG_64BIT
 static int __split_linear_mapping_pmd(pud_t *pudp,
-				      unsigned long vaddr, unsigned long end)
+				      unsigned long vaddr, unsigned long end,
+				      bool force_pte)
 {
 	pmd_t *pmdp;
 	unsigned long next;
@@ -104,7 +105,7 @@ static int __split_linear_mapping_pmd(pud_t *pudp,
 	do {
 		next = pmd_addr_end(vaddr, end);
 
-		if (next - vaddr >= PMD_SIZE &&
+		if (!force_pte && next - vaddr >= PMD_SIZE &&
 		    vaddr <= (vaddr & PMD_MASK) && end >= next)
 			continue;
 
@@ -133,7 +134,8 @@ static int __split_linear_mapping_pmd(pud_t *pudp,
 }
 
 static int __split_linear_mapping_pud(p4d_t *p4dp,
-				      unsigned long vaddr, unsigned long end)
+				      unsigned long vaddr, unsigned long end,
+				      bool force_pte)
 {
 	pud_t *pudp;
 	unsigned long next;
@@ -144,7 +146,7 @@ static int __split_linear_mapping_pud(p4d_t *p4dp,
 	do {
 		next = pud_addr_end(vaddr, end);
 
-		if (next - vaddr >= PUD_SIZE &&
+		if (!force_pte && next - vaddr >= PUD_SIZE &&
 		    vaddr <= (vaddr & PUD_MASK) && end >= next)
 			continue;
 
@@ -169,7 +171,7 @@ static int __split_linear_mapping_pud(p4d_t *p4dp,
 			set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE));
 		}
 
-		ret = __split_linear_mapping_pmd(pudp, vaddr, next);
+		ret = __split_linear_mapping_pmd(pudp, vaddr, next, force_pte);
 		if (ret)
 			return ret;
 	} while (pudp++, vaddr = next, vaddr != end);
@@ -178,7 +180,8 @@ static int __split_linear_mapping_pud(p4d_t *p4dp,
 }
 
 static int __split_linear_mapping_p4d(pgd_t *pgdp,
-				      unsigned long vaddr, unsigned long end)
+				      unsigned long vaddr, unsigned long end,
+				      bool force_pte)
 {
 	p4d_t *p4dp;
 	unsigned long next;
@@ -190,10 +193,10 @@ static int __split_linear_mapping_p4d(pgd_t *pgdp,
 		next = p4d_addr_end(vaddr, end);
 
 		/*
-		 * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't
-		 * need to split, we'll change the protections on the whole P4D.
+		 * Unless PTE mappings are required, change the protections of a
+		 * fully covered P4D mapping without splitting it.
 		 */
-		if (next - vaddr >= P4D_SIZE &&
+		if (!force_pte && next - vaddr >= P4D_SIZE &&
 		    vaddr <= (vaddr & P4D_MASK) && end >= next)
 			continue;
 
@@ -228,7 +231,7 @@ static int __split_linear_mapping_p4d(pgd_t *pgdp,
 			set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE));
 		}
 
-		ret = __split_linear_mapping_pud(p4dp, vaddr, next);
+		ret = __split_linear_mapping_pud(p4dp, vaddr, next, force_pte);
 		if (ret)
 			return ret;
 	} while (p4dp++, vaddr = next, vaddr != end);
@@ -238,7 +241,7 @@ static int __split_linear_mapping_p4d(pgd_t *pgdp,
 
 static int __split_linear_mapping_pgd(pgd_t *pgdp,
 				      unsigned long vaddr,
-				      unsigned long end)
+				      unsigned long end, bool force_pte)
 {
 	unsigned long next;
 	int ret;
@@ -246,7 +249,7 @@ static int __split_linear_mapping_pgd(pgd_t *pgdp,
 	do {
 		next = pgd_addr_end(vaddr, end);
 		/* We never use PGD mappings for the linear mapping */
-		ret = __split_linear_mapping_p4d(pgdp, vaddr, next);
+		ret = __split_linear_mapping_p4d(pgdp, vaddr, next, force_pte);
 		if (ret)
 			return ret;
 	} while (pgdp++, vaddr = next, vaddr != end);
@@ -254,9 +257,30 @@ static int __split_linear_mapping_pgd(pgd_t *pgdp,
 	return 0;
 }
 
-static int split_linear_mapping(unsigned long start, unsigned long end)
+static int split_linear_mapping(unsigned long start, unsigned long end,
+				bool force_pte)
 {
-	return __split_linear_mapping_pgd(pgd_offset_k(start), start, end);
+	return __split_linear_mapping_pgd(pgd_offset_k(start), start, end, force_pte);
+}
+
+/* Split the direct map into base pages without changing its permissions. */
+int set_memory_4k(unsigned long addr, int numpages)
+{
+	unsigned long end = addr + PAGE_SIZE * numpages;
+	int ret;
+
+	if (!numpages)
+		return 0;
+	if (!is_linear_mapping(addr) || !is_linear_mapping(end - 1))
+		return -EINVAL;
+
+	mmap_write_lock(&init_mm);
+	ret = split_linear_mapping(addr, end, true);
+	mmap_write_unlock(&init_mm);
+
+	/* A failed allocation may still have split an upper-level mapping. */
+	flush_tlb_all();
+	return ret;
 }
 #endif	/* CONFIG_64BIT */
 
@@ -296,7 +320,7 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
 			lm_start = (unsigned long)page_address(area->pages[i]);
 			lm_end = lm_start + PAGE_SIZE;
 
-			ret = split_linear_mapping(lm_start, lm_end);
+			ret = split_linear_mapping(lm_start, lm_end, false);
 			if (ret)
 				goto unlock;
 
@@ -314,7 +338,7 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
 			lm_end = end;
 		}
 
-		ret = split_linear_mapping(lm_start, lm_end);
+		ret = split_linear_mapping(lm_start, lm_end, false);
 		if (ret)
 			goto unlock;
 
-- 
2.43.0


_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv