[RFC PATCH v1 43/57] arm64: Clean up simple cases of CONFIG_ARM64_*K_PAGES

[RFC PATCH v1 43/57] arm64: Clean up simple cases of CONFIG_ARM64_*K_PAGES

[Ryan Roberts]

There are a number of places that define macros conditionally depending
on which of the CONFIG_ARM64_*K_PAGES macros are defined. But in
preparation for supporting boot-time page size selection, we will no
longer be able to make these decisions at compile time.

So let's refactor the code to check the size of PAGE_SIZE using the
ternary operator. This approach will still resolve to compile-time
constants when configured for a compile-time page size, but it will also
work when we turn PAGE_SIZE into a run-time value. Additionally,
IS_ENABLED(CONFIG_ARM64_*K_PAGES) instances are also converted to test
the size of PAGE_SIZE.

Additionally modify ARM64_HAS_VA52 capability detection to use a custom
match function, which chooses which feature register and field to check
based on PAGE_SIZE. The compiler will eliminate the other page sizes
when selecting a compile time page size, but will also now cope with
seting page size at boot time.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/include/asm/kvm_arm.h     | 21 ++++-------
 arch/arm64/include/asm/kvm_pgtable.h |  6 +---
 arch/arm64/include/asm/memory.h      |  7 ++--
 arch/arm64/include/asm/processor.h   | 10 +++---
 arch/arm64/include/asm/sparsemem.h   | 11 ++----
 arch/arm64/include/asm/sysreg.h      | 54 ++++++++++++++++++----------
 arch/arm64/kernel/cpufeature.c       | 43 +++++++++++++---------
 arch/arm64/mm/fixmap.c               |  2 +-
 arch/arm64/mm/init.c                 | 20 +++++------
 arch/arm64/mm/kasan_init.c           |  8 ++---
 arch/arm64/mm/mmu.c                  |  2 +-
 drivers/irqchip/irq-gic-v3-its.c     |  2 +-
 12 files changed, 94 insertions(+), 92 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index d81cc746e0ebd..08155dc17ad17 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -189,22 +189,13 @@
  * Entry_Level = 4 - Number_of_levels.
  *
  */
-#ifdef CONFIG_ARM64_64K_PAGES
+#define VTCR_EL2_TGRAN							\
+	(PAGE_SIZE == SZ_64K ?						\
+		VTCR_EL2_TG0_64K :					\
+		(PAGE_SIZE == SZ_16K ? VTCR_EL2_TG0_16K : VTCR_EL2_TG0_4K))
 
-#define VTCR_EL2_TGRAN			VTCR_EL2_TG0_64K
-#define VTCR_EL2_TGRAN_SL0_BASE		3UL
-
-#elif defined(CONFIG_ARM64_16K_PAGES)
-
-#define VTCR_EL2_TGRAN			VTCR_EL2_TG0_16K
-#define VTCR_EL2_TGRAN_SL0_BASE		3UL
-
-#else	/* 4K */
-
-#define VTCR_EL2_TGRAN			VTCR_EL2_TG0_4K
-#define VTCR_EL2_TGRAN_SL0_BASE		2UL
-
-#endif
+#define VTCR_EL2_TGRAN_SL0_BASE						\
+	(PAGE_SIZE == SZ_64K ? 3UL : (PAGE_SIZE == SZ_16K ? 3UL : 2UL))
 
 #define VTCR_EL2_LVLS_TO_SL0(levels)	\
 	((VTCR_EL2_TGRAN_SL0_BASE - (4 - (levels))) << VTCR_EL2_SL0_SHIFT)
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 19278dfe79782..796614bf59e78 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -20,11 +20,7 @@
  *  - 16K (level 2):	32MB
  *  - 64K (level 2):	512MB
  */
-#ifdef CONFIG_ARM64_4K_PAGES
-#define KVM_PGTABLE_MIN_BLOCK_LEVEL	1
-#else
-#define KVM_PGTABLE_MIN_BLOCK_LEVEL	2
-#endif
+#define KVM_PGTABLE_MIN_BLOCK_LEVEL	(PAGE_SIZE == SZ_4K ? 1 : 2)
 
 #define kvm_lpa2_is_enabled()		system_supports_lpa2()
 
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 54fb014eba058..6aa97fa22dc30 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -188,11 +188,8 @@
 #define MT_S2_FWB_NORMAL_NC	5
 #define MT_S2_FWB_DEVICE_nGnRE	1
 
-#ifdef CONFIG_ARM64_4K_PAGES
-#define IOREMAP_MAX_ORDER	(PUD_SHIFT)
-#else
-#define IOREMAP_MAX_ORDER	(PMD_SHIFT)
-#endif
+#define IOREMAP_MAX_ORDER						\
+	(PAGE_SIZE == SZ_4K ? PUD_SHIFT : PMD_SHIFT)
 
 /*
  *  Open-coded (swapper_pg_dir - reserved_pg_dir) as this cannot be calculated
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index f77371232d8c6..444694a4e6733 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -55,15 +55,15 @@
 #define TASK_SIZE_MAX		(UL(1) << VA_BITS)
 
 #ifdef CONFIG_COMPAT
-#if defined(CONFIG_ARM64_64K_PAGES) && defined(CONFIG_KUSER_HELPERS)
+#if defined(CONFIG_KUSER_HELPERS)
 /*
- * With CONFIG_ARM64_64K_PAGES enabled, the last page is occupied
- * by the compat vectors page.
+ * With 64K pages in use, the last page is occupied by the compat vectors page.
  */
-#define TASK_SIZE_32		UL(0x100000000)
+#define TASK_SIZE_32							\
+	(PAGE_SIZE == SZ_64K ? UL(0x100000000) : (UL(0x100000000) - PAGE_SIZE))
 #else
 #define TASK_SIZE_32		(UL(0x100000000) - PAGE_SIZE)
-#endif /* CONFIG_ARM64_64K_PAGES */
+#endif /* CONFIG_KUSER_HELPERS */
 #define TASK_SIZE		(test_thread_flag(TIF_32BIT) ? \
 				TASK_SIZE_32 : TASK_SIZE_64)
 #define TASK_SIZE_OF(tsk)	(test_tsk_thread_flag(tsk, TIF_32BIT) ? \
diff --git a/arch/arm64/include/asm/sparsemem.h b/arch/arm64/include/asm/sparsemem.h
index 8a8acc220371c..a05fdd54014f7 100644
--- a/arch/arm64/include/asm/sparsemem.h
+++ b/arch/arm64/include/asm/sparsemem.h
@@ -11,19 +11,12 @@
  * Section size must be at least 512MB for 64K base
  * page size config. Otherwise it will be less than
  * MAX_PAGE_ORDER and the build process will fail.
- */
-#ifdef CONFIG_ARM64_64K_PAGES
-#define SECTION_SIZE_BITS 29
-
-#else
-
-/*
+ *
  * Section size must be at least 128MB for 4K base
  * page size config. Otherwise PMD based huge page
  * entries could not be created for vmemmap mappings.
  * 16K follows 4K for simplicity.
  */
-#define SECTION_SIZE_BITS 27
-#endif /* CONFIG_ARM64_64K_PAGES */
+#define SECTION_SIZE_BITS	(PAGE_SIZE == SZ_64K ? 29 : 27)
 
 #endif
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 4a9ea103817e8..cbcf861bbf2a6 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -10,10 +10,12 @@
 #define __ASM_SYSREG_H
 
 #include <linux/bits.h>
+#include <linux/sizes.h>
 #include <linux/stringify.h>
 #include <linux/kasan-tags.h>
 
 #include <asm/gpr-num.h>
+#include <asm/page-def.h>
 
 /*
  * ARMv8 ARM reserves the following encoding for system registers:
@@ -913,24 +915,40 @@
 #define ID_AA64MMFR0_EL1_PARANGE_MAX	ID_AA64MMFR0_EL1_PARANGE_48
 #endif
 
-#if defined(CONFIG_ARM64_4K_PAGES)
-#define ID_AA64MMFR0_EL1_TGRAN_SHIFT		ID_AA64MMFR0_EL1_TGRAN4_SHIFT
-#define ID_AA64MMFR0_EL1_TGRAN_LPA2		ID_AA64MMFR0_EL1_TGRAN4_52_BIT
-#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
-#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX
-#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT		ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT
-#elif defined(CONFIG_ARM64_16K_PAGES)
-#define ID_AA64MMFR0_EL1_TGRAN_SHIFT		ID_AA64MMFR0_EL1_TGRAN16_SHIFT
-#define ID_AA64MMFR0_EL1_TGRAN_LPA2		ID_AA64MMFR0_EL1_TGRAN16_52_BIT
-#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN
-#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX
-#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT		ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT
-#elif defined(CONFIG_ARM64_64K_PAGES)
-#define ID_AA64MMFR0_EL1_TGRAN_SHIFT		ID_AA64MMFR0_EL1_TGRAN64_SHIFT
-#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN
-#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX
-#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT		ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
-#endif
+#define ID_AA64MMFR0_EL1_TGRAN_SHIFT					\
+	(PAGE_SIZE == SZ_4K ?						\
+		ID_AA64MMFR0_EL1_TGRAN4_SHIFT :				\
+		(PAGE_SIZE == SZ_16K ?					\
+			ID_AA64MMFR0_EL1_TGRAN16_SHIFT :		\
+			ID_AA64MMFR0_EL1_TGRAN64_SHIFT))
+
+#define ID_AA64MMFR0_EL1_TGRAN_LPA2					\
+	(PAGE_SIZE == SZ_4K ?						\
+		ID_AA64MMFR0_EL1_TGRAN4_52_BIT :			\
+		(PAGE_SIZE == SZ_16K ?					\
+			ID_AA64MMFR0_EL1_TGRAN16_52_BIT :		\
+			-1))
+
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN				\
+	(PAGE_SIZE == SZ_4K ?						\
+		ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN :			\
+		(PAGE_SIZE == SZ_16K ?					\
+			ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN :	\
+			ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN))
+
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX				\
+	(PAGE_SIZE == SZ_4K ?						\
+		ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX :			\
+		(PAGE_SIZE == SZ_16K ?					\
+			ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX :	\
+			ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX))
+
+#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT					\
+	(PAGE_SIZE == SZ_4K ?						\
+		ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT :			\
+		(PAGE_SIZE == SZ_16K ?					\
+			ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT :		\
+			ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT))
 
 #define CPACR_EL1_FPEN_EL1EN	(BIT(20)) /* enable EL1 access */
 #define CPACR_EL1_FPEN_EL0EN	(BIT(21)) /* enable EL0 access, if EL1EN set */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 646ecd3069fdd..7705c9c0e7142 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1831,11 +1831,13 @@ static bool has_nv1(const struct arm64_cpu_capabilities *entry, int scope)
 		  is_midr_in_range_list(read_cpuid_id(), nv1_ni_list)));
 }
 
-#if defined(ID_AA64MMFR0_EL1_TGRAN_LPA2) && defined(ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_LPA2)
 static bool has_lpa2_at_stage1(u64 mmfr0)
 {
 	unsigned int tgran;
 
+	if (PAGE_SIZE == SZ_64K)
+		return false;
+
 	tgran = cpuid_feature_extract_unsigned_field(mmfr0,
 					ID_AA64MMFR0_EL1_TGRAN_SHIFT);
 	return tgran == ID_AA64MMFR0_EL1_TGRAN_LPA2;
@@ -1845,6 +1847,9 @@ static bool has_lpa2_at_stage2(u64 mmfr0)
 {
 	unsigned int tgran;
 
+	if (PAGE_SIZE == SZ_64K)
+		return false;
+
 	tgran = cpuid_feature_extract_unsigned_field(mmfr0,
 					ID_AA64MMFR0_EL1_TGRAN_2_SHIFT);
 	return tgran == ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_LPA2;
@@ -1857,10 +1862,26 @@ static bool has_lpa2(const struct arm64_cpu_capabilities *entry, int scope)
 	mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 	return has_lpa2_at_stage1(mmfr0) && has_lpa2_at_stage2(mmfr0);
 }
-#else
-static bool has_lpa2(const struct arm64_cpu_capabilities *entry, int scope)
+
+#ifdef CONFIG_ARM64_VA_BITS_52
+static bool has_va52(const struct arm64_cpu_capabilities *entry, int scope)
 {
-	return false;
+	const struct arm64_cpu_capabilities entry_64k = {
+		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, VARange, 52)
+	};
+	const struct arm64_cpu_capabilities entry_16k = {
+		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)
+	};
+	const struct arm64_cpu_capabilities entry_4k = {
+		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)
+	};
+
+	if (PAGE_SIZE == SZ_64K)
+		return has_cpuid_feature(&entry_64k, scope);
+	else if (PAGE_SIZE == SZ_16K)
+		return has_cpuid_feature(&entry_16k, scope);
+	else
+		return has_cpuid_feature(&entry_4k, scope);
 }
 #endif
 
@@ -2847,20 +2868,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 	},
 #ifdef CONFIG_ARM64_VA_BITS_52
 	{
+		.desc = "52-bit Virtual Addressing",
 		.capability = ARM64_HAS_VA52,
 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
-		.matches = has_cpuid_feature,
-#ifdef CONFIG_ARM64_64K_PAGES
-		.desc = "52-bit Virtual Addressing (LVA)",
-		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, VARange, 52)
-#else
-		.desc = "52-bit Virtual Addressing (LPA2)",
-#ifdef CONFIG_ARM64_4K_PAGES
-		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)
-#else
-		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)
-#endif
-#endif
+		.matches = has_va52,
 	},
 #endif
 	{
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index de1e09d986ad2..15ce3253ad359 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -82,7 +82,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
 		 * share the top level pgd entry, which should only happen on
 		 * 16k/4 levels configurations.
 		 */
-		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
+		BUG_ON(PAGE_SIZE != SZ_16K);
 	}
 
 	if (p4d_none(p4d))
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 9b5ab6818f7f3..42eb246949072 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -73,13 +73,10 @@ phys_addr_t __ro_after_init arm64_dma_phys_limit;
  * (64k granule), or a multiple that can be mapped using contiguous bits
  * in the page tables: 32 * PMD_SIZE (16k granule)
  */
-#if defined(CONFIG_ARM64_4K_PAGES)
-#define ARM64_MEMSTART_SHIFT		PUD_SHIFT
-#elif defined(CONFIG_ARM64_16K_PAGES)
-#define ARM64_MEMSTART_SHIFT		CONT_PMD_SHIFT
-#else
-#define ARM64_MEMSTART_SHIFT		PMD_SHIFT
-#endif
+#define ARM64_MEMSTART_SHIFT						\
+	(PAGE_SIZE == SZ_4K ?						\
+		PUD_SHIFT :						\
+		(PAGE_SIZE == SZ_16K ? CONT_PMD_SHIFT : PMD_SHIFT))
 
 /*
  * sparsemem vmemmap imposes an additional requirement on the alignment of
@@ -87,11 +84,10 @@ phys_addr_t __ro_after_init arm64_dma_phys_limit;
  * has a direct correspondence, and needs to appear sufficiently aligned
  * in the virtual address space.
  */
-#if ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS
-#define ARM64_MEMSTART_ALIGN	(1UL << SECTION_SIZE_BITS)
-#else
-#define ARM64_MEMSTART_ALIGN	(1UL << ARM64_MEMSTART_SHIFT)
-#endif
+#define ARM64_MEMSTART_ALIGN						\
+	(ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS ?			\
+		(1UL << SECTION_SIZE_BITS) :				\
+		(1UL << ARM64_MEMSTART_SHIFT))
 
 static void __init arch_reserve_crashkernel(void)
 {
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index b65a29440a0c9..9af897fb3c432 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -178,10 +178,10 @@ static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
 	} while (pgdp++, addr = next, addr != end);
 }
 
-#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS > 4
+#if CONFIG_PGTABLE_LEVELS > 4
 #define SHADOW_ALIGN	P4D_SIZE
 #else
-#define SHADOW_ALIGN	PUD_SIZE
+#define SHADOW_ALIGN	(PAGE_SIZE == SZ_64K ? P4D_SIZE : PUD_SIZE)
 #endif
 
 /*
@@ -243,8 +243,8 @@ static int __init root_level_idx(u64 addr)
 	 * not implemented. This means we need to index the table as usual,
 	 * instead of masking off bits based on vabits_actual.
 	 */
-	u64 vabits = IS_ENABLED(CONFIG_ARM64_64K_PAGES) ? VA_BITS
-							: vabits_actual;
+	u64 vabits = PAGE_SIZE == SZ_64K ? VA_BITS
+					 : vabits_actual;
 	int shift = (ARM64_HW_PGTABLE_LEVELS(vabits) - 1) * (PAGE_SHIFT - 3);
 
 	return (addr & ~_PAGE_OFFSET(vabits)) >> (shift + PAGE_SHIFT);
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index d4d30eaefb4cd..a528787c1e550 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1179,7 +1179,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
 {
 	WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
 
-	if (!IS_ENABLED(CONFIG_ARM64_4K_PAGES))
+	if (PAGE_SIZE != SZ_4K)
 		return vmemmap_populate_basepages(start, end, node, altmap);
 	else
 		return vmemmap_populate_hugepages(start, end, node, altmap);
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index fdec478ba5e70..b745579b4b9f3 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -2323,7 +2323,7 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser,
 	baser_phys = virt_to_phys(base);
 
 	/* Check if the physical address of the memory is above 48bits */
-	if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
+	if (PAGE_SIZE == SZ_64K && (baser_phys >> 48)) {
 
 		/* 52bit PA is supported only when PageSize=64K */
 		if (psz != SZ_64K) {
-- 
2.43.0

[RFC PATCH v1 44/57] arm64: Align sections to PAGE_SIZE_MAX

[Ryan Roberts]

Increase alignment of sections in nvhe hyp, vdso and final vmlinux image
from PAGE_SIZE to PAGE_SIZE_MAX. For compile-time PAGE_SIZE,
PAGE_SIZE_MAX == PAGE_SIZE so there is no change. For boot-time
PAGE_SIZE, PAGE_SIZE_MAX is the largest selectable page size.

For a boot-time page size build, image size is comparable to a 64K page
size compile-time build. In future, it may be desirable to optimize
run-time memory consumption by freeing unused padding pages when the
boot-time selected page size is less than PAGE_SIZE_MAX.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/include/asm/memory.h     |  4 +--
 arch/arm64/kernel/vdso-wrap.S       |  4 +--
 arch/arm64/kernel/vdso.c            |  7 +++---
 arch/arm64/kernel/vdso/vdso.lds.S   |  4 +--
 arch/arm64/kernel/vdso32-wrap.S     |  4 +--
 arch/arm64/kernel/vdso32/vdso.lds.S |  4 +--
 arch/arm64/kernel/vmlinux.lds.S     | 38 ++++++++++++++---------------
 arch/arm64/kvm/hyp/nvhe/hyp.lds.S   |  2 +-
 8 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 6aa97fa22dc30..5393a859183f7 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -195,13 +195,13 @@
  *  Open-coded (swapper_pg_dir - reserved_pg_dir) as this cannot be calculated
  *  until link time.
  */
-#define RESERVED_SWAPPER_OFFSET	(PAGE_SIZE)
+#define RESERVED_SWAPPER_OFFSET	(PAGE_SIZE_MAX)
 
 /*
  *  Open-coded (swapper_pg_dir - tramp_pg_dir) as this cannot be calculated
  *  until link time.
  */
-#define TRAMP_SWAPPER_OFFSET	(2 * PAGE_SIZE)
+#define TRAMP_SWAPPER_OFFSET	(2 * PAGE_SIZE_MAX)
 
 #ifndef __ASSEMBLY__
 
diff --git a/arch/arm64/kernel/vdso-wrap.S b/arch/arm64/kernel/vdso-wrap.S
index c4b1990bf2be0..79fa77628199b 100644
--- a/arch/arm64/kernel/vdso-wrap.S
+++ b/arch/arm64/kernel/vdso-wrap.S
@@ -13,10 +13,10 @@
 
 	.globl vdso_start, vdso_end
 	.section .rodata
-	.balign PAGE_SIZE
+	.balign PAGE_SIZE_MAX
 vdso_start:
 	.incbin "arch/arm64/kernel/vdso/vdso.so"
-	.balign PAGE_SIZE
+	.balign PAGE_SIZE_MAX
 vdso_end:
 
 	.previous
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 89b6e78400023..1efe98909a2e0 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -195,7 +195,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
 
 	vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
 	/* Be sure to map the data page */
-	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
+	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE_MAX;
 
 	vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
 	if (IS_ERR_VALUE(vdso_base)) {
@@ -203,7 +203,8 @@ static int __setup_additional_pages(enum vdso_abi abi,
 		goto up_fail;
 	}
 
-	ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
+	ret = _install_special_mapping(mm, vdso_base,
+				       VVAR_NR_PAGES * PAGE_SIZE_MAX,
 				       VM_READ|VM_MAYREAD|VM_PFNMAP,
 				       vdso_info[abi].dm);
 	if (IS_ERR(ret))
@@ -212,7 +213,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
 	if (system_supports_bti_kernel())
 		gp_flags = VM_ARM64_BTI;
 
-	vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
+	vdso_base += VVAR_NR_PAGES * PAGE_SIZE_MAX;
 	mm->context.vdso = (void *)vdso_base;
 	ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
 				       VM_READ|VM_EXEC|gp_flags|
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index 45354f2ddf706..f7d1537a689e8 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -18,9 +18,9 @@ OUTPUT_ARCH(aarch64)
 
 SECTIONS
 {
-	PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
+	PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE_MAX);
 #ifdef CONFIG_TIME_NS
-	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
+	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE_MAX);
 #endif
 	. = VDSO_LBASE + SIZEOF_HEADERS;
 
diff --git a/arch/arm64/kernel/vdso32-wrap.S b/arch/arm64/kernel/vdso32-wrap.S
index e72ac7bc4c04f..1c6069d6c457e 100644
--- a/arch/arm64/kernel/vdso32-wrap.S
+++ b/arch/arm64/kernel/vdso32-wrap.S
@@ -10,10 +10,10 @@
 
 	.globl vdso32_start, vdso32_end
 	.section .rodata
-	.balign PAGE_SIZE
+	.balign PAGE_SIZE_MAX
 vdso32_start:
 	.incbin "arch/arm64/kernel/vdso32/vdso.so"
-	.balign PAGE_SIZE
+	.balign PAGE_SIZE_MAX
 vdso32_end:
 
 	.previous
diff --git a/arch/arm64/kernel/vdso32/vdso.lds.S b/arch/arm64/kernel/vdso32/vdso.lds.S
index 8d95d7d35057d..c46d18a69d1ce 100644
--- a/arch/arm64/kernel/vdso32/vdso.lds.S
+++ b/arch/arm64/kernel/vdso32/vdso.lds.S
@@ -18,9 +18,9 @@ OUTPUT_ARCH(arm)
 
 SECTIONS
 {
-	PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
+	PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE_MAX);
 #ifdef CONFIG_TIME_NS
-	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE);
+	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE_MAX);
 #endif
 	. = VDSO_LBASE + SIZEOF_HEADERS;
 
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 7f3f6d709ae73..1ef6dea13b57c 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -15,16 +15,16 @@
 
 #define HYPERVISOR_DATA_SECTIONS				\
 	HYP_SECTION_NAME(.rodata) : {				\
-		. = ALIGN(PAGE_SIZE);				\
+		. = ALIGN(PAGE_SIZE_MAX);			\
 		__hyp_rodata_start = .;				\
 		*(HYP_SECTION_NAME(.data..ro_after_init))	\
 		*(HYP_SECTION_NAME(.rodata))			\
-		. = ALIGN(PAGE_SIZE);				\
+		. = ALIGN(PAGE_SIZE_MAX);			\
 		__hyp_rodata_end = .;				\
 	}
 
 #define HYPERVISOR_PERCPU_SECTION				\
-	. = ALIGN(PAGE_SIZE);					\
+	. = ALIGN(PAGE_SIZE_MAX);				\
 	HYP_SECTION_NAME(.data..percpu) : {			\
 		*(HYP_SECTION_NAME(.data..percpu))		\
 	}
@@ -39,7 +39,7 @@
 #define BSS_FIRST_SECTIONS					\
 	__hyp_bss_start = .;					\
 	*(HYP_SECTION_NAME(.bss))				\
-	. = ALIGN(PAGE_SIZE);					\
+	. = ALIGN(PAGE_SIZE_MAX);				\
 	__hyp_bss_end = .;
 
 /*
@@ -48,7 +48,7 @@
  * between them, which can in some cases cause the linker to misalign them. To
  * work around the issue, force a page alignment for __bss_start.
  */
-#define SBSS_ALIGN			PAGE_SIZE
+#define SBSS_ALIGN			PAGE_SIZE_MAX
 #else /* CONFIG_KVM */
 #define HYPERVISOR_EXTABLE
 #define HYPERVISOR_DATA_SECTIONS
@@ -75,14 +75,14 @@ ENTRY(_text)
 jiffies = jiffies_64;
 
 #define HYPERVISOR_TEXT					\
-	. = ALIGN(PAGE_SIZE);				\
+	. = ALIGN(PAGE_SIZE_MAX);			\
 	__hyp_idmap_text_start = .;			\
 	*(.hyp.idmap.text)				\
 	__hyp_idmap_text_end = .;			\
 	__hyp_text_start = .;				\
 	*(.hyp.text)					\
 	HYPERVISOR_EXTABLE				\
-	. = ALIGN(PAGE_SIZE);				\
+	. = ALIGN(PAGE_SIZE_MAX);			\
 	__hyp_text_end = .;
 
 #define IDMAP_TEXT					\
@@ -113,11 +113,11 @@ jiffies = jiffies_64;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
 #define TRAMP_TEXT					\
-	. = ALIGN(PAGE_SIZE);				\
+	. = ALIGN(PAGE_SIZE_MAX);			\
 	__entry_tramp_text_start = .;			\
 	*(.entry.tramp.text)				\
-	. = ALIGN(PAGE_SIZE);				\
 	__entry_tramp_text_end = .;			\
+	. = ALIGN(PAGE_SIZE_MAX);			\
 	*(.entry.tramp.rodata)
 #else
 #define TRAMP_TEXT
@@ -187,7 +187,7 @@ SECTIONS
 	_etext = .;			/* End of text section */
 
 	/* everything from this point to __init_begin will be marked RO NX */
-	RO_DATA(PAGE_SIZE)
+	RO_DATA(PAGE_SIZE_MAX)
 
 	HYPERVISOR_DATA_SECTIONS
 
@@ -206,22 +206,22 @@ SECTIONS
 		HIBERNATE_TEXT
 		KEXEC_TEXT
 		IDMAP_TEXT
-		. = ALIGN(PAGE_SIZE);
+		. = ALIGN(PAGE_SIZE_MAX);
 	}
 
 	idmap_pg_dir = .;
-	. += PAGE_SIZE;
+	. += PAGE_SIZE_MAX;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
 	tramp_pg_dir = .;
-	. += PAGE_SIZE;
+	. += PAGE_SIZE_MAX;
 #endif
 
 	reserved_pg_dir = .;
-	. += PAGE_SIZE;
+	. += PAGE_SIZE_MAX;
 
 	swapper_pg_dir = .;
-	. += PAGE_SIZE;
+	. += PAGE_SIZE_MAX;
 
 	. = ALIGN(SEGMENT_ALIGN);
 	__init_begin = .;
@@ -290,7 +290,7 @@ SECTIONS
 
 	_data = .;
 	_sdata = .;
-	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
+	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE_MAX, THREAD_ALIGN)
 
 	/*
 	 * Data written with the MMU off but read with the MMU on requires
@@ -317,7 +317,7 @@ SECTIONS
 	/* start of zero-init region */
 	BSS_SECTION(SBSS_ALIGN, 0, 0)
 
-	. = ALIGN(PAGE_SIZE);
+	. = ALIGN(PAGE_SIZE_MAX);
 	init_pg_dir = .;
 	. += INIT_DIR_SIZE_MAX;
 	init_pg_end = .;
@@ -356,7 +356,7 @@ SECTIONS
  * former is page-aligned, but the latter may not be with 16K or 64K pages, so
  * it should also not cross a page boundary.
  */
-ASSERT(__hyp_idmap_text_end - __hyp_idmap_text_start <= PAGE_SIZE,
+ASSERT(__hyp_idmap_text_end - __hyp_idmap_text_start <= SZ_4K,
 	"HYP init code too big")
 ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
 	"ID map text too big or misaligned")
@@ -367,7 +367,7 @@ ASSERT(__hibernate_exit_text_start == swsusp_arch_suspend_exit,
        "Hibernate exit text does not start with swsusp_arch_suspend_exit")
 #endif
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
-ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
+ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3 * SZ_4K,
 	"Entry trampoline text too big")
 #endif
 #ifdef CONFIG_KVM
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index f4562f417d3fc..74c7c21626270 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -21,7 +21,7 @@ SECTIONS {
 	 * .hyp..data..percpu needs to be page aligned to maintain the same
 	 * alignment for when linking into vmlinux.
 	 */
-	. = ALIGN(PAGE_SIZE);
+	. = ALIGN(PAGE_SIZE_MAX);
 	BEGIN_HYP_SECTION(.data..percpu)
 		PERCPU_INPUT(L1_CACHE_BYTES)
 	END_HYP_SECTION
-- 
2.43.0

[RFC PATCH v1 48/57] arm64: Convert switch to if for non-const comparison values

[Ryan Roberts]

When we enable boot-time page size, some macros are no longer
compile-time constants. Where these macros are used as cases in switch
statements, the switch statements no longer compile.

Let's convert these to if/else blocks, which can handle the runtime
values.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/kvm/mmu.c        | 32 +++++++++++++++-----------------
 arch/arm64/mm/hugetlbpage.c | 34 +++++++++++-----------------------
 2 files changed, 26 insertions(+), 40 deletions(-)

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index a509b63bd4dd5..248a2d7ad6dbb 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1487,29 +1487,27 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 		vma_shift = get_vma_page_shift(vma, hva);
 	}
 
-	switch (vma_shift) {
 #ifndef __PAGETABLE_PMD_FOLDED
-	case PUD_SHIFT:
-		if (fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE))
-			break;
-		fallthrough;
+	if (vma_shift == PUD_SHIFT) {
+		if (!fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE))
+			vma_shift = PMD_SHIFT;
+	}
 #endif
-	case CONT_PMD_SHIFT:
+	if (vma_shift == CONT_PMD_SHIFT) {
 		vma_shift = PMD_SHIFT;
-		fallthrough;
-	case PMD_SHIFT:
-		if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE))
-			break;
-		fallthrough;
-	case CONT_PTE_SHIFT:
+	}
+	if (vma_shift == PMD_SHIFT) {
+		if (!fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE))
+			vma_shift = PAGE_SHIFT;
+	}
+	if (vma_shift == CONT_PTE_SHIFT) {
 		vma_shift = PAGE_SHIFT;
 		force_pte = true;
-		fallthrough;
-	case PAGE_SHIFT:
-		break;
-	default:
-		WARN_ONCE(1, "Unknown vma_shift %d", vma_shift);
 	}
+	if (vma_shift != PUD_SHIFT &&
+	    vma_shift != PMD_SHIFT &&
+	    vma_shift != PAGE_SHIFT)
+		WARN_ONCE(1, "Unknown vma_shift %d", vma_shift);
 
 	vma_pagesize = 1UL << vma_shift;
 
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 5f1e2103888b7..bc98c20655bba 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -51,16 +51,12 @@ void __init arm64_hugetlb_cma_reserve(void)
 
 static bool __hugetlb_valid_size(unsigned long size)
 {
-	switch (size) {
 #ifndef __PAGETABLE_PMD_FOLDED
-	case PUD_SIZE:
+	if (size == PUD_SIZE)
 		return pud_sect_supported();
 #endif
-	case CONT_PMD_SIZE:
-	case PMD_SIZE:
-	case CONT_PTE_SIZE:
+	if (size == CONT_PMD_SIZE || size == PMD_SIZE || size == CONT_PTE_SIZE)
 		return true;
-	}
 
 	return false;
 }
@@ -104,24 +100,20 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
 
 	*pgsize = size;
 
-	switch (size) {
 #ifndef __PAGETABLE_PMD_FOLDED
-	case PUD_SIZE:
+	if (size == PUD_SIZE) {
 		if (pud_sect_supported())
 			contig_ptes = 1;
-		break;
+	} else
 #endif
-	case PMD_SIZE:
+	if (size == PMD_SIZE) {
 		contig_ptes = 1;
-		break;
-	case CONT_PMD_SIZE:
+	} else if (size == CONT_PMD_SIZE) {
 		*pgsize = PMD_SIZE;
 		contig_ptes = CONT_PMDS;
-		break;
-	case CONT_PTE_SIZE:
+	} else if (size == CONT_PTE_SIZE) {
 		*pgsize = PAGE_SIZE;
 		contig_ptes = CONT_PTES;
-		break;
 	}
 
 	return contig_ptes;
@@ -339,20 +331,16 @@ unsigned long hugetlb_mask_last_page(struct hstate *h)
 {
 	unsigned long hp_size = huge_page_size(h);
 
-	switch (hp_size) {
 #ifndef __PAGETABLE_PMD_FOLDED
-	case PUD_SIZE:
+	if (hp_size == PUD_SIZE)
 		return PGDIR_SIZE - PUD_SIZE;
 #endif
-	case CONT_PMD_SIZE:
+	if (hp_size == CONT_PMD_SIZE)
 		return PUD_SIZE - CONT_PMD_SIZE;
-	case PMD_SIZE:
+	if (hp_size == PMD_SIZE)
 		return PUD_SIZE - PMD_SIZE;
-	case CONT_PTE_SIZE:
+	if (hp_size == CONT_PTE_SIZE)
 		return PMD_SIZE - CONT_PTE_SIZE;
-	default:
-		break;
-	}
 
 	return 0UL;
 }
-- 
2.43.0

[RFC PATCH v1 52/57] arm64: Remove PAGE_SIZE from assembly code

[Ryan Roberts]

Remove usage of PAGE_SHIFT, PAGE_SIZE and PAGE_MASK macros from assembly
code since these are no longer compile-time constants when boot-time
page size is in use.

For the most part, they are replaced with run-time lookups based on the
value of TG0. This is done outside of loops so while there is a cost of
a few extra instructions, performance should not be impacted.

However, invalid_host_el2_vect requires that the page shift be an
immediate since it has no registers to spare. So for this, let's use
alternatives patching. This code is guarranteed not to run until after
patching is complete.

__pi_copy_page has no registers to spare to hold the page size, and we
want to avoid having to reload it on every iteration of the loop. Since
I couldn't provably conclude that the function is not called prior to
alternatives patching, I opted to make a copy of the function for each
page size and branch to the right one at the start.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/include/asm/assembler.h  | 18 +++++++++++++---
 arch/arm64/kernel/hibernate-asm.S   |  6 ++++--
 arch/arm64/kernel/relocate_kernel.S | 10 ++++++---
 arch/arm64/kvm/hyp/nvhe/host.S      | 10 ++++++++-
 arch/arm64/lib/clear_page.S         |  7 ++++--
 arch/arm64/lib/copy_page.S          | 33 +++++++++++++++++++++--------
 arch/arm64/lib/mte.S                | 27 +++++++++++++++++------
 7 files changed, 85 insertions(+), 26 deletions(-)

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 77c2d707adb1a..6424fd6be1cbe 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -495,9 +495,11 @@ alternative_endif
 .Lskip_\@:
 	.endm
 /*
- * copy_page - copy src to dest using temp registers t1-t8
+ * copy_page - copy src to dest using temp registers t1-t9
  */
-	.macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
+	.macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req t9:req
+	get_page_size \t9
+	sub	\t9, \t9, #1			// (PAGE_SIZE - 1) in \t9
 9998:	ldp	\t1, \t2, [\src]
 	ldp	\t3, \t4, [\src, #16]
 	ldp	\t5, \t6, [\src, #32]
@@ -508,7 +510,7 @@ alternative_endif
 	stnp	\t5, \t6, [\dest, #32]
 	stnp	\t7, \t8, [\dest, #48]
 	add	\dest, \dest, #64
-	tst	\src, #(PAGE_SIZE - 1)
+	tst	\src, \t9
 	b.ne	9998b
 	.endm
 
@@ -911,4 +913,14 @@ alternative_cb_end
 	.macro	tgran_lpa2, val, tg0
 	value_for_page_size \val, \tg0, ID_AA64MMFR0_EL1_TGRAN4_52_BIT, ID_AA64MMFR0_EL1_TGRAN16_52_BIT, -1
 	.endm
+
+	.macro	get_page_size, val
+	get_tg0 \val
+	value_for_page_size \val, \val, SZ_4K, SZ_16K, SZ_64K
+	.endm
+
+	.macro	get_page_mask, val
+	get_tg0 \val
+	value_for_page_size \val, \val, (~(SZ_4K-1)), (~(SZ_16K-1)), (~(SZ_64K-1))
+	.endm
 #endif	/* __ASM_ASSEMBLER_H */
diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S
index 0e1d9c3c6a933..375b2fcf82e84 100644
--- a/arch/arm64/kernel/hibernate-asm.S
+++ b/arch/arm64/kernel/hibernate-asm.S
@@ -57,6 +57,8 @@ SYM_CODE_START(swsusp_arch_suspend_exit)
 	mov	x24, x4
 	mov	x25, x5
 
+	get_page_size x12
+
 	/* walk the restore_pblist and use copy_page() to over-write memory */
 	mov	x19, x3
 
@@ -64,9 +66,9 @@ SYM_CODE_START(swsusp_arch_suspend_exit)
 	mov	x0, x10
 	ldr	x1, [x19, #HIBERN_PBE_ADDR]
 
-	copy_page	x0, x1, x2, x3, x4, x5, x6, x7, x8, x9
+	copy_page	x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x11
 
-	add	x1, x10, #PAGE_SIZE
+	add	x1, x10, x12
 	/* Clean the copied page to PoU - based on caches_clean_inval_pou() */
 	raw_dcache_line_size x2, x3
 	sub	x3, x2, #1
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index 413f899e4ac63..bc4f37fba6c74 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -46,6 +46,10 @@ SYM_CODE_START(arm64_relocate_new_kernel)
 	ldr	x27, [x0, #KIMAGE_ARCH_EL2_VECTORS]
 	ldr	x26, [x0, #KIMAGE_ARCH_DTB_MEM]
 
+	/* Grab page size values. */
+	get_page_size x10			/* x10 = PAGE_SIZE */
+	get_page_mask x11			/* x11 = PAGE_MASK */
+
 	/* Setup the list loop variables. */
 	ldr	x18, [x0, #KIMAGE_ARCH_ZERO_PAGE] /* x18 = zero page for BBM */
 	ldr	x17, [x0, #KIMAGE_ARCH_TTBR1]	/* x17 = linear map copy */
@@ -54,7 +58,7 @@ SYM_CODE_START(arm64_relocate_new_kernel)
 	raw_dcache_line_size x15, x1		/* x15 = dcache line size */
 	break_before_make_ttbr_switch	x18, x17, x1, x2 /* set linear map */
 .Lloop:
-	and	x12, x16, PAGE_MASK		/* x12 = addr */
+	and	x12, x16, x11			/* x12 = addr */
 	sub	x12, x12, x22			/* Convert x12 to virt */
 	/* Test the entry flags. */
 .Ltest_source:
@@ -62,8 +66,8 @@ SYM_CODE_START(arm64_relocate_new_kernel)
 
 	/* Invalidate dest page to PoC. */
 	mov	x19, x13
-	copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
-	add	x1, x19, #PAGE_SIZE
+	copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8, x9
+	add	x1, x19, x10
 	dcache_by_myline_op civac, sy, x19, x1, x15, x20
 	b	.Lnext
 .Ltest_indirection:
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 3d610fc51f4d3..2b0d583fcf1af 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -193,7 +193,15 @@ SYM_FUNC_END(__host_hvc)
 	 */
 	add	sp, sp, x0			// sp' = sp + x0
 	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
-	tbz	x0, #PAGE_SHIFT, .L__hyp_sp_overflow\@
+alternative_if ARM64_USE_PAGE_SIZE_4K
+	tbz	x0, #ARM64_PAGE_SHIFT_4K, .L__hyp_sp_overflow\@
+alternative_else_nop_endif
+alternative_if ARM64_USE_PAGE_SIZE_16K
+	tbz	x0, #ARM64_PAGE_SHIFT_16K, .L__hyp_sp_overflow\@
+alternative_else_nop_endif
+alternative_if ARM64_USE_PAGE_SIZE_64K
+	tbz	x0, #ARM64_PAGE_SHIFT_64K, .L__hyp_sp_overflow\@
+alternative_else_nop_endif
 	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
 	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
 
diff --git a/arch/arm64/lib/clear_page.S b/arch/arm64/lib/clear_page.S
index ebde40e7fa2b2..b6f2cb8d704cc 100644
--- a/arch/arm64/lib/clear_page.S
+++ b/arch/arm64/lib/clear_page.S
@@ -15,6 +15,9 @@
  *	x0 - dest
  */
 SYM_FUNC_START(__pi_clear_page)
+	get_page_size x3
+	sub	x3, x3, #1	/* (PAGE_SIZE - 1) in x3 */
+
 	mrs	x1, dczid_el0
 	tbnz	x1, #4, 2f	/* Branch if DC ZVA is prohibited */
 	and	w1, w1, #0xf
@@ -23,7 +26,7 @@ SYM_FUNC_START(__pi_clear_page)
 
 1:	dc	zva, x0
 	add	x0, x0, x1
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	1b
 	ret
 
@@ -32,7 +35,7 @@ SYM_FUNC_START(__pi_clear_page)
 	stnp	xzr, xzr, [x0, #32]
 	stnp	xzr, xzr, [x0, #48]
 	add	x0, x0, #64
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	2b
 	ret
 SYM_FUNC_END(__pi_clear_page)
diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S
index 6a56d7cf309da..6c19b03ab4d69 100644
--- a/arch/arm64/lib/copy_page.S
+++ b/arch/arm64/lib/copy_page.S
@@ -10,14 +10,7 @@
 #include <asm/cpufeature.h>
 #include <asm/alternative.h>
 
-/*
- * Copy a page from src to dest (both are page aligned)
- *
- * Parameters:
- *	x0 - dest
- *	x1 - src
- */
-SYM_FUNC_START(__pi_copy_page)
+	.macro	copy_page_body, page_size
 	ldp	x2, x3, [x1]
 	ldp	x4, x5, [x1, #16]
 	ldp	x6, x7, [x1, #32]
@@ -30,7 +23,7 @@ SYM_FUNC_START(__pi_copy_page)
 	add	x0, x0, #256
 	add	x1, x1, #128
 1:
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, #(\page_size - 1)
 
 	stnp	x2, x3, [x0, #-256]
 	ldp	x2, x3, [x1]
@@ -62,7 +55,29 @@ SYM_FUNC_START(__pi_copy_page)
 	stnp	x12, x13, [x0, #80 - 256]
 	stnp	x14, x15, [x0, #96 - 256]
 	stnp	x16, x17, [x0, #112 - 256]
+	.endm
 
+/*
+ * Copy a page from src to dest (both are page aligned)
+ *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ */
+SYM_FUNC_START(__pi_copy_page)
+	get_tg0	x2
+.Lsz_64k:
+	cmp	x2, #TCR_TG0_64K
+	b.ne	.Lsz_16k
+	copy_page_body SZ_64K
+	ret
+.Lsz_16k:
+	cmp	x2, #TCR_TG0_16K
+	b.ne	.Lsz_4k
+	copy_page_body SZ_16K
+	ret
+.Lsz_4k:
+	copy_page_body SZ_4K
 	ret
 SYM_FUNC_END(__pi_copy_page)
 SYM_FUNC_ALIAS(copy_page, __pi_copy_page)
diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index 5018ac03b6bf3..b4f6f5be0ec79 100644
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -28,10 +28,13 @@
  *   x0 - address of the page to be cleared
  */
 SYM_FUNC_START(mte_clear_page_tags)
+	get_page_size x3
+	sub	x3, x3, #1		// (PAGE_SIZE - 1) in x3
+
 	multitag_transfer_size x1, x2
 1:	stgm	xzr, [x0]
 	add	x0, x0, x1
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	1b
 	ret
 SYM_FUNC_END(mte_clear_page_tags)
@@ -43,6 +46,9 @@ SYM_FUNC_END(mte_clear_page_tags)
  *	x0 - address to the beginning of the page
  */
 SYM_FUNC_START(mte_zero_clear_page_tags)
+	get_page_size x3
+	sub	x3, x3, #1		// (PAGE_SIZE - 1) in x3
+
 	and	x0, x0, #(1 << MTE_TAG_SHIFT) - 1	// clear the tag
 	mrs	x1, dczid_el0
 	tbnz	x1, #4, 2f	// Branch if DC GZVA is prohibited
@@ -52,12 +58,12 @@ SYM_FUNC_START(mte_zero_clear_page_tags)
 
 1:	dc	gzva, x0
 	add	x0, x0, x1
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	1b
 	ret
 
 2:	stz2g	x0, [x0], #(MTE_GRANULE_SIZE * 2)
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	2b
 	ret
 SYM_FUNC_END(mte_zero_clear_page_tags)
@@ -68,6 +74,9 @@ SYM_FUNC_END(mte_zero_clear_page_tags)
  *   x1 - address of the source page
  */
 SYM_FUNC_START(mte_copy_page_tags)
+	get_page_size x7
+	sub	x7, x7, #1		// (PAGE_SIZE - 1) in x7
+
 	mov	x2, x0
 	mov	x3, x1
 	multitag_transfer_size x5, x6
@@ -75,7 +84,7 @@ SYM_FUNC_START(mte_copy_page_tags)
 	stgm	x4, [x2]
 	add	x2, x2, x5
 	add	x3, x3, x5
-	tst	x2, #(PAGE_SIZE - 1)
+	tst	x2, x7
 	b.ne	1b
 	ret
 SYM_FUNC_END(mte_copy_page_tags)
@@ -137,6 +146,9 @@ SYM_FUNC_END(mte_copy_tags_to_user)
  *   x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
  */
 SYM_FUNC_START(mte_save_page_tags)
+	get_page_size x3
+	sub	x3, x3, #1		// (PAGE_SIZE - 1) in x3
+
 	multitag_transfer_size x7, x5
 1:
 	mov	x2, #0
@@ -149,7 +161,7 @@ SYM_FUNC_START(mte_save_page_tags)
 
 	str	x2, [x1], #8
 
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	1b
 
 	ret
@@ -161,6 +173,9 @@ SYM_FUNC_END(mte_save_page_tags)
  *   x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
  */
 SYM_FUNC_START(mte_restore_page_tags)
+	get_page_size x3
+	sub	x3, x3, #1		// (PAGE_SIZE - 1) in x3
+
 	multitag_transfer_size x7, x5
 1:
 	ldr	x2, [x1], #8
@@ -170,7 +185,7 @@ SYM_FUNC_START(mte_restore_page_tags)
 	tst	x0, #0xFF
 	b.ne	2b
 
-	tst	x0, #(PAGE_SIZE - 1)
+	tst	x0, x3
 	b.ne	1b
 
 	ret
-- 
2.43.0

[RFC PATCH v1 53/57] arm64: Runtime-fold pmd level

[Ryan Roberts]

For a given VA size, the number of levels of lookup depends on the page
size. With boot-time page size selection, we therefore don't know how
many levels of lookup we require until boot time. So we need to
runtime-fold some levels of lookup.

We already have code to runtime-fold p4d and pud levels; that exists for
LPA2 fallback paths and can be repurposed for our needs. But pmd level
also needs to support runtime folding; for example, 16K/36-bit and
64K/42-bit configs require only 2 levels.

So let's add the required code. However, note that until we actually add
the boot-time page size config, pgtable_l3_enabled() simply returns the
compile-time determined answer.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/include/asm/pgalloc.h |  16 +++-
 arch/arm64/include/asm/pgtable.h | 123 +++++++++++++++++++++++--------
 arch/arm64/include/asm/tlb.h     |   3 +
 arch/arm64/kernel/cpufeature.c   |   4 +-
 arch/arm64/kvm/mmu.c             |   9 +--
 arch/arm64/mm/fixmap.c           |   2 +-
 arch/arm64/mm/hugetlbpage.c      |  16 ++--
 arch/arm64/mm/init.c             |   2 +-
 arch/arm64/mm/mmu.c              |   2 +-
 arch/arm64/mm/ptdump.c           |   3 +-
 10 files changed, 126 insertions(+), 54 deletions(-)

diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 8ff5f2a2579e4..51cc2f32931d2 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -15,6 +15,7 @@
 
 #define __HAVE_ARCH_PGD_FREE
 #define __HAVE_ARCH_PUD_FREE
+#define __HAVE_ARCH_PMD_FREE
 #include <asm-generic/pgalloc.h>
 
 #define PGD_SIZE	(PTRS_PER_PGD * sizeof(pgd_t))
@@ -23,7 +24,8 @@
 
 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 {
-	set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot));
+	if (pgtable_l3_enabled())
+		set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot));
 }
 
 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
@@ -33,6 +35,18 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
 	pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN;
 	__pud_populate(pudp, __pa(pmdp), pudval);
 }
+
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+	struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
+
+	if (!pgtable_l3_enabled())
+		return;
+
+	BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
+	pagetable_pmd_dtor(ptdesc);
+	pagetable_free(ptdesc);
+}
 #else
 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 {
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index fd47f70a42396..8ead41da715b0 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -672,15 +672,21 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 #define pmd_leaf_size(pmd)	(pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
 #define pte_leaf_size(pte)	(pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE)
 
-#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
-static inline bool pud_sect(pud_t pud) { return false; }
-static inline bool pud_table(pud_t pud) { return true; }
-#else
-#define pud_sect(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
-				 PUD_TYPE_SECT)
-#define pud_table(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
-				 PUD_TYPE_TABLE)
-#endif
+static inline bool pgtable_l3_enabled(void);
+
+static inline bool pud_sect(pud_t pud)
+{
+	if (PAGE_SIZE == SZ_64K || !pgtable_l3_enabled())
+		return false;
+	return (pud_val(pud) & PUD_TYPE_MASK) == PUD_TYPE_SECT;
+}
+
+static inline bool pud_table(pud_t pud)
+{
+	if (PAGE_SIZE == SZ_64K || !pgtable_l3_enabled())
+		return true;
+	return (pud_val(pud) & PUD_TYPE_MASK) == PUD_TYPE_TABLE;
+}
 
 extern pgd_t init_pg_dir[];
 extern pgd_t init_pg_end[];
@@ -699,12 +705,10 @@ static inline bool in_swapper_pgdir(void *addr)
 
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
-#ifdef __PAGETABLE_PMD_FOLDED
-	if (in_swapper_pgdir(pmdp)) {
+	if (!pgtable_l3_enabled() && in_swapper_pgdir(pmdp)) {
 		set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
 		return;
 	}
-#endif /* __PAGETABLE_PMD_FOLDED */
 
 	WRITE_ONCE(*pmdp, pmd);
 
@@ -749,20 +753,27 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 
 #if CONFIG_PGTABLE_LEVELS > 2
 
+static __always_inline bool pgtable_l3_enabled(void)
+{
+	return true;
+}
+
+static inline bool mm_pmd_folded(const struct mm_struct *mm)
+{
+	return !pgtable_l3_enabled();
+}
+#define mm_pmd_folded  mm_pmd_folded
+
 #define pmd_ERROR(e)	\
 	pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
 
-#define pud_none(pud)		(!pud_val(pud))
-#define pud_bad(pud)		(!pud_table(pud))
-#define pud_present(pud)	pte_present(pud_pte(pud))
-#ifndef __PAGETABLE_PMD_FOLDED
-#define pud_leaf(pud)		(pud_present(pud) && !pud_table(pud))
-#else
-#define pud_leaf(pud)		false
-#endif
-#define pud_valid(pud)		pte_valid(pud_pte(pud))
-#define pud_user(pud)		pte_user(pud_pte(pud))
-#define pud_user_exec(pud)	pte_user_exec(pud_pte(pud))
+#define pud_none(pud)		(pgtable_l3_enabled() && !pud_val(pud))
+#define pud_bad(pud)		(pgtable_l3_enabled() && !pud_table(pud))
+#define pud_present(pud)	(!pgtable_l3_enabled() || pte_present(pud_pte(pud)))
+#define pud_leaf(pud)		(pgtable_l3_enabled() && pte_present(pud_pte(pud)) && !pud_table(pud))
+#define pud_valid(pud)		(pgtable_l3_enabled() && pte_valid(pud_pte(pud)))
+#define pud_user(pud)		(pgtable_l3_enabled() && pte_user(pud_pte(pud)))
+#define pud_user_exec(pud)	(pgtable_l3_enabled() && pte_user_exec(pud_pte(pud)))
 
 static inline bool pgtable_l4_enabled(void);
 
@@ -783,7 +794,8 @@ static inline void set_pud(pud_t *pudp, pud_t pud)
 
 static inline void pud_clear(pud_t *pudp)
 {
-	set_pud(pudp, __pud(0));
+	if (pgtable_l3_enabled())
+		set_pud(pudp, __pud(0));
 }
 
 static inline phys_addr_t pud_page_paddr(pud_t pud)
@@ -791,25 +803,74 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)
 	return __pud_to_phys(pud);
 }
 
+#define pmd_index(addr)		(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+
+static inline pmd_t *pud_to_folded_pmd(pud_t *pudp, unsigned long addr)
+{
+	return (pmd_t *)pudp;
+}
+
 static inline pmd_t *pud_pgtable(pud_t pud)
 {
 	return (pmd_t *)__va(pud_page_paddr(pud));
 }
 
-/* Find an entry in the second-level page table. */
-#define pmd_offset_phys(dir, addr)	(pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
+static inline phys_addr_t pmd_offset_phys(pud_t *pudp, unsigned long addr)
+{
+	BUG_ON(!pgtable_l3_enabled());
+
+	return pud_page_paddr(READ_ONCE(*pudp)) + pmd_index(addr) * sizeof(pmd_t);
+}
+
+static inline pmd_t *pmd_offset_lockless(pud_t *pudp, pud_t pud,
+					 unsigned long addr)
+{
+	if (!pgtable_l3_enabled())
+		return pud_to_folded_pmd(pudp, addr);
+	return (pmd_t *)__va(pud_page_paddr(pud)) + pmd_index(addr);
+}
+#define pmd_offset_lockless pmd_offset_lockless
 
-#define pmd_set_fixmap(addr)		((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
-#define pmd_set_fixmap_offset(pud, addr)	pmd_set_fixmap(pmd_offset_phys(pud, addr))
-#define pmd_clear_fixmap()		clear_fixmap(FIX_PMD)
+static inline pmd_t *pmd_offset(pud_t *pudp, unsigned long addr)
+{
+	return pmd_offset_lockless(pudp, READ_ONCE(*pudp), addr);
+}
+#define pmd_offset pmd_offset
 
-#define pud_page(pud)			phys_to_page(__pud_to_phys(pud))
+static inline pmd_t *pmd_set_fixmap(unsigned long addr)
+{
+	if (!pgtable_l3_enabled())
+		return NULL;
+	return (pmd_t *)set_fixmap_offset(FIX_PMD, addr);
+}
+
+static inline pmd_t *pmd_set_fixmap_offset(pud_t *pudp, unsigned long addr)
+{
+	if (!pgtable_l3_enabled())
+		return pud_to_folded_pmd(pudp, addr);
+	return pmd_set_fixmap(pmd_offset_phys(pudp, addr));
+}
+
+static inline void pmd_clear_fixmap(void)
+{
+	if (pgtable_l3_enabled())
+		clear_fixmap(FIX_PMD);
+}
 
 /* use ONLY for statically allocated translation tables */
-#define pmd_offset_kimg(dir,addr)	((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
+static inline pmd_t *pmd_offset_kimg(pud_t *pudp, u64 addr)
+{
+	if (!pgtable_l3_enabled())
+		return pud_to_folded_pmd(pudp, addr);
+	return (pmd_t *)__phys_to_kimg(pmd_offset_phys(pudp, addr));
+}
+
+#define pud_page(pud)			phys_to_page(__pud_to_phys(pud))
 
 #else
 
+static inline bool pgtable_l3_enabled(void) { return false; }
+
 #define pud_valid(pud)		false
 #define pud_page_paddr(pud)	({ BUILD_BUG(); 0; })
 #define pud_user_exec(pud)	pud_user(pud) /* Always 0 with folding */
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index a947c6e784ed2..527630f0803c6 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -92,6 +92,9 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 {
 	struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
 
+	if (!pgtable_l3_enabled())
+		return;
+
 	pagetable_pmd_dtor(ptdesc);
 	tlb_remove_ptdesc(tlb, ptdesc);
 }
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index e5618423bb99d..663cc76569a27 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1923,8 +1923,10 @@ static int __init __kpti_install_ng_mappings(void *__unused)
 
 	if (levels == 5 && !pgtable_l5_enabled())
 		levels = 4;
-	else if (levels == 4 && !pgtable_l4_enabled())
+	if (levels == 4 && !pgtable_l4_enabled())
 		levels = 3;
+	if (levels == 3 && !pgtable_l3_enabled())
+		levels = 2;
 
 	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
 
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 248a2d7ad6dbb..146ecdaaaf647 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1370,12 +1370,11 @@ static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva)
 
 	pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start);
 
-#ifndef __PAGETABLE_PMD_FOLDED
-	if ((hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) &&
+	if (pgtable_l3_enabled() &&
+	    (hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) &&
 	    ALIGN_DOWN(hva, PUD_SIZE) >= vma->vm_start &&
 	    ALIGN(hva, PUD_SIZE) <= vma->vm_end)
 		return PUD_SHIFT;
-#endif
 
 	if ((hva & (PMD_SIZE - 1)) == (pa & (PMD_SIZE - 1)) &&
 	    ALIGN_DOWN(hva, PMD_SIZE) >= vma->vm_start &&
@@ -1487,12 +1486,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 		vma_shift = get_vma_page_shift(vma, hva);
 	}
 
-#ifndef __PAGETABLE_PMD_FOLDED
-	if (vma_shift == PUD_SHIFT) {
+	if (pgtable_l3_enabled() && vma_shift == PUD_SHIFT) {
 		if (!fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE))
 			vma_shift = PMD_SHIFT;
 	}
-#endif
 	if (vma_shift == CONT_PMD_SHIFT) {
 		vma_shift = PMD_SHIFT;
 	}
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index a0dcf2375ccb4..f2c6678046a96 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -87,7 +87,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
 	p4d_t p4d = READ_ONCE(*p4dp);
 	pud_t *pudp;
 
-	if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) &&
+	if (ptg_pgtable_levels > 3 && !p4d_none(p4d) &&
 	    p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) {
 		/*
 		 * We only end up here if the kernel mapping and the fixmap
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index bc98c20655bba..2add0839179e3 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -51,10 +51,9 @@ void __init arm64_hugetlb_cma_reserve(void)
 
 static bool __hugetlb_valid_size(unsigned long size)
 {
-#ifndef __PAGETABLE_PMD_FOLDED
-	if (size == PUD_SIZE)
+	if (pgtable_l3_enabled() && size == PUD_SIZE)
 		return pud_sect_supported();
-#endif
+
 	if (size == CONT_PMD_SIZE || size == PMD_SIZE || size == CONT_PTE_SIZE)
 		return true;
 
@@ -100,13 +99,10 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
 
 	*pgsize = size;
 
-#ifndef __PAGETABLE_PMD_FOLDED
-	if (size == PUD_SIZE) {
+	if (pgtable_l3_enabled() && size == PUD_SIZE) {
 		if (pud_sect_supported())
 			contig_ptes = 1;
-	} else
-#endif
-	if (size == PMD_SIZE) {
+	} else if (size == PMD_SIZE) {
 		contig_ptes = 1;
 	} else if (size == CONT_PMD_SIZE) {
 		*pgsize = PMD_SIZE;
@@ -331,10 +327,8 @@ unsigned long hugetlb_mask_last_page(struct hstate *h)
 {
 	unsigned long hp_size = huge_page_size(h);
 
-#ifndef __PAGETABLE_PMD_FOLDED
-	if (hp_size == PUD_SIZE)
+	if (pgtable_l3_enabled() && hp_size == PUD_SIZE)
 		return PGDIR_SIZE - PUD_SIZE;
-#endif
 	if (hp_size == CONT_PMD_SIZE)
 		return PUD_SIZE - CONT_PMD_SIZE;
 	if (hp_size == PMD_SIZE)
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 4d24034418b39..62587104f30d8 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -396,7 +396,7 @@ void __init mem_init(void)
 	 * scratch using the virtual address range and page size.
 	 */
 	VM_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) !=
-		  CONFIG_PGTABLE_LEVELS);
+		  ptg_pgtable_levels);
 
 	if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
 		extern int sysctl_overcommit_memory;
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index ad7fd3fda705a..b78a341cd9e70 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1046,7 +1046,7 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
 		free_empty_pte_table(pmdp, addr, next, floor, ceiling);
 	} while (addr = next, addr < end);
 
-	if (CONFIG_PGTABLE_LEVELS <= 2)
+	if (!pgtable_l3_enabled())
 		return;
 
 	if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK))
diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c
index 6986827e0d645..045a4188afc10 100644
--- a/arch/arm64/mm/ptdump.c
+++ b/arch/arm64/mm/ptdump.c
@@ -230,7 +230,8 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
 
 	/* check if the current level has been folded dynamically */
 	if ((level == 1 && mm_p4d_folded(st->mm)) ||
-	    (level == 2 && mm_pud_folded(st->mm)))
+	    (level == 2 && mm_pud_folded(st->mm)) ||
+	    (level == 3 && mm_pmd_folded(st->mm)))
 		level = 0;
 
 	if (level >= 0)
-- 
2.43.0

[RFC PATCH v1 56/57] arm64: Determine THREAD_SIZE at boot-time

[Ryan Roberts]

Since THREAD_SIZE depends on PAGE_SIZE when stacks are vmapped, we must
defer the decision on THREAD_SIZE until we have selected PAGE_SIZE at
boot.

The one wrinkle is entry.S's requirement to have THREAD_SHIFT as an
immediate in order to check that the stack has not overflowed without
clobbering any registers, early in the exception handler. Solve this by
patching alternatives. During early boot, all 3 options are NOPs until
the alternative is patched in. So we forgo overflow checking until
boot-cpu patching is complete.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/include/asm/assembler.h |  5 +++
 arch/arm64/include/asm/efi.h       |  2 +-
 arch/arm64/include/asm/memory.h    | 51 +++++++++++++++++++++++++-----
 arch/arm64/kernel/efi.c            |  2 +-
 arch/arm64/kernel/entry.S          | 10 +++++-
 arch/arm64/kernel/head.S           |  3 +-
 arch/arm64/kernel/vmlinux.lds.S    |  4 +--
 arch/arm64/kvm/hyp/nvhe/hyp.lds.S  |  2 +-
 8 files changed, 64 insertions(+), 15 deletions(-)

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 0cfa7c3efd214..745328e7768b7 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -928,4 +928,9 @@ alternative_cb_end
 	get_tg0 \val
 	value_for_page_size \val, \val, (~(SZ_4K-1)), (~(SZ_16K-1)), (~(SZ_64K-1))
 	.endm
+
+	.macro	get_task_size, val
+	get_tg0 \val
+	value_for_page_size \val, \val, (1 << THREAD_SHIFT_4K), (1 << THREAD_SHIFT_16K), (1 << THREAD_SHIFT_64K)
+	.endm
 #endif	/* __ASM_ASSEMBLER_H */
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index bcd5622aa0968..913f599c14e40 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -68,7 +68,7 @@ void arch_efi_call_virt_teardown(void);
  * kernel need greater alignment than we require the segments to be padded to.
  */
 #define EFI_KIMG_ALIGN	\
-	(SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN)
+	(SEGMENT_ALIGN > THREAD_ALIGN_MAX ? SEGMENT_ALIGN : THREAD_ALIGN_MAX)
 
 /*
  * On arm64, we have to ensure that the initrd ends up in the linear region,
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 5393a859183f7..e28f5700ef022 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -110,23 +110,56 @@
 #define PAGE_END		(_PAGE_END(VA_BITS_MIN))
 #endif /* CONFIG_KASAN */
 
-#define MIN_THREAD_SHIFT	(14 + KASAN_THREAD_SHIFT)
+#define IDEAL_THREAD_SHIFT	(14 + KASAN_THREAD_SHIFT)
 
 /*
  * VMAP'd stacks are allocated at page granularity, so we must ensure that such
  * stacks are a multiple of page size.
  */
-#if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT)
-#define THREAD_SHIFT		PAGE_SHIFT
+
+#if defined(CONFIG_VMAP_STACK)
+#define THREAD_SHIFT							\
+	(IDEAL_THREAD_SHIFT < PAGE_SHIFT ? PAGE_SHIFT : IDEAL_THREAD_SHIFT)
+#if (IDEAL_THREAD_SHIFT < PAGE_SHIFT_MIN)
+#define THREAD_SHIFT_MIN	PAGE_SHIFT_MIN
 #else
-#define THREAD_SHIFT		MIN_THREAD_SHIFT
+#define THREAD_SHIFT_MIN	IDEAL_THREAD_SHIFT
 #endif
-
-#if THREAD_SHIFT >= PAGE_SHIFT
-#define THREAD_SIZE_ORDER	(THREAD_SHIFT - PAGE_SHIFT)
+#if (IDEAL_THREAD_SHIFT < PAGE_SHIFT_MAX)
+#define THREAD_SHIFT_MAX	PAGE_SHIFT_MAX
+#else
+#define THREAD_SHIFT_MAX	IDEAL_THREAD_SHIFT
+#endif
+#if (IDEAL_THREAD_SHIFT < ARM64_PAGE_SHIFT_4K)
+#define THREAD_SHIFT_4K		ARM64_PAGE_SHIFT_4K
+#else
+#define THREAD_SHIFT_4K		IDEAL_THREAD_SHIFT
+#endif
+#if (IDEAL_THREAD_SHIFT < ARM64_PAGE_SHIFT_16K)
+#define THREAD_SHIFT_16K	ARM64_PAGE_SHIFT_16K
+#else
+#define THREAD_SHIFT_16K	IDEAL_THREAD_SHIFT
+#endif
+#if (IDEAL_THREAD_SHIFT < ARM64_PAGE_SHIFT_64K)
+#define THREAD_SHIFT_64K	ARM64_PAGE_SHIFT_64K
+#else
+#define THREAD_SHIFT_64K	IDEAL_THREAD_SHIFT
 #endif
+#else
+#define THREAD_SHIFT		IDEAL_THREAD_SHIFT
+#define THREAD_SHIFT_MIN	IDEAL_THREAD_SHIFT
+#define THREAD_SHIFT_MAX	IDEAL_THREAD_SHIFT
+#define THREAD_SHIFT_4K		IDEAL_THREAD_SHIFT
+#define THREAD_SHIFT_16K	IDEAL_THREAD_SHIFT
+#define THREAD_SHIFT_64K	IDEAL_THREAD_SHIFT
+#endif
+
+#define THREAD_SIZE_ORDER						\
+	(PAGE_SHIFT < THREAD_SHIFT ? THREAD_SHIFT - PAGE_SHIFT : 0)
 
 #define THREAD_SIZE		(UL(1) << THREAD_SHIFT)
+#define THREAD_SIZE_MIN		(UL(1) << THREAD_SHIFT_MIN)
+#define THREAD_SIZE_MAX		(UL(1) << THREAD_SHIFT_MAX)
 
 /*
  * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by
@@ -135,11 +168,13 @@
  */
 #ifdef CONFIG_VMAP_STACK
 #define THREAD_ALIGN		(2 * THREAD_SIZE)
+#define THREAD_ALIGN_MAX	(2 * THREAD_SIZE_MAX)
 #else
 #define THREAD_ALIGN		THREAD_SIZE
+#define THREAD_ALIGN_MAX	THREAD_SIZE_MAX
 #endif
 
-#define IRQ_STACK_SIZE		THREAD_SIZE
+#define IRQ_STACK_SIZE		THREAD_SIZE_MIN
 
 #define OVERFLOW_STACK_SIZE	SZ_4K
 
diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index 712718aed5dd9..ebc44b7e83199 100644
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -197,7 +197,7 @@ bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg)
 }
 
 /* EFI requires 8 KiB of stack space for runtime services */
-static_assert(THREAD_SIZE >= SZ_8K);
+static_assert(THREAD_SIZE_MIN >= SZ_8K);
 
 static int __init arm64_efi_rt_init(void)
 {
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index ba47dc8672c04..1ab65e406b62e 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -62,7 +62,15 @@
 	 */
 	add	sp, sp, x0			// sp' = sp + x0
 	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
-	tbnz	x0, #THREAD_SHIFT, 0f
+alternative_if ARM64_USE_PAGE_SIZE_4K
+	tbnz	x0, #THREAD_SHIFT_4K, 0f
+alternative_else_nop_endif
+alternative_if ARM64_USE_PAGE_SIZE_16K
+	tbnz	x0, #THREAD_SHIFT_16K, 0f
+alternative_else_nop_endif
+alternative_if ARM64_USE_PAGE_SIZE_64K
+	tbnz	x0, #THREAD_SHIFT_64K, 0f
+alternative_else_nop_endif
 	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
 	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
 	b	el\el\ht\()_\regsize\()_\label
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 761b7f5633e15..2530ee5cee548 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -198,7 +198,8 @@ SYM_CODE_END(preserve_boot_args)
 	msr	sp_el0, \tsk
 
 	ldr	\tmp1, [\tsk, #TSK_STACK]
-	add	sp, \tmp1, #THREAD_SIZE
+	get_task_size \tmp2
+	add	sp, \tmp1, \tmp2
 	sub	sp, sp, #PT_REGS_SIZE
 
 	stp	xzr, xzr, [sp, #S_STACKFRAME]
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 09fcc234c0f77..937900a458a89 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -60,11 +60,11 @@
 #define RO_EXCEPTION_TABLE_ALIGN	4
 #define RUNTIME_DISCARD_EXIT
 
+#include <asm/memory.h>
 #include <asm-generic/vmlinux.lds.h>
 #include <asm/cache.h>
 #include <asm/kernel-pgtable.h>
 #include <asm/kexec.h>
-#include <asm/memory.h>
 #include <asm/page.h>
 
 #include "image.h"
@@ -292,7 +292,7 @@ SECTIONS
 
 	_data = .;
 	_sdata = .;
-	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE_MAX, THREAD_ALIGN)
+	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE_MAX, THREAD_ALIGN_MAX)
 
 	/*
 	 * Data written with the MMU off but read with the MMU on requires
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 74c7c21626270..fe1fbfa8f8f05 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -7,9 +7,9 @@
  */
 
 #include <asm/hyp_image.h>
+#include <asm/memory.h>
 #include <asm-generic/vmlinux.lds.h>
 #include <asm/cache.h>
-#include <asm/memory.h>
 
 SECTIONS {
 	HYP_SECTION(.idmap.text)
-- 
2.43.0

[RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Ryan Roberts]

Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
selected instead of a page size. When selected, the resulting kernel's
page size can be configured at boot via the command line.

For now, boot-time page size kernels are limited to 48-bit VA, since
more work is required to support LPA2. Additionally MMAP_RND_BITS and
SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
work could be implemented to be able to configure these at boot time for
optimial page size-specific values.

Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---

***NOTE***
Any confused maintainers may want to read the cover note here for context:
https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/

 arch/arm64/Kconfig                         | 26 ++++++++++---
 arch/arm64/include/asm/kvm_hyp.h           | 11 ++++++
 arch/arm64/include/asm/pgtable-geometry.h  | 22 ++++++++++-
 arch/arm64/include/asm/pgtable-hwdef.h     |  6 +--
 arch/arm64/include/asm/pgtable.h           | 10 ++++-
 arch/arm64/include/asm/sparsemem.h         |  4 ++
 arch/arm64/kernel/image-vars.h             | 11 ++++++
 arch/arm64/kernel/image.h                  |  4 ++
 arch/arm64/kernel/pi/map_kernel.c          | 45 ++++++++++++++++++++++
 arch/arm64/kvm/arm.c                       | 10 +++++
 arch/arm64/kvm/hyp/nvhe/Makefile           |  1 +
 arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c | 16 ++++++++
 arch/arm64/mm/Makefile                     |  1 +
 arch/arm64/mm/pgd.c                        | 10 +++--
 arch/arm64/mm/pgtable-geometry.c           | 24 ++++++++++++
 drivers/firmware/efi/libstub/arm64.c       |  3 +-
 16 files changed, 187 insertions(+), 17 deletions(-)
 create mode 100644 arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
 create mode 100644 arch/arm64/mm/pgtable-geometry.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a2f8ff354ca67..573d308741169 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -121,6 +121,7 @@ config ARM64
 	select BUILDTIME_TABLE_SORT
 	select CLONE_BACKWARDS
 	select COMMON_CLK
+	select CONSTRUCTORS if ARM64_BOOT_TIME_PAGE_SIZE
 	select CPU_PM if (SUSPEND || CPU_IDLE)
 	select CPUMASK_OFFSTACK if NR_CPUS > 256
 	select CRC32
@@ -284,18 +285,20 @@ config MMU
 
 config ARM64_CONT_PTE_SHIFT
 	int
+	depends on !ARM64_BOOT_TIME_PAGE_SIZE
 	default 5 if PAGE_SIZE_64KB
 	default 7 if PAGE_SIZE_16KB
 	default 4
 
 config ARM64_CONT_PMD_SHIFT
 	int
+	depends on !ARM64_BOOT_TIME_PAGE_SIZE
 	default 5 if PAGE_SIZE_64KB
 	default 5 if PAGE_SIZE_16KB
 	default 4
 
 config ARCH_MMAP_RND_BITS_MIN
-	default 14 if PAGE_SIZE_64KB
+	default 14 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
 	default 16 if PAGE_SIZE_16KB
 	default 18
 
@@ -306,15 +309,15 @@ config ARCH_MMAP_RND_BITS_MAX
 	default 24 if ARM64_VA_BITS=39
 	default 27 if ARM64_VA_BITS=42
 	default 30 if ARM64_VA_BITS=47
-	default 29 if ARM64_VA_BITS=48 && ARM64_64K_PAGES
+	default 29 if ARM64_VA_BITS=48 && (ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE)
 	default 31 if ARM64_VA_BITS=48 && ARM64_16K_PAGES
 	default 33 if ARM64_VA_BITS=48
-	default 14 if ARM64_64K_PAGES
+	default 14 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
 	default 16 if ARM64_16K_PAGES
 	default 18
 
 config ARCH_MMAP_RND_COMPAT_BITS_MIN
-	default 7 if ARM64_64K_PAGES
+	default 7 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
 	default 9 if ARM64_16K_PAGES
 	default 11
 
@@ -362,6 +365,7 @@ config FIX_EARLYCON_MEM
 
 config PGTABLE_LEVELS
 	int
+	default 4 if ARM64_BOOT_TIME_PAGE_SIZE # Advertise max supported levels
 	default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36
 	default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42
 	default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_VA_BITS_52)
@@ -1316,6 +1320,14 @@ config ARM64_64K_PAGES
 	  look-up. AArch32 emulation requires applications compiled
 	  with 64K aligned segments.
 
+config ARM64_BOOT_TIME_PAGE_SIZE
+	bool "Boot-time selection"
+	select HAVE_PAGE_SIZE_64KB # Advertise largest page size to core
+	help
+	  Select desired page size (4KB, 16KB or 64KB) at boot-time via the
+	  kernel command line option "arm64.pagesize=4k", "arm64.pagesize=16k"
+	  or "arm64.pagesize=64k".
+
 endchoice
 
 choice
@@ -1348,6 +1360,7 @@ config ARM64_VA_BITS_48
 config ARM64_VA_BITS_52
 	bool "52-bit"
 	depends on ARM64_PAN || !ARM64_SW_TTBR0_PAN
+	depends on !ARM64_BOOT_TIME_PAGE_SIZE
 	help
 	  Enable 52-bit virtual addressing for userspace when explicitly
 	  requested via a hint to mmap(). The kernel will also use 52-bit
@@ -1588,9 +1601,10 @@ config XEN
 # 4K  |       27          |      12      |       15             |         10              |
 # 16K |       27          |      14      |       13             |         11              |
 # 64K |       29          |      16      |       13             |         13              |
+# BOOT|       29          |    16 (max)  |       13             |         13              |
 config ARCH_FORCE_MAX_ORDER
 	int
-	default "13" if ARM64_64K_PAGES
+	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
 	default "11" if ARM64_16K_PAGES
 	default "10"
 	help
@@ -1663,7 +1677,7 @@ config ARM64_TAGGED_ADDR_ABI
 
 menuconfig COMPAT
 	bool "Kernel support for 32-bit EL0"
-	depends on ARM64_4K_PAGES || EXPERT
+	depends on ARM64_4K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE || EXPERT
 	select HAVE_UID16
 	select OLD_SIGSUSPEND3
 	select COMPAT_OLD_SIGACTION
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index c838309e4ec47..9397a14642afa 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -145,4 +145,15 @@ extern unsigned long kvm_nvhe_sym(__icache_flags);
 extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
 extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
 
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+extern int kvm_nvhe_sym(ptg_page_shift);
+extern int kvm_nvhe_sym(ptg_pmd_shift);
+extern int kvm_nvhe_sym(ptg_pud_shift);
+extern int kvm_nvhe_sym(ptg_p4d_shift);
+extern int kvm_nvhe_sym(ptg_pgdir_shift);
+extern int kvm_nvhe_sym(ptg_cont_pte_shift);
+extern int kvm_nvhe_sym(ptg_cont_pmd_shift);
+extern int kvm_nvhe_sym(ptg_pgtable_levels);
+#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
+
 #endif /* __ARM64_KVM_HYP_H__ */
diff --git a/arch/arm64/include/asm/pgtable-geometry.h b/arch/arm64/include/asm/pgtable-geometry.h
index 62fe125909c08..18a8c8d499ecc 100644
--- a/arch/arm64/include/asm/pgtable-geometry.h
+++ b/arch/arm64/include/asm/pgtable-geometry.h
@@ -6,16 +6,33 @@
 #define ARM64_PAGE_SHIFT_16K	14
 #define ARM64_PAGE_SHIFT_64K	16
 
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+#define PAGE_SHIFT_MIN		ARM64_PAGE_SHIFT_4K
+#define PAGE_SHIFT_MAX		ARM64_PAGE_SHIFT_64K
+#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
 #define PAGE_SHIFT_MIN		CONFIG_PAGE_SHIFT
+#define PAGE_SHIFT_MAX		CONFIG_PAGE_SHIFT
+#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
+
 #define PAGE_SIZE_MIN		(_AC(1, UL) << PAGE_SHIFT_MIN)
 #define PAGE_MASK_MIN		(~(PAGE_SIZE_MIN-1))
-
-#define PAGE_SHIFT_MAX		CONFIG_PAGE_SHIFT
 #define PAGE_SIZE_MAX		(_AC(1, UL) << PAGE_SHIFT_MAX)
 #define PAGE_MASK_MAX		(~(PAGE_SIZE_MAX-1))
 
 #include <asm-generic/pgtable-geometry.h>
 
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+#ifndef __ASSEMBLY__
+extern int ptg_page_shift;
+extern int ptg_pmd_shift;
+extern int ptg_pud_shift;
+extern int ptg_p4d_shift;
+extern int ptg_pgdir_shift;
+extern int ptg_cont_pte_shift;
+extern int ptg_cont_pmd_shift;
+extern int ptg_pgtable_levels;
+#endif /* __ASSEMBLY__ */
+#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
 #define ptg_page_shift		CONFIG_PAGE_SHIFT
 #define ptg_pmd_shift		ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
 #define ptg_pud_shift		ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
@@ -24,5 +41,6 @@
 #define ptg_cont_pte_shift	(CONFIG_ARM64_CONT_PTE_SHIFT + PAGE_SHIFT)
 #define ptg_cont_pmd_shift	(CONFIG_ARM64_CONT_PMD_SHIFT + PMD_SHIFT)
 #define ptg_pgtable_levels	CONFIG_PGTABLE_LEVELS
+#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
 
 #endif /* ASM_PGTABLE_GEOMETRY_H */
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index ca8bcbc1fe220..da5404617acbf 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -52,7 +52,7 @@
 #define PMD_SHIFT		ptg_pmd_shift
 #define PMD_SIZE		(_AC(1, UL) << PMD_SHIFT)
 #define PMD_MASK		(~(PMD_SIZE-1))
-#define PTRS_PER_PMD		(1 << (PAGE_SHIFT - 3))
+#define PTRS_PER_PMD		(ptg_pgtable_levels > 2 ? (1 << (PAGE_SHIFT - 3)) : 1)
 #define MAX_PTRS_PER_PMD	(1 << (PAGE_SHIFT_MAX - 3))
 #endif
 
@@ -63,7 +63,7 @@
 #define PUD_SHIFT		ptg_pud_shift
 #define PUD_SIZE		(_AC(1, UL) << PUD_SHIFT)
 #define PUD_MASK		(~(PUD_SIZE-1))
-#define PTRS_PER_PUD		(1 << (PAGE_SHIFT - 3))
+#define PTRS_PER_PUD		(ptg_pgtable_levels > 3 ? (1 << (PAGE_SHIFT - 3)) : 1)
 #define MAX_PTRS_PER_PUD	(1 << (PAGE_SHIFT_MAX - 3))
 #endif
 
@@ -71,7 +71,7 @@
 #define P4D_SHIFT		ptg_p4d_shift
 #define P4D_SIZE		(_AC(1, UL) << P4D_SHIFT)
 #define P4D_MASK		(~(P4D_SIZE-1))
-#define PTRS_PER_P4D		(1 << (PAGE_SHIFT - 3))
+#define PTRS_PER_P4D		(ptg_pgtable_levels > 4 ? (1 << (PAGE_SHIFT - 3)) : 1)
 #define MAX_PTRS_PER_P4D	(1 << (PAGE_SHIFT_MAX - 3))
 #endif
 
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 8ead41da715b0..ad9f75f5cc29a 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -755,7 +755,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 
 static __always_inline bool pgtable_l3_enabled(void)
 {
-	return true;
+	return ptg_pgtable_levels > 2;
 }
 
 static inline bool mm_pmd_folded(const struct mm_struct *mm)
@@ -888,6 +888,8 @@ static inline bool pgtable_l3_enabled(void) { return false; }
 
 static __always_inline bool pgtable_l4_enabled(void)
 {
+	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
+		return ptg_pgtable_levels > 3;
 	if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
 		return true;
 	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
@@ -935,6 +937,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
 
 static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
 {
+	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
+		return (pud_t *)p4dp;
 	return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
 }
 
@@ -1014,6 +1018,8 @@ static inline bool pgtable_l4_enabled(void) { return false; }
 
 static __always_inline bool pgtable_l5_enabled(void)
 {
+	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
+		return ptg_pgtable_levels > 4;
 	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
 		return vabits_actual == VA_BITS;
 	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
@@ -1059,6 +1065,8 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
 
 static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
 {
+	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
+		return (p4d_t *)pgdp;
 	return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr);
 }
 
diff --git a/arch/arm64/include/asm/sparsemem.h b/arch/arm64/include/asm/sparsemem.h
index a05fdd54014f7..2daf1263ba638 100644
--- a/arch/arm64/include/asm/sparsemem.h
+++ b/arch/arm64/include/asm/sparsemem.h
@@ -17,6 +17,10 @@
  * entries could not be created for vmemmap mappings.
  * 16K follows 4K for simplicity.
  */
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+#define SECTION_SIZE_BITS	29
+#else
 #define SECTION_SIZE_BITS	(PAGE_SIZE == SZ_64K ? 29 : 27)
+#endif
 
 #endif
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index a168f3337446f..9968320f83bc4 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -36,6 +36,17 @@ PROVIDE(__pi___memcpy			= __pi_memcpy);
 PROVIDE(__pi___memmove			= __pi_memmove);
 PROVIDE(__pi___memset			= __pi_memset);
 
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+PROVIDE(__pi_ptg_page_shift		= ptg_page_shift);
+PROVIDE(__pi_ptg_pmd_shift		= ptg_pmd_shift);
+PROVIDE(__pi_ptg_pud_shift		= ptg_pud_shift);
+PROVIDE(__pi_ptg_p4d_shift		= ptg_p4d_shift);
+PROVIDE(__pi_ptg_pgdir_shift		= ptg_pgdir_shift);
+PROVIDE(__pi_ptg_cont_pte_shift		= ptg_cont_pte_shift);
+PROVIDE(__pi_ptg_cont_pmd_shift		= ptg_cont_pmd_shift);
+PROVIDE(__pi_ptg_pgtable_levels		= ptg_pgtable_levels);
+#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
+
 PROVIDE(__pi_id_aa64isar1_override	= id_aa64isar1_override);
 PROVIDE(__pi_id_aa64isar2_override	= id_aa64isar2_override);
 PROVIDE(__pi_id_aa64mmfr0_override	= id_aa64mmfr0_override);
diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
index 7bc3ba8979019..01502fc3b891b 100644
--- a/arch/arm64/kernel/image.h
+++ b/arch/arm64/kernel/image.h
@@ -47,7 +47,11 @@
 #define __HEAD_FLAG_BE		ARM64_IMAGE_FLAG_LE
 #endif
 
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+#define __HEAD_FLAG_PAGE_SIZE	0
+#else
 #define __HEAD_FLAG_PAGE_SIZE	((PAGE_SHIFT - 10) / 2)
+#endif
 
 #define __HEAD_FLAG_PHYS_BASE	1
 
diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c
index deb8cd50b0b0c..22b3c70e04f9c 100644
--- a/arch/arm64/kernel/pi/map_kernel.c
+++ b/arch/arm64/kernel/pi/map_kernel.c
@@ -221,6 +221,49 @@ static void __init map_fdt(u64 fdt, int page_shift)
 	dsb(ishst);
 }
 
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+static void __init ptg_init(int page_shift)
+{
+	ptg_pgtable_levels =
+		__ARM64_HW_PGTABLE_LEVELS(page_shift, CONFIG_ARM64_VA_BITS);
+
+	ptg_pgdir_shift = __ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift,
+				4 - ptg_pgtable_levels);
+
+	ptg_p4d_shift = ptg_pgtable_levels >= 5 ?
+				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 0) :
+				ptg_pgdir_shift;
+
+	ptg_pud_shift = ptg_pgtable_levels >= 4 ?
+				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 1) :
+				ptg_pgdir_shift;
+
+	ptg_pmd_shift = ptg_pgtable_levels >= 3 ?
+				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 2) :
+				ptg_pgdir_shift;
+
+	ptg_page_shift = page_shift;
+
+	switch (page_shift) {
+	case ARM64_PAGE_SHIFT_64K:
+		ptg_cont_pte_shift = ptg_page_shift + 5;
+		ptg_cont_pmd_shift = ptg_pmd_shift + 5;
+		break;
+	case ARM64_PAGE_SHIFT_16K:
+		ptg_cont_pte_shift = ptg_page_shift + 7;
+		ptg_cont_pmd_shift = ptg_pmd_shift + 5;
+		break;
+	default: /* ARM64_PAGE_SHIFT_4K */
+		ptg_cont_pte_shift = ptg_page_shift + 4;
+		ptg_cont_pmd_shift = ptg_pmd_shift + 4;
+	}
+}
+#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
+static inline void ptg_init(int page_shift)
+{
+}
+#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
+
 asmlinkage void __init early_map_kernel(u64 boot_status, void *fdt)
 {
 	static char const chosen_str[] __initconst = "/chosen";
@@ -247,6 +290,8 @@ asmlinkage void __init early_map_kernel(u64 boot_status, void *fdt)
 	if (!page_shift)
 		page_shift = early_page_shift;
 
+	ptg_init(page_shift);
+
 	if (va_bits > 48) {
 		u64 page_size = early_page_size(page_shift);
 
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 9bef7638342ef..c835a50b8b768 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -2424,6 +2424,16 @@ static void kvm_hyp_init_symbols(void)
 	kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
 	kvm_nvhe_sym(__icache_flags) = __icache_flags;
 	kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
+#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
+	kvm_nvhe_sym(ptg_page_shift) = ptg_page_shift;
+	kvm_nvhe_sym(ptg_pmd_shift) = ptg_pmd_shift;
+	kvm_nvhe_sym(ptg_pud_shift) = ptg_pud_shift;
+	kvm_nvhe_sym(ptg_p4d_shift) = ptg_p4d_shift;
+	kvm_nvhe_sym(ptg_pgdir_shift) = ptg_pgdir_shift;
+	kvm_nvhe_sym(ptg_cont_pte_shift) = ptg_cont_pte_shift;
+	kvm_nvhe_sym(ptg_cont_pmd_shift) = ptg_cont_pmd_shift;
+	kvm_nvhe_sym(ptg_pgtable_levels) = ptg_pgtable_levels;
+#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
 }
 
 static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index b43426a493df5..a8fcbb84c7996 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -27,6 +27,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
 	 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
 hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
 	 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
+hyp-obj-$(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE) += pgtable-geometry.o
 hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o
 hyp-obj-y += $(lib-objs)
 
diff --git a/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c b/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
new file mode 100644
index 0000000000000..17f807450a31a
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 ARM Ltd.
+ */
+
+#include <linux/cache.h>
+#include <asm/pgtable-geometry.h>
+
+int ptg_page_shift __ro_after_init;
+int ptg_pmd_shift __ro_after_init;
+int ptg_pud_shift __ro_after_init;
+int ptg_p4d_shift __ro_after_init;
+int ptg_pgdir_shift __ro_after_init;
+int ptg_cont_pte_shift __ro_after_init;
+int ptg_cont_pmd_shift __ro_after_init;
+int ptg_pgtable_levels __ro_after_init;
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 60454256945b8..2ba30d06b35fe 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -3,6 +3,7 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
 				   cache.o copypage.o flush.o \
 				   ioremap.o mmap.o pgd.o mmu.o \
 				   context.o proc.o pageattr.o fixmap.o
+obj-$(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE) += pgtable-geometry.o
 obj-$(CONFIG_ARM64_CONTPTE)	+= contpte.o
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
 obj-$(CONFIG_PTDUMP_CORE)	+= ptdump.o
diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 4b106510358b1..c052d0dcb0c69 100644
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -21,10 +21,12 @@ static bool pgdir_is_page_size(void)
 {
 	if (PGD_SIZE == PAGE_SIZE)
 		return true;
-	if (CONFIG_PGTABLE_LEVELS == 4)
-		return !pgtable_l4_enabled();
-	if (CONFIG_PGTABLE_LEVELS == 5)
-		return !pgtable_l5_enabled();
+	if (!IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE)) {
+		if (CONFIG_PGTABLE_LEVELS == 4)
+			return !pgtable_l4_enabled();
+		if (CONFIG_PGTABLE_LEVELS == 5)
+			return !pgtable_l5_enabled();
+	}
 	return false;
 }
 
diff --git a/arch/arm64/mm/pgtable-geometry.c b/arch/arm64/mm/pgtable-geometry.c
new file mode 100644
index 0000000000000..ba50637f1e9d0
--- /dev/null
+++ b/arch/arm64/mm/pgtable-geometry.c
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 ARM Ltd.
+ */
+
+#include <linux/cache.h>
+#include <asm/pgtable-geometry.h>
+
+/*
+ * TODO: These should be __ro_after_init, but we need to write to them from the
+ * pi code where they are mapped in the early page table as read-only.
+ * __ro_after_init doesn't become writable until later when the swapper pgtable
+ * is fully set up. We should update the early page table to map __ro_after_init
+ * as read-write.
+ */
+
+int ptg_page_shift __read_mostly;
+int ptg_pmd_shift __read_mostly;
+int ptg_pud_shift __read_mostly;
+int ptg_p4d_shift __read_mostly;
+int ptg_pgdir_shift __read_mostly;
+int ptg_cont_pte_shift __read_mostly;
+int ptg_cont_pmd_shift __read_mostly;
+int ptg_pgtable_levels __read_mostly;
diff --git a/drivers/firmware/efi/libstub/arm64.c b/drivers/firmware/efi/libstub/arm64.c
index e57cd3de0a00f..8db9dba7d5423 100644
--- a/drivers/firmware/efi/libstub/arm64.c
+++ b/drivers/firmware/efi/libstub/arm64.c
@@ -68,7 +68,8 @@ efi_status_t check_platform_features(void)
 		efi_novamap = true;
 
 	/* UEFI mandates support for 4 KB granularity, no need to check */
-	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES))
+	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES) ||
+	    IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
 		return EFI_SUCCESS;
 
 	tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf;
-- 
2.43.0

Re: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Zi Yan]

[-- Attachment #1: Type: text/plain, Size: 1561 bytes --]

On 14 Oct 2024, at 6:59, Ryan Roberts wrote:

> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
> selected instead of a page size. When selected, the resulting kernel's
> page size can be configured at boot via the command line.
>
> For now, boot-time page size kernels are limited to 48-bit VA, since
> more work is required to support LPA2. Additionally MMAP_RND_BITS and
> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
> work could be implemented to be able to configure these at boot time for
> optimial page size-specific values.
>
> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
> ---

<snip>

>
> @@ -1588,9 +1601,10 @@ config XEN
>  # 4K  |       27          |      12      |       15             |         10              |
>  # 16K |       27          |      14      |       13             |         11              |
>  # 64K |       29          |      16      |       13             |         13              |
> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>  config ARCH_FORCE_MAX_ORDER
>  	int
> -	default "13" if ARM64_64K_PAGES
> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>  	default "11" if ARM64_16K_PAGES
>  	default "10"
>  	help

So boot-time page size kernel always has the highest MAX_PAGE_ORDER, which
means the section size increases for 4KB and 16KB page sizes. Any downside
for this?

Is there any plan (not in this patchset) to support boot-time MAX_PAGE_ORDER
to keep section size the same?

Best Regards,
Yan, Zi

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 854 bytes --]

RE: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Michael Kelley]

From: Ryan Roberts <ryan.roberts@arm.com> Sent: Monday, October 14, 2024 3:59 AM
> 
> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
> selected instead of a page size. When selected, the resulting kernel's
> page size can be configured at boot via the command line.
> 
> For now, boot-time page size kernels are limited to 48-bit VA, since
> more work is required to support LPA2. Additionally MMAP_RND_BITS and
> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
> work could be implemented to be able to configure these at boot time for
> optimial page size-specific values.
> 
> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
> ---
> 
> ***NOTE***
> Any confused maintainers may want to read the cover note here for context:
> https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/
> 
>  arch/arm64/Kconfig                         | 26 ++++++++++---
>  arch/arm64/include/asm/kvm_hyp.h           | 11 ++++++
>  arch/arm64/include/asm/pgtable-geometry.h  | 22 ++++++++++-
>  arch/arm64/include/asm/pgtable-hwdef.h     |  6 +--
>  arch/arm64/include/asm/pgtable.h           | 10 ++++-
>  arch/arm64/include/asm/sparsemem.h         |  4 ++
>  arch/arm64/kernel/image-vars.h             | 11 ++++++
>  arch/arm64/kernel/image.h                  |  4 ++
>  arch/arm64/kernel/pi/map_kernel.c          | 45 ++++++++++++++++++++++
>  arch/arm64/kvm/arm.c                       | 10 +++++
>  arch/arm64/kvm/hyp/nvhe/Makefile           |  1 +
>  arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c | 16 ++++++++
>  arch/arm64/mm/Makefile                     |  1 +
>  arch/arm64/mm/pgd.c                        | 10 +++--
>  arch/arm64/mm/pgtable-geometry.c           | 24 ++++++++++++
>  drivers/firmware/efi/libstub/arm64.c       |  3 +-
>  16 files changed, 187 insertions(+), 17 deletions(-)
>  create mode 100644 arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
>  create mode 100644 arch/arm64/mm/pgtable-geometry.c
> 
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index a2f8ff354ca67..573d308741169 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -121,6 +121,7 @@ config ARM64
>  	select BUILDTIME_TABLE_SORT
>  	select CLONE_BACKWARDS
>  	select COMMON_CLK
> +	select CONSTRUCTORS if ARM64_BOOT_TIME_PAGE_SIZE
>  	select CPU_PM if (SUSPEND || CPU_IDLE)
>  	select CPUMASK_OFFSTACK if NR_CPUS > 256
>  	select CRC32
> @@ -284,18 +285,20 @@ config MMU
> 
>  config ARM64_CONT_PTE_SHIFT
>  	int
> +	depends on !ARM64_BOOT_TIME_PAGE_SIZE
>  	default 5 if PAGE_SIZE_64KB
>  	default 7 if PAGE_SIZE_16KB
>  	default 4
> 
>  config ARM64_CONT_PMD_SHIFT
>  	int
> +	depends on !ARM64_BOOT_TIME_PAGE_SIZE
>  	default 5 if PAGE_SIZE_64KB
>  	default 5 if PAGE_SIZE_16KB
>  	default 4
> 
>  config ARCH_MMAP_RND_BITS_MIN
> -	default 14 if PAGE_SIZE_64KB
> +	default 14 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>  	default 16 if PAGE_SIZE_16KB
>  	default 18
> 
> @@ -306,15 +309,15 @@ config ARCH_MMAP_RND_BITS_MAX
>  	default 24 if ARM64_VA_BITS=39
>  	default 27 if ARM64_VA_BITS=42
>  	default 30 if ARM64_VA_BITS=47
> -	default 29 if ARM64_VA_BITS=48 && ARM64_64K_PAGES
> +	default 29 if ARM64_VA_BITS=48 && (ARM64_64K_PAGES ||
> ARM64_BOOT_TIME_PAGE_SIZE)
>  	default 31 if ARM64_VA_BITS=48 && ARM64_16K_PAGES
>  	default 33 if ARM64_VA_BITS=48
> -	default 14 if ARM64_64K_PAGES
> +	default 14 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>  	default 16 if ARM64_16K_PAGES
>  	default 18
> 
>  config ARCH_MMAP_RND_COMPAT_BITS_MIN
> -	default 7 if ARM64_64K_PAGES
> +	default 7 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>  	default 9 if ARM64_16K_PAGES
>  	default 11
> 
> @@ -362,6 +365,7 @@ config FIX_EARLYCON_MEM
> 
>  config PGTABLE_LEVELS
>  	int
> +	default 4 if ARM64_BOOT_TIME_PAGE_SIZE # Advertise max supported levels
>  	default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36
>  	default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42
>  	default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 ||
> ARM64_VA_BITS_52)
> @@ -1316,6 +1320,14 @@ config ARM64_64K_PAGES
>  	  look-up. AArch32 emulation requires applications compiled
>  	  with 64K aligned segments.
> 
> +config ARM64_BOOT_TIME_PAGE_SIZE
> +	bool "Boot-time selection"
> +	select HAVE_PAGE_SIZE_64KB # Advertise largest page size to core
> +	help
> +	  Select desired page size (4KB, 16KB or 64KB) at boot-time via the
> +	  kernel command line option "arm64.pagesize=4k", "arm64.pagesize=16k"
> +	  or "arm64.pagesize=64k".
> +
>  endchoice
> 
>  choice
> @@ -1348,6 +1360,7 @@ config ARM64_VA_BITS_48
>  config ARM64_VA_BITS_52
>  	bool "52-bit"
>  	depends on ARM64_PAN || !ARM64_SW_TTBR0_PAN
> +	depends on !ARM64_BOOT_TIME_PAGE_SIZE
>  	help
>  	  Enable 52-bit virtual addressing for userspace when explicitly
>  	  requested via a hint to mmap(). The kernel will also use 52-bit
> @@ -1588,9 +1601,10 @@ config XEN
>  # 4K  |       27          |      12      |       15             |         10              |
>  # 16K |       27          |      14      |       13             |         11              |
>  # 64K |       29          |      16      |       13             |         13              |
> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>  config ARCH_FORCE_MAX_ORDER
>  	int
> -	default "13" if ARM64_64K_PAGES
> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>  	default "11" if ARM64_16K_PAGES
>  	default "10"
>  	help
> @@ -1663,7 +1677,7 @@ config ARM64_TAGGED_ADDR_ABI
> 
>  menuconfig COMPAT
>  	bool "Kernel support for 32-bit EL0"
> -	depends on ARM64_4K_PAGES || EXPERT
> +	depends on ARM64_4K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE || EXPERT
>  	select HAVE_UID16
>  	select OLD_SIGSUSPEND3
>  	select COMPAT_OLD_SIGACTION
> diff --git a/arch/arm64/include/asm/kvm_hyp.h
> b/arch/arm64/include/asm/kvm_hyp.h
> index c838309e4ec47..9397a14642afa 100644
> --- a/arch/arm64/include/asm/kvm_hyp.h
> +++ b/arch/arm64/include/asm/kvm_hyp.h
> @@ -145,4 +145,15 @@ extern unsigned long kvm_nvhe_sym(__icache_flags);
>  extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
>  extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
> 
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +extern int kvm_nvhe_sym(ptg_page_shift);
> +extern int kvm_nvhe_sym(ptg_pmd_shift);
> +extern int kvm_nvhe_sym(ptg_pud_shift);
> +extern int kvm_nvhe_sym(ptg_p4d_shift);
> +extern int kvm_nvhe_sym(ptg_pgdir_shift);
> +extern int kvm_nvhe_sym(ptg_cont_pte_shift);
> +extern int kvm_nvhe_sym(ptg_cont_pmd_shift);
> +extern int kvm_nvhe_sym(ptg_pgtable_levels);
> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
> +
>  #endif /* __ARM64_KVM_HYP_H__ */
> diff --git a/arch/arm64/include/asm/pgtable-geometry.h
> b/arch/arm64/include/asm/pgtable-geometry.h
> index 62fe125909c08..18a8c8d499ecc 100644
> --- a/arch/arm64/include/asm/pgtable-geometry.h
> +++ b/arch/arm64/include/asm/pgtable-geometry.h
> @@ -6,16 +6,33 @@
>  #define ARM64_PAGE_SHIFT_16K	14
>  #define ARM64_PAGE_SHIFT_64K	16
> 
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +#define PAGE_SHIFT_MIN		ARM64_PAGE_SHIFT_4K
> +#define PAGE_SHIFT_MAX		ARM64_PAGE_SHIFT_64K
> +#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>  #define PAGE_SHIFT_MIN		CONFIG_PAGE_SHIFT
> +#define PAGE_SHIFT_MAX		CONFIG_PAGE_SHIFT
> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
> +
>  #define PAGE_SIZE_MIN		(_AC(1, UL) << PAGE_SHIFT_MIN)
>  #define PAGE_MASK_MIN		(~(PAGE_SIZE_MIN-1))
> -
> -#define PAGE_SHIFT_MAX		CONFIG_PAGE_SHIFT
>  #define PAGE_SIZE_MAX		(_AC(1, UL) << PAGE_SHIFT_MAX)
>  #define PAGE_MASK_MAX		(~(PAGE_SIZE_MAX-1))
> 
>  #include <asm-generic/pgtable-geometry.h>
> 
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +#ifndef __ASSEMBLY__
> +extern int ptg_page_shift;
> +extern int ptg_pmd_shift;
> +extern int ptg_pud_shift;
> +extern int ptg_p4d_shift;
> +extern int ptg_pgdir_shift;
> +extern int ptg_cont_pte_shift;
> +extern int ptg_cont_pmd_shift;
> +extern int ptg_pgtable_levels;
> +#endif /* __ASSEMBLY__ */
> +#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>  #define ptg_page_shift		CONFIG_PAGE_SHIFT
>  #define ptg_pmd_shift		ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
>  #define ptg_pud_shift		ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
> @@ -24,5 +41,6 @@
>  #define ptg_cont_pte_shift	(CONFIG_ARM64_CONT_PTE_SHIFT + PAGE_SHIFT)
>  #define ptg_cont_pmd_shift	(CONFIG_ARM64_CONT_PMD_SHIFT + PMD_SHIFT)
>  #define ptg_pgtable_levels	CONFIG_PGTABLE_LEVELS
> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
> 
>  #endif /* ASM_PGTABLE_GEOMETRY_H */
> diff --git a/arch/arm64/include/asm/pgtable-hwdef.h
> b/arch/arm64/include/asm/pgtable-hwdef.h
> index ca8bcbc1fe220..da5404617acbf 100644
> --- a/arch/arm64/include/asm/pgtable-hwdef.h
> +++ b/arch/arm64/include/asm/pgtable-hwdef.h
> @@ -52,7 +52,7 @@
>  #define PMD_SHIFT		ptg_pmd_shift
>  #define PMD_SIZE		(_AC(1, UL) << PMD_SHIFT)
>  #define PMD_MASK		(~(PMD_SIZE-1))
> -#define PTRS_PER_PMD		(1 << (PAGE_SHIFT - 3))
> +#define PTRS_PER_PMD		(ptg_pgtable_levels > 2 ? (1 << (PAGE_SHIFT -
> 3)) : 1)
>  #define MAX_PTRS_PER_PMD	(1 << (PAGE_SHIFT_MAX - 3))
>  #endif
> 
> @@ -63,7 +63,7 @@
>  #define PUD_SHIFT		ptg_pud_shift
>  #define PUD_SIZE		(_AC(1, UL) << PUD_SHIFT)
>  #define PUD_MASK		(~(PUD_SIZE-1))
> -#define PTRS_PER_PUD		(1 << (PAGE_SHIFT - 3))
> +#define PTRS_PER_PUD		(ptg_pgtable_levels > 3 ? (1 << (PAGE_SHIFT -
> 3)) : 1)
>  #define MAX_PTRS_PER_PUD	(1 << (PAGE_SHIFT_MAX - 3))
>  #endif
> 
> @@ -71,7 +71,7 @@
>  #define P4D_SHIFT		ptg_p4d_shift
>  #define P4D_SIZE		(_AC(1, UL) << P4D_SHIFT)
>  #define P4D_MASK		(~(P4D_SIZE-1))
> -#define PTRS_PER_P4D		(1 << (PAGE_SHIFT - 3))
> +#define PTRS_PER_P4D		(ptg_pgtable_levels > 4 ? (1 << (PAGE_SHIFT -
> 3)) : 1)
>  #define MAX_PTRS_PER_P4D	(1 << (PAGE_SHIFT_MAX - 3))
>  #endif
> 
> diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
> index 8ead41da715b0..ad9f75f5cc29a 100644
> --- a/arch/arm64/include/asm/pgtable.h
> +++ b/arch/arm64/include/asm/pgtable.h
> @@ -755,7 +755,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
> 
>  static __always_inline bool pgtable_l3_enabled(void)
>  {
> -	return true;
> +	return ptg_pgtable_levels > 2;
>  }
> 
>  static inline bool mm_pmd_folded(const struct mm_struct *mm)
> @@ -888,6 +888,8 @@ static inline bool pgtable_l3_enabled(void) { return false; }
> 
>  static __always_inline bool pgtable_l4_enabled(void)
>  {
> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
> +		return ptg_pgtable_levels > 3;
>  	if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
>  		return true;
>  	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
> @@ -935,6 +937,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
> 
>  static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
>  {
> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
> +		return (pud_t *)p4dp;
>  	return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
>  }
> 
> @@ -1014,6 +1018,8 @@ static inline bool pgtable_l4_enabled(void) { return false; }
> 
>  static __always_inline bool pgtable_l5_enabled(void)
>  {
> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
> +		return ptg_pgtable_levels > 4;
>  	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
>  		return vabits_actual == VA_BITS;
>  	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
> @@ -1059,6 +1065,8 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
> 
>  static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
>  {
> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
> +		return (p4d_t *)pgdp;
>  	return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr);
>  }
> 
> diff --git a/arch/arm64/include/asm/sparsemem.h
> b/arch/arm64/include/asm/sparsemem.h
> index a05fdd54014f7..2daf1263ba638 100644
> --- a/arch/arm64/include/asm/sparsemem.h
> +++ b/arch/arm64/include/asm/sparsemem.h
> @@ -17,6 +17,10 @@
>   * entries could not be created for vmemmap mappings.
>   * 16K follows 4K for simplicity.
>   */
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +#define SECTION_SIZE_BITS	29
> +#else
>  #define SECTION_SIZE_BITS	(PAGE_SIZE == SZ_64K ? 29 : 27)
> +#endif
> 
>  #endif
> diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
> index a168f3337446f..9968320f83bc4 100644
> --- a/arch/arm64/kernel/image-vars.h
> +++ b/arch/arm64/kernel/image-vars.h
> @@ -36,6 +36,17 @@ PROVIDE(__pi___memcpy			=
> __pi_memcpy);
>  PROVIDE(__pi___memmove			= __pi_memmove);
>  PROVIDE(__pi___memset			= __pi_memset);
> 
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +PROVIDE(__pi_ptg_page_shift		= ptg_page_shift);
> +PROVIDE(__pi_ptg_pmd_shift		= ptg_pmd_shift);
> +PROVIDE(__pi_ptg_pud_shift		= ptg_pud_shift);
> +PROVIDE(__pi_ptg_p4d_shift		= ptg_p4d_shift);
> +PROVIDE(__pi_ptg_pgdir_shift		= ptg_pgdir_shift);
> +PROVIDE(__pi_ptg_cont_pte_shift		= ptg_cont_pte_shift);
> +PROVIDE(__pi_ptg_cont_pmd_shift		= ptg_cont_pmd_shift);
> +PROVIDE(__pi_ptg_pgtable_levels		= ptg_pgtable_levels);
> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
> +
>  PROVIDE(__pi_id_aa64isar1_override	= id_aa64isar1_override);
>  PROVIDE(__pi_id_aa64isar2_override	= id_aa64isar2_override);
>  PROVIDE(__pi_id_aa64mmfr0_override	= id_aa64mmfr0_override);
> diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
> index 7bc3ba8979019..01502fc3b891b 100644
> --- a/arch/arm64/kernel/image.h
> +++ b/arch/arm64/kernel/image.h
> @@ -47,7 +47,11 @@
>  #define __HEAD_FLAG_BE		ARM64_IMAGE_FLAG_LE
>  #endif
> 
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +#define __HEAD_FLAG_PAGE_SIZE	0
> +#else
>  #define __HEAD_FLAG_PAGE_SIZE	((PAGE_SHIFT - 10) / 2)
> +#endif
> 
>  #define __HEAD_FLAG_PHYS_BASE	1
> 
> diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c
> index deb8cd50b0b0c..22b3c70e04f9c 100644
> --- a/arch/arm64/kernel/pi/map_kernel.c
> +++ b/arch/arm64/kernel/pi/map_kernel.c
> @@ -221,6 +221,49 @@ static void __init map_fdt(u64 fdt, int page_shift)
>  	dsb(ishst);
>  }
> 
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +static void __init ptg_init(int page_shift)
> +{
> +	ptg_pgtable_levels =
> +		__ARM64_HW_PGTABLE_LEVELS(page_shift,
> CONFIG_ARM64_VA_BITS);
> +
> +	ptg_pgdir_shift = __ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift,
> +				4 - ptg_pgtable_levels);
> +
> +	ptg_p4d_shift = ptg_pgtable_levels >= 5 ?
> +				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 0) :
> +				ptg_pgdir_shift;
> +
> +	ptg_pud_shift = ptg_pgtable_levels >= 4 ?
> +				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 1) :
> +				ptg_pgdir_shift;
> +
> +	ptg_pmd_shift = ptg_pgtable_levels >= 3 ?
> +				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 2) :
> +				ptg_pgdir_shift;
> +
> +	ptg_page_shift = page_shift;
> +
> +	switch (page_shift) {
> +	case ARM64_PAGE_SHIFT_64K:
> +		ptg_cont_pte_shift = ptg_page_shift + 5;
> +		ptg_cont_pmd_shift = ptg_pmd_shift + 5;
> +		break;
> +	case ARM64_PAGE_SHIFT_16K:
> +		ptg_cont_pte_shift = ptg_page_shift + 7;
> +		ptg_cont_pmd_shift = ptg_pmd_shift + 5;
> +		break;
> +	default: /* ARM64_PAGE_SHIFT_4K */
> +		ptg_cont_pte_shift = ptg_page_shift + 4;
> +		ptg_cont_pmd_shift = ptg_pmd_shift + 4;
> +	}
> +}
> +#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
> +static inline void ptg_init(int page_shift)
> +{
> +}
> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
> +
>  asmlinkage void __init early_map_kernel(u64 boot_status, void *fdt)
>  {
>  	static char const chosen_str[] __initconst = "/chosen";
> @@ -247,6 +290,8 @@ asmlinkage void __init early_map_kernel(u64 boot_status, void
> *fdt)
>  	if (!page_shift)
>  		page_shift = early_page_shift;
> 
> +	ptg_init(page_shift);
> +
>  	if (va_bits > 48) {
>  		u64 page_size = early_page_size(page_shift);
> 
> diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
> index 9bef7638342ef..c835a50b8b768 100644
> --- a/arch/arm64/kvm/arm.c
> +++ b/arch/arm64/kvm/arm.c
> @@ -2424,6 +2424,16 @@ static void kvm_hyp_init_symbols(void)
>  	kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) =
> read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
>  	kvm_nvhe_sym(__icache_flags) = __icache_flags;
>  	kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
> +	kvm_nvhe_sym(ptg_page_shift) = ptg_page_shift;
> +	kvm_nvhe_sym(ptg_pmd_shift) = ptg_pmd_shift;
> +	kvm_nvhe_sym(ptg_pud_shift) = ptg_pud_shift;
> +	kvm_nvhe_sym(ptg_p4d_shift) = ptg_p4d_shift;
> +	kvm_nvhe_sym(ptg_pgdir_shift) = ptg_pgdir_shift;
> +	kvm_nvhe_sym(ptg_cont_pte_shift) = ptg_cont_pte_shift;
> +	kvm_nvhe_sym(ptg_cont_pmd_shift) = ptg_cont_pmd_shift;
> +	kvm_nvhe_sym(ptg_pgtable_levels) = ptg_pgtable_levels;
> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>  }
> 
>  static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
> diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile
> b/arch/arm64/kvm/hyp/nvhe/Makefile
> index b43426a493df5..a8fcbb84c7996 100644
> --- a/arch/arm64/kvm/hyp/nvhe/Makefile
> +++ b/arch/arm64/kvm/hyp/nvhe/Makefile
> @@ -27,6 +27,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-
> init.o host.o
>  	 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
>  hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
>  	 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
> +hyp-obj-$(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE) += pgtable-geometry.o
>  hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o
>  hyp-obj-y += $(lib-objs)
> 
> diff --git a/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
> b/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
> new file mode 100644
> index 0000000000000..17f807450a31a
> --- /dev/null
> +++ b/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
> @@ -0,0 +1,16 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (C) 2024 ARM Ltd.
> + */
> +
> +#include <linux/cache.h>
> +#include <asm/pgtable-geometry.h>
> +
> +int ptg_page_shift __ro_after_init;
> +int ptg_pmd_shift __ro_after_init;
> +int ptg_pud_shift __ro_after_init;
> +int ptg_p4d_shift __ro_after_init;
> +int ptg_pgdir_shift __ro_after_init;
> +int ptg_cont_pte_shift __ro_after_init;
> +int ptg_cont_pmd_shift __ro_after_init;
> +int ptg_pgtable_levels __ro_after_init;
> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> index 60454256945b8..2ba30d06b35fe 100644
> --- a/arch/arm64/mm/Makefile
> +++ b/arch/arm64/mm/Makefile
> @@ -3,6 +3,7 @@ obj-y				:= dma-mapping.o extable.o
> fault.o init.o \
>  				   cache.o copypage.o flush.o \
>  				   ioremap.o mmap.o pgd.o mmu.o \
>  				   context.o proc.o pageattr.o fixmap.o
> +obj-$(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE) += pgtable-geometry.o
>  obj-$(CONFIG_ARM64_CONTPTE)	+= contpte.o
>  obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
>  obj-$(CONFIG_PTDUMP_CORE)	+= ptdump.o
> diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
> index 4b106510358b1..c052d0dcb0c69 100644
> --- a/arch/arm64/mm/pgd.c
> +++ b/arch/arm64/mm/pgd.c
> @@ -21,10 +21,12 @@ static bool pgdir_is_page_size(void)
>  {
>  	if (PGD_SIZE == PAGE_SIZE)
>  		return true;
> -	if (CONFIG_PGTABLE_LEVELS == 4)
> -		return !pgtable_l4_enabled();
> -	if (CONFIG_PGTABLE_LEVELS == 5)
> -		return !pgtable_l5_enabled();
> +	if (!IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE)) {
> +		if (CONFIG_PGTABLE_LEVELS == 4)
> +			return !pgtable_l4_enabled();
> +		if (CONFIG_PGTABLE_LEVELS == 5)
> +			return !pgtable_l5_enabled();
> +	}
>  	return false;
>  }
> 
> diff --git a/arch/arm64/mm/pgtable-geometry.c b/arch/arm64/mm/pgtable-
> geometry.c
> new file mode 100644
> index 0000000000000..ba50637f1e9d0
> --- /dev/null
> +++ b/arch/arm64/mm/pgtable-geometry.c
> @@ -0,0 +1,24 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (C) 2024 ARM Ltd.
> + */
> +
> +#include <linux/cache.h>
> +#include <asm/pgtable-geometry.h>
> +
> +/*
> + * TODO: These should be __ro_after_init, but we need to write to them from the
> + * pi code where they are mapped in the early page table as read-only.
> + * __ro_after_init doesn't become writable until later when the swapper pgtable
> + * is fully set up. We should update the early page table to map __ro_after_init
> + * as read-write.
> + */
> +
> +int ptg_page_shift __read_mostly;
> +int ptg_pmd_shift __read_mostly;

I found that ptg_page_shift and ptg_pmd_shift need
EXPORT_SYMBOL_GPL for cases where code compiled
as a module is using PAGE_SIZE/PAGE_SHIFT or
PMD_SIZE/PMD_SHIFT. Some of the others below
might also need EXPORT_SYMBOL_GPL.

Michael

> +int ptg_pud_shift __read_mostly;
> +int ptg_p4d_shift __read_mostly;
> +int ptg_pgdir_shift __read_mostly;
> +int ptg_cont_pte_shift __read_mostly;
> +int ptg_cont_pmd_shift __read_mostly;
> +int ptg_pgtable_levels __read_mostly;
> diff --git a/drivers/firmware/efi/libstub/arm64.c
> b/drivers/firmware/efi/libstub/arm64.c
> index e57cd3de0a00f..8db9dba7d5423 100644
> --- a/drivers/firmware/efi/libstub/arm64.c
> +++ b/drivers/firmware/efi/libstub/arm64.c
> @@ -68,7 +68,8 @@ efi_status_t check_platform_features(void)
>  		efi_novamap = true;
> 
>  	/* UEFI mandates support for 4 KB granularity, no need to check */
> -	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES))
> +	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES) ||
> +	    IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
>  		return EFI_SUCCESS;
> 
>  	tg = (read_cpuid(ID_AA64MMFR0_EL1) >>
> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf;
> --
> 2.43.0
> 

Re: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Ryan Roberts]

On 15/10/2024 18:42, Zi Yan wrote:
> On 14 Oct 2024, at 6:59, Ryan Roberts wrote:
> 
>> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
>> selected instead of a page size. When selected, the resulting kernel's
>> page size can be configured at boot via the command line.
>>
>> For now, boot-time page size kernels are limited to 48-bit VA, since
>> more work is required to support LPA2. Additionally MMAP_RND_BITS and
>> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
>> work could be implemented to be able to configure these at boot time for
>> optimial page size-specific values.
>>
>> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
>> ---
> 
> <snip>
> 
>>
>> @@ -1588,9 +1601,10 @@ config XEN
>>  # 4K  |       27          |      12      |       15             |         10              |
>>  # 16K |       27          |      14      |       13             |         11              |
>>  # 64K |       29          |      16      |       13             |         13              |
>> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>>  config ARCH_FORCE_MAX_ORDER
>>  	int
>> -	default "13" if ARM64_64K_PAGES
>> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>  	default "11" if ARM64_16K_PAGES
>>  	default "10"
>>  	help
> 
> So boot-time page size kernel always has the highest MAX_PAGE_ORDER, which
> means the section size increases for 4KB and 16KB page sizes. Any downside
> for this?

I guess there is some cost to the buddy when MAX_PAGE_ORDER is larger than it
needs to be - I expect you can explain those details much better than I can. I'm
just setting it to the worst case for now as it was the easiest solution for the
initial series.

> 
> Is there any plan (not in this patchset) to support boot-time MAX_PAGE_ORDER
> to keep section size the same?

Yes absolutely. I should have documented MAX_PAGE_ORDER in the commit log along
with the comments for MMAP_RND_BITS and SECTION_SIZE_BITS - that was an
oversight and I'll fix it in the next version. I plan to look at making all 3
values boot-time configurable in future (although I have no idea at this point
how involved that will be).

Thanks,
Ryan

> 
> Best Regards,
> Yan, Zi

Re: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Ryan Roberts]

On 15/10/2024 18:52, Michael Kelley wrote:
> From: Ryan Roberts <ryan.roberts@arm.com> Sent: Monday, October 14, 2024 3:59 AM
>>
>> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
>> selected instead of a page size. When selected, the resulting kernel's
>> page size can be configured at boot via the command line.
>>
>> For now, boot-time page size kernels are limited to 48-bit VA, since
>> more work is required to support LPA2. Additionally MMAP_RND_BITS and
>> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
>> work could be implemented to be able to configure these at boot time for
>> optimial page size-specific values.
>>
>> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
>> ---
>>
>> ***NOTE***
>> Any confused maintainers may want to read the cover note here for context:
>> https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/
>>
>>  arch/arm64/Kconfig                         | 26 ++++++++++---
>>  arch/arm64/include/asm/kvm_hyp.h           | 11 ++++++
>>  arch/arm64/include/asm/pgtable-geometry.h  | 22 ++++++++++-
>>  arch/arm64/include/asm/pgtable-hwdef.h     |  6 +--
>>  arch/arm64/include/asm/pgtable.h           | 10 ++++-
>>  arch/arm64/include/asm/sparsemem.h         |  4 ++
>>  arch/arm64/kernel/image-vars.h             | 11 ++++++
>>  arch/arm64/kernel/image.h                  |  4 ++
>>  arch/arm64/kernel/pi/map_kernel.c          | 45 ++++++++++++++++++++++
>>  arch/arm64/kvm/arm.c                       | 10 +++++
>>  arch/arm64/kvm/hyp/nvhe/Makefile           |  1 +
>>  arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c | 16 ++++++++
>>  arch/arm64/mm/Makefile                     |  1 +
>>  arch/arm64/mm/pgd.c                        | 10 +++--
>>  arch/arm64/mm/pgtable-geometry.c           | 24 ++++++++++++
>>  drivers/firmware/efi/libstub/arm64.c       |  3 +-
>>  16 files changed, 187 insertions(+), 17 deletions(-)
>>  create mode 100644 arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
>>  create mode 100644 arch/arm64/mm/pgtable-geometry.c
>>
>> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
>> index a2f8ff354ca67..573d308741169 100644
>> --- a/arch/arm64/Kconfig
>> +++ b/arch/arm64/Kconfig
>> @@ -121,6 +121,7 @@ config ARM64
>>  	select BUILDTIME_TABLE_SORT
>>  	select CLONE_BACKWARDS
>>  	select COMMON_CLK
>> +	select CONSTRUCTORS if ARM64_BOOT_TIME_PAGE_SIZE
>>  	select CPU_PM if (SUSPEND || CPU_IDLE)
>>  	select CPUMASK_OFFSTACK if NR_CPUS > 256
>>  	select CRC32
>> @@ -284,18 +285,20 @@ config MMU
>>
>>  config ARM64_CONT_PTE_SHIFT
>>  	int
>> +	depends on !ARM64_BOOT_TIME_PAGE_SIZE
>>  	default 5 if PAGE_SIZE_64KB
>>  	default 7 if PAGE_SIZE_16KB
>>  	default 4
>>
>>  config ARM64_CONT_PMD_SHIFT
>>  	int
>> +	depends on !ARM64_BOOT_TIME_PAGE_SIZE
>>  	default 5 if PAGE_SIZE_64KB
>>  	default 5 if PAGE_SIZE_16KB
>>  	default 4
>>
>>  config ARCH_MMAP_RND_BITS_MIN
>> -	default 14 if PAGE_SIZE_64KB
>> +	default 14 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>  	default 16 if PAGE_SIZE_16KB
>>  	default 18
>>
>> @@ -306,15 +309,15 @@ config ARCH_MMAP_RND_BITS_MAX
>>  	default 24 if ARM64_VA_BITS=39
>>  	default 27 if ARM64_VA_BITS=42
>>  	default 30 if ARM64_VA_BITS=47
>> -	default 29 if ARM64_VA_BITS=48 && ARM64_64K_PAGES
>> +	default 29 if ARM64_VA_BITS=48 && (ARM64_64K_PAGES ||
>> ARM64_BOOT_TIME_PAGE_SIZE)
>>  	default 31 if ARM64_VA_BITS=48 && ARM64_16K_PAGES
>>  	default 33 if ARM64_VA_BITS=48
>> -	default 14 if ARM64_64K_PAGES
>> +	default 14 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>  	default 16 if ARM64_16K_PAGES
>>  	default 18
>>
>>  config ARCH_MMAP_RND_COMPAT_BITS_MIN
>> -	default 7 if ARM64_64K_PAGES
>> +	default 7 if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>  	default 9 if ARM64_16K_PAGES
>>  	default 11
>>
>> @@ -362,6 +365,7 @@ config FIX_EARLYCON_MEM
>>
>>  config PGTABLE_LEVELS
>>  	int
>> +	default 4 if ARM64_BOOT_TIME_PAGE_SIZE # Advertise max supported levels
>>  	default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36
>>  	default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42
>>  	default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 ||
>> ARM64_VA_BITS_52)
>> @@ -1316,6 +1320,14 @@ config ARM64_64K_PAGES
>>  	  look-up. AArch32 emulation requires applications compiled
>>  	  with 64K aligned segments.
>>
>> +config ARM64_BOOT_TIME_PAGE_SIZE
>> +	bool "Boot-time selection"
>> +	select HAVE_PAGE_SIZE_64KB # Advertise largest page size to core
>> +	help
>> +	  Select desired page size (4KB, 16KB or 64KB) at boot-time via the
>> +	  kernel command line option "arm64.pagesize=4k", "arm64.pagesize=16k"
>> +	  or "arm64.pagesize=64k".
>> +
>>  endchoice
>>
>>  choice
>> @@ -1348,6 +1360,7 @@ config ARM64_VA_BITS_48
>>  config ARM64_VA_BITS_52
>>  	bool "52-bit"
>>  	depends on ARM64_PAN || !ARM64_SW_TTBR0_PAN
>> +	depends on !ARM64_BOOT_TIME_PAGE_SIZE
>>  	help
>>  	  Enable 52-bit virtual addressing for userspace when explicitly
>>  	  requested via a hint to mmap(). The kernel will also use 52-bit
>> @@ -1588,9 +1601,10 @@ config XEN
>>  # 4K  |       27          |      12      |       15             |         10              |
>>  # 16K |       27          |      14      |       13             |         11              |
>>  # 64K |       29          |      16      |       13             |         13              |
>> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>>  config ARCH_FORCE_MAX_ORDER
>>  	int
>> -	default "13" if ARM64_64K_PAGES
>> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>  	default "11" if ARM64_16K_PAGES
>>  	default "10"
>>  	help
>> @@ -1663,7 +1677,7 @@ config ARM64_TAGGED_ADDR_ABI
>>
>>  menuconfig COMPAT
>>  	bool "Kernel support for 32-bit EL0"
>> -	depends on ARM64_4K_PAGES || EXPERT
>> +	depends on ARM64_4K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE || EXPERT
>>  	select HAVE_UID16
>>  	select OLD_SIGSUSPEND3
>>  	select COMPAT_OLD_SIGACTION
>> diff --git a/arch/arm64/include/asm/kvm_hyp.h
>> b/arch/arm64/include/asm/kvm_hyp.h
>> index c838309e4ec47..9397a14642afa 100644
>> --- a/arch/arm64/include/asm/kvm_hyp.h
>> +++ b/arch/arm64/include/asm/kvm_hyp.h
>> @@ -145,4 +145,15 @@ extern unsigned long kvm_nvhe_sym(__icache_flags);
>>  extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
>>  extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
>>
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +extern int kvm_nvhe_sym(ptg_page_shift);
>> +extern int kvm_nvhe_sym(ptg_pmd_shift);
>> +extern int kvm_nvhe_sym(ptg_pud_shift);
>> +extern int kvm_nvhe_sym(ptg_p4d_shift);
>> +extern int kvm_nvhe_sym(ptg_pgdir_shift);
>> +extern int kvm_nvhe_sym(ptg_cont_pte_shift);
>> +extern int kvm_nvhe_sym(ptg_cont_pmd_shift);
>> +extern int kvm_nvhe_sym(ptg_pgtable_levels);
>> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>> +
>>  #endif /* __ARM64_KVM_HYP_H__ */
>> diff --git a/arch/arm64/include/asm/pgtable-geometry.h
>> b/arch/arm64/include/asm/pgtable-geometry.h
>> index 62fe125909c08..18a8c8d499ecc 100644
>> --- a/arch/arm64/include/asm/pgtable-geometry.h
>> +++ b/arch/arm64/include/asm/pgtable-geometry.h
>> @@ -6,16 +6,33 @@
>>  #define ARM64_PAGE_SHIFT_16K	14
>>  #define ARM64_PAGE_SHIFT_64K	16
>>
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +#define PAGE_SHIFT_MIN		ARM64_PAGE_SHIFT_4K
>> +#define PAGE_SHIFT_MAX		ARM64_PAGE_SHIFT_64K
>> +#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>>  #define PAGE_SHIFT_MIN		CONFIG_PAGE_SHIFT
>> +#define PAGE_SHIFT_MAX		CONFIG_PAGE_SHIFT
>> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>> +
>>  #define PAGE_SIZE_MIN		(_AC(1, UL) << PAGE_SHIFT_MIN)
>>  #define PAGE_MASK_MIN		(~(PAGE_SIZE_MIN-1))
>> -
>> -#define PAGE_SHIFT_MAX		CONFIG_PAGE_SHIFT
>>  #define PAGE_SIZE_MAX		(_AC(1, UL) << PAGE_SHIFT_MAX)
>>  #define PAGE_MASK_MAX		(~(PAGE_SIZE_MAX-1))
>>
>>  #include <asm-generic/pgtable-geometry.h>
>>
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +#ifndef __ASSEMBLY__
>> +extern int ptg_page_shift;
>> +extern int ptg_pmd_shift;
>> +extern int ptg_pud_shift;
>> +extern int ptg_p4d_shift;
>> +extern int ptg_pgdir_shift;
>> +extern int ptg_cont_pte_shift;
>> +extern int ptg_cont_pmd_shift;
>> +extern int ptg_pgtable_levels;
>> +#endif /* __ASSEMBLY__ */
>> +#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>>  #define ptg_page_shift		CONFIG_PAGE_SHIFT
>>  #define ptg_pmd_shift		ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
>>  #define ptg_pud_shift		ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
>> @@ -24,5 +41,6 @@
>>  #define ptg_cont_pte_shift	(CONFIG_ARM64_CONT_PTE_SHIFT + PAGE_SHIFT)
>>  #define ptg_cont_pmd_shift	(CONFIG_ARM64_CONT_PMD_SHIFT + PMD_SHIFT)
>>  #define ptg_pgtable_levels	CONFIG_PGTABLE_LEVELS
>> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>>
>>  #endif /* ASM_PGTABLE_GEOMETRY_H */
>> diff --git a/arch/arm64/include/asm/pgtable-hwdef.h
>> b/arch/arm64/include/asm/pgtable-hwdef.h
>> index ca8bcbc1fe220..da5404617acbf 100644
>> --- a/arch/arm64/include/asm/pgtable-hwdef.h
>> +++ b/arch/arm64/include/asm/pgtable-hwdef.h
>> @@ -52,7 +52,7 @@
>>  #define PMD_SHIFT		ptg_pmd_shift
>>  #define PMD_SIZE		(_AC(1, UL) << PMD_SHIFT)
>>  #define PMD_MASK		(~(PMD_SIZE-1))
>> -#define PTRS_PER_PMD		(1 << (PAGE_SHIFT - 3))
>> +#define PTRS_PER_PMD		(ptg_pgtable_levels > 2 ? (1 << (PAGE_SHIFT -
>> 3)) : 1)
>>  #define MAX_PTRS_PER_PMD	(1 << (PAGE_SHIFT_MAX - 3))
>>  #endif
>>
>> @@ -63,7 +63,7 @@
>>  #define PUD_SHIFT		ptg_pud_shift
>>  #define PUD_SIZE		(_AC(1, UL) << PUD_SHIFT)
>>  #define PUD_MASK		(~(PUD_SIZE-1))
>> -#define PTRS_PER_PUD		(1 << (PAGE_SHIFT - 3))
>> +#define PTRS_PER_PUD		(ptg_pgtable_levels > 3 ? (1 << (PAGE_SHIFT -
>> 3)) : 1)
>>  #define MAX_PTRS_PER_PUD	(1 << (PAGE_SHIFT_MAX - 3))
>>  #endif
>>
>> @@ -71,7 +71,7 @@
>>  #define P4D_SHIFT		ptg_p4d_shift
>>  #define P4D_SIZE		(_AC(1, UL) << P4D_SHIFT)
>>  #define P4D_MASK		(~(P4D_SIZE-1))
>> -#define PTRS_PER_P4D		(1 << (PAGE_SHIFT - 3))
>> +#define PTRS_PER_P4D		(ptg_pgtable_levels > 4 ? (1 << (PAGE_SHIFT -
>> 3)) : 1)
>>  #define MAX_PTRS_PER_P4D	(1 << (PAGE_SHIFT_MAX - 3))
>>  #endif
>>
>> diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
>> index 8ead41da715b0..ad9f75f5cc29a 100644
>> --- a/arch/arm64/include/asm/pgtable.h
>> +++ b/arch/arm64/include/asm/pgtable.h
>> @@ -755,7 +755,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
>>
>>  static __always_inline bool pgtable_l3_enabled(void)
>>  {
>> -	return true;
>> +	return ptg_pgtable_levels > 2;
>>  }
>>
>>  static inline bool mm_pmd_folded(const struct mm_struct *mm)
>> @@ -888,6 +888,8 @@ static inline bool pgtable_l3_enabled(void) { return false; }
>>
>>  static __always_inline bool pgtable_l4_enabled(void)
>>  {
>> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
>> +		return ptg_pgtable_levels > 3;
>>  	if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
>>  		return true;
>>  	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
>> @@ -935,6 +937,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
>>
>>  static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
>>  {
>> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
>> +		return (pud_t *)p4dp;
>>  	return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
>>  }
>>
>> @@ -1014,6 +1018,8 @@ static inline bool pgtable_l4_enabled(void) { return false; }
>>
>>  static __always_inline bool pgtable_l5_enabled(void)
>>  {
>> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
>> +		return ptg_pgtable_levels > 4;
>>  	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
>>  		return vabits_actual == VA_BITS;
>>  	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
>> @@ -1059,6 +1065,8 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
>>
>>  static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
>>  {
>> +	if (IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
>> +		return (p4d_t *)pgdp;
>>  	return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr);
>>  }
>>
>> diff --git a/arch/arm64/include/asm/sparsemem.h
>> b/arch/arm64/include/asm/sparsemem.h
>> index a05fdd54014f7..2daf1263ba638 100644
>> --- a/arch/arm64/include/asm/sparsemem.h
>> +++ b/arch/arm64/include/asm/sparsemem.h
>> @@ -17,6 +17,10 @@
>>   * entries could not be created for vmemmap mappings.
>>   * 16K follows 4K for simplicity.
>>   */
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +#define SECTION_SIZE_BITS	29
>> +#else
>>  #define SECTION_SIZE_BITS	(PAGE_SIZE == SZ_64K ? 29 : 27)
>> +#endif
>>
>>  #endif
>> diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
>> index a168f3337446f..9968320f83bc4 100644
>> --- a/arch/arm64/kernel/image-vars.h
>> +++ b/arch/arm64/kernel/image-vars.h
>> @@ -36,6 +36,17 @@ PROVIDE(__pi___memcpy			=
>> __pi_memcpy);
>>  PROVIDE(__pi___memmove			= __pi_memmove);
>>  PROVIDE(__pi___memset			= __pi_memset);
>>
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +PROVIDE(__pi_ptg_page_shift		= ptg_page_shift);
>> +PROVIDE(__pi_ptg_pmd_shift		= ptg_pmd_shift);
>> +PROVIDE(__pi_ptg_pud_shift		= ptg_pud_shift);
>> +PROVIDE(__pi_ptg_p4d_shift		= ptg_p4d_shift);
>> +PROVIDE(__pi_ptg_pgdir_shift		= ptg_pgdir_shift);
>> +PROVIDE(__pi_ptg_cont_pte_shift		= ptg_cont_pte_shift);
>> +PROVIDE(__pi_ptg_cont_pmd_shift		= ptg_cont_pmd_shift);
>> +PROVIDE(__pi_ptg_pgtable_levels		= ptg_pgtable_levels);
>> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>> +
>>  PROVIDE(__pi_id_aa64isar1_override	= id_aa64isar1_override);
>>  PROVIDE(__pi_id_aa64isar2_override	= id_aa64isar2_override);
>>  PROVIDE(__pi_id_aa64mmfr0_override	= id_aa64mmfr0_override);
>> diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
>> index 7bc3ba8979019..01502fc3b891b 100644
>> --- a/arch/arm64/kernel/image.h
>> +++ b/arch/arm64/kernel/image.h
>> @@ -47,7 +47,11 @@
>>  #define __HEAD_FLAG_BE		ARM64_IMAGE_FLAG_LE
>>  #endif
>>
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +#define __HEAD_FLAG_PAGE_SIZE	0
>> +#else
>>  #define __HEAD_FLAG_PAGE_SIZE	((PAGE_SHIFT - 10) / 2)
>> +#endif
>>
>>  #define __HEAD_FLAG_PHYS_BASE	1
>>
>> diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c
>> index deb8cd50b0b0c..22b3c70e04f9c 100644
>> --- a/arch/arm64/kernel/pi/map_kernel.c
>> +++ b/arch/arm64/kernel/pi/map_kernel.c
>> @@ -221,6 +221,49 @@ static void __init map_fdt(u64 fdt, int page_shift)
>>  	dsb(ishst);
>>  }
>>
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +static void __init ptg_init(int page_shift)
>> +{
>> +	ptg_pgtable_levels =
>> +		__ARM64_HW_PGTABLE_LEVELS(page_shift,
>> CONFIG_ARM64_VA_BITS);
>> +
>> +	ptg_pgdir_shift = __ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift,
>> +				4 - ptg_pgtable_levels);
>> +
>> +	ptg_p4d_shift = ptg_pgtable_levels >= 5 ?
>> +				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 0) :
>> +				ptg_pgdir_shift;
>> +
>> +	ptg_pud_shift = ptg_pgtable_levels >= 4 ?
>> +				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 1) :
>> +				ptg_pgdir_shift;
>> +
>> +	ptg_pmd_shift = ptg_pgtable_levels >= 3 ?
>> +				__ARM64_HW_PGTABLE_LEVEL_SHIFT(page_shift, 2) :
>> +				ptg_pgdir_shift;
>> +
>> +	ptg_page_shift = page_shift;
>> +
>> +	switch (page_shift) {
>> +	case ARM64_PAGE_SHIFT_64K:
>> +		ptg_cont_pte_shift = ptg_page_shift + 5;
>> +		ptg_cont_pmd_shift = ptg_pmd_shift + 5;
>> +		break;
>> +	case ARM64_PAGE_SHIFT_16K:
>> +		ptg_cont_pte_shift = ptg_page_shift + 7;
>> +		ptg_cont_pmd_shift = ptg_pmd_shift + 5;
>> +		break;
>> +	default: /* ARM64_PAGE_SHIFT_4K */
>> +		ptg_cont_pte_shift = ptg_page_shift + 4;
>> +		ptg_cont_pmd_shift = ptg_pmd_shift + 4;
>> +	}
>> +}
>> +#else /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>> +static inline void ptg_init(int page_shift)
>> +{
>> +}
>> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>> +
>>  asmlinkage void __init early_map_kernel(u64 boot_status, void *fdt)
>>  {
>>  	static char const chosen_str[] __initconst = "/chosen";
>> @@ -247,6 +290,8 @@ asmlinkage void __init early_map_kernel(u64 boot_status, void
>> *fdt)
>>  	if (!page_shift)
>>  		page_shift = early_page_shift;
>>
>> +	ptg_init(page_shift);
>> +
>>  	if (va_bits > 48) {
>>  		u64 page_size = early_page_size(page_shift);
>>
>> diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
>> index 9bef7638342ef..c835a50b8b768 100644
>> --- a/arch/arm64/kvm/arm.c
>> +++ b/arch/arm64/kvm/arm.c
>> @@ -2424,6 +2424,16 @@ static void kvm_hyp_init_symbols(void)
>>  	kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) =
>> read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
>>  	kvm_nvhe_sym(__icache_flags) = __icache_flags;
>>  	kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
>> +#ifdef CONFIG_ARM64_BOOT_TIME_PAGE_SIZE
>> +	kvm_nvhe_sym(ptg_page_shift) = ptg_page_shift;
>> +	kvm_nvhe_sym(ptg_pmd_shift) = ptg_pmd_shift;
>> +	kvm_nvhe_sym(ptg_pud_shift) = ptg_pud_shift;
>> +	kvm_nvhe_sym(ptg_p4d_shift) = ptg_p4d_shift;
>> +	kvm_nvhe_sym(ptg_pgdir_shift) = ptg_pgdir_shift;
>> +	kvm_nvhe_sym(ptg_cont_pte_shift) = ptg_cont_pte_shift;
>> +	kvm_nvhe_sym(ptg_cont_pmd_shift) = ptg_cont_pmd_shift;
>> +	kvm_nvhe_sym(ptg_pgtable_levels) = ptg_pgtable_levels;
>> +#endif /* CONFIG_ARM64_BOOT_TIME_PAGE_SIZE */
>>  }
>>
>>  static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
>> diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile
>> b/arch/arm64/kvm/hyp/nvhe/Makefile
>> index b43426a493df5..a8fcbb84c7996 100644
>> --- a/arch/arm64/kvm/hyp/nvhe/Makefile
>> +++ b/arch/arm64/kvm/hyp/nvhe/Makefile
>> @@ -27,6 +27,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-
>> init.o host.o
>>  	 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
>>  hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
>>  	 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
>> +hyp-obj-$(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE) += pgtable-geometry.o
>>  hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o
>>  hyp-obj-y += $(lib-objs)
>>
>> diff --git a/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
>> b/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
>> new file mode 100644
>> index 0000000000000..17f807450a31a
>> --- /dev/null
>> +++ b/arch/arm64/kvm/hyp/nvhe/pgtable-geometry.c
>> @@ -0,0 +1,16 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/*
>> + * Copyright (C) 2024 ARM Ltd.
>> + */
>> +
>> +#include <linux/cache.h>
>> +#include <asm/pgtable-geometry.h>
>> +
>> +int ptg_page_shift __ro_after_init;
>> +int ptg_pmd_shift __ro_after_init;
>> +int ptg_pud_shift __ro_after_init;
>> +int ptg_p4d_shift __ro_after_init;
>> +int ptg_pgdir_shift __ro_after_init;
>> +int ptg_cont_pte_shift __ro_after_init;
>> +int ptg_cont_pmd_shift __ro_after_init;
>> +int ptg_pgtable_levels __ro_after_init;
>> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
>> index 60454256945b8..2ba30d06b35fe 100644
>> --- a/arch/arm64/mm/Makefile
>> +++ b/arch/arm64/mm/Makefile
>> @@ -3,6 +3,7 @@ obj-y				:= dma-mapping.o extable.o
>> fault.o init.o \
>>  				   cache.o copypage.o flush.o \
>>  				   ioremap.o mmap.o pgd.o mmu.o \
>>  				   context.o proc.o pageattr.o fixmap.o
>> +obj-$(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE) += pgtable-geometry.o
>>  obj-$(CONFIG_ARM64_CONTPTE)	+= contpte.o
>>  obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
>>  obj-$(CONFIG_PTDUMP_CORE)	+= ptdump.o
>> diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
>> index 4b106510358b1..c052d0dcb0c69 100644
>> --- a/arch/arm64/mm/pgd.c
>> +++ b/arch/arm64/mm/pgd.c
>> @@ -21,10 +21,12 @@ static bool pgdir_is_page_size(void)
>>  {
>>  	if (PGD_SIZE == PAGE_SIZE)
>>  		return true;
>> -	if (CONFIG_PGTABLE_LEVELS == 4)
>> -		return !pgtable_l4_enabled();
>> -	if (CONFIG_PGTABLE_LEVELS == 5)
>> -		return !pgtable_l5_enabled();
>> +	if (!IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE)) {
>> +		if (CONFIG_PGTABLE_LEVELS == 4)
>> +			return !pgtable_l4_enabled();
>> +		if (CONFIG_PGTABLE_LEVELS == 5)
>> +			return !pgtable_l5_enabled();
>> +	}
>>  	return false;
>>  }
>>
>> diff --git a/arch/arm64/mm/pgtable-geometry.c b/arch/arm64/mm/pgtable-
>> geometry.c
>> new file mode 100644
>> index 0000000000000..ba50637f1e9d0
>> --- /dev/null
>> +++ b/arch/arm64/mm/pgtable-geometry.c
>> @@ -0,0 +1,24 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/*
>> + * Copyright (C) 2024 ARM Ltd.
>> + */
>> +
>> +#include <linux/cache.h>
>> +#include <asm/pgtable-geometry.h>
>> +
>> +/*
>> + * TODO: These should be __ro_after_init, but we need to write to them from the
>> + * pi code where they are mapped in the early page table as read-only.
>> + * __ro_after_init doesn't become writable until later when the swapper pgtable
>> + * is fully set up. We should update the early page table to map __ro_after_init
>> + * as read-write.
>> + */
>> +
>> +int ptg_page_shift __read_mostly;
>> +int ptg_pmd_shift __read_mostly;
> 
> I found that ptg_page_shift and ptg_pmd_shift need
> EXPORT_SYMBOL_GPL for cases where code compiled
> as a module is using PAGE_SIZE/PAGE_SHIFT or
> PMD_SIZE/PMD_SHIFT. Some of the others below
> might also need EXPORT_SYMBOL_GPL.

Ahh good spot - thanks! I'll fix this in the next version.

I wonder if these should really be EXPORT_SYMBOL() and not limited to GPL? I
guess having access to PAGE_SIZE is a pretty fundamental thing? Anybody know the
policy here?

Thanks,
Ryan

> 
> Michael
> 
>> +int ptg_pud_shift __read_mostly;
>> +int ptg_p4d_shift __read_mostly;
>> +int ptg_pgdir_shift __read_mostly;
>> +int ptg_cont_pte_shift __read_mostly;
>> +int ptg_cont_pmd_shift __read_mostly;
>> +int ptg_pgtable_levels __read_mostly;
>> diff --git a/drivers/firmware/efi/libstub/arm64.c
>> b/drivers/firmware/efi/libstub/arm64.c
>> index e57cd3de0a00f..8db9dba7d5423 100644
>> --- a/drivers/firmware/efi/libstub/arm64.c
>> +++ b/drivers/firmware/efi/libstub/arm64.c
>> @@ -68,7 +68,8 @@ efi_status_t check_platform_features(void)
>>  		efi_novamap = true;
>>
>>  	/* UEFI mandates support for 4 KB granularity, no need to check */
>> -	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES))
>> +	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES) ||
>> +	    IS_ENABLED(CONFIG_ARM64_BOOT_TIME_PAGE_SIZE))
>>  		return EFI_SUCCESS;
>>
>>  	tg = (read_cpuid(ID_AA64MMFR0_EL1) >>
>> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf;
>> --
>> 2.43.0
>>
> 

Re: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Zi Yan]

[-- Attachment #1: Type: text/plain, Size: 3722 bytes --]

On 16 Oct 2024, at 4:14, Ryan Roberts wrote:

> On 15/10/2024 18:42, Zi Yan wrote:
>> On 14 Oct 2024, at 6:59, Ryan Roberts wrote:
>>
>>> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
>>> selected instead of a page size. When selected, the resulting kernel's
>>> page size can be configured at boot via the command line.
>>>
>>> For now, boot-time page size kernels are limited to 48-bit VA, since
>>> more work is required to support LPA2. Additionally MMAP_RND_BITS and
>>> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
>>> work could be implemented to be able to configure these at boot time for
>>> optimial page size-specific values.
>>>
>>> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
>>> ---
>>
>> <snip>
>>
>>>
>>> @@ -1588,9 +1601,10 @@ config XEN
>>>  # 4K  |       27          |      12      |       15             |         10              |
>>>  # 16K |       27          |      14      |       13             |         11              |
>>>  # 64K |       29          |      16      |       13             |         13              |
>>> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>>>  config ARCH_FORCE_MAX_ORDER
>>>  	int
>>> -	default "13" if ARM64_64K_PAGES
>>> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>>  	default "11" if ARM64_16K_PAGES
>>>  	default "10"
>>>  	help
>>
>> So boot-time page size kernel always has the highest MAX_PAGE_ORDER, which
>> means the section size increases for 4KB and 16KB page sizes. Any downside
>> for this?
>
> I guess there is some cost to the buddy when MAX_PAGE_ORDER is larger than it
> needs to be - I expect you can explain those details much better than I can. I'm
> just setting it to the worst case for now as it was the easiest solution for the
> initial series.

From my past experience (around 5.19), the perf impact (using vm-scalability)
seems very small due to MAX_PAGE_ORDER increases [1] (I made MAX_PAGE_ORDER
a boot time variable and increased it to 20 for my 1GB THP experiments).

Larger MAX_PAGE_ORDER means larger section size and larger mem_block size,
so the granularity of memory hotplug also increases. In this case:
1. ARM64 4KB: mem_block size increases from 4MB to 32MB,
2. ARM64 16KB: mem_block size increases from 32MB to 128MB,
3. ARM64 64KB: mem_block size keeps the same, 512MB.

DavidH was concerned about large mem_block size before. He might have some
opinion on this.


>
>>
>> Is there any plan (not in this patchset) to support boot-time MAX_PAGE_ORDER
>> to keep section size the same?
>
> Yes absolutely. I should have documented MAX_PAGE_ORDER in the commit log along
> with the comments for MMAP_RND_BITS and SECTION_SIZE_BITS - that was an
> oversight and I'll fix it in the next version. I plan to look at making all 3
> values boot-time configurable in future (although I have no idea at this point
> how involved that will be).

In [1], I tried to make MAX_PAGE_ORDER a boot time variable,
but for a different purpose, allocating 1GB THP. I needed some additional
changes in my patchset, since I assumed MAX_PAGE_ORDER can go beyond
section size, which makes things a little bit complicated. For your case,
I assume you are not planning to make MAX_PAGE_ORDER bigger than section
size, then I should be able to revive my patchset with fewer changes.

In terms of SECTION_SIZE_BITS, why do you want to make it a boot time variable?
Since it decides the minimum memory hotplug size, I assume we should keep
it unchanged or as small as possible to make virtual machine memory usage
efficient.


[1] https://lore.kernel.org/linux-mm/20220811231643.1012912-1-zi.yan@sent.com/


Best Regards,
Yan, Zi

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 854 bytes --]

Re: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Ryan Roberts]

On 16/10/2024 15:21, Zi Yan wrote:
> On 16 Oct 2024, at 4:14, Ryan Roberts wrote:
> 
>> On 15/10/2024 18:42, Zi Yan wrote:
>>> On 14 Oct 2024, at 6:59, Ryan Roberts wrote:
>>>
>>>> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
>>>> selected instead of a page size. When selected, the resulting kernel's
>>>> page size can be configured at boot via the command line.
>>>>
>>>> For now, boot-time page size kernels are limited to 48-bit VA, since
>>>> more work is required to support LPA2. Additionally MMAP_RND_BITS and
>>>> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
>>>> work could be implemented to be able to configure these at boot time for
>>>> optimial page size-specific values.
>>>>
>>>> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
>>>> ---
>>>
>>> <snip>
>>>
>>>>
>>>> @@ -1588,9 +1601,10 @@ config XEN
>>>>  # 4K  |       27          |      12      |       15             |         10              |
>>>>  # 16K |       27          |      14      |       13             |         11              |
>>>>  # 64K |       29          |      16      |       13             |         13              |
>>>> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>>>>  config ARCH_FORCE_MAX_ORDER
>>>>  	int
>>>> -	default "13" if ARM64_64K_PAGES
>>>> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>>>  	default "11" if ARM64_16K_PAGES
>>>>  	default "10"
>>>>  	help
>>>
>>> So boot-time page size kernel always has the highest MAX_PAGE_ORDER, which
>>> means the section size increases for 4KB and 16KB page sizes. Any downside
>>> for this?
>>
>> I guess there is some cost to the buddy when MAX_PAGE_ORDER is larger than it
>> needs to be - I expect you can explain those details much better than I can. I'm
>> just setting it to the worst case for now as it was the easiest solution for the
>> initial series.
> 
> From my past experience (around 5.19), the perf impact (using vm-scalability)
> seems very small due to MAX_PAGE_ORDER increases [1] (I made MAX_PAGE_ORDER
> a boot time variable and increased it to 20 for my 1GB THP experiments).
> 
> Larger MAX_PAGE_ORDER means larger section size and larger mem_block size,
> so the granularity of memory hotplug also increases. In this case:
> 1. ARM64 4KB: mem_block size increases from 4MB to 32MB,
> 2. ARM64 16KB: mem_block size increases from 32MB to 128MB,
> 3. ARM64 64KB: mem_block size keeps the same, 512MB.
> 
> DavidH was concerned about large mem_block size before. He might have some
> opinion on this.
> 
> 
>>
>>>
>>> Is there any plan (not in this patchset) to support boot-time MAX_PAGE_ORDER
>>> to keep section size the same?
>>
>> Yes absolutely. I should have documented MAX_PAGE_ORDER in the commit log along
>> with the comments for MMAP_RND_BITS and SECTION_SIZE_BITS - that was an
>> oversight and I'll fix it in the next version. I plan to look at making all 3
>> values boot-time configurable in future (although I have no idea at this point
>> how involved that will be).
> 
> In [1], I tried to make MAX_PAGE_ORDER a boot time variable,
> but for a different purpose, allocating 1GB THP. I needed some additional
> changes in my patchset, since I assumed MAX_PAGE_ORDER can go beyond
> section size, which makes things a little bit complicated. For your case,
> I assume you are not planning to make MAX_PAGE_ORDER bigger than section
> size, then I should be able to revive my patchset with fewer changes.

Yes correct; no need to make it bigger than section size. Thanks for the patch,
I'll certainly use it as a base when I get there or if you're interested in
doing it then even better ;-)

But I don't think this is urgent. For now, boot-time page size is a new Kconfig
for arm64. It still supports the compile-time page size options. So having a
larger MAX_PAGE_ORDER than strictly necessary doesn't represent a regression,
just a limitation of boot-time page size config - something we can optimize later.

> 
> In terms of SECTION_SIZE_BITS, why do you want to make it a boot time variable?
> Since it decides the minimum memory hotplug size, I assume we should keep
> it unchanged or as small as possible to make virtual machine memory usage
> efficient.

When I say "boot-time variable" I just mean something that the arch can
configure at boot based on the selected page size. I'm not proposing to allow
the user to set it via the command line. That means we need to rid the code of
any assumptions that it is compile time constant (e.g. c preprocessor usage of
the value, etc). The same goes for MAX_PAGE_ORDER and the MMAP_RND_BITS stuff.

> 
> 
> [1] https://lore.kernel.org/linux-mm/20220811231643.1012912-1-zi.yan@sent.com/
> 
> 
> Best Regards,
> Yan, Zi

Re: [RFC PATCH v1 57/57] arm64: Enable boot-time page size selection

[Zi Yan]

[-- Attachment #1: Type: text/plain, Size: 5026 bytes --]

On 16 Oct 2024, at 10:31, Ryan Roberts wrote:

> On 16/10/2024 15:21, Zi Yan wrote:
>> On 16 Oct 2024, at 4:14, Ryan Roberts wrote:
>>
>>> On 15/10/2024 18:42, Zi Yan wrote:
>>>> On 14 Oct 2024, at 6:59, Ryan Roberts wrote:
>>>>
>>>>> Introduce a new Kconfig, ARM64_BOOT_TIME_PAGE_SIZE, which can be
>>>>> selected instead of a page size. When selected, the resulting kernel's
>>>>> page size can be configured at boot via the command line.
>>>>>
>>>>> For now, boot-time page size kernels are limited to 48-bit VA, since
>>>>> more work is required to support LPA2. Additionally MMAP_RND_BITS and
>>>>> SECTION_SIZE_BITS are configured for the worst case (64K pages). Future
>>>>> work could be implemented to be able to configure these at boot time for
>>>>> optimial page size-specific values.
>>>>>
>>>>> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
>>>>> ---
>>>>
>>>> <snip>
>>>>
>>>>>
>>>>> @@ -1588,9 +1601,10 @@ config XEN
>>>>>  # 4K  |       27          |      12      |       15             |         10              |
>>>>>  # 16K |       27          |      14      |       13             |         11              |
>>>>>  # 64K |       29          |      16      |       13             |         13              |
>>>>> +# BOOT|       29          |    16 (max)  |       13             |         13              |
>>>>>  config ARCH_FORCE_MAX_ORDER
>>>>>  	int
>>>>> -	default "13" if ARM64_64K_PAGES
>>>>> +	default "13" if ARM64_64K_PAGES || ARM64_BOOT_TIME_PAGE_SIZE
>>>>>  	default "11" if ARM64_16K_PAGES
>>>>>  	default "10"
>>>>>  	help
>>>>
>>>> So boot-time page size kernel always has the highest MAX_PAGE_ORDER, which
>>>> means the section size increases for 4KB and 16KB page sizes. Any downside
>>>> for this?
>>>
>>> I guess there is some cost to the buddy when MAX_PAGE_ORDER is larger than it
>>> needs to be - I expect you can explain those details much better than I can. I'm
>>> just setting it to the worst case for now as it was the easiest solution for the
>>> initial series.
>>
>> From my past experience (around 5.19), the perf impact (using vm-scalability)
>> seems very small due to MAX_PAGE_ORDER increases [1] (I made MAX_PAGE_ORDER
>> a boot time variable and increased it to 20 for my 1GB THP experiments).
>>
>> Larger MAX_PAGE_ORDER means larger section size and larger mem_block size,
>> so the granularity of memory hotplug also increases. In this case:
>> 1. ARM64 4KB: mem_block size increases from 4MB to 32MB,
>> 2. ARM64 16KB: mem_block size increases from 32MB to 128MB,
>> 3. ARM64 64KB: mem_block size keeps the same, 512MB.
>>
>> DavidH was concerned about large mem_block size before. He might have some
>> opinion on this.
>>
>>
>>>
>>>>
>>>> Is there any plan (not in this patchset) to support boot-time MAX_PAGE_ORDER
>>>> to keep section size the same?
>>>
>>> Yes absolutely. I should have documented MAX_PAGE_ORDER in the commit log along
>>> with the comments for MMAP_RND_BITS and SECTION_SIZE_BITS - that was an
>>> oversight and I'll fix it in the next version. I plan to look at making all 3
>>> values boot-time configurable in future (although I have no idea at this point
>>> how involved that will be).
>>
>> In [1], I tried to make MAX_PAGE_ORDER a boot time variable,
>> but for a different purpose, allocating 1GB THP. I needed some additional
>> changes in my patchset, since I assumed MAX_PAGE_ORDER can go beyond
>> section size, which makes things a little bit complicated. For your case,
>> I assume you are not planning to make MAX_PAGE_ORDER bigger than section
>> size, then I should be able to revive my patchset with fewer changes.
>
> Yes correct; no need to make it bigger than section size. Thanks for the patch,
> I'll certainly use it as a base when I get there or if you're interested in
> doing it then even better ;-)
>
> But I don't think this is urgent. For now, boot-time page size is a new Kconfig
> for arm64. It still supports the compile-time page size options. So having a
> larger MAX_PAGE_ORDER than strictly necessary doesn't represent a regression,
> just a limitation of boot-time page size config - something we can optimize later.

Sure. I will revisit my boot time MAX_PAGE_ORDER patchset when this patchset
settles. Glad to help. :)

>
>>
>> In terms of SECTION_SIZE_BITS, why do you want to make it a boot time variable?
>> Since it decides the minimum memory hotplug size, I assume we should keep
>> it unchanged or as small as possible to make virtual machine memory usage
>> efficient.
>
> When I say "boot-time variable" I just mean something that the arch can
> configure at boot based on the selected page size. I'm not proposing to allow
> the user to set it via the command line. That means we need to rid the code of
> any assumptions that it is compile time constant (e.g. c preprocessor usage of
> the value, etc). The same goes for MAX_PAGE_ORDER and the MMAP_RND_BITS stuff.
>

Got it.

>> [1] https://lore.kernel.org/linux-mm/20220811231643.1012912-1-zi.yan@sent.com/


Best Regards,
Yan, Zi

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 854 bytes --]

Re: [RFC PATCH v1 44/57] arm64: Align sections to PAGE_SIZE_MAX

[Thomas Weißschuh]

On 2024-10-14 11:58:51+0100, Ryan Roberts wrote:
> Increase alignment of sections in nvhe hyp, vdso and final vmlinux image
> from PAGE_SIZE to PAGE_SIZE_MAX. For compile-time PAGE_SIZE,
> PAGE_SIZE_MAX == PAGE_SIZE so there is no change. For boot-time
> PAGE_SIZE, PAGE_SIZE_MAX is the largest selectable page size.
> 
> For a boot-time page size build, image size is comparable to a 64K page
> size compile-time build. In future, it may be desirable to optimize
> run-time memory consumption by freeing unused padding pages when the
> boot-time selected page size is less than PAGE_SIZE_MAX.
> 
> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
> ---
> 
> ***NOTE***
> Any confused maintainers may want to read the cover note here for context:
> https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/
> 
>  arch/arm64/include/asm/memory.h     |  4 +--
>  arch/arm64/kernel/vdso-wrap.S       |  4 +--
>  arch/arm64/kernel/vdso.c            |  7 +++---
>  arch/arm64/kernel/vdso/vdso.lds.S   |  4 +--
>  arch/arm64/kernel/vdso32-wrap.S     |  4 +--
>  arch/arm64/kernel/vdso32/vdso.lds.S |  4 +--
>  arch/arm64/kernel/vmlinux.lds.S     | 38 ++++++++++++++---------------
>  arch/arm64/kvm/hyp/nvhe/hyp.lds.S   |  2 +-
>  8 files changed, 34 insertions(+), 33 deletions(-)

> diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
> index 89b6e78400023..1efe98909a2e0 100644
> --- a/arch/arm64/kernel/vdso.c
> +++ b/arch/arm64/kernel/vdso.c
> @@ -195,7 +195,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
>  
>  	vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
>  	/* Be sure to map the data page */
> -	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
> +	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE_MAX;
>  
>  	vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
>  	if (IS_ERR_VALUE(vdso_base)) {
> @@ -203,7 +203,8 @@ static int __setup_additional_pages(enum vdso_abi abi,
>  		goto up_fail;
>  	}
>  
> -	ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
> +	ret = _install_special_mapping(mm, vdso_base,
> +				       VVAR_NR_PAGES * PAGE_SIZE_MAX,
>  				       VM_READ|VM_MAYREAD|VM_PFNMAP,
>  				       vdso_info[abi].dm);
>  	if (IS_ERR(ret))
> @@ -212,7 +213,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
>  	if (system_supports_bti_kernel())
>  		gp_flags = VM_ARM64_BTI;
>  
> -	vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
> +	vdso_base += VVAR_NR_PAGES * PAGE_SIZE_MAX;
>  	mm->context.vdso = (void *)vdso_base;
>  	ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
>  				       VM_READ|VM_EXEC|gp_flags|

> diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
> index 45354f2ddf706..f7d1537a689e8 100644
> --- a/arch/arm64/kernel/vdso/vdso.lds.S
> +++ b/arch/arm64/kernel/vdso/vdso.lds.S
> @@ -18,9 +18,9 @@ OUTPUT_ARCH(aarch64)
>  
>  SECTIONS
>  {
> -	PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
> +	PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE_MAX);
>  #ifdef CONFIG_TIME_NS
> -	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
> +	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE_MAX);

This looks like it also needs a change to vvar_fault() in vdso.c.
The symbols are now always PAGE_SIZE_MAX apart, while vvar_fault() works
in page offsets (vmf->pgoff) that are based on the runtime PAGE_SIZE and
it expects hardcoded offsets.

As test you can use tools/testing/selftests/timens/timens.

(I can't test this right now, so it's only a suspicion)

>  #endif
>  	. = VDSO_LBASE + SIZEOF_HEADERS;

> diff --git a/arch/arm64/kernel/vdso32/vdso.lds.S b/arch/arm64/kernel/vdso32/vdso.lds.S
> index 8d95d7d35057d..c46d18a69d1ce 100644
> --- a/arch/arm64/kernel/vdso32/vdso.lds.S
> +++ b/arch/arm64/kernel/vdso32/vdso.lds.S
> @@ -18,9 +18,9 @@ OUTPUT_ARCH(arm)
>  
>  SECTIONS
>  {
> -	PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
> +	PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE_MAX);
>  #ifdef CONFIG_TIME_NS
> -	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE);
> +	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE_MAX);
>  #endif
>  	. = VDSO_LBASE + SIZEOF_HEADERS;

Re: [RFC PATCH v1 44/57] arm64: Align sections to PAGE_SIZE_MAX

[Ryan Roberts]

On 19/10/2024 15:16, Thomas Weißschuh wrote:
> On 2024-10-14 11:58:51+0100, Ryan Roberts wrote:
>> Increase alignment of sections in nvhe hyp, vdso and final vmlinux image
>> from PAGE_SIZE to PAGE_SIZE_MAX. For compile-time PAGE_SIZE,
>> PAGE_SIZE_MAX == PAGE_SIZE so there is no change. For boot-time
>> PAGE_SIZE, PAGE_SIZE_MAX is the largest selectable page size.
>>
>> For a boot-time page size build, image size is comparable to a 64K page
>> size compile-time build. In future, it may be desirable to optimize
>> run-time memory consumption by freeing unused padding pages when the
>> boot-time selected page size is less than PAGE_SIZE_MAX.
>>
>> Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
>> ---
>>
>> ***NOTE***
>> Any confused maintainers may want to read the cover note here for context:
>> https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/
>>
>>  arch/arm64/include/asm/memory.h     |  4 +--
>>  arch/arm64/kernel/vdso-wrap.S       |  4 +--
>>  arch/arm64/kernel/vdso.c            |  7 +++---
>>  arch/arm64/kernel/vdso/vdso.lds.S   |  4 +--
>>  arch/arm64/kernel/vdso32-wrap.S     |  4 +--
>>  arch/arm64/kernel/vdso32/vdso.lds.S |  4 +--
>>  arch/arm64/kernel/vmlinux.lds.S     | 38 ++++++++++++++---------------
>>  arch/arm64/kvm/hyp/nvhe/hyp.lds.S   |  2 +-
>>  8 files changed, 34 insertions(+), 33 deletions(-)
> 
>> diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
>> index 89b6e78400023..1efe98909a2e0 100644
>> --- a/arch/arm64/kernel/vdso.c
>> +++ b/arch/arm64/kernel/vdso.c
>> @@ -195,7 +195,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
>>  
>>  	vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
>>  	/* Be sure to map the data page */
>> -	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
>> +	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE_MAX;
>>  
>>  	vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
>>  	if (IS_ERR_VALUE(vdso_base)) {
>> @@ -203,7 +203,8 @@ static int __setup_additional_pages(enum vdso_abi abi,
>>  		goto up_fail;
>>  	}
>>  
>> -	ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
>> +	ret = _install_special_mapping(mm, vdso_base,
>> +				       VVAR_NR_PAGES * PAGE_SIZE_MAX,
>>  				       VM_READ|VM_MAYREAD|VM_PFNMAP,
>>  				       vdso_info[abi].dm);
>>  	if (IS_ERR(ret))
>> @@ -212,7 +213,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
>>  	if (system_supports_bti_kernel())
>>  		gp_flags = VM_ARM64_BTI;
>>  
>> -	vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
>> +	vdso_base += VVAR_NR_PAGES * PAGE_SIZE_MAX;
>>  	mm->context.vdso = (void *)vdso_base;
>>  	ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
>>  				       VM_READ|VM_EXEC|gp_flags|
> 
>> diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
>> index 45354f2ddf706..f7d1537a689e8 100644
>> --- a/arch/arm64/kernel/vdso/vdso.lds.S
>> +++ b/arch/arm64/kernel/vdso/vdso.lds.S
>> @@ -18,9 +18,9 @@ OUTPUT_ARCH(aarch64)
>>  
>>  SECTIONS
>>  {
>> -	PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
>> +	PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE_MAX);
>>  #ifdef CONFIG_TIME_NS
>> -	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
>> +	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE_MAX);
> 
> This looks like it also needs a change to vvar_fault() in vdso.c.
> The symbols are now always PAGE_SIZE_MAX apart, while vvar_fault() works
> in page offsets (vmf->pgoff) that are based on the runtime PAGE_SIZE and
> it expects hardcoded offsets.
> 
> As test you can use tools/testing/selftests/timens/timens.
> 
> (I can't test this right now, so it's only a suspicion)

Ahh good spot - that test does infact fail.

This fixes the problem:

---8<---
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 1efe98909a2e0..d2049ba6b19f5 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -151,10 +151,11 @@ int vdso_join_timens(struct task_struct *task, struct
time_namespace *ns)
 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
                             struct vm_area_struct *vma, struct vm_fault *vmf)
 {
+       pgoff_t pgmaxoff = vmf->pgoff >> (PAGE_SHIFT_MAX - PAGE_SHIFT);
        struct page *timens_page = find_timens_vvar_page(vma);
        unsigned long pfn;

-       switch (vmf->pgoff) {
+       switch (pgmaxoff) {
        case VVAR_DATA_PAGE_OFFSET:
                if (timens_page)
                        pfn = page_to_pfn(timens_page);
---8<---

I'll include it in the next version.

Thanks,
Ryan

> 
>>  #endif
>>  	. = VDSO_LBASE + SIZEOF_HEADERS;
> 
>> diff --git a/arch/arm64/kernel/vdso32/vdso.lds.S b/arch/arm64/kernel/vdso32/vdso.lds.S
>> index 8d95d7d35057d..c46d18a69d1ce 100644
>> --- a/arch/arm64/kernel/vdso32/vdso.lds.S
>> +++ b/arch/arm64/kernel/vdso32/vdso.lds.S
>> @@ -18,9 +18,9 @@ OUTPUT_ARCH(arm)
>>  
>>  SECTIONS
>>  {
>> -	PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
>> +	PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE_MAX);
>>  #ifdef CONFIG_TIME_NS
>> -	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE);
>> +	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE_MAX);
>>  #endif
>>  	. = VDSO_LBASE + SIZEOF_HEADERS;