[RFC PATCH 0/5] KVM: arm64: Use TTBR1_EL2 for hVHE runtime mappings

[RFC PATCH 0/5] KVM: arm64: Use TTBR1_EL2 for hVHE runtime mappings

[Aneesh Kumar K.V (Arm)]

hVHE currently disables TTBR1_EL2 and places both the identity mapping
and the hyp runtime mappings in TTBR0_EL2. As a result, it continues to
use the legacy nVHE virtual-address layout even though hVHE uses the
EL2&0 translation regime and can retain canonical kernel-image addresses.

This series moves the hVHE runtime mappings to TTBR1_EL2 and reserves
TTBR0_EL2 for the identity mapping used during MMU and page-table
transitions. Conventional VHE is unaffected, and nVHE continues to use
the existing TTBR0-only layout, including when protected mode is enabled.

The existing code does not consistently distinguish kernel-image symbols
from linear-map addresses because both currently use the same EL2
address transformation. The first four patches make this distinction
explicit by:

  - adding helpers for kernel-image symbol addresses;
  - splitting the hyp mapping APIs by address type;
  - providing separate symbol and linear VA-to-PA conversions; and
  - clarifying the private hyp VA allocation terminology.

The final patch implements the TTBR1 layout. Hyp symbols retain their
linked kernel-image VAs, while pools, per-CPU regions, SVE state and shared
memory continue to use linear-map addresses. A separate TTBR0 page table
contains the idmap, and both roots are installed during initial EL2 setup
and pKVM finalization.

The resulting layouts are:

  nVHE:
    TTBR0_EL2: idmap and runtime mappings
    TTBR1_EL2: unused

  hVHE:
    TTBR0_EL2: idmap
    TTBR1_EL2: runtime mappings

Cc: Marc Zyngier <maz@kernel.org> 
Cc: Oliver Upton <oupton@kernel.org>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Steffen Eiden <seiden@linux.ibm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Zenghui Yu <yuzenghui@huawei.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>

Aneesh Kumar K.V (Arm) (5):
  KVM: arm64: Make hyp symbol address conversion explicit
  KVM: arm64: Split hyp mapping APIs by address type
  KVM: arm64: Split hyp VA-to-PA conversion by address type
  KVM: arm64: Rename the hyp private VA allocation base
  KVM: arm64: Use TTBR1_EL2 for hVHE runtime mappings

 arch/arm64/include/asm/kvm_asm.h         |  13 +--
 arch/arm64/include/asm/kvm_host.h        |   2 +
 arch/arm64/include/asm/kvm_hyp.h         |   4 +-
 arch/arm64/include/asm/kvm_mmu.h         |  89 ++++++++++++++--
 arch/arm64/include/asm/kvm_pgtable.h     |  27 +++++
 arch/arm64/include/asm/kvm_pkvm.h        |   3 +
 arch/arm64/kernel/asm-offsets.c          |   5 +-
 arch/arm64/kvm/arm.c                     |  92 +++++++++-------
 arch/arm64/kvm/hyp/include/nvhe/memory.h |  13 +--
 arch/arm64/kvm/hyp/include/nvhe/mm.h     |   9 +-
 arch/arm64/kvm/hyp/nvhe/early_alloc.c    |   1 +
 arch/arm64/kvm/hyp/nvhe/events.c         |   2 +-
 arch/arm64/kvm/hyp/nvhe/host.S           |   4 +-
 arch/arm64/kvm/hyp/nvhe/hyp-init.S       |  29 ++++-
 arch/arm64/kvm/hyp/nvhe/mem_protect.c    |  30 +++---
 arch/arm64/kvm/hyp/nvhe/mm.c             | 110 +++++++++++++------
 arch/arm64/kvm/hyp/nvhe/psci-relay.c     |  12 +--
 arch/arm64/kvm/hyp/nvhe/setup.c          |  55 +++++++---
 arch/arm64/kvm/hyp/pgtable.c             |  62 ++++++++++-
 arch/arm64/kvm/hyp_trace.c               |   2 +-
 arch/arm64/kvm/mmu.c                     | 128 ++++++++++++++++-------
 arch/arm64/kvm/pkvm.c                    |   3 +
 arch/arm64/kvm/va_layout.c               |  32 +++++-
 23 files changed, 546 insertions(+), 181 deletions(-)


base-commit: a13c140cc289c0b7b3770bce5b3ad42ab35074aa
-- 
2.43.0

[RFC PATCH 1/5] KVM: arm64: Make hyp symbol address conversion explicit

[Aneesh Kumar K.V (Arm)]

kvm_ksym_ref() and kvm_ksym_ref_nvhe() have confusing names which do not
clearly describe the address conversion they perform.

Replace them with kvm_hyp_kimg_kaddr() for selecting the kernel address
of a hyp symbol and kern_sym_hyp_va() for obtaining its hyp VA. Use
lm_alias() directly where a linear-map address is required.

This is a naming cleanup with no functional change.

Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 arch/arm64/include/asm/kvm_asm.h | 10 ---------
 arch/arm64/include/asm/kvm_mmu.h | 33 +++++++++++++++++++++++++++
 arch/arm64/kvm/arm.c             | 38 ++++++++++++++++----------------
 arch/arm64/kvm/va_layout.c       |  2 +-
 4 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 043495f7fc78..6e0cc007bfe1 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -233,16 +233,6 @@ struct kvm_nvhe_stacktrace_info {
 	unsigned long pc;
 };
 
-/* Translate a kernel address @ptr into its equivalent linear mapping */
-#define kvm_ksym_ref(ptr)						\
-	({								\
-		void *val = (ptr);					\
-		if (!is_kernel_in_hyp_mode())				\
-			val = lm_alias((ptr));				\
-		val;							\
-	 })
-#define kvm_ksym_ref_nvhe(sym)	kvm_ksym_ref(kvm_nvhe_sym(sym))
-
 struct kvm;
 struct kvm_vcpu;
 struct kvm_s2_mmu;
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6eae7e7e2a68..6b708ff176fd 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -107,6 +107,28 @@ u32 kvm_hyp_va_bits(void);
 void kvm_apply_hyp_relocations(void);
 
 #define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset)
+/**
+ * kvm_hyp_kimg_kaddr - Select the kernel address used for an EL2 mapping
+ * @ptr: Address within the kernel image
+ *
+ * Return @ptr when EL2 uses the kernel-image address directly, as with
+ * VHE. Otherwise return the corresponding linear-map alias used as input
+ * to the legacy nVHE VA conversion.
+ *
+ * Return: A kernel address suitable as the basis for an EL2 virtual address.
+ */
+static __always_inline void *kvm_hyp_kimg_kaddr(void *ptr)
+{
+	/*
+	 * With VHE, the host runs at EL2 using the kernel mapping, so
+	 * kernel-image VAs need no translation.
+	 * Only legacy nVHE requires a linear-map alias.
+	 */
+	if (is_kernel_in_hyp_mode())
+		return ptr;
+
+	return lm_alias(ptr);
+}
 
 /*
  * Convert a kernel VA into a HYP VA.
@@ -140,6 +162,17 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
 
 #define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))
 
+/**
+ * kern_sym_hyp_va - Convert a kernel image symbol address to its runtime hyp VA
+ * @ptr: Address of a symbol in the kernel image
+ *
+ * Return: The runtime hyp virtual address corresponding to @ptr.
+ */
+static __always_inline void *kern_sym_hyp_va(void *ptr)
+{
+	return kern_hyp_va(kvm_hyp_kimg_kaddr(ptr));
+}
+
 extern u32 __hyp_va_bits;
 
 /*
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 50adfff75be8..e1613186efb0 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -2105,10 +2105,10 @@ static int kvm_init_vector_slots(void)
 	int err;
 	void *base;
 
-	base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
+	base = kern_sym_hyp_va(__kvm_hyp_vector);
 	kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
 
-	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
+	base = kern_sym_hyp_va(__bp_harden_hyp_vecs);
 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
 
 	if (kvm_system_needs_idmapped_vectors() &&
@@ -2136,7 +2136,7 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
 	 * Also drop the KASAN tag which gets in the way...
 	 */
 	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
-			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
+			    (unsigned long)kvm_hyp_kimg_kaddr(CHOOSE_NVHE_SYM(__per_cpu_start));
 
 	params->mair_el2 = read_sysreg(mair_el1);
 
@@ -2532,13 +2532,13 @@ static void __init teardown_hyp_mode(void)
 
 static int __init do_pkvm_init(u32 hyp_va_bits)
 {
-	void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
+	void *per_cpu_base = kvm_nvhe_sym(kvm_arm_hyp_percpu_base);
 	int ret;
 
 	preempt_disable();
 	cpu_hyp_init_context();
 	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
-				kern_hyp_va(per_cpu_base),
+				kern_sym_hyp_va(per_cpu_base),
 				hyp_va_bits);
 	cpu_hyp_init_features();
 
@@ -2611,8 +2611,8 @@ static void kvm_hyp_init_symbols(void)
 	 * Flush entire BSS since part of its data containing init symbols is read
 	 * while the MMU is off.
 	 */
-	kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start),
-				kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start));
+	kvm_flush_dcache_to_poc(lm_alias(__hyp_bss_start),
+				lm_alias(__hyp_bss_end) - lm_alias(__hyp_bss_start));
 }
 
 static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
@@ -2772,29 +2772,29 @@ static int __init init_hyp_mode(void)
 	/*
 	 * Map the Hyp-code called directly from the host
 	 */
-	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
-				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
+	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_text_start),
+				  kvm_hyp_kimg_kaddr(__hyp_text_end), PAGE_HYP_EXEC);
 	if (err) {
 		kvm_err("Cannot map world-switch code\n");
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start),
-				  kvm_ksym_ref(__hyp_data_end), PAGE_HYP);
+	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_data_start),
+				  kvm_hyp_kimg_kaddr(__hyp_data_end), PAGE_HYP);
 	if (err) {
 		kvm_err("Cannot map .hyp.data section\n");
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
-				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
+	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_rodata_start),
+				  kvm_hyp_kimg_kaddr(__hyp_rodata_end), PAGE_HYP_RO);
 	if (err) {
 		kvm_err("Cannot map .hyp.rodata section\n");
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
-				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
+	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__start_rodata),
+				  kvm_hyp_kimg_kaddr(__end_rodata), PAGE_HYP_RO);
 	if (err) {
 		kvm_err("Cannot map rodata section\n");
 		goto out_err;
@@ -2805,15 +2805,15 @@ static int __init init_hyp_mode(void)
 	 * section thanks to an assertion in the linker script. Map it RW and
 	 * the rest of .bss RO.
 	 */
-	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
-				  kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
+	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_start),
+				  kvm_hyp_kimg_kaddr(__hyp_bss_end), PAGE_HYP);
 	if (err) {
 		kvm_err("Cannot map hyp bss section: %d\n", err);
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
-				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
+	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_end),
+				  kvm_hyp_kimg_kaddr(__bss_stop), PAGE_HYP_RO);
 	if (err) {
 		kvm_err("Cannot map bss section\n");
 		goto out_err;
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 2346f9435a71..63bb58a7868a 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -223,7 +223,7 @@ void kvm_patch_vector_branch(struct alt_instr *alt,
 	/*
 	 * Compute HYP VA by using the same computation as kern_hyp_va()
 	 */
-	addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
+	addr = __early_kern_hyp_va((u64)kvm_hyp_kimg_kaddr(__kvm_hyp_vector));
 
 	/* Use PC[10:7] to branch to the same vector in KVM */
 	addr |= ((u64)origptr & GENMASK_ULL(10, 7));
-- 
2.43.0

[RFC PATCH 2/5] KVM: arm64: Split hyp mapping APIs by address type

[Aneesh Kumar K.V (Arm)]

Kernel-image symbols and linear-map addresses may refer to the same
physical memory, but they are not interchangeable when constructing the
EL2 address space. A kernel-image symbol requires __pa_symbol() for its
physical address and kern_sym_hyp_va() for its runtime hyp VA, whereas a
linear-map address requires __pa() and kern_hyp_va().

The existing create_hyp_mappings() and pkvm_create_mappings() interfaces
do not express this distinction. Host callers prepare symbol addresses
before invoking create_hyp_mappings(), while the common implementation
attempts to infer the physical address from the resulting pointer. The
pKVM implementation similarly assumes that every input can be translated
using the linear hyp VA-to-PA offset.

This ambiguity works while hyp symbols and linear allocations share the
same fixed-offset translation. A later patch introduces an hVHE TTBR1
layout which retains kernel-image VAs for hyp symbols. That layout
requires symbol addresses to be distinguished from linear-map addresses.

Replace create_hyp_mappings() with explicit create_hyp_symbol_mappings()
and create_hyp_linear_mappings() interfaces. Resolve the appropriate hyp
VA and PA in each wrapper, then pass those values to a common mapping
helper. This also removes the need for the generic
kernel-address-to-physical-address classifier.

Apply the same separation to the pKVM mapping code. Introduce symbol and
linear-map wrappers around an internal helper which receives an already
resolved hyp VA range and starting PA. Retain a locked linear variant for
callers which already hold the hyp page-table lock.

Update all callers to use the interface matching the address type they
provide.

Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 arch/arm64/include/asm/kvm_mmu.h      |  5 +-
 arch/arm64/kvm/arm.c                  | 29 +++++------
 arch/arm64/kvm/hyp/include/nvhe/mm.h  |  8 ++-
 arch/arm64/kvm/hyp/nvhe/mem_protect.c | 16 ++++--
 arch/arm64/kvm/hyp/nvhe/mm.c          | 46 ++++++++++++-----
 arch/arm64/kvm/hyp/nvhe/setup.c       | 18 ++++---
 arch/arm64/kvm/hyp_trace.c            |  2 +-
 arch/arm64/kvm/mmu.c                  | 71 +++++++++++++++------------
 8 files changed, 122 insertions(+), 73 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6b708ff176fd..8c2972aa83b0 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -190,7 +190,10 @@ extern u32 __hyp_va_bits;
 
 int kvm_share_hyp(void *from, void *to);
 void kvm_unshare_hyp(void *from, void *to);
-int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
+int create_hyp_linear_mappings(void *from, void *to,
+			       enum kvm_pgtable_prot prot);
+int create_hyp_symbol_mappings(void *from, void *to,
+			       enum kvm_pgtable_prot prot);
 int __create_hyp_mappings(unsigned long start, unsigned long size,
 			  unsigned long phys, enum kvm_pgtable_prot prot);
 int hyp_alloc_private_va_range(size_t size, unsigned long *haddr);
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index e1613186efb0..e498bb780382 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -2620,7 +2620,7 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
 	void *addr = phys_to_virt(hyp_mem_base);
 	int ret;
 
-	ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
+	ret = create_hyp_linear_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
 	if (ret)
 		return ret;
 
@@ -2772,29 +2772,29 @@ static int __init init_hyp_mode(void)
 	/*
 	 * Map the Hyp-code called directly from the host
 	 */
-	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_text_start),
-				  kvm_hyp_kimg_kaddr(__hyp_text_end), PAGE_HYP_EXEC);
+	err = create_hyp_symbol_mappings(__hyp_text_start, __hyp_text_end,
+					 PAGE_HYP_EXEC);
 	if (err) {
 		kvm_err("Cannot map world-switch code\n");
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_data_start),
-				  kvm_hyp_kimg_kaddr(__hyp_data_end), PAGE_HYP);
+	err = create_hyp_symbol_mappings(__hyp_data_start, __hyp_data_end,
+					 PAGE_HYP);
 	if (err) {
 		kvm_err("Cannot map .hyp.data section\n");
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_rodata_start),
-				  kvm_hyp_kimg_kaddr(__hyp_rodata_end), PAGE_HYP_RO);
+	err = create_hyp_symbol_mappings(__hyp_rodata_start, __hyp_rodata_end,
+					 PAGE_HYP_RO);
 	if (err) {
 		kvm_err("Cannot map .hyp.rodata section\n");
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__start_rodata),
-				  kvm_hyp_kimg_kaddr(__end_rodata), PAGE_HYP_RO);
+	err = create_hyp_symbol_mappings(__start_rodata, __end_rodata,
+					 PAGE_HYP_RO);
 	if (err) {
 		kvm_err("Cannot map rodata section\n");
 		goto out_err;
@@ -2805,15 +2805,15 @@ static int __init init_hyp_mode(void)
 	 * section thanks to an assertion in the linker script. Map it RW and
 	 * the rest of .bss RO.
 	 */
-	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_start),
-				  kvm_hyp_kimg_kaddr(__hyp_bss_end), PAGE_HYP);
+	err = create_hyp_symbol_mappings(__hyp_bss_start, __hyp_bss_end,
+					 PAGE_HYP);
 	if (err) {
 		kvm_err("Cannot map hyp bss section: %d\n", err);
 		goto out_err;
 	}
 
-	err = create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_end),
-				  kvm_hyp_kimg_kaddr(__bss_stop), PAGE_HYP_RO);
+	err = create_hyp_symbol_mappings(__hyp_bss_end, __bss_stop,
+					 PAGE_HYP_RO);
 	if (err) {
 		kvm_err("Cannot map bss section\n");
 		goto out_err;
@@ -2846,7 +2846,8 @@ static int __init init_hyp_mode(void)
 		char *percpu_end = percpu_begin + nvhe_percpu_size();
 
 		/* Map Hyp percpu pages */
-		err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
+		err = create_hyp_linear_mappings(percpu_begin, percpu_end,
+						 PAGE_HYP);
 		if (err) {
 			kvm_err("Cannot map hyp percpu region\n");
 			goto out_err;
diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h
index 6e83ce35c2f2..c41152c84705 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/mm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h
@@ -23,8 +23,12 @@ int hyp_create_idmap(u32 hyp_va_bits);
 int hyp_map_vectors(void);
 int hyp_back_vmemmap(phys_addr_t back);
 int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);
-int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
-int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);
+int pkvm_create_linear_mappings(void *from, void *to,
+				enum kvm_pgtable_prot prot);
+int pkvm_create_symbol_mappings(void *from, void *to,
+				enum kvm_pgtable_prot prot);
+int pkvm_create_linear_mappings_locked(void *from, void *to,
+				       enum kvm_pgtable_prot prot);
 int __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
 				  enum kvm_pgtable_prot prot,
 				  unsigned long *haddr);
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 4e329e39a695..7f815345d6a5 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -1109,7 +1109,7 @@ int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
 		goto unlock;
 
 	__hyp_set_page_state_range(phys, size, PKVM_PAGE_OWNED);
-	WARN_ON(pkvm_create_mappings_locked(virt, virt + size, PAGE_HYP));
+	WARN_ON(pkvm_create_linear_mappings_locked(virt, virt + size, PAGE_HYP));
 	WARN_ON(host_stage2_set_owner_locked(phys, size, PKVM_ID_HYP));
 
 unlock:
@@ -1175,12 +1175,18 @@ int hyp_pin_shared_mem(void *from, void *to)
 		goto unlock;
 
 	for (cur = start; cur < end; cur += PAGE_SIZE) {
+		void *addr = (void *)cur;
+
 		p = hyp_virt_to_page(cur);
 		hyp_page_ref_inc(p);
-		if (p->refcount == 1)
-			WARN_ON(pkvm_create_mappings_locked((void *)cur,
-							    (void *)cur + PAGE_SIZE,
-							    PAGE_HYP));
+		if (p->refcount == 1) {
+			int err;
+
+			err = pkvm_create_linear_mappings_locked(addr,
+								 addr + PAGE_SIZE,
+								 PAGE_HYP);
+			WARN_ON(err);
+		}
 	}
 
 unlock:
diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
index 3b0bee496bff..9150ff94b01a 100644
--- a/arch/arm64/kvm/hyp/nvhe/mm.c
+++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -110,23 +110,21 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
 	return err;
 }
 
-int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot)
+static int __pkvm_create_mappings_locked(unsigned long start,
+					 unsigned long end,
+					 phys_addr_t phys,
+					 enum kvm_pgtable_prot prot)
 {
-	unsigned long start = (unsigned long)from;
-	unsigned long end = (unsigned long)to;
-	unsigned long virt_addr;
-	phys_addr_t phys;
-
 	hyp_assert_lock_held(&pkvm_pgd_lock);
 
 	start = start & PAGE_MASK;
 	end = PAGE_ALIGN(end);
+	phys = ALIGN_DOWN(phys, PAGE_SIZE);
 
-	for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
+	for (; start < end; start += PAGE_SIZE, phys += PAGE_SIZE) {
 		int err;
 
-		phys = hyp_virt_to_phys((void *)virt_addr);
-		err = kvm_pgtable_hyp_map(&pkvm_pgtable, virt_addr, PAGE_SIZE,
+		err = kvm_pgtable_hyp_map(&pkvm_pgtable, start, PAGE_SIZE,
 					  phys, prot);
 		if (err)
 			return err;
@@ -135,12 +133,38 @@ int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot
 	return 0;
 }
 
-int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
+int pkvm_create_linear_mappings_locked(void *from, void *to,
+				       enum kvm_pgtable_prot prot)
+{
+	unsigned long start = (unsigned long)from & PAGE_MASK;
+	phys_addr_t phys = hyp_virt_to_phys((void *)start);
+
+	return __pkvm_create_mappings_locked(start, (unsigned long)to,
+					     phys, prot);
+}
+
+int pkvm_create_linear_mappings(void *from, void *to,
+				enum kvm_pgtable_prot prot)
+{
+	int ret;
+
+	hyp_spin_lock(&pkvm_pgd_lock);
+	ret = pkvm_create_linear_mappings_locked(from, to, prot);
+	hyp_spin_unlock(&pkvm_pgd_lock);
+
+	return ret;
+}
+
+int pkvm_create_symbol_mappings(void *from, void *to,
+				enum kvm_pgtable_prot prot)
 {
+	unsigned long start = (unsigned long)from & PAGE_MASK;
+	phys_addr_t phys = __hyp_pa(start);
 	int ret;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
-	ret = pkvm_create_mappings_locked(from, to, prot);
+	ret = __pkvm_create_mappings_locked(start, (unsigned long)to,
+					    phys, prot);
 	hyp_spin_unlock(&pkvm_pgd_lock);
 
 	return ret;
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 75b00c323310..66a3d01cd79e 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -86,7 +86,7 @@ static int pkvm_create_host_sve_mappings(void)
 
 		start = kern_hyp_va(sve_regs);
 		end = start + PAGE_ALIGN(pkvm_host_sve_state_size());
-		ret = pkvm_create_mappings(start, end, PAGE_HYP);
+		ret = pkvm_create_linear_mappings(start, end, PAGE_HYP);
 		if (ret)
 			return ret;
 	}
@@ -121,23 +121,27 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
 	if (ret)
 		return ret;
 
-	ret = pkvm_create_mappings(__hyp_text_start, __hyp_text_end, PAGE_HYP_EXEC);
+	ret = pkvm_create_symbol_mappings(__hyp_text_start, __hyp_text_end,
+					  PAGE_HYP_EXEC);
 	if (ret)
 		return ret;
 
-	ret = pkvm_create_mappings(__hyp_data_start, __hyp_data_end, PAGE_HYP);
+	ret = pkvm_create_symbol_mappings(__hyp_data_start, __hyp_data_end,
+					  PAGE_HYP);
 	if (ret)
 		return ret;
 
-	ret = pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_HYP_RO);
+	ret = pkvm_create_symbol_mappings(__hyp_rodata_start, __hyp_rodata_end,
+					  PAGE_HYP_RO);
 	if (ret)
 		return ret;
 
-	ret = pkvm_create_mappings(__hyp_bss_start, __hyp_bss_end, PAGE_HYP);
+	ret = pkvm_create_symbol_mappings(__hyp_bss_start, __hyp_bss_end,
+					  PAGE_HYP);
 	if (ret)
 		return ret;
 
-	ret = pkvm_create_mappings(virt, virt + size, PAGE_HYP);
+	ret = pkvm_create_linear_mappings(virt, virt + size, PAGE_HYP);
 	if (ret)
 		return ret;
 
@@ -146,7 +150,7 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
 
 		start = (void *)kern_hyp_va(per_cpu_base[i]);
 		end = start + PAGE_ALIGN(hyp_percpu_size);
-		ret = pkvm_create_mappings(start, end, PAGE_HYP);
+		ret = pkvm_create_linear_mappings(start, end, PAGE_HYP);
 		if (ret)
 			return ret;
 
diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c
index 2411b4c32932..0cf073682173 100644
--- a/arch/arm64/kvm/hyp_trace.c
+++ b/arch/arm64/kvm/hyp_trace.c
@@ -142,7 +142,7 @@ static int __map_hyp(void *start, size_t size)
 	if (is_protected_kvm_enabled())
 		return 0;
 
-	return create_hyp_mappings(start, start + size, PAGE_HYP);
+	return create_hyp_linear_mappings(start, start + size, PAGE_HYP);
 }
 
 static int __share_page(unsigned long va)
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 6c941aaa10c6..154d53dca619 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -436,17 +436,6 @@ int __create_hyp_mappings(unsigned long start, unsigned long size,
 	return err;
 }
 
-static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
-{
-	if (!is_vmalloc_addr(kaddr)) {
-		BUG_ON(!virt_addr_valid(kaddr));
-		return __pa(kaddr);
-	} else {
-		return page_to_phys(vmalloc_to_page(kaddr)) +
-		       offset_in_page(kaddr);
-	}
-}
-
 struct hyp_shared_pfn {
 	u64 pfn;
 	int count;
@@ -559,7 +548,7 @@ int kvm_share_hyp(void *from, void *to)
 		return -EINVAL;
 
 	if (kvm_host_owns_hyp_mappings())
-		return create_hyp_mappings(from, to, PAGE_HYP);
+		return create_hyp_linear_mappings(from, to, PAGE_HYP);
 
 	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
 	end = PAGE_ALIGN(__pa(to));
@@ -608,23 +597,10 @@ void kvm_unshare_hyp(void *from, void *to)
 	}
 }
 
-/**
- * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
- * @from:	The virtual kernel start address of the range
- * @to:		The virtual kernel end address of the range (exclusive)
- * @prot:	The protection to be applied to this range
- *
- * The same virtual address as the kernel virtual address is also used
- * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
- * physical pages.
- */
-int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
+static int __create_hyp_mapping_range(unsigned long start, unsigned long end,
+				      phys_addr_t phys,
+				      enum kvm_pgtable_prot prot)
 {
-	phys_addr_t phys_addr;
-	unsigned long virt_addr;
-	unsigned long start = kern_hyp_va((unsigned long)from);
-	unsigned long end = kern_hyp_va((unsigned long)to);
-
 	if (is_kernel_in_hyp_mode())
 		return 0;
 
@@ -633,13 +609,12 @@ int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
 
 	start = start & PAGE_MASK;
 	end = PAGE_ALIGN(end);
+	phys = ALIGN_DOWN(phys, PAGE_SIZE);
 
-	for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
+	for (; start < end; start += PAGE_SIZE, phys += PAGE_SIZE) {
 		int err;
 
-		phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
-		err = __create_hyp_mappings(virt_addr, PAGE_SIZE, phys_addr,
-					    prot);
+		err = __create_hyp_mappings(start, PAGE_SIZE, phys, prot);
 		if (err)
 			return err;
 	}
@@ -647,6 +622,38 @@ int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
 	return 0;
 }
 
+/**
+ * create_hyp_linear_mappings - map a host linear-map range at EL2
+ * @from:	The host linear-map start address
+ * @to:		The host linear-map end address (exclusive)
+ * @prot:	The protection to apply to the mapping
+ */
+int create_hyp_linear_mappings(void *from, void *to,
+			       enum kvm_pgtable_prot prot)
+{
+	unsigned long start = kern_hyp_va((unsigned long)from);
+	unsigned long end = kern_hyp_va((unsigned long)to);
+	phys_addr_t phys = __pa((unsigned long)from & PAGE_MASK);
+
+	return __create_hyp_mapping_range(start, end, phys, prot);
+}
+
+/**
+ * create_hyp_symbol_mappings - map a kernel-image symbol range at EL2
+ * @from:	The kernel-image symbol at the start of the range
+ * @to:		The kernel-image symbol at the end of the range (exclusive)
+ * @prot:	The protection to apply to the mapping
+ */
+int create_hyp_symbol_mappings(void *from, void *to,
+			       enum kvm_pgtable_prot prot)
+{
+	unsigned long start = (unsigned long)kern_sym_hyp_va(from);
+	unsigned long end = (unsigned long)kern_sym_hyp_va(to);
+	phys_addr_t phys = __pa_symbol((unsigned long)from & PAGE_MASK);
+
+	return __create_hyp_mapping_range(start, end, phys, prot);
+}
+
 static int __hyp_alloc_private_va_range(unsigned long base)
 {
 	lockdep_assert_held(&kvm_hyp_pgd_mutex);
-- 
2.43.0

[RFC PATCH 3/5] KVM: arm64: Split hyp VA-to-PA conversion by address type

[Aneesh Kumar K.V (Arm)]

The existing hyp VA-to-PA conversion uses one physvirt offset for both
kernel-image symbols and linear-map addresses. This only works while both
address types use the same EL2 VA transformation.

With hVHE TTBR1, hyp symbols retain their kernel-image VAs while pools,
per-CPU data and shared memory remain linear-map-derived. Introduce a
separate symbol physvirt offset and explicit symbol and linear conversion
helpers.

Update every caller according to its address type and remove the
ambiguous hyp_pa and __hyp_pa helpers. The offsets remain same as
hyp_physvirt_offset, making this preparatory for the TTBR1 layout.

Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 arch/arm64/include/asm/kvm_host.h        |  2 ++
 arch/arm64/include/asm/kvm_mmu.h         | 17 ++++++++++++-----
 arch/arm64/kvm/arm.c                     |  2 +-
 arch/arm64/kvm/hyp/include/nvhe/memory.h | 13 +++++++------
 arch/arm64/kvm/hyp/nvhe/early_alloc.c    |  1 +
 arch/arm64/kvm/hyp/nvhe/events.c         |  2 +-
 arch/arm64/kvm/hyp/nvhe/host.S           |  4 ++--
 arch/arm64/kvm/hyp/nvhe/mem_protect.c    | 14 +++++++-------
 arch/arm64/kvm/hyp/nvhe/mm.c             | 11 ++++++-----
 arch/arm64/kvm/hyp/nvhe/psci-relay.c     | 12 ++++++------
 arch/arm64/kvm/hyp/nvhe/setup.c          |  4 ++--
 arch/arm64/kvm/va_layout.c               |  2 ++
 12 files changed, 49 insertions(+), 35 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index bae2c4f92ef5..d92e0d066de3 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -827,6 +827,8 @@ extern struct kvm_host_psci_config kvm_nvhe_sym(kvm_host_psci_config);
 
 extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
 #define hyp_physvirt_offset CHOOSE_NVHE_SYM(hyp_physvirt_offset)
+extern s64 kvm_nvhe_sym(hyp_symbol_physvirt_offset);
+#define hyp_symbol_physvirt_offset CHOOSE_NVHE_SYM(hyp_symbol_physvirt_offset)
 
 extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS];
 #define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 8c2972aa83b0..bcc8e28985f7 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -54,15 +54,21 @@
 #include <asm/alternative.h>
 
 /*
- * Convert a hypervisor VA to a PA
+ * Convert an address in the hypervisor linear map to a PA
  * reg: hypervisor address to be converted in place
  * tmp: temporary register
  */
-.macro hyp_pa reg, tmp
+.macro hyp_linear_pa reg, tmp
 	ldr_l	\tmp, hyp_physvirt_offset
 	add	\reg, \reg, \tmp
 .endm
 
+/* Convert a hypervisor kernel-image VA to a PA. */
+.macro hyp_symbol_pa reg, tmp
+	ldr_l	\tmp, hyp_symbol_physvirt_offset
+	add	\reg, \reg, \tmp
+.endm
+
 /*
  * Convert a hypervisor VA to a kernel image address
  * reg: hypervisor address to be converted in place
@@ -74,8 +80,8 @@
  * specific registers encoded in the instructions).
  */
 .macro hyp_kimg_va reg, tmp
-	/* Convert hyp VA -> PA. */
-	hyp_pa	\reg, \tmp
+	/* Convert hyp kernel-image VA -> PA. */
+	hyp_symbol_pa	\reg, \tmp
 
 	/* Load kimage_voffset. */
 alternative_cb ARM64_ALWAYS_SYSTEM, kvm_get_kimage_voffset
@@ -106,7 +112,6 @@ void kvm_compute_layout(void);
 u32 kvm_hyp_va_bits(void);
 void kvm_apply_hyp_relocations(void);
 
-#define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset)
 /**
  * kvm_hyp_kimg_kaddr - Select the kernel address used for an EL2 mapping
  * @ptr: Address within the kernel image
@@ -130,6 +135,8 @@ static __always_inline void *kvm_hyp_kimg_kaddr(void *ptr)
 	return lm_alias(ptr);
 }
 
+#define __hyp_linear_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset)
+#define __hyp_symbol_pa(x) (((phys_addr_t)(x)) + hyp_symbol_physvirt_offset)
 /*
  * Convert a kernel VA into a HYP VA.
  *
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index e498bb780382..924f33c1a9d4 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -2835,7 +2835,7 @@ static int __init init_hyp_mode(void)
 		/*
 		 * Save the stack PA in nvhe_init_params. This will be needed
 		 * to recreate the stack mapping in protected nVHE mode.
-		 * __hyp_pa() won't do the right thing there, since the stack
+		 * __hyp_linear_pa() won't do the right thing there, since the stack
 		 * has been mapped in the flexible private VA space.
 		 */
 		params->stack_pa = __pa(stack_base);
diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/include/nvhe/memory.h
index b50712d47f6d..a7c3e70f7256 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/memory.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h
@@ -73,16 +73,17 @@ struct hyp_page {
 extern u64 __hyp_vmemmap;
 #define hyp_vmemmap ((struct hyp_page *)__hyp_vmemmap)
 
-#define __hyp_va(phys)	((void *)((phys_addr_t)(phys) - hyp_physvirt_offset))
+#define __hyp_linear_va(phys) \
+	((void *)((phys_addr_t)(phys) - hyp_physvirt_offset))
 
 static inline void *hyp_phys_to_virt(phys_addr_t phys)
 {
-	return __hyp_va(phys);
+	return __hyp_linear_va(phys);
 }
 
 static inline phys_addr_t hyp_virt_to_phys(void *addr)
 {
-	return __hyp_pa(addr);
+	return __hyp_linear_pa(addr);
 }
 
 #define hyp_phys_to_pfn(phys)	((phys) >> PAGE_SHIFT)
@@ -94,12 +95,12 @@ static inline struct hyp_page *hyp_phys_to_page(phys_addr_t phys)
 	return &hyp_vmemmap[hyp_phys_to_pfn(phys)];
 }
 
-#define hyp_virt_to_page(virt)	hyp_phys_to_page(__hyp_pa(virt))
-#define hyp_virt_to_pfn(virt)	hyp_phys_to_pfn(__hyp_pa(virt))
+#define hyp_virt_to_page(virt)	hyp_phys_to_page(__hyp_linear_pa(virt))
+#define hyp_virt_to_pfn(virt)	hyp_phys_to_pfn(__hyp_linear_pa(virt))
 
 #define hyp_page_to_pfn(page)	((struct hyp_page *)(page) - hyp_vmemmap)
 #define hyp_page_to_phys(page)  hyp_pfn_to_phys((hyp_page_to_pfn(page)))
-#define hyp_page_to_virt(page)	__hyp_va(hyp_page_to_phys(page))
+#define hyp_page_to_virt(page)	__hyp_linear_va(hyp_page_to_phys(page))
 #define hyp_page_to_pool(page)	(((struct hyp_page *)page)->pool)
 
 static inline enum pkvm_page_state get_host_state(struct hyp_page *p)
diff --git a/arch/arm64/kvm/hyp/nvhe/early_alloc.c b/arch/arm64/kvm/hyp/nvhe/early_alloc.c
index 00de04153cc6..6002c61ae62c 100644
--- a/arch/arm64/kvm/hyp/nvhe/early_alloc.c
+++ b/arch/arm64/kvm/hyp/nvhe/early_alloc.c
@@ -11,6 +11,7 @@
 
 struct kvm_pgtable_mm_ops hyp_early_alloc_mm_ops;
 s64 __ro_after_init hyp_physvirt_offset;
+s64 __ro_after_init hyp_symbol_physvirt_offset;
 
 static unsigned long base;
 static unsigned long end;
diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c
index add9383aadb5..4204cf81f0e4 100644
--- a/arch/arm64/kvm/hyp/nvhe/events.c
+++ b/arch/arm64/kvm/hyp/nvhe/events.c
@@ -17,7 +17,7 @@ int __tracing_enable_event(unsigned short id, bool enable)
 	if (event_id >= __hyp_event_ids_end)
 		return -EINVAL;
 
-	enabled = hyp_fixmap_map(__hyp_pa(&event_id->enabled));
+	enabled = hyp_fixmap_map(__hyp_symbol_pa(&event_id->enabled));
 	atomic_set(enabled, enable);
 	hyp_fixmap_unmap();
 
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 9393fe3ea6a1..079788f152d1 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -133,7 +133,7 @@ SYM_FUNC_START(__hyp_do_panic)
 	mrs	x0, esr_el2
 	mov	x4, x3
 	mov	x3, x2
-	hyp_pa	x3, x6
+	hyp_symbol_pa	x3, x6
 	get_vcpu_ptr x5, x6
 	mrs	x6, far_el2
 	mrs	x7, hpfar_el2
@@ -163,7 +163,7 @@ alternative_else_nop_endif
 	 * Preserve x0-x4, which may contain stub parameters.
 	 */
 	adr_l	x5, __kvm_handle_stub_hvc
-	hyp_pa	x5, x6
+	hyp_symbol_pa	x5, x6
 	br	x5
 SYM_FUNC_END(__host_hvc)
 
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 7f815345d6a5..593359d04053 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -175,7 +175,7 @@ int kvm_host_prepare_stage2(void *pgt_pool_base)
 	if (ret)
 		return ret;
 
-	mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);
+	mmu->pgd_phys = __hyp_linear_pa(host_mmu.pgt.pgd);
 	mmu->pgt = &host_mmu.pgt;
 	atomic64_set(&mmu->vmid.id, 0);
 
@@ -243,9 +243,9 @@ static void __apply_guest_page(void *va, size_t size,
 		void *map;
 
 		if (IS_ALIGNED((unsigned long)va, PMD_SIZE) && size >= PMD_SIZE)
-			map = hyp_fixblock_map(__hyp_pa(va), &map_size);
+			map = hyp_fixblock_map(__hyp_linear_pa(va), &map_size);
 		else
-			map = hyp_fixmap_map(__hyp_pa(va));
+			map = hyp_fixmap_map(__hyp_linear_pa(va));
 
 		func(map, map_size);
 
@@ -300,7 +300,7 @@ int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd)
 	if (ret)
 		return ret;
 
-	vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
+	vm->kvm.arch.mmu.pgd_phys = __hyp_linear_pa(vm->pgt.pgd);
 
 	return 0;
 }
@@ -1092,7 +1092,7 @@ int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
 {
 	u64 phys = hyp_pfn_to_phys(pfn);
 	u64 size = PAGE_SIZE * nr_pages;
-	void *virt = __hyp_va(phys);
+	void *virt = __hyp_linear_va(phys);
 	int ret;
 
 	if (!pfn_range_is_valid(pfn, nr_pages))
@@ -1123,7 +1123,7 @@ int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages)
 {
 	u64 phys = hyp_pfn_to_phys(pfn);
 	u64 size = PAGE_SIZE * nr_pages;
-	u64 virt = (u64)__hyp_va(phys);
+	u64 virt = (u64)__hyp_linear_va(phys);
 	int ret;
 
 	if (!pfn_range_is_valid(pfn, nr_pages))
@@ -1158,7 +1158,7 @@ int hyp_pin_shared_mem(void *from, void *to)
 {
 	u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
 	u64 end = PAGE_ALIGN((u64)to);
-	u64 phys = __hyp_pa(start);
+	u64 phys = __hyp_linear_pa(start);
 	u64 size = end - start;
 	struct hyp_page *p;
 	int ret;
diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
index 9150ff94b01a..061fca95a78a 100644
--- a/arch/arm64/kvm/hyp/nvhe/mm.c
+++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -159,7 +159,7 @@ int pkvm_create_symbol_mappings(void *from, void *to,
 				enum kvm_pgtable_prot prot)
 {
 	unsigned long start = (unsigned long)from & PAGE_MASK;
-	phys_addr_t phys = __hyp_pa(start);
+	phys_addr_t phys = __hyp_symbol_pa(start);
 	int ret;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
@@ -242,7 +242,7 @@ int hyp_map_vectors(void)
 		return 0;
 	}
 
-	phys = __hyp_pa(__bp_harden_hyp_vecs);
+	phys = __hyp_symbol_pa(__bp_harden_hyp_vecs);
 	ret = __pkvm_create_private_mapping(phys, __BP_HARDEN_HYP_VECS_SZ,
 					    PAGE_HYP_EXEC, &bp_base);
 	if (ret)
@@ -418,7 +418,8 @@ int hyp_create_fixmap(void)
 			return ret;
 
 		ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, PAGE_SIZE,
-					  __hyp_pa(__hyp_bss_start), PAGE_HYP);
+					  __hyp_symbol_pa(__hyp_bss_start),
+					  PAGE_HYP);
 		if (ret)
 			return ret;
 
@@ -434,10 +435,10 @@ int hyp_create_idmap(u32 hyp_va_bits)
 {
 	unsigned long start, end;
 
-	start = hyp_virt_to_phys((void *)__hyp_idmap_text_start);
+	start = __hyp_symbol_pa(__hyp_idmap_text_start);
 	start = ALIGN_DOWN(start, PAGE_SIZE);
 
-	end = hyp_virt_to_phys((void *)__hyp_idmap_text_end);
+	end = __hyp_symbol_pa(__hyp_idmap_text_end);
 	end = ALIGN(end, PAGE_SIZE);
 
 	/*
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index e20db999e328..91b658ac74ee 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -139,8 +139,8 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt)
 	wmb();
 
 	ret = psci_call(func_id, mpidr,
-			__hyp_pa(&kvm_hyp_cpu_entry),
-			__hyp_pa(init_params));
+			__hyp_symbol_pa(&kvm_hyp_cpu_entry),
+			__hyp_linear_pa(init_params));
 
 	/* If successful, the lock will be released by the target CPU. */
 	if (ret != PSCI_RET_SUCCESS)
@@ -173,8 +173,8 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
 	 * point if it is a deep sleep state.
 	 */
 	return psci_call(func_id, power_state,
-			 __hyp_pa(&kvm_hyp_cpu_resume),
-			 __hyp_pa(init_params));
+			 __hyp_symbol_pa(&kvm_hyp_cpu_resume),
+			 __hyp_linear_pa(init_params));
 }
 
 static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
@@ -197,8 +197,8 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
 
 	/* Will only return on error. */
 	return psci_call(func_id,
-			 __hyp_pa(&kvm_hyp_cpu_resume),
-			 __hyp_pa(init_params), 0);
+			 __hyp_symbol_pa(&kvm_hyp_cpu_resume),
+			 __hyp_linear_pa(init_params), 0);
 }
 
 static void __noreturn __kvm_host_psci_cpu_entry(unsigned long pc, unsigned long r0)
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 66a3d01cd79e..2ef1972cc3dd 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -169,7 +169,7 @@ static void update_nvhe_init_params(void)
 
 	for (i = 0; i < hyp_nr_cpus; i++) {
 		params = per_cpu_ptr(&kvm_init_params, i);
-		params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
+		params->pgd_pa = __hyp_linear_pa(pkvm_pgtable.pgd);
 		dcache_clean_inval_poc((unsigned long)params,
 				    (unsigned long)params + sizeof(*params));
 	}
@@ -371,7 +371,7 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_bas
 
 	/* Jump in the idmap page to switch to the new page-tables */
 	params = this_cpu_ptr(&kvm_init_params);
-	fn = (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd);
+	fn = (typeof(fn))__hyp_symbol_pa(__pkvm_init_switch_pgd);
 	fn(params->pgd_pa, params->stack_hyp_va, __pkvm_init_finalise);
 
 	unreachable();
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 63bb58a7868a..75a89ad8fecf 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -44,6 +44,8 @@ static void init_hyp_physvirt_offset(void)
 	kern_va = (u64)lm_alias(__hyp_text_start);
 	hyp_va = __early_kern_hyp_va(kern_va);
 	hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
+
+	hyp_symbol_physvirt_offset = hyp_physvirt_offset;
 }
 
 /*
-- 
2.43.0

[RFC PATCH 4/5] KVM: arm64: Rename the hyp private VA allocation base

[Aneesh Kumar K.V (Arm)]

io_map_base is not used for I/O mappings. It is the allocation base for
private hyp VAs used by stacks, fixmaps and others.

Rename it to __hyp_private_va_base in the hyp code

No functional change in this patch.

Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 arch/arm64/kvm/hyp/nvhe/mm.c | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
index 061fca95a78a..3233a7c70f7c 100644
--- a/arch/arm64/kvm/hyp/nvhe/mm.c
+++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -24,7 +24,7 @@ hyp_spinlock_t pkvm_pgd_lock;
 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
 unsigned int hyp_memblock_nr;
 
-static u64 __io_map_base;
+static u64 __hyp_private_va_base;
 
 struct hyp_fixmap_slot {
 	u64 addr;
@@ -50,7 +50,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size)
 
 	hyp_assert_lock_held(&pkvm_pgd_lock);
 
-	if (!start || start < __io_map_base)
+	if (!start || start < __hyp_private_va_base)
 		return -EINVAL;
 
 	/* The allocated size is always a multiple of PAGE_SIZE */
@@ -60,7 +60,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size)
 	if (cur > __hyp_vmemmap)
 		return -ENOMEM;
 
-	__io_map_base = cur;
+	__hyp_private_va_base = cur;
 
 	return 0;
 }
@@ -70,8 +70,8 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size)
  * @size:	The size of the VA range to reserve.
  * @haddr:	The hypervisor virtual start address of the allocation.
  *
- * The private virtual address (VA) range is allocated above __io_map_base
- * and aligned based on the order of @size.
+ * The private virtual address (VA) range is allocated from
+ * __hyp_private_va_base and aligned based on the order of @size.
  *
  * Return: 0 on success or negative error code on failure.
  */
@@ -81,7 +81,7 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr)
 	int ret;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
-	addr = __io_map_base;
+	addr = __hyp_private_va_base;
 	ret = __pkvm_alloc_private_va_range(addr, size);
 	hyp_spin_unlock(&pkvm_pgd_lock);
 
@@ -365,7 +365,7 @@ static int create_fixblock(void)
 		return -EINVAL;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
-	addr = ALIGN(__io_map_base, PMD_SIZE);
+	addr = ALIGN(__hyp_private_va_base, PMD_SIZE);
 	ret = __pkvm_alloc_private_va_range(addr, PMD_SIZE);
 	if (ret)
 		goto unlock;
@@ -446,12 +446,13 @@ int hyp_create_idmap(u32 hyp_va_bits)
 	 * memory -- see va_layout.c for more details. The other half of the VA
 	 * space contains the trampoline page, and needs some care. Split that
 	 * second half in two and find the quarter of VA space not conflicting
-	 * with the idmap to place the IOs and the vmemmap. IOs use the lower
-	 * half of the quarter and the vmemmap the upper half.
+	 * with the idmap to place private mappings and the vmemmap. Private
+	 * mappings use the lower half of the quarter and the vmemmap the upper
+	 * half.
 	 */
-	__io_map_base = start & BIT(hyp_va_bits - 2);
-	__io_map_base ^= BIT(hyp_va_bits - 2);
-	__hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3);
+	__hyp_private_va_base = start & BIT(hyp_va_bits - 2);
+	__hyp_private_va_base ^= BIT(hyp_va_bits - 2);
+	__hyp_vmemmap = __hyp_private_va_base | BIT(hyp_va_bits - 3);
 
 	return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
 }
@@ -464,13 +465,13 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr)
 
 	hyp_spin_lock(&pkvm_pgd_lock);
 
-	prev_base = __io_map_base;
+	prev_base = __hyp_private_va_base;
 	/*
 	 * Efficient stack verification using the NVHE_STACK_SHIFT bit implies
 	 * an alignment of our allocation on the order of the size.
 	 */
 	size = NVHE_STACK_SIZE * 2;
-	addr = ALIGN(__io_map_base, size);
+	addr = ALIGN(__hyp_private_va_base, size);
 
 	ret = __pkvm_alloc_private_va_range(addr, size);
 	if (!ret) {
@@ -486,7 +487,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr)
 		ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE,
 					  NVHE_STACK_SIZE, phys, PAGE_HYP);
 		if (ret)
-			__io_map_base = prev_base;
+			__hyp_private_va_base = prev_base;
 	}
 	hyp_spin_unlock(&pkvm_pgd_lock);
 
-- 
2.43.0

[RFC PATCH 5/5] KVM: arm64: Use TTBR1_EL2 for hVHE runtime mappings

[Aneesh Kumar K.V (Arm)]

hVHE currently disables TTBR1_EL2 and places both the identity mapping
and runtime hyp mappings in TTBR0_EL2. This unnecessarily retains the
legacy nVHE VA layout even though hVHE can use canonical kernel-image
addresses through the EL2&0 translation regime.

Place hVHE runtime mappings in TTBR1_EL2 and reserve TTBR0_EL2 for the
identity mapping used during MMU and page-table transitions. Select this
layout based on hVHE support independently of protected mode, while
retaining the existing TTBR0-only layout for nVHE and protected nVHE.

Keep hyp symbols at their linked kernel-image VAs, disable the legacy VA
relocations for hVHE, and configure TCR_EL2 for both translation ranges.
Allocate separate runtime and idmap page tables during initial setup and
pKVM finalization, pass both roots through the per-CPU initialization
parameters, and install both during page-table transitions.

Add support for walking canonical TTBR1 hyp page tables, including
handling the wrap at the top of the address space. Use this when fixing
page-table reference counts and destroying hyp page tables.

Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
 arch/arm64/include/asm/kvm_asm.h     |  3 +-
 arch/arm64/include/asm/kvm_hyp.h     |  4 +-
 arch/arm64/include/asm/kvm_mmu.h     | 46 ++++++++++++++++++---
 arch/arm64/include/asm/kvm_pgtable.h | 27 ++++++++++++
 arch/arm64/include/asm/kvm_pkvm.h    |  3 ++
 arch/arm64/kernel/asm-offsets.c      |  5 ++-
 arch/arm64/kvm/arm.c                 | 47 +++++++++++++--------
 arch/arm64/kvm/hyp/include/nvhe/mm.h |  1 +
 arch/arm64/kvm/hyp/nvhe/hyp-init.S   | 29 ++++++++++---
 arch/arm64/kvm/hyp/nvhe/mm.c         | 24 +++++++++++
 arch/arm64/kvm/hyp/nvhe/setup.c      | 35 ++++++++++++++--
 arch/arm64/kvm/hyp/pgtable.c         | 62 +++++++++++++++++++++++++---
 arch/arm64/kvm/mmu.c                 | 57 +++++++++++++++++++++++--
 arch/arm64/kvm/pkvm.c                |  3 ++
 arch/arm64/kvm/va_layout.c           | 32 +++++++++++---
 15 files changed, 328 insertions(+), 50 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 6e0cc007bfe1..e42baadf7990 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -210,7 +210,8 @@ struct kvm_nvhe_init_params {
 	unsigned long tpidr_el2;
 	unsigned long stack_hyp_va;
 	unsigned long stack_pa;
-	phys_addr_t pgd_pa;
+	phys_addr_t ttbr0_pgd_pa;
+	phys_addr_t ttbr1_pgd_pa;
 	unsigned long hcr_el2;
 	unsigned long vttbr;
 	unsigned long vtcr;
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 4974492744cc..5cc196f84958 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -131,8 +131,8 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,
 #endif
 
 #ifdef __KVM_NVHE_HYPERVISOR__
-void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
-		void (*fn)(void));
+void __pkvm_init_switch_pgd(phys_addr_t ttbr0, phys_addr_t ttbr1,
+			    unsigned long sp, void (*fn)(void));
 int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_base, u32 hyp_va_bits);
 void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
 #endif
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index bcc8e28985f7..30ac5cecaa93 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -105,6 +105,7 @@ alternative_cb_end
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_host.h>
 #include <asm/kvm_nested.h>
+#include <asm/virt.h>
 
 void kvm_update_va_mask(struct alt_instr *alt,
 			__le32 *origptr, __le32 *updptr, int nr_inst);
@@ -112,13 +113,28 @@ void kvm_compute_layout(void);
 u32 kvm_hyp_va_bits(void);
 void kvm_apply_hyp_relocations(void);
 
+static __always_inline bool kvm_hyp_init_uses_ttbr1(void)
+{
+	BUILD_BUG_ON(__is_defined(__KVM_NVHE_HYPERVISOR__));
+
+	return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE);
+}
+
+static __always_inline bool kvm_hyp_uses_ttbr1(void)
+{
+	return cpus_have_final_cap(ARM64_KVM_HVHE);
+}
+
 /**
  * kvm_hyp_kimg_kaddr - Select the kernel address used for an EL2 mapping
  * @ptr: Address within the kernel image
  *
- * Return @ptr when EL2 uses the kernel-image address directly, as with
- * VHE. Otherwise return the corresponding linear-map alias used as input
- * to the legacy nVHE VA conversion.
+ * Return @ptr when EL2 uses the kernel-image address directly, as with VHE
+ * or hVHE TTBR1. Otherwise return the corresponding linear-map alias used
+ * as input to the legacy nVHE VA conversion.
+ *
+ * This helper relies on finalized CPU capabilities and must not be used from
+ * earlier boot stages.
  *
  * Return: A kernel address suitable as the basis for an EL2 virtual address.
  */
@@ -126,10 +142,11 @@ static __always_inline void *kvm_hyp_kimg_kaddr(void *ptr)
 {
 	/*
 	 * With VHE, the host runs at EL2 using the kernel mapping, so
-	 * kernel-image VAs need no translation.
-	 * Only legacy nVHE requires a linear-map alias.
+	 * kernel-image VAs need no translation. hVHE similarly retains
+	 * kernel-image VAs in its TTBR1 EL2 mapping. Only legacy nVHE
+	 * requires a linear-map alias.
 	 */
-	if (is_kernel_in_hyp_mode())
+	if (is_kernel_in_hyp_mode() || kvm_hyp_uses_ttbr1())
 		return ptr;
 
 	return lm_alias(ptr);
@@ -137,6 +154,22 @@ static __always_inline void *kvm_hyp_kimg_kaddr(void *ptr)
 
 #define __hyp_linear_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset)
 #define __hyp_symbol_pa(x) (((phys_addr_t)(x)) + hyp_symbol_physvirt_offset)
+
+static __always_inline unsigned long kvm_hyp_ttbr1_private_start(void)
+{
+	return PAGE_ALIGN((unsigned long)__hyp_bss_end);
+}
+
+static __always_inline unsigned long kvm_hyp_ttbr1_private_end(void)
+{
+	return VMALLOC_END;
+}
+
+static __always_inline unsigned long kvm_hyp_ttbr1_vmemmap_base(void)
+{
+	return kvm_hyp_ttbr1_private_start() + SZ_1G;
+}
+
 /*
  * Convert a kernel VA into a HYP VA.
  *
@@ -228,6 +261,7 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu);
 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu);
 
 phys_addr_t kvm_mmu_get_httbr(void);
+phys_addr_t kvm_mmu_get_hyp_ttbr1(void);
 phys_addr_t kvm_get_idmap_vector(void);
 int __init kvm_mmu_init(u32 hyp_va_bits);
 
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 41a8687938eb..1d51dcaaac0d 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -255,6 +255,14 @@ enum kvm_pgtable_stage2_flags {
 	KVM_PGTABLE_S2_AS_S1			= BIT(1),
 };
 
+/**
+ * enum kvm_pgtable_hyp_flags - Hypervisor page-table flags.
+ * @KVM_PGTABLE_HYP_TTBR1:	Walk TTBR1 canonical virtual addresses.
+ */
+enum kvm_pgtable_hyp_flags {
+	KVM_PGTABLE_HYP_TTBR1			= BIT(0),
+};
+
 /**
  * enum kvm_pgtable_prot - Page-table permissions and attributes.
  * @KVM_PGTABLE_PROT_UX:	Unprivileged execute permission.
@@ -443,6 +451,7 @@ static inline bool kvm_pgtable_walk_lock_held(void)
  * @start_level:	Level at which the page-table walk starts.
  * @pgd:		Pointer to the first top-level entry of the page-table.
  * @mm_ops:		Memory management callbacks.
+ * @hyp_flags:		Hypervisor page-table flags.
  * @mmu:		Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
  * @flags:		Stage-2 page-table flags.
  * @force_pte_cb:	Function that returns true if page level mappings must
@@ -457,6 +466,9 @@ struct kvm_pgtable {
 			kvm_pteref_t				pgd;
 			struct kvm_pgtable_mm_ops		*mm_ops;
 
+			/* Hyp only */
+			enum kvm_pgtable_hyp_flags		hyp_flags;
+
 			/* Stage-2 only */
 			enum kvm_pgtable_stage2_flags		flags;
 			kvm_pgtable_force_pte_cb_t		force_pte_cb;
@@ -465,6 +477,11 @@ struct kvm_pgtable {
 	struct kvm_s2_mmu					*mmu;
 };
 
+static inline void kvm_pgtable_hyp_enable_ttbr1(struct kvm_pgtable *pgt)
+{
+	pgt->hyp_flags |= KVM_PGTABLE_HYP_TTBR1;
+}
+
 /**
  * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.
  * @pgt:	Uninitialised page-table structure to initialise.
@@ -851,6 +868,16 @@ int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,
 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
 		     struct kvm_pgtable_walker *walker);
 
+/**
+ * kvm_pgtable_hyp_walk() - Walk the entire usable VA range of a hyp pgtable.
+ * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
+ * @walker:	Walker callback description.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int kvm_pgtable_hyp_walk(struct kvm_pgtable *pgt,
+			 struct kvm_pgtable_walker *walker);
+
 /**
  * kvm_pgtable_get_leaf() - Walk a page-table and retrieve the leaf entry
  *			    with its level.
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index 74fedd9c5ff0..e632fe4ec43c 100644
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -139,6 +139,9 @@ static inline unsigned long hyp_s1_pgtable_pages(void)
 	/* Allow 1 GiB for private mappings */
 	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
 
+	/* Allow a separate TTBR0 idmap page-table. */
+	res += __hyp_pgtable_max_pages(1);
+
 	return res;
 }
 
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index b6367ff3a49c..f95883771d34 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -120,7 +120,10 @@ int main(void)
   DEFINE(NVHE_INIT_TCR_EL2,	offsetof(struct kvm_nvhe_init_params, tcr_el2));
   DEFINE(NVHE_INIT_TPIDR_EL2,	offsetof(struct kvm_nvhe_init_params, tpidr_el2));
   DEFINE(NVHE_INIT_STACK_HYP_VA,	offsetof(struct kvm_nvhe_init_params, stack_hyp_va));
-  DEFINE(NVHE_INIT_PGD_PA,	offsetof(struct kvm_nvhe_init_params, pgd_pa));
+  DEFINE(NVHE_INIT_TTBR0_PGD_PA,
+	 offsetof(struct kvm_nvhe_init_params, ttbr0_pgd_pa));
+  DEFINE(NVHE_INIT_TTBR1_PGD_PA,
+	 offsetof(struct kvm_nvhe_init_params, ttbr1_pgd_pa));
   DEFINE(NVHE_INIT_HCR_EL2,	offsetof(struct kvm_nvhe_init_params, hcr_el2));
   DEFINE(NVHE_INIT_VTTBR,	offsetof(struct kvm_nvhe_init_params, vttbr));
   DEFINE(NVHE_INIT_VTCR,	offsetof(struct kvm_nvhe_init_params, vtcr));
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 924f33c1a9d4..f2ea75461c1f 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -2124,38 +2124,51 @@ static int kvm_init_vector_slots(void)
 	return 0;
 }
 
+static unsigned long __init kvm_compute_hyp_tcr(u32 hyp_va_bits)
+{
+	unsigned long tcr = read_sysreg(tcr_el1);
+
+	if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
+		tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK |
+			 TCR_T1SZ_MASK |
+			 TCR_EPD0_MASK | TCR_EPD1_MASK);
+
+		if (kvm_hyp_uses_ttbr1())
+			tcr |= TCR_T1SZ(hyp_va_bits);
+		else
+			tcr |= TCR_EPD1_MASK;
+	} else {
+		unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
+
+		tcr &= TCR_EL2_MASK;
+		tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
+		if (lpa2_is_enabled())
+			tcr |= TCR_EL2_DS;
+	}
+
+	return tcr | TCR_T0SZ(hyp_va_bits);
+}
+
 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
 {
 	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
-	unsigned long tcr;
 
 	/*
 	 * Calculate the raw per-cpu offset without a translation from the
 	 * kernel's mapping to the linear mapping, and store it in tpidr_el2
 	 * so that we can use adr_l to access per-cpu variables in EL2.
+	 * In TTBR1 mode, this is the delta from the canonical hyp symbol to
+	 * the linear-map VA at which the allocated per-cpu copy is mapped.
 	 * Also drop the KASAN tag which gets in the way...
 	 */
 	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
 			    (unsigned long)kvm_hyp_kimg_kaddr(CHOOSE_NVHE_SYM(__per_cpu_start));
 
 	params->mair_el2 = read_sysreg(mair_el1);
+	params->tcr_el2 = kvm_compute_hyp_tcr(hyp_va_bits);
 
-	tcr = read_sysreg(tcr_el1);
-	if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
-		tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);
-		tcr |= TCR_EPD1_MASK;
-	} else {
-		unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
-
-		tcr &= TCR_EL2_MASK;
-		tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
-		if (lpa2_is_enabled())
-			tcr |= TCR_EL2_DS;
-	}
-	tcr |= TCR_T0SZ(hyp_va_bits);
-	params->tcr_el2 = tcr;
-
-	params->pgd_pa = kvm_mmu_get_httbr();
+	params->ttbr0_pgd_pa = kvm_mmu_get_httbr();
+	params->ttbr1_pgd_pa = kvm_mmu_get_hyp_ttbr1();
 	if (is_protected_kvm_enabled())
 		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
 	else
diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h
index c41152c84705..46f008ee6916 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/mm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h
@@ -11,6 +11,7 @@
 #include <nvhe/spinlock.h>
 
 extern struct kvm_pgtable pkvm_pgtable;
+extern struct kvm_pgtable pkvm_idmap_pgtable;
 extern hyp_spinlock_t pkvm_pgd_lock;
 
 int hyp_create_fixmap(void);
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 89cb553be1e5..07fc29c34fb3 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -127,13 +127,22 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init)
 	ldr	x1, [x0, #NVHE_INIT_VTCR]
 	msr	vtcr_el2, x1
 
-	ldr	x1, [x0, #NVHE_INIT_PGD_PA]
+	ldr	x1, [x0, #NVHE_INIT_TTBR0_PGD_PA]
 	phys_to_ttbr x2, x1
 alternative_if ARM64_HAS_CNP
 	orr	x2, x2, #TTBRx_EL1_CnP
 alternative_else_nop_endif
 	msr	ttbr0_el2, x2
 
+	ldr	x1, [x0, #NVHE_INIT_TTBR1_PGD_PA]
+	cbz	x1, 2f
+	phys_to_ttbr x2, x1
+alternative_if ARM64_HAS_CNP
+	orr	x2, x2, #TTBRx_EL1_CnP
+alternative_else_nop_endif
+	msr_s	SYS_TTBR1_EL2, x2
+2:
+
 	ldr	x0, [x0, #NVHE_INIT_TCR_EL2]
 	msr	tcr_el2, x0
 
@@ -261,13 +270,15 @@ reset:
 SYM_CODE_END(__kvm_handle_stub_hvc)
 
 /*
- * void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
- *                             void (*fn)(void));
+ * void __pkvm_init_switch_pgd(phys_addr_t ttbr0, phys_addr_t ttbr1,
+ *                             unsigned long sp, void (*fn)(void));
  *
  * SYM_TYPED_FUNC_START() allows C to call this ID-mapped function indirectly
  * using a physical pointer without triggering a kCFI failure.
  */
 SYM_TYPED_FUNC_START(__pkvm_init_switch_pgd)
+	mov	x6, x3
+
 	/* Turn the MMU off */
 	pre_disable_mmu_workaround
 	mrs	x3, sctlr_el2
@@ -284,14 +295,22 @@ alternative_if ARM64_HAS_CNP
 alternative_else_nop_endif
 	msr	ttbr0_el2, x5
 
+	cbz	x1, 1f
+	phys_to_ttbr x5, x1
+alternative_if ARM64_HAS_CNP
+	orr	x5, x5, #TTBRx_EL1_CnP
+alternative_else_nop_endif
+	msr_s	SYS_TTBR1_EL2, x5
+1:
+
 	/* Set the new stack pointer */
-	mov	sp, x1
+	mov	sp, x2
 
 	/* And turn the MMU back on! */
 	dsb	nsh
 	isb
 	set_sctlr_el2	x3
-	ret	x2
+	ret	x6
 SYM_FUNC_END(__pkvm_init_switch_pgd)
 
 	.popsection
diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
index 3233a7c70f7c..c856b2399960 100644
--- a/arch/arm64/kvm/hyp/nvhe/mm.c
+++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -19,6 +19,7 @@
 #include <nvhe/spinlock.h>
 
 struct kvm_pgtable pkvm_pgtable;
+struct kvm_pgtable pkvm_idmap_pgtable;
 hyp_spinlock_t pkvm_pgd_lock;
 
 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
@@ -431,6 +432,22 @@ int hyp_create_fixmap(void)
 	return create_fixblock();
 }
 
+static int pkvm_map_idmap_text(unsigned long start, unsigned long end)
+{
+	unsigned long idmap_start, idmap_end;
+
+	idmap_start = __hyp_symbol_pa(__hyp_idmap_text_start);
+	idmap_start = ALIGN_DOWN(idmap_start, PAGE_SIZE);
+	idmap_end = __hyp_symbol_pa(__hyp_idmap_text_end);
+	idmap_end = ALIGN(idmap_end, PAGE_SIZE);
+
+	if (WARN_ON(start != idmap_start || end != idmap_end))
+		return -EINVAL;
+
+	return kvm_pgtable_hyp_map(&pkvm_idmap_pgtable, start,
+				   end - start, start, PAGE_HYP_EXEC);
+}
+
 int hyp_create_idmap(u32 hyp_va_bits)
 {
 	unsigned long start, end;
@@ -441,6 +458,13 @@ int hyp_create_idmap(u32 hyp_va_bits)
 	end = __hyp_symbol_pa(__hyp_idmap_text_end);
 	end = ALIGN(end, PAGE_SIZE);
 
+	if (kvm_hyp_uses_ttbr1()) {
+		__hyp_private_va_base = kvm_hyp_ttbr1_private_start();
+		__hyp_vmemmap = kvm_hyp_ttbr1_vmemmap_base();
+
+		return pkvm_map_idmap_text(start, end);
+	}
+
 	/*
 	 * One half of the VA space is reserved to linearly map portions of
 	 * memory -- see va_layout.c for more details. The other half of the VA
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 2ef1972cc3dd..45b8f7213e39 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -109,6 +109,15 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
 	if (ret)
 		return ret;
 
+	if (kvm_hyp_uses_ttbr1()) {
+		kvm_pgtable_hyp_enable_ttbr1(&pkvm_pgtable);
+
+		ret = kvm_pgtable_hyp_init(&pkvm_idmap_pgtable, hyp_va_bits,
+					   &hyp_early_alloc_mm_ops);
+		if (ret)
+			return ret;
+	}
+
 	ret = hyp_create_idmap(hyp_va_bits);
 	if (ret)
 		return ret;
@@ -169,12 +178,27 @@ static void update_nvhe_init_params(void)
 
 	for (i = 0; i < hyp_nr_cpus; i++) {
 		params = per_cpu_ptr(&kvm_init_params, i);
-		params->pgd_pa = __hyp_linear_pa(pkvm_pgtable.pgd);
+		params->ttbr0_pgd_pa = kvm_hyp_uses_ttbr1() ?
+					__hyp_linear_pa(pkvm_idmap_pgtable.pgd) :
+					__hyp_linear_pa(pkvm_pgtable.pgd);
+		params->ttbr1_pgd_pa = kvm_hyp_uses_ttbr1() ?
+					__hyp_linear_pa(pkvm_pgtable.pgd) : 0;
 		dcache_clean_inval_poc((unsigned long)params,
 				    (unsigned long)params + sizeof(*params));
 	}
 }
 
+static bool pkvm_init_ttbrs_valid(struct kvm_nvhe_init_params *params)
+{
+	if (!params->ttbr0_pgd_pa)
+		return false;
+
+	if (kvm_hyp_uses_ttbr1())
+		return params->ttbr1_pgd_pa;
+
+	return !params->ttbr1_pgd_pa;
+}
+
 static void *hyp_zalloc_hyp_page(void *arg)
 {
 	return hyp_alloc_pages(&hpool, 0);
@@ -284,8 +308,7 @@ static int fix_hyp_pgtable_refcnt(void)
 		.arg	= pkvm_pgtable.mm_ops,
 	};
 
-	return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits),
-				&walker);
+	return kvm_pgtable_hyp_walk(&pkvm_pgtable, &walker);
 }
 
 void __noreturn __pkvm_init_finalise(void)
@@ -371,8 +394,12 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_bas
 
 	/* Jump in the idmap page to switch to the new page-tables */
 	params = this_cpu_ptr(&kvm_init_params);
+	if (WARN_ON(!pkvm_init_ttbrs_valid(params)))
+		return -EINVAL;
+
 	fn = (typeof(fn))__hyp_symbol_pa(__pkvm_init_switch_pgd);
-	fn(params->pgd_pa, params->stack_hyp_va, __pkvm_init_finalise);
+	fn(params->ttbr0_pgd_pa, params->ttbr1_pgd_pa,
+	   params->stack_hyp_va, __pkvm_init_finalise);
 
 	unreachable();
 }
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 91a7dfad6686..ca07fc2a55ec 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -10,6 +10,7 @@
 #include <linux/bitfield.h>
 #include <asm/kvm_pgtable.h>
 #include <asm/stage2_pgtable.h>
+#include <asm/kvm_mmu.h>
 
 struct kvm_pgtable_walk_data {
 	struct kvm_pgtable_walker	*walker;
@@ -71,6 +72,31 @@ static u32 kvm_pgd_pages(u32 ia_bits, s8 start_level)
 	return kvm_pgd_page_idx(&pgt, -1ULL) + 1;
 }
 
+static bool kvm_pgtable_walk_range_valid(struct kvm_pgtable *pgt,
+					 struct kvm_pgtable_walk_data *data)
+{
+	u64 limit = BIT(pgt->ia_bits);
+	u64 mask = limit - 1;
+	u64 last = data->end - 1;
+
+	if (data->addr > data->end)
+		return false;
+
+	if (!(pgt->hyp_flags & KVM_PGTABLE_HYP_TTBR1))
+		return data->addr <= limit && data->end <= limit;
+
+	/*
+	 * TTBR1 hyp walks use canonical virtual addresses supplied by the
+	 * caller. The index helpers extract the relevant bits for each level;
+	 * the masked comparison here only rejects ranges that cross out of the
+	 * single TTBR1 address window described by ia_bits.
+	 */
+	if ((data->addr | mask) != ~0ULL || (last | mask) != ~0ULL)
+		return false;
+
+	return (data->addr & mask) <= (last & mask);
+}
+
 static bool kvm_pte_table(kvm_pte_t pte, s8 level)
 {
 	if (level == KVM_PGTABLE_LAST_LEVEL)
@@ -193,8 +219,13 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
 		goto out;
 
 	if (!table) {
-		data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level));
-		data->addr += kvm_granule_size(level);
+		u64 granule = kvm_granule_size(level);
+		u64 addr = ALIGN_DOWN(data->addr, granule);
+
+		if (granule > data->end - addr)
+			data->addr = data->end;
+		else
+			data->addr = addr + granule;
 		goto out;
 	}
 
@@ -241,9 +272,8 @@ static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_da
 {
 	u32 idx;
 	int ret = 0;
-	u64 limit = BIT(pgt->ia_bits);
 
-	if (data->addr > limit || data->end > limit)
+	if (!kvm_pgtable_walk_range_valid(pgt, data))
 		return -ERANGE;
 
 	if (!pgt->pgd)
@@ -281,6 +311,26 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
 	return r;
 }
 
+int kvm_pgtable_hyp_walk(struct kvm_pgtable *pgt,
+			 struct kvm_pgtable_walker *walker)
+{
+	u64 size = BIT(pgt->ia_bits);
+	u64 addr = 0;
+
+	if (pgt->hyp_flags & KVM_PGTABLE_HYP_TTBR1) {
+		addr = -size;
+
+		/*
+		 * The exclusive end of the TTBR1 range cannot be represented in
+		 * a u64. Exclude the last page, which cannot be mapped through
+		 * kvm_pgtable_walk() for the same reason.
+		 */
+		size -= PAGE_SIZE;
+	}
+
+	return kvm_pgtable_walk(pgt, addr, size, walker);
+}
+
 struct leaf_walk_data {
 	kvm_pte_t	pte;
 	s8		level;
@@ -539,6 +589,7 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
 	pgt->ia_bits		= va_bits;
 	pgt->start_level	= start_level;
 	pgt->mm_ops		= mm_ops;
+	pgt->hyp_flags		= 0;
 	pgt->mmu		= NULL;
 	pgt->force_pte_cb	= NULL;
 
@@ -568,7 +619,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)
 		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
 	};
 
-	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
+	WARN_ON(kvm_pgtable_hyp_walk(pgt, &walker));
 	pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
 	pgt->pgd = NULL;
 }
@@ -1589,6 +1640,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
 	pgt->start_level	= start_level;
 	pgt->mm_ops		= mm_ops;
 	pgt->mmu		= mmu;
+	pgt->hyp_flags		= 0;
 	pgt->flags		= flags;
 	pgt->force_pte_cb	= force_pte_cb;
 
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 154d53dca619..46fd7040bca7 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -20,11 +20,13 @@
 #include <asm/kvm_pkvm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
+#include <asm/sections.h>
 #include <asm/virt.h>
 
 #include "trace.h"
 
 static struct kvm_pgtable *hyp_pgtable;
+static struct kvm_pgtable *hyp_idmap_pgtable;
 static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
 
 static unsigned long __ro_after_init hyp_idmap_start;
@@ -392,6 +394,11 @@ static void stage2_flush_vm(struct kvm *kvm)
 void __init free_hyp_pgds(void)
 {
 	mutex_lock(&kvm_hyp_pgd_mutex);
+	if (hyp_idmap_pgtable) {
+		kvm_pgtable_hyp_destroy(hyp_idmap_pgtable);
+		kfree(hyp_idmap_pgtable);
+		hyp_idmap_pgtable = NULL;
+	}
 	if (hyp_pgtable) {
 		kvm_pgtable_hyp_destroy(hyp_pgtable);
 		kfree(hyp_pgtable);
@@ -2481,6 +2488,17 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
 
 phys_addr_t kvm_mmu_get_httbr(void)
 {
+	if (hyp_idmap_pgtable)
+		return __pa(hyp_idmap_pgtable->pgd);
+
+	return __pa(hyp_pgtable->pgd);
+}
+
+phys_addr_t kvm_mmu_get_hyp_ttbr1(void)
+{
+	if (!kvm_hyp_uses_ttbr1())
+		return 0;
+
 	return __pa(hyp_pgtable->pgd);
 }
 
@@ -2491,9 +2509,15 @@ phys_addr_t kvm_get_idmap_vector(void)
 
 static int kvm_map_idmap_text(void)
 {
+	struct kvm_pgtable *pgt = hyp_idmap_pgtable ?: hyp_pgtable;
 	unsigned long size = hyp_idmap_end - hyp_idmap_start;
-	int err = __create_hyp_mappings(hyp_idmap_start, size, hyp_idmap_start,
-					PAGE_HYP_EXEC);
+	int err;
+
+	if (WARN_ON(kvm_hyp_uses_ttbr1() && !hyp_idmap_pgtable))
+		return -EINVAL;
+
+	err = kvm_pgtable_hyp_map(pgt, hyp_idmap_start, size, hyp_idmap_start,
+				  PAGE_HYP_EXEC);
 	if (err)
 		kvm_err("Failed to idmap %lx-%lx\n",
 			hyp_idmap_start, hyp_idmap_end);
@@ -2559,14 +2583,39 @@ int __init kvm_mmu_init(u32 hyp_va_bits)
 	if (err)
 		goto out_free_pgtable;
 
+	if (kvm_hyp_uses_ttbr1()) {
+		kvm_pgtable_hyp_enable_ttbr1(hyp_pgtable);
+
+		hyp_idmap_pgtable = kzalloc_obj(*hyp_idmap_pgtable);
+		if (!hyp_idmap_pgtable) {
+			kvm_err("Hyp mode idmap page-table not allocated\n");
+			err = -ENOMEM;
+			goto out_destroy_pgtable;
+		}
+
+		err = kvm_pgtable_hyp_init(hyp_idmap_pgtable, hyp_va_bits,
+					   &kvm_hyp_mm_ops);
+		if (err)
+			goto out_free_idmap_pgtable;
+	}
+
 	err = kvm_map_idmap_text();
 	if (err)
-		goto out_destroy_pgtable;
+		goto out_destroy_idmap_pgtable;
 
-	io_map_base = hyp_idmap_start;
+	if (kvm_hyp_uses_ttbr1())
+		io_map_base = kvm_hyp_ttbr1_private_end();
+	else
+		io_map_base = hyp_idmap_start;
 	__hyp_va_bits = hyp_va_bits;
 	return 0;
 
+out_destroy_idmap_pgtable:
+	if (hyp_idmap_pgtable)
+		kvm_pgtable_hyp_destroy(hyp_idmap_pgtable);
+out_free_idmap_pgtable:
+	kfree(hyp_idmap_pgtable);
+	hyp_idmap_pgtable = NULL;
 out_destroy_pgtable:
 	kvm_pgtable_hyp_destroy(hyp_pgtable);
 out_free_pgtable:
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 053e4f733e4b..1352c89d2bd8 100644
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -292,6 +292,9 @@ static int __init finalize_pkvm(void)
 	ret = pkvm_drop_host_privileges();
 	if (ret)
 		pr_err("Failed to finalize Hyp protection: %d\n", ret);
+	else
+		kvm_info("pKVM finalized in %s mode\n",
+			 kvm_hyp_uses_ttbr1() ? "hVHE" : "nVHE");
 
 	return ret;
 }
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 75a89ad8fecf..1e1816995fe4 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -33,19 +33,29 @@ static u64 __early_kern_hyp_va(u64 addr)
 	return addr;
 }
 
+static u64 __early_kern_sym_hyp_va(void *ptr)
+{
+	if (is_kernel_in_hyp_mode() || kvm_hyp_init_uses_ttbr1())
+		return  (u64)ptr;
+
+	return __early_kern_hyp_va((u64)lm_alias(ptr));
+}
+
 /*
  * Store a hyp VA <-> PA offset into a EL2-owned variable.
  */
 static void init_hyp_physvirt_offset(void)
 {
-	u64 kern_va, hyp_va;
+	u64 kern_va, hyp_va, sym_va;
 
 	/* Compute the offset from the hyp VA and PA of a random symbol. */
 	kern_va = (u64)lm_alias(__hyp_text_start);
 	hyp_va = __early_kern_hyp_va(kern_va);
 	hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
 
-	hyp_symbol_physvirt_offset = hyp_physvirt_offset;
+	sym_va = __early_kern_sym_hyp_va(__hyp_text_start);
+	hyp_symbol_physvirt_offset = (s64)__pa_symbol(__hyp_text_start) -
+				     (s64)sym_va;
 }
 
 /*
@@ -87,6 +97,14 @@ __init void kvm_compute_layout(void)
 	u64 hyp_va_msb;
 	u32 hyp_va_bits = kvm_hyp_va_bits();
 
+	if (kvm_hyp_init_uses_ttbr1()) {
+		tag_lsb = 0;
+		tag_val = 0;
+		va_mask = ~0ULL;
+		init_hyp_physvirt_offset();
+		return;
+	}
+
 	/* Where is my RAM region? */
 	hyp_va_msb  = idmap_addr & BIT(hyp_va_bits - 1);
 	hyp_va_msb ^= BIT(hyp_va_bits - 1);
@@ -117,6 +135,9 @@ __init void kvm_apply_hyp_relocations(void)
 	int32_t *begin = (int32_t *)__hyp_reloc_begin;
 	int32_t *end = (int32_t *)__hyp_reloc_end;
 
+	if (kvm_hyp_init_uses_ttbr1())
+		return;
+
 	for (rel = begin; rel < end; ++rel) {
 		uintptr_t *ptr, kimg_va;
 
@@ -187,14 +208,15 @@ void __init kvm_update_va_mask(struct alt_instr *alt,
 		u32 rd, rn, insn, oinsn;
 
 		/*
-		 * VHE doesn't need any address translation, let's NOP
+		 * VHE/hVHE doesn't need any address translation, let's NOP
 		 * everything.
 		 *
 		 * Alternatively, if the tag is zero (because the layout
 		 * dictates it and we don't have any spare bits in the
 		 * address), NOP everything after masking the kernel VA.
 		 */
-		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
+		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) ||
+		    kvm_hyp_init_uses_ttbr1() || (!tag_val && i > 0)) {
 			updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
 			continue;
 		}
@@ -225,7 +247,7 @@ void kvm_patch_vector_branch(struct alt_instr *alt,
 	/*
 	 * Compute HYP VA by using the same computation as kern_hyp_va()
 	 */
-	addr = __early_kern_hyp_va((u64)kvm_hyp_kimg_kaddr(__kvm_hyp_vector));
+	addr = __early_kern_sym_hyp_va(__kvm_hyp_vector);
 
 	/* Use PC[10:7] to branch to the same vector in KVM */
 	addr |= ((u64)origptr & GENMASK_ULL(10, 7));
-- 
2.43.0