* [PATCH v2 01/20] KVM: arm64: Remove __activate_vm wrapper
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 02/20] KVM: arm64: Remove hyp_panic arguments Andrew Scull
` (18 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
The __activate_vm wrapper serves no useful function and has a misleading
name as it simply calls __load_guest_stage2 and does not touch
HCR_EL2.VM so remove it.
Also rename __deactivate_vm to __load_host_stage2 to match naming
pattern.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/kvm/hyp/include/hyp/switch.h | 5 -----
arch/arm64/kvm/hyp/nvhe/switch.c | 8 ++++----
arch/arm64/kvm/hyp/vhe/switch.c | 10 +++++-----
3 files changed, 9 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 426ef65601dd..d07777d857a3 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -122,11 +122,6 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
}
}
-static inline void __activate_vm(struct kvm_s2_mmu *mmu)
-{
- __load_guest_stage2(mmu);
-}
-
static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar)
{
u64 par, tmp;
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 341be2f2f312..2940d220096d 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -93,7 +93,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
}
-static void __deactivate_vm(struct kvm_vcpu *vcpu)
+static void __load_host_stage2(void)
{
write_sysreg(0, vttbr_el2);
}
@@ -194,7 +194,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
__sysreg32_restore_state(vcpu);
__sysreg_restore_state_nvhe(guest_ctxt);
- __activate_vm(kern_hyp_va(vcpu->arch.hw_mmu));
+ __load_guest_stage2(kern_hyp_va(vcpu->arch.hw_mmu));
__activate_traps(vcpu);
__hyp_vgic_restore_state(vcpu);
@@ -219,7 +219,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
__hyp_vgic_save_state(vcpu);
__deactivate_traps(vcpu);
- __deactivate_vm(vcpu);
+ __load_host_stage2();
__sysreg_restore_state_nvhe(host_ctxt);
@@ -253,7 +253,7 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
if (read_sysreg(vttbr_el2)) {
__timer_disable_traps(vcpu);
__deactivate_traps(vcpu);
- __deactivate_vm(vcpu);
+ __load_host_stage2();
__sysreg_restore_state_nvhe(host_ctxt);
}
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index c52d714e0d75..30bb76066cc0 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -120,12 +120,12 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
* HCR_EL2.TGE.
*
* We have already configured the guest's stage 1 translation in
- * kvm_vcpu_load_sysregs_vhe above. We must now call __activate_vm
- * before __activate_traps, because __activate_vm configures
- * stage 2 translation, and __activate_traps clear HCR_EL2.TGE
- * (among other things).
+ * kvm_vcpu_load_sysregs_vhe above. We must now call
+ * __load_guest_stage2 before __activate_traps, because
+ * __load_guest_stage2 configures stage 2 translation, and
+ * __activate_traps clear HCR_EL2.TGE (among other things).
*/
- __activate_vm(vcpu->arch.hw_mmu);
+ __load_guest_stage2(vcpu->arch.hw_mmu);
__activate_traps(vcpu);
sysreg_restore_guest_state_vhe(guest_ctxt);
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 02/20] KVM: arm64: Remove hyp_panic arguments
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
2020-08-20 10:34 ` [PATCH v2 01/20] KVM: arm64: Remove __activate_vm wrapper Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 03/20] KVM: arm64: Remove kvm_host_data_t typedef Andrew Scull
` (17 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
hyp_panic is able to find all the context it needs from within itself so
remove the argument. The __hyp_panic wrapper becomes redundant so is
also removed.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_hyp.h | 2 +-
arch/arm64/kvm/hyp/hyp-entry.S | 11 +++--------
arch/arm64/kvm/hyp/nvhe/switch.c | 4 +++-
arch/arm64/kvm/hyp/vhe/switch.c | 4 +++-
4 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 46689e7db46c..3de99b323061 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -89,7 +89,7 @@ void deactivate_traps_vhe_put(void);
u64 __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt);
-void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt);
+void __noreturn hyp_panic(void);
#ifdef __KVM_NVHE_HYPERVISOR__
void __noreturn __hyp_do_panic(unsigned long, ...);
#endif
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 689fccbc9de7..b0923e9f9291 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -148,7 +148,7 @@ el2_sync:
/* if this was something else, then panic! */
tst x0, #PSR_IL_BIT
- b.eq __hyp_panic
+ b.eq hyp_panic
/* Let's attempt a recovery from the illegal exception return */
get_vcpu_ptr x1, x0
@@ -176,7 +176,7 @@ el2_error:
cmp x0, x1
adr x1, abort_guest_exit_end
ccmp x0, x1, #4, ne
- b.ne __hyp_panic
+ b.ne hyp_panic
mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
eret
sb
@@ -193,12 +193,7 @@ SYM_FUNC_START(__hyp_do_panic)
SYM_FUNC_END(__hyp_do_panic)
#endif
-SYM_CODE_START(__hyp_panic)
- get_host_ctxt x0, x1
- b hyp_panic
-SYM_CODE_END(__hyp_panic)
-
-.macro invalid_vector label, target = __hyp_panic
+.macro invalid_vector label, target = hyp_panic
.align 2
SYM_CODE_START(\label)
b \target
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 2940d220096d..f969a6dc4741 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -242,11 +242,13 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
return exit_code;
}
-void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
+void __noreturn hyp_panic(void)
{
u64 spsr = read_sysreg_el2(SYS_SPSR);
u64 elr = read_sysreg_el2(SYS_ELR);
u64 par = read_sysreg(par_el1);
+ struct kvm_cpu_context *host_ctxt =
+ &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
struct kvm_vcpu *vcpu = host_ctxt->__hyp_running_vcpu;
unsigned long str_va;
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 30bb76066cc0..099bfa78011c 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -208,8 +208,10 @@ static void __hyp_call_panic(u64 spsr, u64 elr, u64 par,
}
NOKPROBE_SYMBOL(__hyp_call_panic);
-void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
+void __noreturn hyp_panic(void)
{
+ struct kvm_cpu_context *host_ctxt =
+ &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
u64 spsr = read_sysreg_el2(SYS_SPSR);
u64 elr = read_sysreg_el2(SYS_ELR);
u64 par = read_sysreg(par_el1);
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 03/20] KVM: arm64: Remove kvm_host_data_t typedef
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
2020-08-20 10:34 ` [PATCH v2 01/20] KVM: arm64: Remove __activate_vm wrapper Andrew Scull
2020-08-20 10:34 ` [PATCH v2 02/20] KVM: arm64: Remove hyp_panic arguments Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 04/20] KVM: arm64: Restrict symbol aliasing to outside nVHE Andrew Scull
` (16 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
The kvm_host_data_t typedef is used inconsistently and goes against the
kernel's coding style. Remove it in favour of the full struct specifier.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_host.h | 4 +---
arch/arm64/kvm/arm.c | 4 ++--
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 65568b23868a..ec1a80796999 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -262,8 +262,6 @@ struct kvm_host_data {
struct kvm_pmu_events pmu_events;
};
-typedef struct kvm_host_data kvm_host_data_t;
-
struct vcpu_reset_state {
unsigned long pc;
unsigned long r0;
@@ -565,7 +563,7 @@ void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
-DECLARE_PER_CPU(kvm_host_data_t, kvm_host_data);
+DECLARE_PER_CPU(struct kvm_host_data, kvm_host_data);
static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt)
{
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 691d21e4c717..6b09c0f3365c 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -46,7 +46,7 @@
__asm__(".arch_extension virt");
#endif
-DEFINE_PER_CPU(kvm_host_data_t, kvm_host_data);
+DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
/* The VMID used in the VTTBR */
@@ -1538,7 +1538,7 @@ static int init_hyp_mode(void)
}
for_each_possible_cpu(cpu) {
- kvm_host_data_t *cpu_data;
+ struct kvm_host_data *cpu_data;
cpu_data = per_cpu_ptr(&kvm_host_data, cpu);
err = create_hyp_mappings(cpu_data, cpu_data + 1, PAGE_HYP);
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 04/20] KVM: arm64: Restrict symbol aliasing to outside nVHE
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (2 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 03/20] KVM: arm64: Remove kvm_host_data_t typedef Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 05/20] KVM: arm64: Save chosen hyp vector to a percpu variable Andrew Scull
` (15 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, David Brazdil, will,
julien.thierry.kdev
nVHE symbols are prefixed but this is sometimes hidden from the host by
aliasing the non-prefixed symbol to the prefixed version with a macro.
This runs into problems if nVHE tries to use the symbol as it becomes
doubly prefixed. Avoid this by omitting the aliasing macro for nVHE.
Cc: David Brazdil <dbrazdil@google.com>
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_asm.h | 3 +++
1 file changed, 3 insertions(+)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index fb1a922b31ba..413911d6446a 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -99,8 +99,11 @@ struct kvm_s2_mmu;
DECLARE_KVM_NVHE_SYM(__kvm_hyp_init);
DECLARE_KVM_HYP_SYM(__kvm_hyp_vector);
+
+#ifndef __KVM_NVHE_HYPERVISOR__
#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
+#endif
#ifdef CONFIG_KVM_INDIRECT_VECTORS
extern atomic_t arm64_el2_vector_last_slot;
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 05/20] KVM: arm64: Save chosen hyp vector to a percpu variable
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (3 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 04/20] KVM: arm64: Restrict symbol aliasing to outside nVHE Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 06/20] KVM: arm64: nVHE: Use separate vector for the host Andrew Scull
` (14 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
Introduce a percpu variable to hold the address of the selected hyp
vector that will be used with guests. This avoids the selection process
each time a guest is being entered and can be used by nVHE when a
separate vector is introduced for the host.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_hyp.h | 2 ++
arch/arm64/kvm/arm.c | 5 ++++-
arch/arm64/kvm/hyp/vhe/switch.c | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 3de99b323061..1e2491da324e 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -12,6 +12,8 @@
#include <asm/alternative.h>
#include <asm/sysreg.h>
+DECLARE_PER_CPU(unsigned long, kvm_hyp_vector);
+
#define read_sysreg_elx(r,nvh,vh) \
({ \
u64 reg; \
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 6b09c0f3365c..d779814f0da6 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -47,6 +47,7 @@ __asm__(".arch_extension virt");
#endif
DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
+DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
/* The VMID used in the VTTBR */
@@ -1276,7 +1277,7 @@ static void cpu_init_hyp_mode(void)
pgd_ptr = kvm_mmu_get_httbr();
hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE;
- vector_ptr = (unsigned long)kvm_get_hyp_vector();
+ vector_ptr = __this_cpu_read(kvm_hyp_vector);
/*
* Call initialization code, and switch to the full blown HYP code.
@@ -1309,6 +1310,8 @@ static void cpu_hyp_reinit(void)
cpu_hyp_reset();
+ __this_cpu_write(kvm_hyp_vector, (unsigned long)kvm_get_hyp_vector());
+
if (is_kernel_in_hyp_mode())
kvm_timer_init_vhe();
else
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 099bfa78011c..20ee901fd4ff 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -59,7 +59,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
write_sysreg(val, cpacr_el1);
- write_sysreg(kvm_get_hyp_vector(), vbar_el1);
+ write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1);
}
NOKPROBE_SYMBOL(__activate_traps);
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 06/20] KVM: arm64: nVHE: Use separate vector for the host
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (4 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 05/20] KVM: arm64: Save chosen hyp vector to a percpu variable Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 07/20] KVM: arm64: nVHE: Don't consume host SErrors with ESB Andrew Scull
` (13 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
The host is treated differently from the guests when an exception is
taken so introduce a separate vector that is specialized for the host.
This also allows the nVHE specific code to move out of hyp-entry.S and
into nvhe/host.S.
The host is only expected to make HVC calls and anything else is
considered invalid and results in a panic.
Hyp initialization is now passed the vector that is used for the host
and it is swapped for the guest vector during the context switch.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_asm.h | 2 +
arch/arm64/kernel/image-vars.h | 1 +
arch/arm64/kvm/arm.c | 11 +++-
arch/arm64/kvm/hyp/hyp-entry.S | 65 -------------------
arch/arm64/kvm/hyp/nvhe/Makefile | 2 +-
arch/arm64/kvm/hyp/nvhe/host.S | 104 +++++++++++++++++++++++++++++++
arch/arm64/kvm/hyp/nvhe/switch.c | 3 +
7 files changed, 121 insertions(+), 67 deletions(-)
create mode 100644 arch/arm64/kvm/hyp/nvhe/host.S
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 413911d6446a..81f29a2c361a 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -98,10 +98,12 @@ struct kvm_vcpu;
struct kvm_s2_mmu;
DECLARE_KVM_NVHE_SYM(__kvm_hyp_init);
+DECLARE_KVM_NVHE_SYM(__kvm_hyp_host_vector);
DECLARE_KVM_HYP_SYM(__kvm_hyp_vector);
#ifndef __KVM_NVHE_HYPERVISOR__
#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
+#define __kvm_hyp_host_vector CHOOSE_NVHE_SYM(__kvm_hyp_host_vector)
#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
#endif
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 9e897c500237..7e93b0c426d4 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -71,6 +71,7 @@ KVM_NVHE_ALIAS(kvm_update_va_mask);
/* Global kernel state accessed by nVHE hyp code. */
KVM_NVHE_ALIAS(arm64_ssbd_callback_required);
KVM_NVHE_ALIAS(kvm_host_data);
+KVM_NVHE_ALIAS(kvm_hyp_vector);
KVM_NVHE_ALIAS(kvm_vgic_global_state);
/* Kernel constant needed to compute idmap addresses. */
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index d779814f0da6..c6f76fdabecb 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1277,7 +1277,7 @@ static void cpu_init_hyp_mode(void)
pgd_ptr = kvm_mmu_get_httbr();
hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE;
- vector_ptr = __this_cpu_read(kvm_hyp_vector);
+ vector_ptr = (unsigned long)kern_hyp_va(kvm_ksym_ref(__kvm_hyp_host_vector));
/*
* Call initialization code, and switch to the full blown HYP code.
@@ -1542,6 +1542,7 @@ static int init_hyp_mode(void)
for_each_possible_cpu(cpu) {
struct kvm_host_data *cpu_data;
+ unsigned long *vector;
cpu_data = per_cpu_ptr(&kvm_host_data, cpu);
err = create_hyp_mappings(cpu_data, cpu_data + 1, PAGE_HYP);
@@ -1550,6 +1551,14 @@ static int init_hyp_mode(void)
kvm_err("Cannot map host CPU state: %d\n", err);
goto out_err;
}
+
+ vector = per_cpu_ptr(&kvm_hyp_vector, cpu);
+ err = create_hyp_mappings(vector, vector + 1, PAGE_HYP);
+
+ if (err) {
+ kvm_err("Cannot map hyp guest vector address\n");
+ goto out_err;
+ }
}
err = hyp_map_aux_data();
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index b0923e9f9291..6e14873680a8 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -12,25 +12,10 @@
#include <asm/cpufeature.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
-#include <asm/kvm_mmu.h>
#include <asm/mmu.h>
.text
-.macro do_el2_call
- /*
- * Shuffle the parameters before calling the function
- * pointed to in x0. Assumes parameters in x[1,2,3].
- */
- str lr, [sp, #-16]!
- mov lr, x0
- mov x0, x1
- mov x1, x2
- mov x2, x3
- blr lr
- ldr lr, [sp], #16
-.endm
-
el1_sync: // Guest trapped into EL2
mrs x0, esr_el2
@@ -39,43 +24,6 @@ el1_sync: // Guest trapped into EL2
ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
b.ne el1_trap
-#ifdef __KVM_NVHE_HYPERVISOR__
- mrs x1, vttbr_el2 // If vttbr is valid, the guest
- cbnz x1, el1_hvc_guest // called HVC
-
- /* Here, we're pretty sure the host called HVC. */
- ldp x0, x1, [sp], #16
-
- /* Check for a stub HVC call */
- cmp x0, #HVC_STUB_HCALL_NR
- b.hs 1f
-
- /*
- * Compute the idmap address of __kvm_handle_stub_hvc and
- * jump there. Since we use kimage_voffset, do not use the
- * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
- * (by loading it from the constant pool).
- *
- * Preserve x0-x4, which may contain stub parameters.
- */
- ldr x5, =__kvm_handle_stub_hvc
- ldr_l x6, kimage_voffset
-
- /* x5 = __pa(x5) */
- sub x5, x5, x6
- br x5
-
-1:
- /*
- * Perform the EL2 call
- */
- kern_hyp_va x0
- do_el2_call
-
- eret
- sb
-#endif /* __KVM_NVHE_HYPERVISOR__ */
-
el1_hvc_guest:
/*
* Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
@@ -181,18 +129,6 @@ el2_error:
eret
sb
-#ifdef __KVM_NVHE_HYPERVISOR__
-SYM_FUNC_START(__hyp_do_panic)
- mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
- PSR_MODE_EL1h)
- msr spsr_el2, lr
- ldr lr, =panic
- msr elr_el2, lr
- eret
- sb
-SYM_FUNC_END(__hyp_do_panic)
-#endif
-
.macro invalid_vector label, target = hyp_panic
.align 2
SYM_CODE_START(\label)
@@ -205,7 +141,6 @@ SYM_CODE_END(\label)
invalid_vector el2t_irq_invalid
invalid_vector el2t_fiq_invalid
invalid_vector el2t_error_invalid
- invalid_vector el2h_sync_invalid
invalid_vector el2h_irq_invalid
invalid_vector el2h_fiq_invalid
invalid_vector el1_fiq_invalid
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index aef76487edc2..ddf98eb07b9d 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -6,7 +6,7 @@
asflags-y := -D__KVM_NVHE_HYPERVISOR__
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o
+obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
new file mode 100644
index 000000000000..9c96b9a3b71d
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 - Google Inc
+ * Author: Andrew Scull <ascull@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#include <asm/assembler.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_mmu.h>
+
+ .text
+
+SYM_FUNC_START(__hyp_do_panic)
+ mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
+ PSR_MODE_EL1h)
+ msr spsr_el2, lr
+ ldr lr, =panic
+ msr elr_el2, lr
+ eret
+ sb
+SYM_FUNC_END(__hyp_do_panic)
+
+.macro valid_host_el1_sync_vect
+ .align 7
+ esb
+ stp x0, x1, [sp, #-16]!
+
+ mrs x0, esr_el2
+ lsr x0, x0, #ESR_ELx_EC_SHIFT
+ cmp x0, #ESR_ELx_EC_HVC64
+ b.ne hyp_panic
+
+ ldp x0, x1, [sp], #16
+
+ /* Check for a stub HVC call */
+ cmp x0, #HVC_STUB_HCALL_NR
+ b.hs 1f
+
+ /*
+ * Compute the idmap address of __kvm_handle_stub_hvc and
+ * jump there. Since we use kimage_voffset, do not use the
+ * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
+ * (by loading it from the constant pool).
+ *
+ * Preserve x0-x4, which may contain stub parameters.
+ */
+ ldr x5, =__kvm_handle_stub_hvc
+ ldr_l x6, kimage_voffset
+
+ /* x5 = __pa(x5) */
+ sub x5, x5, x6
+ br x5
+
+1:
+ /*
+ * Shuffle the parameters before calling the function
+ * pointed to in x0. Assumes parameters in x[1,2,3].
+ */
+ kern_hyp_va x0
+ str lr, [sp, #-16]!
+ mov lr, x0
+ mov x0, x1
+ mov x1, x2
+ mov x2, x3
+ blr lr
+ ldr lr, [sp], #16
+
+ eret
+ sb
+.endm
+
+.macro invalid_host_vect
+ .align 7
+ b hyp_panic
+.endm
+
+/*
+ * CONFIG_KVM_INDIRECT_VECTORS is not applied to the host vector because the
+ * host already knows the address of hyp by virtue of loading it there.
+ */
+ .align 11
+SYM_CODE_START(__kvm_hyp_host_vector)
+ invalid_host_vect // Synchronous EL2t
+ invalid_host_vect // IRQ EL2t
+ invalid_host_vect // FIQ EL2t
+ invalid_host_vect // Error EL2t
+
+ invalid_host_vect // Synchronous EL2h
+ invalid_host_vect // IRQ EL2h
+ invalid_host_vect // FIQ EL2h
+ invalid_host_vect // Error EL2h
+
+ valid_host_el1_sync_vect // Synchronous 64-bit EL1
+ invalid_host_vect // IRQ 64-bit EL1
+ invalid_host_vect // FIQ 64-bit EL1
+ invalid_host_vect // Error 64-bit EL1
+
+ invalid_host_vect // Synchronous 32-bit EL1
+ invalid_host_vect // IRQ 32-bit EL1
+ invalid_host_vect // FIQ 32-bit EL1
+ invalid_host_vect // Error 32-bit EL1
+SYM_CODE_END(__kvm_hyp_host_vector)
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index f969a6dc4741..05bb0b472091 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -42,6 +42,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
}
write_sysreg(val, cptr_el2);
+ write_sysreg(__hyp_this_cpu_read(kvm_hyp_vector), vbar_el2);
if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
@@ -60,6 +61,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
static void __deactivate_traps(struct kvm_vcpu *vcpu)
{
+ extern char __kvm_hyp_host_vector[];
u64 mdcr_el2;
___deactivate_traps(vcpu);
@@ -91,6 +93,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
write_sysreg(mdcr_el2, mdcr_el2);
write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2);
write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
+ write_sysreg(__kvm_hyp_host_vector, vbar_el2);
}
static void __load_host_stage2(void)
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 07/20] KVM: arm64: nVHE: Don't consume host SErrors with ESB
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (5 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 06/20] KVM: arm64: nVHE: Use separate vector for the host Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 08/20] KVM: arm64: Preserve host DISR_EL1 Andrew Scull
` (12 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
The ESB at the start of the host vector may cause SErrors to be consumed
to DISR_EL1. However, this is not checked for the host so the SError
could go unhandled.
Remove the ESB so that SErrors are not consumed but are instead left
pending for the host to consume. __guest_enter already defers entry into
a guest if there are any SErrors pending.
Fixes: 0e5b9c085dce ("KVM: arm64: Consume pending SError as early as possible")
Cc: James Morse <james.morse@arm.com>
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/kvm/hyp/nvhe/host.S | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 9c96b9a3b71d..5a7380c342c8 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -24,7 +24,6 @@ SYM_FUNC_END(__hyp_do_panic)
.macro valid_host_el1_sync_vect
.align 7
- esb
stp x0, x1, [sp, #-16]!
mrs x0, esr_el2
@@ -77,6 +76,11 @@ SYM_FUNC_END(__hyp_do_panic)
.endm
/*
+ * The host vector does not use an ESB instruction in order to avoid consuming
+ * SErrors that should only be consumed by the host. Guest entry is deferred by
+ * __guest_enter if there are any pending asynchronous exceptions so hyp will
+ * always return to the host without having consumerd host SErrors.
+ *
* CONFIG_KVM_INDIRECT_VECTORS is not applied to the host vector because the
* host already knows the address of hyp by virtue of loading it there.
*/
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 08/20] KVM: arm64: Preserve host DISR_EL1
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (6 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 07/20] KVM: arm64: nVHE: Don't consume host SErrors with ESB Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 09/20] KVM: arm64: Introduce hyp context Andrew Scull
` (11 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
With the RAS extension available, guest exits can overwrite DISR_EL1
after checking for any guest SErrors. If the host was using DISR_EL1 to
track a deferred SError, it would be lost so save and restore DISR_EL1
for the host.
Cc: James Morse <james.morse@arm.com>
Signed-off-by: Andrew Scull <ascull@google.com>
---
This may not be necessary if it is a safe assumption that the host
doesn't mind that DISR_EL1 is clobbered across the kvm_vcpu_run call.
---
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
index 7a986030145f..a549f8698bd4 100644
--- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
@@ -56,8 +56,12 @@ static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
ctxt->regs.pc = read_sysreg_el2(SYS_ELR);
ctxt->regs.pstate = read_sysreg_el2(SYS_SPSR);
- if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
- ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
+ if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) {
+ if (ctxt->__hyp_running_vcpu)
+ ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_DISR_EL1);
+ else
+ ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
+ }
}
static inline void __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
@@ -152,8 +156,12 @@ static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctx
write_sysreg_el2(ctxt->regs.pc, SYS_ELR);
write_sysreg_el2(pstate, SYS_SPSR);
- if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
- write_sysreg_s(ctxt_sys_reg(ctxt, DISR_EL1), SYS_VDISR_EL2);
+ if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) {
+ if (ctxt->__hyp_running_vcpu)
+ write_sysreg_s(ctxt_sys_reg(ctxt, DISR_EL1), SYS_DISR_EL1);
+ else
+ write_sysreg_s(ctxt_sys_reg(ctxt, DISR_EL1), SYS_VDISR_EL2);
+ }
}
static inline void __sysreg32_save_state(struct kvm_vcpu *vcpu)
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 09/20] KVM: arm64: Introduce hyp context
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (7 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 08/20] KVM: arm64: Preserve host DISR_EL1 Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 10/20] KVM: arm64: Update context references from host to hyp Andrew Scull
` (10 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
During __guest_enter, save and restore from a new hyp context rather
than the host context. This is preparation for separation of the hyp and
host context in nVHE.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_hyp.h | 3 ++-
arch/arm64/kernel/image-vars.h | 1 +
arch/arm64/kvm/arm.c | 10 ++++++++++
arch/arm64/kvm/hyp/entry.S | 10 +++++-----
arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
arch/arm64/kvm/hyp/nvhe/switch.c | 2 +-
arch/arm64/kvm/hyp/vhe/switch.c | 2 +-
7 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 1e2491da324e..0b525e05e5bf 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -12,6 +12,7 @@
#include <asm/alternative.h>
#include <asm/sysreg.h>
+DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
DECLARE_PER_CPU(unsigned long, kvm_hyp_vector);
#define read_sysreg_elx(r,nvh,vh) \
@@ -89,7 +90,7 @@ void activate_traps_vhe_load(struct kvm_vcpu *vcpu);
void deactivate_traps_vhe_put(void);
#endif
-u64 __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt);
+u64 __guest_enter(struct kvm_vcpu *vcpu);
void __noreturn hyp_panic(void);
#ifdef __KVM_NVHE_HYPERVISOR__
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 7e93b0c426d4..ca1d730e143c 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -71,6 +71,7 @@ KVM_NVHE_ALIAS(kvm_update_va_mask);
/* Global kernel state accessed by nVHE hyp code. */
KVM_NVHE_ALIAS(arm64_ssbd_callback_required);
KVM_NVHE_ALIAS(kvm_host_data);
+KVM_NVHE_ALIAS(kvm_hyp_ctxt);
KVM_NVHE_ALIAS(kvm_hyp_vector);
KVM_NVHE_ALIAS(kvm_vgic_global_state);
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index c6f76fdabecb..0e3bdb7cb9d0 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -47,6 +47,7 @@ __asm__(".arch_extension virt");
#endif
DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
+DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
@@ -1542,6 +1543,7 @@ static int init_hyp_mode(void)
for_each_possible_cpu(cpu) {
struct kvm_host_data *cpu_data;
+ struct kvm_cpu_context *hyp_ctxt;
unsigned long *vector;
cpu_data = per_cpu_ptr(&kvm_host_data, cpu);
@@ -1552,6 +1554,14 @@ static int init_hyp_mode(void)
goto out_err;
}
+ hyp_ctxt = per_cpu_ptr(&kvm_hyp_ctxt, cpu);
+ err = create_hyp_mappings(hyp_ctxt, hyp_ctxt + 1, PAGE_HYP);
+
+ if (err) {
+ kvm_err("Cannot map hyp context: %d\n", err);
+ goto out_err;
+ }
+
vector = per_cpu_ptr(&kvm_hyp_vector, cpu);
err = create_hyp_mappings(vector, vector + 1, PAGE_HYP);
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index ee32a7743389..dbf746478f7e 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -57,15 +57,15 @@
.endm
/*
- * u64 __guest_enter(struct kvm_vcpu *vcpu,
- * struct kvm_cpu_context *host_ctxt);
+ * u64 __guest_enter(struct kvm_vcpu *vcpu);
*/
SYM_FUNC_START(__guest_enter)
// x0: vcpu
- // x1: host context
- // x2-x17: clobbered by macros
+ // x1-x17: clobbered by macros
// x29: guest context
+ hyp_adr_this_cpu x1, kvm_hyp_ctxt, x2
+
// Store the host regs
save_callee_saved_regs x1
@@ -148,7 +148,7 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL)
// Store the guest's sp_el0
save_sp_el0 x1, x2
- get_host_ctxt x2, x3
+ hyp_adr_this_cpu x2, kvm_hyp_ctxt, x3
// Macro ptrauth_switch_to_guest format:
// ptrauth_switch_to_host(guest cxt, host cxt, tmp1, tmp2, tmp3)
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index d07777d857a3..e9382c7e100a 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -377,7 +377,7 @@ static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu)
!esr_is_ptrauth_trap(kvm_vcpu_get_esr(vcpu)))
return false;
- ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
+ ctxt = __hyp_this_cpu_ptr(kvm_hyp_ctxt);
__ptrauth_save_key(ctxt, APIA);
__ptrauth_save_key(ctxt, APIB);
__ptrauth_save_key(ctxt, APDA);
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 05bb0b472091..375163b7f777 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -209,7 +209,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
do {
/* Jump in the fire! */
- exit_code = __guest_enter(vcpu, host_ctxt);
+ exit_code = __guest_enter(vcpu);
/* And we're baaack! */
} while (fixup_guest_exit(vcpu, &exit_code));
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 20ee901fd4ff..987b0225436c 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -135,7 +135,7 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
do {
/* Jump in the fire! */
- exit_code = __guest_enter(vcpu, host_ctxt);
+ exit_code = __guest_enter(vcpu);
/* And we're baaack! */
} while (fixup_guest_exit(vcpu, &exit_code));
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 10/20] KVM: arm64: Update context references from host to hyp
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (8 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 09/20] KVM: arm64: Introduce hyp context Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 11/20] KVM: arm64: Restore hyp when panicking in guest context Andrew Scull
` (9 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
Hyp now has its own nominal context for saving and restoring its state
when switching to and from a guest. Update the related comments and
utilities to match the new name.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_ptrauth.h | 6 +++---
arch/arm64/kvm/hyp/entry.S | 22 +++++++++++-----------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_ptrauth.h b/arch/arm64/include/asm/kvm_ptrauth.h
index 0ddf98c3ba9f..0cd0965255d2 100644
--- a/arch/arm64/include/asm/kvm_ptrauth.h
+++ b/arch/arm64/include/asm/kvm_ptrauth.h
@@ -60,7 +60,7 @@
.endm
/*
- * Both ptrauth_switch_to_guest and ptrauth_switch_to_host macros will
+ * Both ptrauth_switch_to_guest and ptrauth_switch_to_hyp macros will
* check for the presence ARM64_HAS_ADDRESS_AUTH, which is defined as
* (ARM64_HAS_ADDRESS_AUTH_ARCH || ARM64_HAS_ADDRESS_AUTH_IMP_DEF) and
* then proceed ahead with the save/restore of Pointer Authentication
@@ -78,7 +78,7 @@ alternative_else_nop_endif
.L__skip_switch\@:
.endm
-.macro ptrauth_switch_to_host g_ctxt, h_ctxt, reg1, reg2, reg3
+.macro ptrauth_switch_to_hyp g_ctxt, h_ctxt, reg1, reg2, reg3
alternative_if_not ARM64_HAS_ADDRESS_AUTH
b .L__skip_switch\@
alternative_else_nop_endif
@@ -96,7 +96,7 @@ alternative_else_nop_endif
#else /* !CONFIG_ARM64_PTR_AUTH */
.macro ptrauth_switch_to_guest g_ctxt, reg1, reg2, reg3
.endm
-.macro ptrauth_switch_to_host g_ctxt, h_ctxt, reg1, reg2, reg3
+.macro ptrauth_switch_to_hyp g_ctxt, h_ctxt, reg1, reg2, reg3
.endm
#endif /* CONFIG_ARM64_PTR_AUTH */
#endif /* __ASSEMBLY__ */
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index dbf746478f7e..e95cd7f2ff86 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -66,16 +66,16 @@ SYM_FUNC_START(__guest_enter)
hyp_adr_this_cpu x1, kvm_hyp_ctxt, x2
- // Store the host regs
+ // Store the hyp regs
save_callee_saved_regs x1
- // Save the host's sp_el0
+ // Save hyp's sp_el0
save_sp_el0 x1, x2
- // Now the host state is stored if we have a pending RAS SError it must
- // affect the host. If any asynchronous exception is pending we defer
- // the guest entry. The DSB isn't necessary before v8.2 as any SError
- // would be fatal.
+ // Now the hyp state is stored if we have a pending RAS SError it must
+ // affect the host or hyp. If any asynchronous exception is pending we
+ // defer the guest entry. The DSB isn't necessary before v8.2 as any
+ // SError would be fatal.
alternative_if ARM64_HAS_RAS_EXTN
dsb nshst
isb
@@ -150,17 +150,17 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL)
hyp_adr_this_cpu x2, kvm_hyp_ctxt, x3
- // Macro ptrauth_switch_to_guest format:
- // ptrauth_switch_to_host(guest cxt, host cxt, tmp1, tmp2, tmp3)
+ // Macro ptrauth_switch_to_hyp format:
+ // ptrauth_switch_to_hyp(guest cxt, host cxt, tmp1, tmp2, tmp3)
// The below macro to save/restore keys is not implemented in C code
// as it may cause Pointer Authentication key signing mismatch errors
// when this feature is enabled for kernel code.
- ptrauth_switch_to_host x1, x2, x3, x4, x5
+ ptrauth_switch_to_hyp x1, x2, x3, x4, x5
- // Restore the hosts's sp_el0
+ // Restore hyp's sp_el0
restore_sp_el0 x2, x3
- // Now restore the host regs
+ // Now restore the hyp regs
restore_callee_saved_regs x2
alternative_if ARM64_HAS_RAS_EXTN
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 11/20] KVM: arm64: Restore hyp when panicking in guest context
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (9 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 10/20] KVM: arm64: Update context references from host to hyp Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 12/20] KVM: arm64: Share context save and restore macros Andrew Scull
` (8 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
If the guest context is loaded when a panic is triggered, restore the
hyp context so e.g. the shadow call stack works when hyp_panic() is
called and SP_EL0 is valid when the host's panic() is called.
Use the hyp context's __hyp_running_vcpu field to track when hyp
transitions to and from the guest vcpu so the exception handlers know
whether the context needs to be restored.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_asm.h | 10 ++++++++++
arch/arm64/kvm/hyp/entry.S | 24 ++++++++++++++++++++++++
arch/arm64/kvm/hyp/hyp-entry.S | 12 +++++-------
arch/arm64/kvm/hyp/nvhe/host.S | 5 +++++
4 files changed, 44 insertions(+), 7 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 81f29a2c361a..106279a6d027 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -198,6 +198,16 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ];
ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
.endm
+.macro get_loaded_vcpu vcpu, ctxt
+ hyp_adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu
+ ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+.endm
+
+.macro set_loaded_vcpu vcpu, ctxt, tmp
+ hyp_adr_this_cpu \ctxt, kvm_hyp_ctxt, \tmp
+ str \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+.endm
+
#endif
#endif /* __ARM_KVM_ASM_H__ */
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index e95cd7f2ff86..872ac89ffc1e 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -86,6 +86,8 @@ alternative_else_nop_endif
ret
1:
+ set_loaded_vcpu x0, x1, x2
+
add x29, x0, #VCPU_CONTEXT
// Macro ptrauth_switch_to_guest format:
@@ -116,6 +118,26 @@ alternative_else_nop_endif
eret
sb
+SYM_INNER_LABEL(__guest_exit_panic, SYM_L_GLOBAL)
+ // x2-x29,lr: vcpu regs
+ // vcpu x0-x1 on the stack
+
+ // If the hyp context is loaded, go straight to hyp_panic
+ get_loaded_vcpu x0, x1
+ cbz x0, hyp_panic
+
+ // The hyp context is saved so make sure it is restored to allow
+ // hyp_panic to run at hyp and, subsequently, panic to run in the host.
+ // This makes use of __guest_exit to avoid duplication but sets the
+ // return address to tail call into hyp_panic. As a side effect, the
+ // current state is saved to the guest context but it will only be
+ // accurate if the guest had been completely restored.
+ hyp_adr_this_cpu x0, kvm_hyp_ctxt, x1
+ adr x1, hyp_panic
+ str x1, [x0, #CPU_XREG_OFFSET(30)]
+
+ get_vcpu_ptr x1, x0
+
SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL)
// x0: return code
// x1: vcpu
@@ -163,6 +185,8 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL)
// Now restore the hyp regs
restore_callee_saved_regs x2
+ set_loaded_vcpu xzr, x1, x2
+
alternative_if ARM64_HAS_RAS_EXTN
// If we have the RAS extensions we can consume a pending error
// without an unmask-SError and isb. The ESB-instruction consumed any
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 6e14873680a8..b7a1ea221f6e 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -96,7 +96,7 @@ el2_sync:
/* if this was something else, then panic! */
tst x0, #PSR_IL_BIT
- b.eq hyp_panic
+ b.eq __guest_exit_panic
/* Let's attempt a recovery from the illegal exception return */
get_vcpu_ptr x1, x0
@@ -105,8 +105,6 @@ el2_sync:
el2_error:
- ldp x0, x1, [sp], #16
-
/*
* Only two possibilities:
* 1) Either we come from the exit path, having just unmasked
@@ -124,12 +122,13 @@ el2_error:
cmp x0, x1
adr x1, abort_guest_exit_end
ccmp x0, x1, #4, ne
- b.ne hyp_panic
+ b.ne __guest_exit_panic
+ ldp x0, x1, [sp], #16
mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
eret
sb
-.macro invalid_vector label, target = hyp_panic
+.macro invalid_vector label, target = __guest_exit_panic
.align 2
SYM_CODE_START(\label)
b \target
@@ -170,10 +169,9 @@ check_preamble_length 661b, 662b
.macro invalid_vect target
.align 7
661:
- b \target
nop
+ stp x0, x1, [sp, #-16]!
662:
- ldp x0, x1, [sp], #16
b \target
check_preamble_length 661b, 662b
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 5a7380c342c8..d4e8b8084020 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -72,6 +72,11 @@ SYM_FUNC_END(__hyp_do_panic)
.macro invalid_host_vect
.align 7
+ /* If a guest is loaded, panic out of it. */
+ stp x0, x1, [sp, #-16]!
+ get_loaded_vcpu x0, x1
+ cbnz x0, __guest_exit_panic
+ add sp, sp, #16
b hyp_panic
.endm
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 12/20] KVM: arm64: Share context save and restore macros
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (10 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 11/20] KVM: arm64: Restore hyp when panicking in guest context Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 13/20] KVM: arm64: nVHE: Switch to hyp context for EL2 Andrew Scull
` (7 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
To avoid duplicating the context save and restore macros, move them into
a shareable header.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_asm.h | 39 ++++++++++++++++++++++++++++++++
arch/arm64/kvm/hyp/entry.S | 39 --------------------------------
2 files changed, 39 insertions(+), 39 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 106279a6d027..358262d4546e 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -208,6 +208,45 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ];
str \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
.endm
+#define CPU_XREG_OFFSET(x) (CPU_USER_PT_REGS + 8*x)
+#define CPU_LR_OFFSET CPU_XREG_OFFSET(30)
+#define CPU_SP_EL0_OFFSET (CPU_LR_OFFSET + 8)
+
+/*
+ * We treat x18 as callee-saved as the host may use it as a platform
+ * register (e.g. for shadow call stack).
+ */
+.macro save_callee_saved_regs ctxt
+ str x18, [\ctxt, #CPU_XREG_OFFSET(18)]
+ stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
+ stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
+ stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
+ stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
+ stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
+ stp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
+.endm
+
+.macro restore_callee_saved_regs ctxt
+ // We require \ctxt is not x18-x28
+ ldr x18, [\ctxt, #CPU_XREG_OFFSET(18)]
+ ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
+ ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
+ ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
+ ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
+ ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
+ ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
+.endm
+
+.macro save_sp_el0 ctxt, tmp
+ mrs \tmp, sp_el0
+ str \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
+.endm
+
+.macro restore_sp_el0 ctxt, tmp
+ ldr \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
+ msr sp_el0, \tmp
+.endm
+
#endif
#endif /* __ARM_KVM_ASM_H__ */
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 872ac89ffc1e..204130db62ee 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -7,7 +7,6 @@
#include <linux/linkage.h>
#include <asm/alternative.h>
-#include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
#include <asm/kvm.h>
@@ -16,46 +15,8 @@
#include <asm/kvm_mmu.h>
#include <asm/kvm_ptrauth.h>
-#define CPU_XREG_OFFSET(x) (CPU_USER_PT_REGS + 8*x)
-#define CPU_SP_EL0_OFFSET (CPU_XREG_OFFSET(30) + 8)
-
.text
-/*
- * We treat x18 as callee-saved as the host may use it as a platform
- * register (e.g. for shadow call stack).
- */
-.macro save_callee_saved_regs ctxt
- str x18, [\ctxt, #CPU_XREG_OFFSET(18)]
- stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
- stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
- stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
- stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
- stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
- stp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
-.endm
-
-.macro restore_callee_saved_regs ctxt
- // We require \ctxt is not x18-x28
- ldr x18, [\ctxt, #CPU_XREG_OFFSET(18)]
- ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
- ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
- ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
- ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
- ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
- ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
-.endm
-
-.macro save_sp_el0 ctxt, tmp
- mrs \tmp, sp_el0
- str \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
-.endm
-
-.macro restore_sp_el0 ctxt, tmp
- ldr \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
- msr sp_el0, \tmp
-.endm
-
/*
* u64 __guest_enter(struct kvm_vcpu *vcpu);
*/
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 13/20] KVM: arm64: nVHE: Switch to hyp context for EL2
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (11 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 12/20] KVM: arm64: Share context save and restore macros Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 14/20] KVM: arm64: nVHE: Handle hyp panics Andrew Scull
` (6 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
Save and restore the host context when switching to and from hyp. This
gives hyp its own context that the host will not see as a step towards a
full trust boundary between the two.
SP_EL0 and pointer authentication keys are currently shared between the
host and hyp so don't need to be switched yet.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +
arch/arm64/kvm/hyp/nvhe/Makefile | 2 +-
arch/arm64/kvm/hyp/nvhe/host.S | 68 ++++++++++++++++++-------
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 35 +++++++++++++
4 files changed, 88 insertions(+), 19 deletions(-)
create mode 100644 arch/arm64/kvm/hyp/nvhe/hyp-main.c
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index e9382c7e100a..01d9dcd9d591 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -368,6 +368,8 @@ static inline bool esr_is_ptrauth_trap(u32 esr)
ctxt_sys_reg(ctxt, key ## KEYHI_EL1) = __val; \
} while(0)
+DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
+
static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *ctxt;
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index ddf98eb07b9d..46c89e8c30bc 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -6,7 +6,7 @@
asflags-y := -D__KVM_NVHE_HYPERVISOR__
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
+obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o hyp-main.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index d4e8b8084020..1062547853db 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -12,6 +12,55 @@
.text
+SYM_FUNC_START(__host_exit)
+ stp x0, x1, [sp, #-16]!
+
+ get_host_ctxt x0, x1
+
+ ALTERNATIVE(nop, SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
+
+ /* Store the guest regs x2 and x3 */
+ stp x2, x3, [x0, #CPU_XREG_OFFSET(2)]
+
+ /* Retrieve the guest regs x0-x1 from the stack */
+ ldp x2, x3, [sp], #16 // x0, x1
+
+ // Store the guest regs x0-x1 and x4-x17
+ stp x2, x3, [x0, #CPU_XREG_OFFSET(0)]
+ stp x4, x5, [x0, #CPU_XREG_OFFSET(4)]
+ stp x6, x7, [x0, #CPU_XREG_OFFSET(6)]
+ stp x8, x9, [x0, #CPU_XREG_OFFSET(8)]
+ stp x10, x11, [x0, #CPU_XREG_OFFSET(10)]
+ stp x12, x13, [x0, #CPU_XREG_OFFSET(12)]
+ stp x14, x15, [x0, #CPU_XREG_OFFSET(14)]
+ stp x16, x17, [x0, #CPU_XREG_OFFSET(16)]
+
+ /* Store the guest regs x18-x29, lr */
+ save_callee_saved_regs x0
+
+ /* Save the host context pointer in x29 across the function call */
+ mov x29, x0
+ bl handle_trap
+
+ /* Restore guest regs x0-x17 */
+ ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)]
+ ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)]
+ ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)]
+ ldp x6, x7, [x29, #CPU_XREG_OFFSET(6)]
+ ldp x8, x9, [x29, #CPU_XREG_OFFSET(8)]
+ ldp x10, x11, [x29, #CPU_XREG_OFFSET(10)]
+ ldp x12, x13, [x29, #CPU_XREG_OFFSET(12)]
+ ldp x14, x15, [x29, #CPU_XREG_OFFSET(14)]
+ ldp x16, x17, [x29, #CPU_XREG_OFFSET(16)]
+
+ /* Restore guest regs x18-x29, lr */
+ restore_callee_saved_regs x29
+
+ /* Do not touch any register after this! */
+ eret
+ sb
+SYM_FUNC_END(__host_exit)
+
SYM_FUNC_START(__hyp_do_panic)
mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
PSR_MODE_EL1h)
@@ -35,7 +84,7 @@ SYM_FUNC_END(__hyp_do_panic)
/* Check for a stub HVC call */
cmp x0, #HVC_STUB_HCALL_NR
- b.hs 1f
+ b.hs __host_exit
/*
* Compute the idmap address of __kvm_handle_stub_hvc and
@@ -51,23 +100,6 @@ SYM_FUNC_END(__hyp_do_panic)
/* x5 = __pa(x5) */
sub x5, x5, x6
br x5
-
-1:
- /*
- * Shuffle the parameters before calling the function
- * pointed to in x0. Assumes parameters in x[1,2,3].
- */
- kern_hyp_va x0
- str lr, [sp, #-16]!
- mov lr, x0
- mov x0, x1
- mov x1, x2
- mov x2, x3
- blr lr
- ldr lr, [sp], #16
-
- eret
- sb
.endm
.macro invalid_host_vect
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
new file mode 100644
index 000000000000..c8938e09f585
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 - Google Inc
+ * Author: Andrew Scull <ascull@google.com>
+ */
+
+#include <hyp/switch.h>
+
+#include <asm/kvm_asm.h>
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_host.h>
+#include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
+
+typedef unsigned long (*hypcall_fn_t)
+ (unsigned long, unsigned long, unsigned long);
+
+void handle_trap(struct kvm_cpu_context *host_ctxt) {
+ u64 esr = read_sysreg_el2(SYS_ESR);
+ hypcall_fn_t func;
+ unsigned long ret;
+
+ if (ESR_ELx_EC(esr) != ESR_ELx_EC_HVC64)
+ hyp_panic();
+
+ /*
+ * __kvm_call_hyp takes a pointer in the host address space and
+ * up to three arguments.
+ */
+ func = (hypcall_fn_t)kern_hyp_va(host_ctxt->regs.regs[0]);
+ ret = func(host_ctxt->regs.regs[1],
+ host_ctxt->regs.regs[2],
+ host_ctxt->regs.regs[3]);
+ host_ctxt->regs.regs[0] = ret;
+}
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 14/20] KVM: arm64: nVHE: Handle hyp panics
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (12 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 13/20] KVM: arm64: nVHE: Switch to hyp context for EL2 Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 15/20] smccc: Cast arguments to unsigned long Andrew Scull
` (5 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
Restore the host context when panicking from hyp to give the best chance
of the panic being clean.
The host requires that registers be preserved such as x18 for the shadow
callstack. If the panic is caused by an exception from EL1, the host
context is still valid so the panic can return straight back to the
host. If the panic comes from EL2 then it's most likely that the hyp
context is active and the host context needs to be restored.
There are windows before and after the host context is saved and
restored that restoration is attempted incorrectly and the panic won't
be clean.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_hyp.h | 2 +-
arch/arm64/kvm/hyp/nvhe/host.S | 79 +++++++++++++++++++++++---------
arch/arm64/kvm/hyp/nvhe/switch.c | 18 ++------
3 files changed, 63 insertions(+), 36 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 0b525e05e5bf..6b664de5ec1f 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -94,7 +94,7 @@ u64 __guest_enter(struct kvm_vcpu *vcpu);
void __noreturn hyp_panic(void);
#ifdef __KVM_NVHE_HYPERVISOR__
-void __noreturn __hyp_do_panic(unsigned long, ...);
+void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);
#endif
#endif /* __ARM64_KVM_HYP_H__ */
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 1062547853db..40620c1c87b8 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -47,6 +47,7 @@ SYM_FUNC_START(__host_exit)
ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)]
ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)]
ldp x6, x7, [x29, #CPU_XREG_OFFSET(6)]
+__host_enter_for_panic:
ldp x8, x9, [x29, #CPU_XREG_OFFSET(8)]
ldp x10, x11, [x29, #CPU_XREG_OFFSET(10)]
ldp x12, x13, [x29, #CPU_XREG_OFFSET(12)]
@@ -57,30 +58,49 @@ SYM_FUNC_START(__host_exit)
restore_callee_saved_regs x29
/* Do not touch any register after this! */
+__host_enter_without_restoring:
eret
sb
SYM_FUNC_END(__host_exit)
+/*
+ * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);
+ */
SYM_FUNC_START(__hyp_do_panic)
+ /* Load the format arguments into x1-7 */
+ mov x6, x3
+ get_vcpu_ptr x7, x3
+ mov x7, xzr
+
+ mrs x3, esr_el2
+ mrs x4, far_el2
+ mrs x5, hpfar_el2
+
+ /* Prepare and exit to the host's panic funciton. */
mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
PSR_MODE_EL1h)
msr spsr_el2, lr
ldr lr, =panic
msr elr_el2, lr
- eret
- sb
+
+ /*
+ * Set the panic format string and enter the host, conditionally
+ * restoring the host context.
+ */
+ cmp x0, xzr
+ ldr x0, =__hyp_panic_string
+ b.eq __host_enter_without_restoring
+ b __host_enter_for_panic
SYM_FUNC_END(__hyp_do_panic)
.macro valid_host_el1_sync_vect
.align 7
stp x0, x1, [sp, #-16]!
-
mrs x0, esr_el2
lsr x0, x0, #ESR_ELx_EC_SHIFT
cmp x0, #ESR_ELx_EC_HVC64
- b.ne hyp_panic
-
ldp x0, x1, [sp], #16
+ b.ne __host_exit
/* Check for a stub HVC call */
cmp x0, #HVC_STUB_HCALL_NR
@@ -102,16 +122,31 @@ SYM_FUNC_END(__hyp_do_panic)
br x5
.endm
-.macro invalid_host_vect
+.macro invalid_host_el2_vect
.align 7
/* If a guest is loaded, panic out of it. */
stp x0, x1, [sp, #-16]!
get_loaded_vcpu x0, x1
cbnz x0, __guest_exit_panic
add sp, sp, #16
+
+ /*
+ * The panic may not be clean if the exception is taken before the host
+ * context has been saved by __host_exit or after the hyp context has
+ * been partially clobbered by __host_enter.
+ */
b hyp_panic
.endm
+.macro invalid_host_el1_vect
+ .align 7
+ mov x0, xzr /* restore_host = false */
+ mrs x1, spsr_el2
+ mrs x2, elr_el2
+ mrs x3, par_el1
+ b __hyp_do_panic
+.endm
+
/*
* The host vector does not use an ESB instruction in order to avoid consuming
* SErrors that should only be consumed by the host. Guest entry is deferred by
@@ -123,23 +158,23 @@ SYM_FUNC_END(__hyp_do_panic)
*/
.align 11
SYM_CODE_START(__kvm_hyp_host_vector)
- invalid_host_vect // Synchronous EL2t
- invalid_host_vect // IRQ EL2t
- invalid_host_vect // FIQ EL2t
- invalid_host_vect // Error EL2t
+ invalid_host_el2_vect // Synchronous EL2t
+ invalid_host_el2_vect // IRQ EL2t
+ invalid_host_el2_vect // FIQ EL2t
+ invalid_host_el2_vect // Error EL2t
- invalid_host_vect // Synchronous EL2h
- invalid_host_vect // IRQ EL2h
- invalid_host_vect // FIQ EL2h
- invalid_host_vect // Error EL2h
+ invalid_host_el2_vect // Synchronous EL2h
+ invalid_host_el2_vect // IRQ EL2h
+ invalid_host_el2_vect // FIQ EL2h
+ invalid_host_el2_vect // Error EL2h
valid_host_el1_sync_vect // Synchronous 64-bit EL1
- invalid_host_vect // IRQ 64-bit EL1
- invalid_host_vect // FIQ 64-bit EL1
- invalid_host_vect // Error 64-bit EL1
-
- invalid_host_vect // Synchronous 32-bit EL1
- invalid_host_vect // IRQ 32-bit EL1
- invalid_host_vect // FIQ 32-bit EL1
- invalid_host_vect // Error 32-bit EL1
+ invalid_host_el1_vect // IRQ 64-bit EL1
+ invalid_host_el1_vect // FIQ 64-bit EL1
+ invalid_host_el1_vect // Error 64-bit EL1
+
+ invalid_host_el1_vect // Synchronous 32-bit EL1
+ invalid_host_el1_vect // IRQ 32-bit EL1
+ invalid_host_el1_vect // FIQ 32-bit EL1
+ invalid_host_el1_vect // Error 32-bit EL1
SYM_CODE_END(__kvm_hyp_host_vector)
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 375163b7f777..b7abe66fde22 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -242,6 +242,8 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
if (system_uses_irq_prio_masking())
gic_write_pmr(GIC_PRIO_IRQOFF);
+ host_ctxt->__hyp_running_vcpu = NULL;
+
return exit_code;
}
@@ -253,25 +255,15 @@ void __noreturn hyp_panic(void)
struct kvm_cpu_context *host_ctxt =
&__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
struct kvm_vcpu *vcpu = host_ctxt->__hyp_running_vcpu;
- unsigned long str_va;
+ bool restore_host = true;
- if (read_sysreg(vttbr_el2)) {
+ if (vcpu) {
__timer_disable_traps(vcpu);
__deactivate_traps(vcpu);
__load_host_stage2();
__sysreg_restore_state_nvhe(host_ctxt);
}
- /*
- * Force the panic string to be loaded from the literal pool,
- * making sure it is a kernel address and not a PC-relative
- * reference.
- */
- asm volatile("ldr %0, =%1" : "=r" (str_va) : "S" (__hyp_panic_string));
-
- __hyp_do_panic(str_va,
- spsr, elr,
- read_sysreg(esr_el2), read_sysreg_el2(SYS_FAR),
- read_sysreg(hpfar_el2), par, vcpu);
+ __hyp_do_panic(restore_host, spsr, elr, par);
unreachable();
}
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 15/20] smccc: Cast arguments to unsigned long
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (13 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 14/20] KVM: arm64: nVHE: Handle hyp panics Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 16/20] KVM: arm64: nVHE: Pass pointers consistently to hyp-init Andrew Scull
` (4 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
To avoid warning about implicit casting, make the casting explicit. This
allows, for example, pointers to be used as arguments as are used in the
KVM hyp interface.
Cc: Sudeep Holla <sudeep.holla@arm.com>
Signed-off-by: Andrew Scull <ascull@google.com>
---
include/linux/arm-smccc.h | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
index 15c706fb0a37..3bb109a35554 100644
--- a/include/linux/arm-smccc.h
+++ b/include/linux/arm-smccc.h
@@ -260,7 +260,7 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
typeof(a1) __a1 = a1; \
struct arm_smccc_res *___res = res; \
register unsigned long r0 asm("r0") = (u32)a0; \
- register unsigned long r1 asm("r1") = __a1; \
+ register unsigned long r1 asm("r1") = (unsigned long)__a1; \
register unsigned long r2 asm("r2"); \
register unsigned long r3 asm("r3")
@@ -269,8 +269,8 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
typeof(a2) __a2 = a2; \
struct arm_smccc_res *___res = res; \
register unsigned long r0 asm("r0") = (u32)a0; \
- register unsigned long r1 asm("r1") = __a1; \
- register unsigned long r2 asm("r2") = __a2; \
+ register unsigned long r1 asm("r1") = (unsigned long)__a1; \
+ register unsigned long r2 asm("r2") = (unsigned long)__a2; \
register unsigned long r3 asm("r3")
#define __declare_arg_3(a0, a1, a2, a3, res) \
@@ -279,29 +279,29 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
typeof(a3) __a3 = a3; \
struct arm_smccc_res *___res = res; \
register unsigned long r0 asm("r0") = (u32)a0; \
- register unsigned long r1 asm("r1") = __a1; \
- register unsigned long r2 asm("r2") = __a2; \
- register unsigned long r3 asm("r3") = __a3
+ register unsigned long r1 asm("r1") = (unsigned long)__a1; \
+ register unsigned long r2 asm("r2") = (unsigned long)__a2; \
+ register unsigned long r3 asm("r3") = (unsigned long)__a3
#define __declare_arg_4(a0, a1, a2, a3, a4, res) \
typeof(a4) __a4 = a4; \
__declare_arg_3(a0, a1, a2, a3, res); \
- register unsigned long r4 asm("r4") = __a4
+ register unsigned long r4 asm("r4") = (unsigned long)__a4
#define __declare_arg_5(a0, a1, a2, a3, a4, a5, res) \
typeof(a5) __a5 = a5; \
__declare_arg_4(a0, a1, a2, a3, a4, res); \
- register unsigned long r5 asm("r5") = __a5
+ register unsigned long r5 asm("r5") = (unsigned long)__a5
#define __declare_arg_6(a0, a1, a2, a3, a4, a5, a6, res) \
typeof(a6) __a6 = a6; \
__declare_arg_5(a0, a1, a2, a3, a4, a5, res); \
- register unsigned long r6 asm("r6") = __a6
+ register unsigned long r6 asm("r6") = (unsigned long)__a6
#define __declare_arg_7(a0, a1, a2, a3, a4, a5, a6, a7, res) \
typeof(a7) __a7 = a7; \
__declare_arg_6(a0, a1, a2, a3, a4, a5, a6, res); \
- register unsigned long r7 asm("r7") = __a7
+ register unsigned long r7 asm("r7") = (unsigned long)__a7
#define ___declare_args(count, ...) __declare_arg_ ## count(__VA_ARGS__)
#define __declare_args(count, ...) ___declare_args(count, __VA_ARGS__)
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 16/20] KVM: arm64: nVHE: Pass pointers consistently to hyp-init
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (14 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 15/20] smccc: Cast arguments to unsigned long Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 17/20] KVM: arm64: nVHE: Migrate hyp interface to SMCCC Andrew Scull
` (3 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
Rather than some being kernel pointer and others being hyp pointers,
standardize on all pointers being hyp pointers.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/kvm/arm.c | 1 +
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 1 -
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 0e3bdb7cb9d0..3f5d00167674 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1278,6 +1278,7 @@ static void cpu_init_hyp_mode(void)
pgd_ptr = kvm_mmu_get_httbr();
hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE;
+ hyp_stack_ptr = kern_hyp_va(hyp_stack_ptr);
vector_ptr = (unsigned long)kern_hyp_va(kvm_ksym_ref(__kvm_hyp_host_vector));
/*
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index d9434e90c06d..abe885e26fe2 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -114,7 +114,6 @@ alternative_else_nop_endif
isb
/* Set the stack and new vectors */
- kern_hyp_va x1
mov sp, x1
msr vbar_el2, x2
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 17/20] KVM: arm64: nVHE: Migrate hyp interface to SMCCC
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (15 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 16/20] KVM: arm64: nVHE: Pass pointers consistently to hyp-init Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 18/20] KVM: arm64: nVHE: Migrate hyp-init " Andrew Scull
` (2 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, David Brazdil, will,
julien.thierry.kdev
Rather than passing arbitrary function pointers to run at hyp, define
and equivalent set of SMCCC functions.
Since the SMCCC functions are strongly tied to the original function
prototypes, it is not expected for the host to ever call an invalid ID
but a warning is raised if this does ever occur.
As __kvm_vcpu_run is used for every switch between the host and a guest,
it is explicitly singled out to be identified before the other function
IDs to improve the performance of the hot path.
Signed-off-by: Andrew Scull <ascull@google.com>
Signed-off-by: David Brazdil <dbrazdil@google.com>
---
arch/arm64/include/asm/kvm_asm.h | 24 ++++++
arch/arm64/include/asm/kvm_host.h | 25 ++++---
arch/arm64/kvm/arm.c | 2 +-
arch/arm64/kvm/hyp.S | 24 ++----
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 113 +++++++++++++++++++++++++----
5 files changed, 145 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 358262d4546e..57db148686ee 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -38,6 +38,30 @@
#define __SMCCC_WORKAROUND_1_SMC_SZ 36
+#define KVM_HOST_SMCCC_ID(id) \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_64, \
+ ARM_SMCCC_OWNER_STANDARD_HYP, \
+ (id))
+
+#define KVM_HOST_SMCCC_FUNC(name) KVM_HOST_SMCCC_ID(__KVM_HOST_SMCCC_FUNC_##name)
+
+#define __KVM_HOST_SMCCC_FUNC___kvm_hyp_init 0
+#define __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context 1
+#define __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa 2
+#define __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid 3
+#define __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_local_vmid 4
+#define __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff 5
+#define __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run 6
+#define __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs 7
+#define __KVM_HOST_SMCCC_FUNC___vgic_v3_get_ich_vtr_el2 8
+#define __KVM_HOST_SMCCC_FUNC___vgic_v3_read_vmcr 9
+#define __KVM_HOST_SMCCC_FUNC___vgic_v3_write_vmcr 10
+#define __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs 11
+#define __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 12
+#define __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs 13
+#define __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs 14
+
#ifndef __ASSEMBLY__
#include <linux/mm.h>
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index ec1a80796999..a2ab4933e211 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -11,6 +11,7 @@
#ifndef __ARM64_KVM_HOST_H__
#define __ARM64_KVM_HOST_H__
+#include <linux/arm-smccc.h>
#include <linux/bitmap.h>
#include <linux/types.h>
#include <linux/jump_label.h>
@@ -479,18 +480,20 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
void kvm_arm_halt_guest(struct kvm *kvm);
void kvm_arm_resume_guest(struct kvm *kvm);
-u64 __kvm_call_hyp(void *hypfn, ...);
+u64 __kvm_call_hyp_init(phys_addr_t pgd_ptr,
+ unsigned long hyp_stack_ptr,
+ unsigned long vector_ptr,
+ unsigned long tpidr_el2);
-#define kvm_call_hyp_nvhe(f, ...) \
- do { \
- DECLARE_KVM_NVHE_SYM(f); \
- __kvm_call_hyp(kvm_ksym_ref_nvhe(f), ##__VA_ARGS__); \
- } while(0)
-
-#define kvm_call_hyp_nvhe_ret(f, ...) \
+#define kvm_call_hyp_nvhe(f, ...) \
({ \
- DECLARE_KVM_NVHE_SYM(f); \
- __kvm_call_hyp(kvm_ksym_ref_nvhe(f), ##__VA_ARGS__); \
+ struct arm_smccc_res res; \
+ \
+ arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \
+ ##__VA_ARGS__, &res); \
+ WARN_ON(res.a0 != SMCCC_RET_SUCCESS); \
+ \
+ res.a1; \
})
/*
@@ -516,7 +519,7 @@ u64 __kvm_call_hyp(void *hypfn, ...);
ret = f(__VA_ARGS__); \
isb(); \
} else { \
- ret = kvm_call_hyp_nvhe_ret(f, ##__VA_ARGS__); \
+ ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
} \
\
ret; \
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 3f5d00167674..661fdae8ea3f 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1288,7 +1288,7 @@ static void cpu_init_hyp_mode(void)
* cpus_have_const_cap() wrapper.
*/
BUG_ON(!system_capabilities_finalized());
- __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2);
+ __kvm_call_hyp_init(pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2);
/*
* Disabling SSBD on a non-VHE system requires us to enable SSBS
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index 3c79a1124af2..12aa426f7559 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -11,24 +11,12 @@
#include <asm/cpufeature.h>
/*
- * u64 __kvm_call_hyp(void *hypfn, ...);
- *
- * This is not really a variadic function in the classic C-way and care must
- * be taken when calling this to ensure parameters are passed in registers
- * only, since the stack will change between the caller and the callee.
- *
- * Call the function with the first argument containing a pointer to the
- * function you wish to call in Hyp mode, and subsequent arguments will be
- * passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the
- * function pointer can be passed). The function being called must be mapped
- * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
- * passed in x0.
- *
- * A function pointer with a value less than 0xfff has a special meaning,
- * and is used to implement hyp stubs in the same way as in
- * arch/arm64/kernel/hyp_stub.S.
+ * u64 __kvm_call_hyp_init(phys_addr_t pgd_ptr,
+ * unsigned long hyp_stack_ptr,
+ * unsigned long vector_ptr,
+ * unsigned long tpidr_el2);
*/
-SYM_FUNC_START(__kvm_call_hyp)
+SYM_FUNC_START(__kvm_call_hyp_init)
hvc #0
ret
-SYM_FUNC_END(__kvm_call_hyp)
+SYM_FUNC_END(__kvm_call_hyp_init)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index c8938e09f585..13093df70c87 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -12,24 +12,111 @@
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
-typedef unsigned long (*hypcall_fn_t)
- (unsigned long, unsigned long, unsigned long);
+#include <kvm/arm_hypercalls.h>
+
+static void handle_host_hcall(unsigned long func_id,
+ struct kvm_cpu_context *host_ctxt)
+{
+ unsigned long ret = 0;
+
+ /*
+ * __kvm_vcpu_run is a hot path of the context switch so identify it
+ * quickly before searching through the other functions IDs.
+ */
+ if (func_id == KVM_HOST_SMCCC_FUNC(__kvm_vcpu_run)) {
+ struct kvm_vcpu *vcpu =
+ (struct kvm_vcpu *)host_ctxt->regs.regs[1];
+
+ ret = __kvm_vcpu_run(vcpu);
+ goto out;
+ }
+
+ switch (func_id) {
+ case KVM_HOST_SMCCC_FUNC(__kvm_flush_vm_context):
+ __kvm_flush_vm_context();
+ break;
+ case KVM_HOST_SMCCC_FUNC(__kvm_tlb_flush_vmid_ipa): {
+ struct kvm_s2_mmu *mmu =
+ (struct kvm_s2_mmu *)host_ctxt->regs.regs[1];
+ phys_addr_t ipa = host_ctxt->regs.regs[2];
+ int level = host_ctxt->regs.regs[3];
+
+ __kvm_tlb_flush_vmid_ipa(mmu, ipa, level);
+ break;
+ }
+ case KVM_HOST_SMCCC_FUNC(__kvm_tlb_flush_vmid): {
+ struct kvm_s2_mmu *mmu =
+ (struct kvm_s2_mmu *)host_ctxt->regs.regs[1];
+
+ __kvm_tlb_flush_vmid(mmu);
+ break;
+ }
+ case KVM_HOST_SMCCC_FUNC(__kvm_tlb_flush_local_vmid): {
+ struct kvm_s2_mmu *mmu =
+ (struct kvm_s2_mmu *)host_ctxt->regs.regs[1];
+
+ __kvm_tlb_flush_local_vmid(mmu);
+ break;
+ }
+ case KVM_HOST_SMCCC_FUNC(__kvm_timer_set_cntvoff): {
+ u64 cntvoff = host_ctxt->regs.regs[1];
+
+ __kvm_timer_set_cntvoff(cntvoff);
+ break;
+ }
+ case KVM_HOST_SMCCC_FUNC(__kvm_enable_ssbs):
+ __kvm_enable_ssbs();
+ break;
+ case KVM_HOST_SMCCC_FUNC(__vgic_v3_get_ich_vtr_el2):
+ ret = __vgic_v3_get_ich_vtr_el2();
+ break;
+ case KVM_HOST_SMCCC_FUNC(__vgic_v3_read_vmcr):
+ ret = __vgic_v3_read_vmcr();
+ break;
+ case KVM_HOST_SMCCC_FUNC(__vgic_v3_write_vmcr): {
+ u32 vmcr = host_ctxt->regs.regs[1];
+
+ __vgic_v3_write_vmcr(vmcr);
+ break;
+ }
+ case KVM_HOST_SMCCC_FUNC(__vgic_v3_init_lrs):
+ __vgic_v3_init_lrs();
+ break;
+ case KVM_HOST_SMCCC_FUNC(__kvm_get_mdcr_el2):
+ ret = __kvm_get_mdcr_el2();
+ break;
+ case KVM_HOST_SMCCC_FUNC(__vgic_v3_save_aprs): {
+ struct vgic_v3_cpu_if *cpu_if =
+ (struct vgic_v3_cpu_if *)host_ctxt->regs.regs[1];
+
+ __vgic_v3_save_aprs(cpu_if);
+ break;
+ }
+ case KVM_HOST_SMCCC_FUNC(__vgic_v3_restore_aprs): {
+ struct vgic_v3_cpu_if *cpu_if =
+ (struct vgic_v3_cpu_if *)host_ctxt->regs.regs[1];
+
+ __vgic_v3_restore_aprs(cpu_if);
+ break;
+ }
+ default:
+ /* Invalid host HVC. */
+ host_ctxt->regs.regs[0] = SMCCC_RET_NOT_SUPPORTED;
+ return;
+ }
+
+out:
+ host_ctxt->regs.regs[0] = SMCCC_RET_SUCCESS;
+ host_ctxt->regs.regs[1] = ret;
+}
void handle_trap(struct kvm_cpu_context *host_ctxt) {
u64 esr = read_sysreg_el2(SYS_ESR);
- hypcall_fn_t func;
- unsigned long ret;
+ unsigned long func_id;
if (ESR_ELx_EC(esr) != ESR_ELx_EC_HVC64)
hyp_panic();
- /*
- * __kvm_call_hyp takes a pointer in the host address space and
- * up to three arguments.
- */
- func = (hypcall_fn_t)kern_hyp_va(host_ctxt->regs.regs[0]);
- ret = func(host_ctxt->regs.regs[1],
- host_ctxt->regs.regs[2],
- host_ctxt->regs.regs[3]);
- host_ctxt->regs.regs[0] = ret;
+ func_id = host_ctxt->regs.regs[0];
+ handle_host_hcall(func_id, host_ctxt);
}
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 18/20] KVM: arm64: nVHE: Migrate hyp-init to SMCCC
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (16 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 17/20] KVM: arm64: nVHE: Migrate hyp interface to SMCCC Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 19/20] KVM: arm64: nVHE: Fix pointers during SMCCC convertion Andrew Scull
2020-08-20 10:34 ` [PATCH v2 20/20] KVM: arm64: nVHE: Handle stub HVCs in the host loop Andrew Scull
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
To complete the transition to SMCCC, the hyp initialization is given a
function ID. This looks neater than comparing the hyp stub function IDs
to the page table physical address.
Some care is taken to only clobber x0-3 before the host context is saved
as only those registers can be clobbered accoring to SMCCC. Fortunately,
only a few acrobatics are needed. The possible new tpidr_el2 is moved to
the argument in x2 so that it can be stashed in tpidr_el2 early to free
up a scratch register. The page table configuration then makes use of
x0-2.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/include/asm/kvm_host.h | 5 ---
arch/arm64/kvm/Makefile | 2 +-
arch/arm64/kvm/arm.c | 5 ++-
arch/arm64/kvm/hyp.S | 22 ----------
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 66 +++++++++++++++++-------------
5 files changed, 43 insertions(+), 57 deletions(-)
delete mode 100644 arch/arm64/kvm/hyp.S
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index a2ab4933e211..f8fe2d863f9b 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -480,11 +480,6 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
void kvm_arm_halt_guest(struct kvm *kvm);
void kvm_arm_resume_guest(struct kvm *kvm);
-u64 __kvm_call_hyp_init(phys_addr_t pgd_ptr,
- unsigned long hyp_stack_ptr,
- unsigned long vector_ptr,
- unsigned long tpidr_el2);
-
#define kvm_call_hyp_nvhe(f, ...) \
({ \
struct arm_smccc_res res; \
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 99977c1972cc..1504c81fbf5d 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -13,7 +13,7 @@ obj-$(CONFIG_KVM) += hyp/
kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
$(KVM)/vfio.o $(KVM)/irqchip.o \
arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \
- inject_fault.o regmap.o va_layout.o hyp.o handle_exit.o \
+ inject_fault.o regmap.o va_layout.o handle_exit.o \
guest.o debug.o reset.o sys_regs.o \
vgic-sys-reg-v3.o fpsimd.o pmu.o \
aarch32.o arch_timer.o \
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 661fdae8ea3f..babd61fbb0c1 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1264,6 +1264,7 @@ static void cpu_init_hyp_mode(void)
unsigned long hyp_stack_ptr;
unsigned long vector_ptr;
unsigned long tpidr_el2;
+ struct arm_smccc_res res;
/* Switch from the HYP stub to our own HYP init vector */
__hyp_set_vectors(kvm_get_idmap_vector());
@@ -1288,7 +1289,9 @@ static void cpu_init_hyp_mode(void)
* cpus_have_const_cap() wrapper.
*/
BUG_ON(!system_capabilities_finalized());
- __kvm_call_hyp_init(pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2);
+ arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init),
+ pgd_ptr, tpidr_el2, hyp_stack_ptr, vector_ptr, &res);
+ WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
/*
* Disabling SSBD on a non-VHE system requires us to enable SSBS
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
deleted file mode 100644
index 12aa426f7559..000000000000
--- a/arch/arm64/kvm/hyp.S
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2012,2013 - ARM Ltd
- * Author: Marc Zyngier <marc.zyngier@arm.com>
- */
-
-#include <linux/linkage.h>
-
-#include <asm/alternative.h>
-#include <asm/assembler.h>
-#include <asm/cpufeature.h>
-
-/*
- * u64 __kvm_call_hyp_init(phys_addr_t pgd_ptr,
- * unsigned long hyp_stack_ptr,
- * unsigned long vector_ptr,
- * unsigned long tpidr_el2);
- */
-SYM_FUNC_START(__kvm_call_hyp_init)
- hvc #0
- ret
-SYM_FUNC_END(__kvm_call_hyp_init)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index abe885e26fe2..47224dc62c51 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -4,11 +4,13 @@
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
+#include <linux/arm-smccc.h>
#include <linux/linkage.h>
#include <asm/alternative.h>
#include <asm/assembler.h>
#include <asm/kvm_arm.h>
+#include <asm/kvm_asm.h>
#include <asm/kvm_mmu.h>
#include <asm/pgtable-hwdef.h>
#include <asm/sysreg.h>
@@ -44,27 +46,37 @@ __invalid:
b .
/*
- * x0: HYP pgd
- * x1: HYP stack
- * x2: HYP vectors
- * x3: per-CPU offset
+ * x0: SMCCC function ID
+ * x1: HYP pgd
+ * x2: per-CPU offset
+ * x3: HYP stack
+ * x4: HYP vectors
*/
__do_hyp_init:
/* Check for a stub HVC call */
cmp x0, #HVC_STUB_HCALL_NR
b.lo __kvm_handle_stub_hvc
- phys_to_ttbr x4, x0
+ /* Set tpidr_el2 for use by HYP to free a register */
+ msr tpidr_el2, x2
+
+ mov x2, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
+ cmp x0, x2
+ b.eq 1f
+ mov x0, #SMCCC_RET_NOT_SUPPORTED
+ eret
+
+1: phys_to_ttbr x0, x1
alternative_if ARM64_HAS_CNP
- orr x4, x4, #TTBR_CNP_BIT
+ orr x0, x0, #TTBR_CNP_BIT
alternative_else_nop_endif
- msr ttbr0_el2, x4
+ msr ttbr0_el2, x0
- mrs x4, tcr_el1
- mov_q x5, TCR_EL2_MASK
- and x4, x4, x5
- mov x5, #TCR_EL2_RES1
- orr x4, x4, x5
+ mrs x0, tcr_el1
+ mov_q x1, TCR_EL2_MASK
+ and x0, x0, x1
+ mov x1, #TCR_EL2_RES1
+ orr x0, x0, x1
/*
* The ID map may be configured to use an extended virtual address
@@ -80,18 +92,18 @@ alternative_else_nop_endif
*
* So use the same T0SZ value we use for the ID map.
*/
- ldr_l x5, idmap_t0sz
- bfi x4, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
+ ldr_l x1, idmap_t0sz
+ bfi x0, x1, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
/*
* Set the PS bits in TCR_EL2.
*/
- tcr_compute_pa_size x4, #TCR_EL2_PS_SHIFT, x5, x6
+ tcr_compute_pa_size x0, #TCR_EL2_PS_SHIFT, x1, x2
- msr tcr_el2, x4
+ msr tcr_el2, x0
- mrs x4, mair_el1
- msr mair_el2, x4
+ mrs x0, mair_el1
+ msr mair_el2, x0
isb
/* Invalidate the stale TLBs from Bootloader */
@@ -103,24 +115,22 @@ alternative_else_nop_endif
* as well as the EE bit on BE. Drop the A flag since the compiler
* is allowed to generate unaligned accesses.
*/
- mov_q x4, (SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A))
-CPU_BE( orr x4, x4, #SCTLR_ELx_EE)
+ mov_q x0, (SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A))
+CPU_BE( orr x0, x0, #SCTLR_ELx_EE)
alternative_if ARM64_HAS_ADDRESS_AUTH
- mov_q x5, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
+ mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)
- orr x4, x4, x5
+ orr x0, x0, x1
alternative_else_nop_endif
- msr sctlr_el2, x4
+ msr sctlr_el2, x0
isb
/* Set the stack and new vectors */
- mov sp, x1
- msr vbar_el2, x2
-
- /* Set tpidr_el2 for use by HYP */
- msr tpidr_el2, x3
+ mov sp, x3
+ msr vbar_el2, x4
/* Hello, World! */
+ mov x0, #SMCCC_RET_SUCCESS
eret
SYM_CODE_END(__kvm_hyp_init)
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 19/20] KVM: arm64: nVHE: Fix pointers during SMCCC convertion
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (17 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 18/20] KVM: arm64: nVHE: Migrate hyp-init " Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
2020-08-20 10:34 ` [PATCH v2 20/20] KVM: arm64: nVHE: Handle stub HVCs in the host loop Andrew Scull
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
The host need not concern itself with the pointer differences for the
hyp interfaces that are shared between VHE and nVHE so leave it to the
hyp to handle.
As the SMCCC function IDs are converted into function calls, it is a
suitable place to also convert any pointer arguments into hyp pointers.
This, additionally, eases the reuse of the handlers in different
contexts.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 12 ++++++------
arch/arm64/kvm/hyp/nvhe/switch.c | 2 --
arch/arm64/kvm/hyp/nvhe/tlb.c | 2 --
arch/arm64/kvm/vgic/vgic-v3.c | 4 ++--
4 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 13093df70c87..78d7afcefbb8 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -27,7 +27,7 @@ static void handle_host_hcall(unsigned long func_id,
struct kvm_vcpu *vcpu =
(struct kvm_vcpu *)host_ctxt->regs.regs[1];
- ret = __kvm_vcpu_run(vcpu);
+ ret = __kvm_vcpu_run(kern_hyp_va(vcpu));
goto out;
}
@@ -41,21 +41,21 @@ static void handle_host_hcall(unsigned long func_id,
phys_addr_t ipa = host_ctxt->regs.regs[2];
int level = host_ctxt->regs.regs[3];
- __kvm_tlb_flush_vmid_ipa(mmu, ipa, level);
+ __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level);
break;
}
case KVM_HOST_SMCCC_FUNC(__kvm_tlb_flush_vmid): {
struct kvm_s2_mmu *mmu =
(struct kvm_s2_mmu *)host_ctxt->regs.regs[1];
- __kvm_tlb_flush_vmid(mmu);
+ __kvm_tlb_flush_vmid(kern_hyp_va(mmu));
break;
}
case KVM_HOST_SMCCC_FUNC(__kvm_tlb_flush_local_vmid): {
struct kvm_s2_mmu *mmu =
(struct kvm_s2_mmu *)host_ctxt->regs.regs[1];
- __kvm_tlb_flush_local_vmid(mmu);
+ __kvm_tlb_flush_local_vmid(kern_hyp_va(mmu));
break;
}
case KVM_HOST_SMCCC_FUNC(__kvm_timer_set_cntvoff): {
@@ -89,14 +89,14 @@ static void handle_host_hcall(unsigned long func_id,
struct vgic_v3_cpu_if *cpu_if =
(struct vgic_v3_cpu_if *)host_ctxt->regs.regs[1];
- __vgic_v3_save_aprs(cpu_if);
+ __vgic_v3_save_aprs(kern_hyp_va(cpu_if));
break;
}
case KVM_HOST_SMCCC_FUNC(__vgic_v3_restore_aprs): {
struct vgic_v3_cpu_if *cpu_if =
(struct vgic_v3_cpu_if *)host_ctxt->regs.regs[1];
- __vgic_v3_restore_aprs(cpu_if);
+ __vgic_v3_restore_aprs(kern_hyp_va(cpu_if));
break;
}
default:
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index b7abe66fde22..0d8187358565 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -176,8 +176,6 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
pmr_sync();
}
- vcpu = kern_hyp_va(vcpu);
-
host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
host_ctxt->__hyp_running_vcpu = vcpu;
guest_ctxt = &vcpu->arch.ctxt;
diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c
index 69eae608d670..544bca3072b7 100644
--- a/arch/arm64/kvm/hyp/nvhe/tlb.c
+++ b/arch/arm64/kvm/hyp/nvhe/tlb.c
@@ -54,7 +54,6 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
__tlb_switch_to_guest(mmu, &cxt);
/*
@@ -108,7 +107,6 @@ void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
dsb(ishst);
/* Switch to requested VMID */
- mmu = kern_hyp_va(mmu);
__tlb_switch_to_guest(mmu, &cxt);
__tlbi(vmalls12e1is);
diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c
index 76e2d85789ed..9cdf39a94a63 100644
--- a/arch/arm64/kvm/vgic/vgic-v3.c
+++ b/arch/arm64/kvm/vgic/vgic-v3.c
@@ -662,7 +662,7 @@ void vgic_v3_load(struct kvm_vcpu *vcpu)
if (likely(cpu_if->vgic_sre))
kvm_call_hyp(__vgic_v3_write_vmcr, cpu_if->vgic_vmcr);
- kvm_call_hyp(__vgic_v3_restore_aprs, kern_hyp_va(cpu_if));
+ kvm_call_hyp(__vgic_v3_restore_aprs, cpu_if);
if (has_vhe())
__vgic_v3_activate_traps(cpu_if);
@@ -686,7 +686,7 @@ void vgic_v3_put(struct kvm_vcpu *vcpu)
vgic_v3_vmcr_sync(vcpu);
- kvm_call_hyp(__vgic_v3_save_aprs, kern_hyp_va(cpu_if));
+ kvm_call_hyp(__vgic_v3_save_aprs, cpu_if);
if (has_vhe())
__vgic_v3_deactivate_traps(cpu_if);
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 20/20] KVM: arm64: nVHE: Handle stub HVCs in the host loop
2020-08-20 10:34 [PATCH v2 00/20] Introduce separate nVHE hyp context Andrew Scull
` (18 preceding siblings ...)
2020-08-20 10:34 ` [PATCH v2 19/20] KVM: arm64: nVHE: Fix pointers during SMCCC convertion Andrew Scull
@ 2020-08-20 10:34 ` Andrew Scull
19 siblings, 0 replies; 21+ messages in thread
From: Andrew Scull @ 2020-08-20 10:34 UTC (permalink / raw)
To: kvmarm
Cc: linux-arm-kernel, kernel-team, suzuki.poulose, maz, Sudeep Holla,
james.morse, Andrew Scull, catalin.marinas, will,
julien.thierry.kdev
Since the host is called from the hyp run loop, we can use that context
to identify calls from the host rather than checking VTTBR_EL2, which
will be used for the host's stage 2 in future.
Moving this to C also allows for more flexibiliy e.g. in applying
policies, such as forbidding HVC_RESET_VECTORS, based on the current
state of the hypervisor and removes the special casing for nVHE in the
exception handler.
Control over arch workaround 2 is made available to the host, the same
as any other vcpu.
Signed-off-by: Andrew Scull <ascull@google.com>
---
arch/arm64/kvm/hyp/hyp-entry.S | 1 -
arch/arm64/kvm/hyp/nvhe/host.S | 26 +--------------
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 13 ++++++--
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 51 +++++++++++++++++++++++++++++-
4 files changed, 62 insertions(+), 29 deletions(-)
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index b7a1ea221f6e..db54f84003ec 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -24,7 +24,6 @@ el1_sync: // Guest trapped into EL2
ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
b.ne el1_trap
-el1_hvc_guest:
/*
* Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
* The workaround has already been applied on the host,
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 40620c1c87b8..68d425b7bd62 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -95,31 +95,7 @@ SYM_FUNC_END(__hyp_do_panic)
.macro valid_host_el1_sync_vect
.align 7
- stp x0, x1, [sp, #-16]!
- mrs x0, esr_el2
- lsr x0, x0, #ESR_ELx_EC_SHIFT
- cmp x0, #ESR_ELx_EC_HVC64
- ldp x0, x1, [sp], #16
- b.ne __host_exit
-
- /* Check for a stub HVC call */
- cmp x0, #HVC_STUB_HCALL_NR
- b.hs __host_exit
-
- /*
- * Compute the idmap address of __kvm_handle_stub_hvc and
- * jump there. Since we use kimage_voffset, do not use the
- * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
- * (by loading it from the constant pool).
- *
- * Preserve x0-x4, which may contain stub parameters.
- */
- ldr x5, =__kvm_handle_stub_hvc
- ldr_l x6, kimage_voffset
-
- /* x5 = __pa(x5) */
- sub x5, x5, x6
- br x5
+ b __host_exit
.endm
.macro invalid_host_el2_vect
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 47224dc62c51..40b5aa95c194 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -16,6 +16,9 @@
#include <asm/sysreg.h>
#include <asm/virt.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_ptrauth.h>
+
.text
.pushsection .hyp.idmap.text, "ax"
@@ -138,6 +141,7 @@ SYM_CODE_START(__kvm_handle_stub_hvc)
cmp x0, #HVC_SOFT_RESTART
b.ne 1f
+SYM_INNER_LABEL(__kvm_handle_stub_hvc_soft_restart, SYM_L_GLOBAL)
/* This is where we're about to jump, staying at EL2 */
msr elr_el2, x1
mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT | PSR_MODE_EL2h)
@@ -150,8 +154,13 @@ SYM_CODE_START(__kvm_handle_stub_hvc)
b reset
1: cmp x0, #HVC_RESET_VECTORS
- b.ne 1f
+ b.ne 3f
+ b 2f
+
+SYM_INNER_LABEL(__kvm_handle_stub_hvc_reset_vectors, SYM_L_GLOBAL)
+ restore_callee_saved_regs x5
+2:
/*
* Set the HVC_RESET_VECTORS return code before entering the common
* path so that we do not clobber x0-x2 in case we are coming via
@@ -172,7 +181,7 @@ reset:
msr vbar_el2, x5
eret
-1: /* Bad stub call */
+3: /* Bad stub call */
mov_q x0, HVC_STUB_ERR
eret
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 78d7afcefbb8..302333ea96eb 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -14,6 +14,52 @@
#include <kvm/arm_hypercalls.h>
+typedef __noreturn unsigned long (*stub_hvc_handler_t)
+ (unsigned long, unsigned long, unsigned long, unsigned long,
+ unsigned long, struct kvm_cpu_context *);
+
+extern char __kvm_handle_stub_hvc_soft_restart[];
+extern char __kvm_handle_stub_hvc_reset_vectors[];
+
+static void handle_stub_hvc(unsigned long func_id,
+ struct kvm_cpu_context *host_ctxt)
+{
+ char *stub_hvc_handler_kern_va;
+ stub_hvc_handler_t stub_hvc_handler;
+
+ /*
+ * The handlers of the supported stub HVCs disable the MMU so they must
+ * be called in the idmap. We compute the idmap address by subtracting
+ * kimage_voffset from the kernel VA handler.
+ */
+ switch (func_id) {
+ case HVC_SOFT_RESTART:
+ asm volatile("ldr %0, =%1"
+ : "=r" (stub_hvc_handler_kern_va)
+ : "S" (__kvm_handle_stub_hvc_soft_restart));
+ break;
+ case HVC_RESET_VECTORS:
+ asm volatile("ldr %0, =%1"
+ : "=r" (stub_hvc_handler_kern_va)
+ : "S" (__kvm_handle_stub_hvc_reset_vectors));
+ break;
+ default:
+ host_ctxt->regs.regs[0] = HVC_STUB_ERR;
+ return;
+ }
+
+ stub_hvc_handler = (stub_hvc_handler_t)
+ (stub_hvc_handler_kern_va - kimage_voffset);
+
+ /* Preserve x0-x4, which may contain stub parameters. */
+ stub_hvc_handler(func_id,
+ host_ctxt->regs.regs[1],
+ host_ctxt->regs.regs[2],
+ host_ctxt->regs.regs[3],
+ host_ctxt->regs.regs[4],
+ host_ctxt);
+}
+
static void handle_host_hcall(unsigned long func_id,
struct kvm_cpu_context *host_ctxt)
{
@@ -118,5 +164,8 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) {
hyp_panic();
func_id = host_ctxt->regs.regs[0];
- handle_host_hcall(func_id, host_ctxt);
+ if (func_id < HVC_STUB_HCALL_NR)
+ handle_stub_hvc(func_id, host_ctxt);
+ else
+ handle_host_hcall(func_id, host_ctxt);
}
--
2.28.0.220.ged08abb693-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 21+ messages in thread