* [PATCH v3 20/41] KVM: arm64: Introduce separate VHE/non-VHE sysreg save/restore functions
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
As we are about to handle system registers quite differently between VHE
and non-VHE systems. In preparation for that, we need to split some of
the handling functions between VHE and non-VHE functionality.
For now, we simply copy the non-VHE functions, but we do change the use
of static keys for VHE and non-VHE functionality now that we have
separate functions.
Reviewed-by: Andrew Jones <drjones@redhat.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/include/asm/kvm_hyp.h | 12 ++++++++----
arch/arm64/kvm/hyp/switch.c | 20 ++++++++++----------
arch/arm64/kvm/hyp/sysreg-sr.c | 40 ++++++++++++++++++++++++++++++++--------
3 files changed, 50 insertions(+), 22 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index a0e5a7038237..998152da9b66 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -132,10 +132,14 @@ int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
void __timer_enable_traps(struct kvm_vcpu *vcpu);
void __timer_disable_traps(struct kvm_vcpu *vcpu);
-void __sysreg_save_host_state(struct kvm_cpu_context *ctxt);
-void __sysreg_restore_host_state(struct kvm_cpu_context *ctxt);
-void __sysreg_save_guest_state(struct kvm_cpu_context *ctxt);
-void __sysreg_restore_guest_state(struct kvm_cpu_context *ctxt);
+void __sysreg_save_host_state_nvhe(struct kvm_cpu_context *ctxt);
+void __sysreg_restore_host_state_nvhe(struct kvm_cpu_context *ctxt);
+void __sysreg_save_guest_state_nvhe(struct kvm_cpu_context *ctxt);
+void __sysreg_restore_guest_state_nvhe(struct kvm_cpu_context *ctxt);
+void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt);
+void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt);
+void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt);
+void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt);
void __sysreg32_save_state(struct kvm_vcpu *vcpu);
void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 6175fcb33ed2..42e0123ecd69 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -348,7 +348,7 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
host_ctxt->__hyp_running_vcpu = vcpu;
guest_ctxt = &vcpu->arch.ctxt;
- __sysreg_save_host_state(host_ctxt);
+ sysreg_save_host_state_vhe(host_ctxt);
__activate_traps(vcpu);
__activate_vm(vcpu->kvm);
@@ -360,7 +360,7 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
*/
__sysreg32_restore_state(vcpu);
- __sysreg_restore_guest_state(guest_ctxt);
+ sysreg_restore_guest_state_vhe(guest_ctxt);
__debug_switch_to_guest(vcpu);
do {
@@ -370,13 +370,13 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
/* And we're baaack! */
} while (fixup_guest_exit(vcpu, &exit_code));
- __sysreg_save_guest_state(guest_ctxt);
+ sysreg_save_guest_state_vhe(guest_ctxt);
__sysreg32_save_state(vcpu);
__vgic_save_state(vcpu);
__deactivate_traps(vcpu);
- __sysreg_restore_host_state(host_ctxt);
+ sysreg_restore_host_state_vhe(host_ctxt);
/*
* This must come after restoring the host sysregs, since a non-VHE
@@ -400,7 +400,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
host_ctxt->__hyp_running_vcpu = vcpu;
guest_ctxt = &vcpu->arch.ctxt;
- __sysreg_save_host_state(host_ctxt);
+ __sysreg_save_host_state_nvhe(host_ctxt);
__activate_traps(vcpu);
__activate_vm(kern_hyp_va(vcpu->kvm));
@@ -413,7 +413,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
*/
__sysreg32_restore_state(vcpu);
- __sysreg_restore_guest_state(guest_ctxt);
+ __sysreg_restore_guest_state_nvhe(guest_ctxt);
__debug_switch_to_guest(vcpu);
do {
@@ -423,7 +423,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
/* And we're baaack! */
} while (fixup_guest_exit(vcpu, &exit_code));
- __sysreg_save_guest_state(guest_ctxt);
+ __sysreg_save_guest_state_nvhe(guest_ctxt);
__sysreg32_save_state(vcpu);
__timer_disable_traps(vcpu);
__vgic_save_state(vcpu);
@@ -431,7 +431,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
__deactivate_traps(vcpu);
__deactivate_vm(vcpu);
- __sysreg_restore_host_state(host_ctxt);
+ __sysreg_restore_host_state_nvhe(host_ctxt);
/*
* This must come after restoring the host sysregs, since a non-VHE
@@ -456,7 +456,7 @@ static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
__timer_disable_traps(vcpu);
__deactivate_traps(vcpu);
__deactivate_vm(vcpu);
- __sysreg_restore_host_state(__host_ctxt);
+ __sysreg_restore_host_state_nvhe(__host_ctxt);
}
/*
@@ -479,7 +479,7 @@ static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
vcpu = host_ctxt->__hyp_running_vcpu;
__deactivate_traps(vcpu);
- __sysreg_restore_host_state(host_ctxt);
+ sysreg_restore_host_state_vhe(host_ctxt);
panic(__hyp_panic_string,
spsr, elr,
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index 72cdbc1f678b..5cbde1016303 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -75,15 +75,27 @@ static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
ctxt->gp_regs.regs.pstate = read_sysreg_el2(spsr);
}
-void __hyp_text __sysreg_save_host_state(struct kvm_cpu_context *ctxt)
+void __hyp_text __sysreg_save_host_state_nvhe(struct kvm_cpu_context *ctxt)
+{
+ __sysreg_save_el1_state(ctxt);
+ __sysreg_save_common_state(ctxt);
+ __sysreg_save_user_state(ctxt);
+}
+
+void __hyp_text __sysreg_save_guest_state_nvhe(struct kvm_cpu_context *ctxt)
+{
+ __sysreg_save_el1_state(ctxt);
+ __sysreg_save_common_state(ctxt);
+ __sysreg_save_user_state(ctxt);
+}
+
+void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
{
- if (!has_vhe())
- __sysreg_save_el1_state(ctxt);
__sysreg_save_common_state(ctxt);
__sysreg_save_user_state(ctxt);
}
-void __hyp_text __sysreg_save_guest_state(struct kvm_cpu_context *ctxt)
+void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
{
__sysreg_save_el1_state(ctxt);
__sysreg_save_common_state(ctxt);
@@ -136,15 +148,27 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
write_sysreg_el2(ctxt->gp_regs.regs.pstate, spsr);
}
-void __hyp_text __sysreg_restore_host_state(struct kvm_cpu_context *ctxt)
+void __hyp_text __sysreg_restore_host_state_nvhe(struct kvm_cpu_context *ctxt)
+{
+ __sysreg_restore_el1_state(ctxt);
+ __sysreg_restore_common_state(ctxt);
+ __sysreg_restore_user_state(ctxt);
+}
+
+void __hyp_text __sysreg_restore_guest_state_nvhe(struct kvm_cpu_context *ctxt)
+{
+ __sysreg_restore_el1_state(ctxt);
+ __sysreg_restore_common_state(ctxt);
+ __sysreg_restore_user_state(ctxt);
+}
+
+void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
{
- if (!has_vhe())
- __sysreg_restore_el1_state(ctxt);
__sysreg_restore_common_state(ctxt);
__sysreg_restore_user_state(ctxt);
}
-void __hyp_text __sysreg_restore_guest_state(struct kvm_cpu_context *ctxt)
+void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
{
__sysreg_restore_el1_state(ctxt);
__sysreg_restore_common_state(ctxt);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 19/41] KVM: arm64: Rewrite sysreg alternatives to static keys
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
As we are about to move calls around in the sysreg save/restore logic,
let's first rewrite the alternative function callers, because it is
going to make the next patches much easier to read.
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/sysreg-sr.c | 17 ++++-------------
1 file changed, 4 insertions(+), 13 deletions(-)
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index 99dd50ce483b..72cdbc1f678b 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -22,9 +22,6 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
-/* Yes, this does nothing, on purpose */
-static void __hyp_text __sysreg_do_nothing(struct kvm_cpu_context *ctxt) { }
-
/*
* Non-VHE: Both host and guest must save everything.
*
@@ -78,13 +75,10 @@ static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
ctxt->gp_regs.regs.pstate = read_sysreg_el2(spsr);
}
-static hyp_alternate_select(__sysreg_call_save_host_state,
- __sysreg_save_el1_state, __sysreg_do_nothing,
- ARM64_HAS_VIRT_HOST_EXTN);
-
void __hyp_text __sysreg_save_host_state(struct kvm_cpu_context *ctxt)
{
- __sysreg_call_save_host_state()(ctxt);
+ if (!has_vhe())
+ __sysreg_save_el1_state(ctxt);
__sysreg_save_common_state(ctxt);
__sysreg_save_user_state(ctxt);
}
@@ -142,13 +136,10 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
write_sysreg_el2(ctxt->gp_regs.regs.pstate, spsr);
}
-static hyp_alternate_select(__sysreg_call_restore_host_state,
- __sysreg_restore_el1_state, __sysreg_do_nothing,
- ARM64_HAS_VIRT_HOST_EXTN);
-
void __hyp_text __sysreg_restore_host_state(struct kvm_cpu_context *ctxt)
{
- __sysreg_call_restore_host_state()(ctxt);
+ if (!has_vhe())
+ __sysreg_restore_el1_state(ctxt);
__sysreg_restore_common_state(ctxt);
__sysreg_restore_user_state(ctxt);
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 18/41] KVM: arm64: Move userspace system registers into separate function
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
There's a semantic difference between the EL1 registers that control
operation of a kernel running in EL1 and EL1 registers that only control
userspace execution in EL0. Since we can defer saving/restoring the
latter, move them into their own function.
We also take this chance to rename the function saving/restoring the
remaining system register to make it clear this function deals with
the EL1 system registers.
No functional change.
Reviewed-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/sysreg-sr.c | 46 +++++++++++++++++++++++++++++++-----------
1 file changed, 34 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index 848a46eb33bf..99dd50ce483b 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -34,18 +34,27 @@ static void __hyp_text __sysreg_do_nothing(struct kvm_cpu_context *ctxt) { }
static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
{
- ctxt->sys_regs[ACTLR_EL1] = read_sysreg(actlr_el1);
- ctxt->sys_regs[TPIDR_EL0] = read_sysreg(tpidr_el0);
- ctxt->sys_regs[TPIDRRO_EL0] = read_sysreg(tpidrro_el0);
ctxt->sys_regs[MDSCR_EL1] = read_sysreg(mdscr_el1);
+
+ /*
+ * The host arm64 Linux uses sp_el0 to point to 'current' and it must
+ * therefore be saved/restored on every entry/exit to/from the guest.
+ */
ctxt->gp_regs.regs.sp = read_sysreg(sp_el0);
}
-static void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt)
+static void __hyp_text __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
+{
+ ctxt->sys_regs[TPIDR_EL0] = read_sysreg(tpidr_el0);
+ ctxt->sys_regs[TPIDRRO_EL0] = read_sysreg(tpidrro_el0);
+}
+
+static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
{
ctxt->sys_regs[MPIDR_EL1] = read_sysreg(vmpidr_el2);
ctxt->sys_regs[CSSELR_EL1] = read_sysreg(csselr_el1);
ctxt->sys_regs[SCTLR_EL1] = read_sysreg_el1(sctlr);
+ ctxt->sys_regs[ACTLR_EL1] = read_sysreg(actlr_el1);
ctxt->sys_regs[CPACR_EL1] = read_sysreg_el1(cpacr);
ctxt->sys_regs[TTBR0_EL1] = read_sysreg_el1(ttbr0);
ctxt->sys_regs[TTBR1_EL1] = read_sysreg_el1(ttbr1);
@@ -70,35 +79,46 @@ static void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt)
}
static hyp_alternate_select(__sysreg_call_save_host_state,
- __sysreg_save_state, __sysreg_do_nothing,
+ __sysreg_save_el1_state, __sysreg_do_nothing,
ARM64_HAS_VIRT_HOST_EXTN);
void __hyp_text __sysreg_save_host_state(struct kvm_cpu_context *ctxt)
{
__sysreg_call_save_host_state()(ctxt);
__sysreg_save_common_state(ctxt);
+ __sysreg_save_user_state(ctxt);
}
void __hyp_text __sysreg_save_guest_state(struct kvm_cpu_context *ctxt)
{
- __sysreg_save_state(ctxt);
+ __sysreg_save_el1_state(ctxt);
__sysreg_save_common_state(ctxt);
+ __sysreg_save_user_state(ctxt);
}
static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
{
- write_sysreg(ctxt->sys_regs[ACTLR_EL1], actlr_el1);
- write_sysreg(ctxt->sys_regs[TPIDR_EL0], tpidr_el0);
- write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0);
write_sysreg(ctxt->sys_regs[MDSCR_EL1], mdscr_el1);
+
+ /*
+ * The host arm64 Linux uses sp_el0 to point to 'current' and it must
+ * therefore be saved/restored on every entry/exit to/from the guest.
+ */
write_sysreg(ctxt->gp_regs.regs.sp, sp_el0);
}
-static void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt)
+static void __hyp_text __sysreg_restore_user_state(struct kvm_cpu_context *ctxt)
+{
+ write_sysreg(ctxt->sys_regs[TPIDR_EL0], tpidr_el0);
+ write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0);
+}
+
+static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
{
write_sysreg(ctxt->sys_regs[MPIDR_EL1], vmpidr_el2);
write_sysreg(ctxt->sys_regs[CSSELR_EL1], csselr_el1);
write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1], sctlr);
+ write_sysreg(ctxt->sys_regs[ACTLR_EL1], actlr_el1);
write_sysreg_el1(ctxt->sys_regs[CPACR_EL1], cpacr);
write_sysreg_el1(ctxt->sys_regs[TTBR0_EL1], ttbr0);
write_sysreg_el1(ctxt->sys_regs[TTBR1_EL1], ttbr1);
@@ -123,19 +143,21 @@ static void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt)
}
static hyp_alternate_select(__sysreg_call_restore_host_state,
- __sysreg_restore_state, __sysreg_do_nothing,
+ __sysreg_restore_el1_state, __sysreg_do_nothing,
ARM64_HAS_VIRT_HOST_EXTN);
void __hyp_text __sysreg_restore_host_state(struct kvm_cpu_context *ctxt)
{
__sysreg_call_restore_host_state()(ctxt);
__sysreg_restore_common_state(ctxt);
+ __sysreg_restore_user_state(ctxt);
}
void __hyp_text __sysreg_restore_guest_state(struct kvm_cpu_context *ctxt)
{
- __sysreg_restore_state(ctxt);
+ __sysreg_restore_el1_state(ctxt);
__sysreg_restore_common_state(ctxt);
+ __sysreg_restore_user_state(ctxt);
}
static void __hyp_text __fpsimd32_save_state(struct kvm_cpu_context *ctxt)
--
2.14.2
^ permalink raw reply related
* [PATCH v3 17/41] KVM: arm64: Remove noop calls to timer save/restore from VHE switch
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
The VHE switch function calls __timer_enable_traps and
__timer_disable_traps which don't do anything on VHE systems.
Therefore, simply remove these calls from the VHE switch function and
make the functions non-conditional as they are now only called from the
non-VHE switch path.
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/switch.c | 2 --
virt/kvm/arm/hyp/timer-sr.c | 44 ++++++++++++++++++++++----------------------
2 files changed, 22 insertions(+), 24 deletions(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 9aadef6966bf..6175fcb33ed2 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -354,7 +354,6 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
__activate_vm(vcpu->kvm);
__vgic_restore_state(vcpu);
- __timer_enable_traps(vcpu);
/*
* We must restore the 32-bit state before the sysregs, thanks
@@ -373,7 +372,6 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
__sysreg_save_guest_state(guest_ctxt);
__sysreg32_save_state(vcpu);
- __timer_disable_traps(vcpu);
__vgic_save_state(vcpu);
__deactivate_traps(vcpu);
diff --git a/virt/kvm/arm/hyp/timer-sr.c b/virt/kvm/arm/hyp/timer-sr.c
index f24404b3c8df..77754a62eb0c 100644
--- a/virt/kvm/arm/hyp/timer-sr.c
+++ b/virt/kvm/arm/hyp/timer-sr.c
@@ -27,34 +27,34 @@ void __hyp_text __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high)
write_sysreg(cntvoff, cntvoff_el2);
}
+/*
+ * Should only be called on non-VHE systems.
+ * VHE systems use EL2 timers and configure EL1 timers in kvm_timer_init_vhe().
+ */
void __hyp_text __timer_disable_traps(struct kvm_vcpu *vcpu)
{
- /*
- * We don't need to do this for VHE since the host kernel runs in EL2
- * with HCR_EL2.TGE ==1, which makes those bits have no impact.
- */
- if (!has_vhe()) {
- u64 val;
+ u64 val;
- /* Allow physical timer/counter access for the host */
- val = read_sysreg(cnthctl_el2);
- val |= CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN;
- write_sysreg(val, cnthctl_el2);
- }
+ /* Allow physical timer/counter access for the host */
+ val = read_sysreg(cnthctl_el2);
+ val |= CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN;
+ write_sysreg(val, cnthctl_el2);
}
+/*
+ * Should only be called on non-VHE systems.
+ * VHE systems use EL2 timers and configure EL1 timers in kvm_timer_init_vhe().
+ */
void __hyp_text __timer_enable_traps(struct kvm_vcpu *vcpu)
{
- if (!has_vhe()) {
- u64 val;
+ u64 val;
- /*
- * Disallow physical timer access for the guest
- * Physical counter access is allowed
- */
- val = read_sysreg(cnthctl_el2);
- val &= ~CNTHCTL_EL1PCEN;
- val |= CNTHCTL_EL1PCTEN;
- write_sysreg(val, cnthctl_el2);
- }
+ /*
+ * Disallow physical timer access for the guest
+ * Physical counter access is allowed
+ */
+ val = read_sysreg(cnthctl_el2);
+ val &= ~CNTHCTL_EL1PCEN;
+ val |= CNTHCTL_EL1PCTEN;
+ write_sysreg(val, cnthctl_el2);
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 16/41] KVM: arm64: Don't deactivate VM on VHE systems
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
There is no need to reset the VTTBR to zero when exiting the guest on
VHE systems. VHE systems don't use stage 2 translations for the EL2&0
translation regime used by the host.
Reviewed-by: Andrew Jones <drjones@redhat.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/switch.c | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 05fba76ec918..9aadef6966bf 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -136,9 +136,8 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
write_sysreg(0, pmuserenr_el0);
}
-static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
+static void __hyp_text __activate_vm(struct kvm *kvm)
{
- struct kvm *kvm = kern_hyp_va(vcpu->kvm);
write_sysreg(kvm->arch.vttbr, vttbr_el2);
}
@@ -352,7 +351,7 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
__sysreg_save_host_state(host_ctxt);
__activate_traps(vcpu);
- __activate_vm(vcpu);
+ __activate_vm(vcpu->kvm);
__vgic_restore_state(vcpu);
__timer_enable_traps(vcpu);
@@ -378,7 +377,6 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
__vgic_save_state(vcpu);
__deactivate_traps(vcpu);
- __deactivate_vm(vcpu);
__sysreg_restore_host_state(host_ctxt);
@@ -407,7 +405,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
__sysreg_save_host_state(host_ctxt);
__activate_traps(vcpu);
- __activate_vm(vcpu);
+ __activate_vm(kern_hyp_va(vcpu->kvm));
__vgic_restore_state(vcpu);
__timer_enable_traps(vcpu);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 15/41] KVM: arm64: Remove kern_hyp_va() use in VHE switch function
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
VHE kernels run completely in EL2 and therefore don't have a notion of
kernel and hyp addresses, they are all just kernel addresses. Therefore
don't call kern_hyp_va() in the VHE switch function.
Reviewed-by: Andrew Jones <drjones@redhat.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/switch.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index accfe9a016f9..05fba76ec918 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -345,9 +345,7 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
struct kvm_cpu_context *guest_ctxt;
u64 exit_code;
- vcpu = kern_hyp_va(vcpu);
-
- host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+ host_ctxt = vcpu->arch.host_cpu_context;
host_ctxt->__hyp_running_vcpu = vcpu;
guest_ctxt = &vcpu->arch.ctxt;
--
2.14.2
^ permalink raw reply related
* [PATCH v3 14/41] KVM: arm64: Introduce VHE-specific kvm_vcpu_run
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
So far this is just a copy of the legacy non-VHE switch function, but we
will start reworking these functions in separate directions to work on
VHE and non-VHE in the most optimal way in later patches.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm/include/asm/kvm_asm.h | 5 +++-
arch/arm/kvm/hyp/switch.c | 2 +-
arch/arm64/include/asm/kvm_asm.h | 4 ++-
arch/arm64/kvm/hyp/switch.c | 58 +++++++++++++++++++++++++++++++++++++++-
virt/kvm/arm/arm.c | 5 +++-
5 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 36dd2962a42d..4ac717276543 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -70,7 +70,10 @@ extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
extern void __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high);
-extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
+/* no VHE on 32-bit :( */
+static inline int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) { return 0; }
+
+extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu);
extern void __init_stage2_translation(void);
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index c3b9799e2e13..7b2bd25e3b10 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -153,7 +153,7 @@ static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
return true;
}
-int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *host_ctxt;
struct kvm_cpu_context *guest_ctxt;
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 6c7599b5cb40..fb91e728207b 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -58,7 +58,9 @@ extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
extern void __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high);
-extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
+extern int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu);
+
+extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu);
extern u64 __vgic_v3_get_ich_vtr_el2(void);
extern u64 __vgic_v3_read_vmcr(void);
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 55ca2e3d42eb..accfe9a016f9 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -338,7 +338,63 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
return false;
}
-int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+/* Switch to the guest for VHE systems running in EL2 */
+int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
+{
+ struct kvm_cpu_context *host_ctxt;
+ struct kvm_cpu_context *guest_ctxt;
+ u64 exit_code;
+
+ vcpu = kern_hyp_va(vcpu);
+
+ host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+ host_ctxt->__hyp_running_vcpu = vcpu;
+ guest_ctxt = &vcpu->arch.ctxt;
+
+ __sysreg_save_host_state(host_ctxt);
+
+ __activate_traps(vcpu);
+ __activate_vm(vcpu);
+
+ __vgic_restore_state(vcpu);
+ __timer_enable_traps(vcpu);
+
+ /*
+ * We must restore the 32-bit state before the sysregs, thanks
+ * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
+ */
+ __sysreg32_restore_state(vcpu);
+ __sysreg_restore_guest_state(guest_ctxt);
+ __debug_switch_to_guest(vcpu);
+
+ do {
+ /* Jump in the fire! */
+ exit_code = __guest_enter(vcpu, host_ctxt);
+
+ /* And we're baaack! */
+ } while (fixup_guest_exit(vcpu, &exit_code));
+
+ __sysreg_save_guest_state(guest_ctxt);
+ __sysreg32_save_state(vcpu);
+ __timer_disable_traps(vcpu);
+ __vgic_save_state(vcpu);
+
+ __deactivate_traps(vcpu);
+ __deactivate_vm(vcpu);
+
+ __sysreg_restore_host_state(host_ctxt);
+
+ /*
+ * This must come after restoring the host sysregs, since a non-VHE
+ * system may enable SPE here and make use of the TTBRs.
+ */
+ __debug_switch_to_host(vcpu);
+
+ return exit_code;
+}
+
+/* Switch to the guest for legacy non-VHE systems */
+int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *host_ctxt;
struct kvm_cpu_context *guest_ctxt;
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 5b1487bd91e8..6bce8f9c55db 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -733,7 +733,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
trace_kvm_entry(*vcpu_pc(vcpu));
guest_enter_irqoff();
- ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
+ if (has_vhe())
+ ret = kvm_vcpu_run_vhe(vcpu);
+ else
+ ret = kvm_call_hyp(__kvm_vcpu_run_nvhe, vcpu);
vcpu->mode = OUTSIDE_GUEST_MODE;
vcpu->stat.exits++;
--
2.14.2
^ permalink raw reply related
* [PATCH v3 13/41] KVM: arm64: Factor out fault info population and gic workarounds
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
The current world-switch function has functionality to detect a number
of cases where we need to fixup some part of the exit condition and
possibly run the guest again, before having restored the host state.
This includes populating missing fault info, emulating GICv2 CPU
interface accesses when mapped at unaligned addresses, and emulating
the GICv3 CPU interface on systems that need it.
As we are about to have an alternative switch function for VHE systems,
but VHE systems still need the same early fixup logic, factor out this
logic into a separate function that can be shared by both switch
functions.
No functional change.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/switch.c | 99 ++++++++++++++++++++++++---------------------
1 file changed, 54 insertions(+), 45 deletions(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 63284647ed11..55ca2e3d42eb 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -270,50 +270,24 @@ static bool __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
}
}
-int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+/*
+ * Return true when we were able to fixup the guest exit and should return to
+ * the guest, false when we should restore the host state and return to the
+ * main run loop.
+ */
+static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
{
- struct kvm_cpu_context *host_ctxt;
- struct kvm_cpu_context *guest_ctxt;
- u64 exit_code;
-
- vcpu = kern_hyp_va(vcpu);
-
- host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
- host_ctxt->__hyp_running_vcpu = vcpu;
- guest_ctxt = &vcpu->arch.ctxt;
-
- __sysreg_save_host_state(host_ctxt);
-
- __activate_traps(vcpu);
- __activate_vm(vcpu);
-
- __vgic_restore_state(vcpu);
- __timer_enable_traps(vcpu);
-
- /*
- * We must restore the 32-bit state before the sysregs, thanks
- * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
- */
- __sysreg32_restore_state(vcpu);
- __sysreg_restore_guest_state(guest_ctxt);
- __debug_switch_to_guest(vcpu);
-
- /* Jump in the fire! */
-again:
- exit_code = __guest_enter(vcpu, host_ctxt);
- /* And we're baaack! */
-
/*
* We're using the raw exception code in order to only process
* the trap if no SError is pending. We will come back to the
* same PC once the SError has been injected, and replay the
* trapping instruction.
*/
- if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
- goto again;
+ if (*exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
+ return true;
if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
- exit_code == ARM_EXCEPTION_TRAP) {
+ *exit_code == ARM_EXCEPTION_TRAP) {
bool valid;
valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
@@ -327,9 +301,9 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
if (ret == 1) {
if (__skip_instr(vcpu))
- goto again;
+ return true;
else
- exit_code = ARM_EXCEPTION_TRAP;
+ *exit_code = ARM_EXCEPTION_TRAP;
}
if (ret == -1) {
@@ -341,29 +315,64 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
*/
if (!__skip_instr(vcpu))
*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
- exit_code = ARM_EXCEPTION_EL1_SERROR;
+ *exit_code = ARM_EXCEPTION_EL1_SERROR;
}
-
- /* 0 falls through to be handler out of EL2 */
}
}
if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
- exit_code == ARM_EXCEPTION_TRAP &&
+ *exit_code == ARM_EXCEPTION_TRAP &&
(kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
int ret = __vgic_v3_perform_cpuif_access(vcpu);
if (ret == 1) {
if (__skip_instr(vcpu))
- goto again;
+ return true;
else
- exit_code = ARM_EXCEPTION_TRAP;
+ *exit_code = ARM_EXCEPTION_TRAP;
}
-
- /* 0 falls through to be handled out of EL2 */
}
+ /* Return to the host kernel and handle the exit */
+ return false;
+}
+
+int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+{
+ struct kvm_cpu_context *host_ctxt;
+ struct kvm_cpu_context *guest_ctxt;
+ u64 exit_code;
+
+ vcpu = kern_hyp_va(vcpu);
+
+ host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+ host_ctxt->__hyp_running_vcpu = vcpu;
+ guest_ctxt = &vcpu->arch.ctxt;
+
+ __sysreg_save_host_state(host_ctxt);
+
+ __activate_traps(vcpu);
+ __activate_vm(vcpu);
+
+ __vgic_restore_state(vcpu);
+ __timer_enable_traps(vcpu);
+
+ /*
+ * We must restore the 32-bit state before the sysregs, thanks
+ * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
+ */
+ __sysreg32_restore_state(vcpu);
+ __sysreg_restore_guest_state(guest_ctxt);
+ __debug_switch_to_guest(vcpu);
+
+ do {
+ /* Jump in the fire! */
+ exit_code = __guest_enter(vcpu, host_ctxt);
+
+ /* And we're baaack! */
+ } while (fixup_guest_exit(vcpu, &exit_code));
+
__sysreg_save_guest_state(guest_ctxt);
__sysreg32_save_state(vcpu);
__timer_disable_traps(vcpu);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 12/41] KVM: arm64: Improve debug register save/restore flow
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
Instead of having multiple calls from the world switch path to the debug
logic, each figuring out if the dirty bit is set and if we should
save/restore the debug registers, let's just provide two hooks to the
debug save/restore functionality, one for switching to the guest
context, and one for switching to the host context, and we get the
benefit of only having to evaluate the dirty flag once on each path,
plus we give the compiler some more room to inline some of this
functionality.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/include/asm/kvm_hyp.h | 10 ++-----
arch/arm64/kvm/hyp/debug-sr.c | 56 +++++++++++++++++++++++++++-------------
arch/arm64/kvm/hyp/switch.c | 6 ++---
3 files changed, 42 insertions(+), 30 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 08d3bb66c8b7..a0e5a7038237 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -139,14 +139,8 @@ void __sysreg_restore_guest_state(struct kvm_cpu_context *ctxt);
void __sysreg32_save_state(struct kvm_vcpu *vcpu);
void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
-void __debug_save_state(struct kvm_vcpu *vcpu,
- struct kvm_guest_debug_arch *dbg,
- struct kvm_cpu_context *ctxt);
-void __debug_restore_state(struct kvm_vcpu *vcpu,
- struct kvm_guest_debug_arch *dbg,
- struct kvm_cpu_context *ctxt);
-void __debug_cond_save_host_state(struct kvm_vcpu *vcpu);
-void __debug_cond_restore_host_state(struct kvm_vcpu *vcpu);
+void __debug_switch_to_guest(struct kvm_vcpu *vcpu);
+void __debug_switch_to_host(struct kvm_vcpu *vcpu);
void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c
index 81b8ad44f9e0..ee87115eb12f 100644
--- a/arch/arm64/kvm/hyp/debug-sr.c
+++ b/arch/arm64/kvm/hyp/debug-sr.c
@@ -106,16 +106,13 @@ static void __hyp_text __debug_restore_spe_nvhe(u64 pmscr_el1)
write_sysreg_s(pmscr_el1, SYS_PMSCR_EL1);
}
-void __hyp_text __debug_save_state(struct kvm_vcpu *vcpu,
- struct kvm_guest_debug_arch *dbg,
- struct kvm_cpu_context *ctxt)
+static void __hyp_text __debug_save_state(struct kvm_vcpu *vcpu,
+ struct kvm_guest_debug_arch *dbg,
+ struct kvm_cpu_context *ctxt)
{
u64 aa64dfr0;
int brps, wrps;
- if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
- return;
-
aa64dfr0 = read_sysreg(id_aa64dfr0_el1);
brps = (aa64dfr0 >> 12) & 0xf;
wrps = (aa64dfr0 >> 20) & 0xf;
@@ -128,16 +125,13 @@ void __hyp_text __debug_save_state(struct kvm_vcpu *vcpu,
ctxt->sys_regs[MDCCINT_EL1] = read_sysreg(mdccint_el1);
}
-void __hyp_text __debug_restore_state(struct kvm_vcpu *vcpu,
- struct kvm_guest_debug_arch *dbg,
- struct kvm_cpu_context *ctxt)
+static void __hyp_text __debug_restore_state(struct kvm_vcpu *vcpu,
+ struct kvm_guest_debug_arch *dbg,
+ struct kvm_cpu_context *ctxt)
{
u64 aa64dfr0;
int brps, wrps;
- if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
- return;
-
aa64dfr0 = read_sysreg(id_aa64dfr0_el1);
brps = (aa64dfr0 >> 12) & 0xf;
@@ -151,10 +145,12 @@ void __hyp_text __debug_restore_state(struct kvm_vcpu *vcpu,
write_sysreg(ctxt->sys_regs[MDCCINT_EL1], mdccint_el1);
}
-void __hyp_text __debug_cond_save_host_state(struct kvm_vcpu *vcpu)
+void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu)
{
- __debug_save_state(vcpu, &vcpu->arch.host_debug_state.regs,
- kern_hyp_va(vcpu->arch.host_cpu_context));
+ struct kvm_cpu_context *host_ctxt;
+ struct kvm_cpu_context *guest_ctxt;
+ struct kvm_guest_debug_arch *host_dbg;
+ struct kvm_guest_debug_arch *guest_dbg;
/*
* Non-VHE: Disable and flush SPE data generation
@@ -162,15 +158,39 @@ void __hyp_text __debug_cond_save_host_state(struct kvm_vcpu *vcpu)
*/
if (!has_vhe())
__debug_save_spe_nvhe(&vcpu->arch.host_debug_state.pmscr_el1);
+
+ if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
+ return;
+
+ host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+ guest_ctxt = &vcpu->arch.ctxt;
+ host_dbg = &vcpu->arch.host_debug_state.regs;
+ guest_dbg = kern_hyp_va(vcpu->arch.debug_ptr);
+
+ __debug_save_state(vcpu, host_dbg, host_ctxt);
+ __debug_restore_state(vcpu, guest_dbg, guest_ctxt);
}
-void __hyp_text __debug_cond_restore_host_state(struct kvm_vcpu *vcpu)
+void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
{
+ struct kvm_cpu_context *host_ctxt;
+ struct kvm_cpu_context *guest_ctxt;
+ struct kvm_guest_debug_arch *host_dbg;
+ struct kvm_guest_debug_arch *guest_dbg;
+
if (!has_vhe())
__debug_restore_spe_nvhe(vcpu->arch.host_debug_state.pmscr_el1);
- __debug_restore_state(vcpu, &vcpu->arch.host_debug_state.regs,
- kern_hyp_va(vcpu->arch.host_cpu_context));
+ if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
+ return;
+
+ host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+ guest_ctxt = &vcpu->arch.ctxt;
+ host_dbg = &vcpu->arch.host_debug_state.regs;
+ guest_dbg = kern_hyp_va(vcpu->arch.debug_ptr);
+
+ __debug_save_state(vcpu, guest_dbg, guest_ctxt);
+ __debug_restore_state(vcpu, host_dbg, host_ctxt);
vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
}
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 29e44a20f5e3..63284647ed11 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -283,7 +283,6 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
guest_ctxt = &vcpu->arch.ctxt;
__sysreg_save_host_state(host_ctxt);
- __debug_cond_save_host_state(vcpu);
__activate_traps(vcpu);
__activate_vm(vcpu);
@@ -297,7 +296,7 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
*/
__sysreg32_restore_state(vcpu);
__sysreg_restore_guest_state(guest_ctxt);
- __debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
+ __debug_switch_to_guest(vcpu);
/* Jump in the fire! */
again:
@@ -375,12 +374,11 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
__sysreg_restore_host_state(host_ctxt);
- __debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
/*
* This must come after restoring the host sysregs, since a non-VHE
* system may enable SPE here and make use of the TTBRs.
*/
- __debug_cond_restore_host_state(vcpu);
+ __debug_switch_to_host(vcpu);
return exit_code;
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 11/41] KVM: arm64: Slightly improve debug save/restore functions
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
The debug save/restore functions can be improved by using the has_vhe()
static key instead of the instruction alternative. Using the static key
uses the same paradigm as we're going to use elsewhere, it makes the
code more readable, and it generates slightly better code (no
stack setups and function calls unless necessary).
We also use a static key on the restore path, because it will be
marginally faster than loading a value from memory.
Finally, we don't have to conditionally clear the debug dirty flag if
it's set, we can just clear it.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/debug-sr.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c
index 406829b6a43e..81b8ad44f9e0 100644
--- a/arch/arm64/kvm/hyp/debug-sr.c
+++ b/arch/arm64/kvm/hyp/debug-sr.c
@@ -65,11 +65,6 @@
default: write_debug(ptr[0], reg, 0); \
}
-static void __hyp_text __debug_save_spe_vhe(u64 *pmscr_el1)
-{
- /* The vcpu can run. but it can't hide. */
-}
-
static void __hyp_text __debug_save_spe_nvhe(u64 *pmscr_el1)
{
u64 reg;
@@ -99,11 +94,7 @@ static void __hyp_text __debug_save_spe_nvhe(u64 *pmscr_el1)
dsb(nsh);
}
-static hyp_alternate_select(__debug_save_spe,
- __debug_save_spe_nvhe, __debug_save_spe_vhe,
- ARM64_HAS_VIRT_HOST_EXTN);
-
-static void __hyp_text __debug_restore_spe(u64 pmscr_el1)
+static void __hyp_text __debug_restore_spe_nvhe(u64 pmscr_el1)
{
if (!pmscr_el1)
return;
@@ -164,17 +155,24 @@ void __hyp_text __debug_cond_save_host_state(struct kvm_vcpu *vcpu)
{
__debug_save_state(vcpu, &vcpu->arch.host_debug_state.regs,
kern_hyp_va(vcpu->arch.host_cpu_context));
- __debug_save_spe()(&vcpu->arch.host_debug_state.pmscr_el1);
+
+ /*
+ * Non-VHE: Disable and flush SPE data generation
+ * VHE: The vcpu can run, but it can't hide.
+ */
+ if (!has_vhe())
+ __debug_save_spe_nvhe(&vcpu->arch.host_debug_state.pmscr_el1);
}
void __hyp_text __debug_cond_restore_host_state(struct kvm_vcpu *vcpu)
{
- __debug_restore_spe(vcpu->arch.host_debug_state.pmscr_el1);
+ if (!has_vhe())
+ __debug_restore_spe_nvhe(vcpu->arch.host_debug_state.pmscr_el1);
+
__debug_restore_state(vcpu, &vcpu->arch.host_debug_state.regs,
kern_hyp_va(vcpu->arch.host_cpu_context));
- if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
- vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
+ vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
}
u32 __hyp_text __kvm_get_mdcr_el2(void)
--
2.14.2
^ permalink raw reply related
* [PATCH v3 10/41] KVM: arm64: Move debug dirty flag calculation out of world switch
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
There is no need to figure out inside the world-switch if we should
save/restore the debug registers or not, we can might as well do that in
the higher level debug setup code, making it easier to optimize down the
line.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/debug.c | 5 +++++
arch/arm64/kvm/hyp/debug-sr.c | 6 ------
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index fa63b28c65e0..feedb877cff8 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -193,6 +193,11 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
if (trap_debug)
vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
+ /* If KDE or MDE are set, perform a full save/restore cycle. */
+ if ((vcpu_sys_reg(vcpu, MDSCR_EL1) & DBG_MDSCR_KDE) ||
+ (vcpu_sys_reg(vcpu, MDSCR_EL1) & DBG_MDSCR_MDE))
+ vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+
trace_kvm_arm_set_dreg32("MDCR_EL2", vcpu->arch.mdcr_el2);
trace_kvm_arm_set_dreg32("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
}
diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c
index 321c9c05dd9e..406829b6a43e 100644
--- a/arch/arm64/kvm/hyp/debug-sr.c
+++ b/arch/arm64/kvm/hyp/debug-sr.c
@@ -162,12 +162,6 @@ void __hyp_text __debug_restore_state(struct kvm_vcpu *vcpu,
void __hyp_text __debug_cond_save_host_state(struct kvm_vcpu *vcpu)
{
- /* If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY is set, perform
- * a full save/restore cycle. */
- if ((vcpu->arch.ctxt.sys_regs[MDSCR_EL1] & DBG_MDSCR_KDE) ||
- (vcpu->arch.ctxt.sys_regs[MDSCR_EL1] & DBG_MDSCR_MDE))
- vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
-
__debug_save_state(vcpu, &vcpu->arch.host_debug_state.regs,
kern_hyp_va(vcpu->arch.host_cpu_context));
__debug_save_spe()(&vcpu->arch.host_debug_state.pmscr_el1);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 09/41] KVM: arm64: Defer restoring host VFP state to vcpu_put
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
Avoid saving the guest VFP registers and restoring the host VFP
registers on every exit from the VM. Only when we're about to run
userspace or other threads in the kernel do we really have to switch the
state back to the host state.
We still initially configure the VFP registers to trap when entering the
VM, but the difference is that we now leave the guest state in the
hardware registers as long as we're running this VCPU, even if we
occasionally trap to the host, and we only restore the host state when
we return to user space or when scheduling another thread.
Reviewed-by: Andrew Jones <drjones@redhat.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/include/asm/kvm_host.h | 3 +++
arch/arm64/kernel/asm-offsets.c | 1 +
arch/arm64/kvm/hyp/entry.S | 3 +++
arch/arm64/kvm/hyp/switch.c | 48 ++++++++++++---------------------------
arch/arm64/kvm/hyp/sysreg-sr.c | 21 ++++++++++++++---
5 files changed, 40 insertions(+), 36 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 0e9e7291a7e6..9e23bc968668 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -213,6 +213,9 @@ struct kvm_vcpu_arch {
/* Guest debug state */
u64 debug_flags;
+ /* 1 if the guest VFP state is loaded into the hardware */
+ u8 guest_vfp_loaded;
+
/*
* We maintain more than a single set of debug registers to support
* debugging the guest from the host and to maintain separate host and
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 612021dce84f..99467327c043 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -133,6 +133,7 @@ int main(void)
DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs));
DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_regs, regs));
DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));
+ DEFINE(VCPU_GUEST_VFP_LOADED, offsetof(struct kvm_vcpu, arch.guest_vfp_loaded));
DEFINE(VCPU_FPEXC32_EL2, offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2]));
DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));
DEFINE(HOST_CONTEXT_VCPU, offsetof(struct kvm_cpu_context, __hyp_running_vcpu));
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index a360ac6e89e9..53652287a236 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -184,6 +184,9 @@ alternative_endif
add x0, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
bl __fpsimd_restore_state
+ mov x0, #1
+ strb w0, [x3, #VCPU_GUEST_VFP_LOADED]
+
// Skip restoring fpexc32 for AArch64 guests
mrs x1, hcr_el2
tbnz x1, #HCR_RW_SHIFT, 1f
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 12dc647a6e5f..29e44a20f5e3 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -24,43 +24,32 @@
#include <asm/fpsimd.h>
#include <asm/debug-monitors.h>
-static bool __hyp_text __fpsimd_enabled_nvhe(void)
-{
- return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
-}
-
-static bool __hyp_text __fpsimd_enabled_vhe(void)
-{
- return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
-}
-
-static hyp_alternate_select(__fpsimd_is_enabled,
- __fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
- ARM64_HAS_VIRT_HOST_EXTN);
-
-bool __hyp_text __fpsimd_enabled(void)
-{
- return __fpsimd_is_enabled()();
-}
-
-static void __hyp_text __activate_traps_vhe(void)
+static void __hyp_text __activate_traps_vhe(struct kvm_vcpu *vcpu)
{
u64 val;
val = read_sysreg(cpacr_el1);
val |= CPACR_EL1_TTA;
- val &= ~(CPACR_EL1_FPEN | CPACR_EL1_ZEN);
+ val &= ~CPACR_EL1_ZEN;
+ if (vcpu->arch.guest_vfp_loaded)
+ val |= CPACR_EL1_FPEN;
+ else
+ val &= ~CPACR_EL1_FPEN;
write_sysreg(val, cpacr_el1);
write_sysreg(__kvm_hyp_vector, vbar_el1);
}
-static void __hyp_text __activate_traps_nvhe(void)
+static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)
{
u64 val;
val = CPTR_EL2_DEFAULT;
- val |= CPTR_EL2_TTA | CPTR_EL2_TFP | CPTR_EL2_TZ;
+ val |= CPTR_EL2_TTA | CPTR_EL2_TZ;
+ if (vcpu->arch.guest_vfp_loaded)
+ val &= ~CPTR_EL2_TFP;
+ else
+ val |= CPTR_EL2_TFP;
write_sysreg(val, cptr_el2);
}
@@ -79,7 +68,8 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
* If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
* it will cause an exception.
*/
- if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {
+ if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd() &&
+ !vcpu->arch.guest_vfp_loaded) {
write_sysreg(1 << 30, fpexc32_el2);
isb();
}
@@ -96,7 +86,7 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
write_sysreg(0, pmselr_el0);
write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
- __activate_traps_arch()();
+ __activate_traps_arch()(vcpu);
}
static void __hyp_text __deactivate_traps_vhe(void)
@@ -284,7 +274,6 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *host_ctxt;
struct kvm_cpu_context *guest_ctxt;
- bool fp_enabled;
u64 exit_code;
vcpu = kern_hyp_va(vcpu);
@@ -376,8 +365,6 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
/* 0 falls through to be handled out of EL2 */
}
- fp_enabled = __fpsimd_enabled();
-
__sysreg_save_guest_state(guest_ctxt);
__sysreg32_save_state(vcpu);
__timer_disable_traps(vcpu);
@@ -388,11 +375,6 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
__sysreg_restore_host_state(host_ctxt);
- if (fp_enabled) {
- __fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
- __fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
- }
-
__debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
/*
* This must come after restoring the host sysregs, since a non-VHE
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index 883a6383cd36..848a46eb33bf 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -138,6 +138,11 @@ void __hyp_text __sysreg_restore_guest_state(struct kvm_cpu_context *ctxt)
__sysreg_restore_common_state(ctxt);
}
+static void __hyp_text __fpsimd32_save_state(struct kvm_cpu_context *ctxt)
+{
+ ctxt->sys_regs[FPEXC32_EL2] = read_sysreg(fpexc32_el2);
+}
+
void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
{
u64 *spsr, *sysreg;
@@ -156,9 +161,6 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
- if (__fpsimd_enabled())
- sysreg[FPEXC32_EL2] = read_sysreg(fpexc32_el2);
-
if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
}
@@ -213,6 +215,19 @@ void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu)
*/
void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu)
{
+ struct kvm_cpu_context *host_ctxt = vcpu->arch.host_cpu_context;
+ struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
+
+ /* Restore host FP/SIMD state */
+ if (vcpu->arch.guest_vfp_loaded) {
+ if (vcpu_el1_is_32bit(vcpu)) {
+ kvm_call_hyp(__fpsimd32_save_state,
+ kern_hyp_va(guest_ctxt));
+ }
+ __fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
+ __fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
+ vcpu->arch.guest_vfp_loaded = 0;
+ }
}
void __hyp_text __kvm_set_tpidr_el2(u64 tpidr_el2)
--
2.14.2
^ permalink raw reply related
* [PATCH v3 08/41] KVM: arm/arm64: Introduce vcpu_el1_is_32bit
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
We have numerous checks around that checks if the HCR_EL2 has the RW bit
set to figure out if we're running an AArch64 or AArch32 VM. In some
cases, directly checking the RW bit (given its unintuitive name), is a
bit confusing, and that's not going to improve as we move logic around
for the following patches that optimize KVM on AArch64 hosts with VHE.
Therefore, introduce a helper, vcpu_el1_is_32bit, and replace existing
direct checks of HCR_EL2.RW with the helper.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/include/asm/kvm_emulate.h | 7 ++++++-
arch/arm64/kvm/hyp/switch.c | 8 ++------
arch/arm64/kvm/hyp/sysreg-sr.c | 5 +++--
arch/arm64/kvm/inject_fault.c | 6 +++---
4 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index b36aaa1fe332..e07bf463ac58 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -45,6 +45,11 @@ void kvm_inject_undef32(struct kvm_vcpu *vcpu);
void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr);
void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr);
+static inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
+{
+ return !(vcpu->arch.hcr_el2 & HCR_RW);
+}
+
static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
{
vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
@@ -58,7 +63,7 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
* For now this is conditional, since no AArch32 feature regs
* are currently virtualised.
*/
- if (vcpu->arch.hcr_el2 & HCR_RW)
+ if (!vcpu_el1_is_32bit(vcpu))
vcpu->arch.hcr_el2 |= HCR_TID3;
}
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 11ec1c6f3b84..12dc647a6e5f 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -70,8 +70,6 @@ static hyp_alternate_select(__activate_traps_arch,
static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
{
- u64 val;
-
/*
* We are about to set CPTR_EL2.TFP to trap all floating point
* register accesses to EL2, however, the ARM ARM clearly states that
@@ -81,13 +79,11 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
* If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
* it will cause an exception.
*/
- val = vcpu->arch.hcr_el2;
-
- if (!(val & HCR_RW) && system_supports_fpsimd()) {
+ if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {
write_sysreg(1 << 30, fpexc32_el2);
isb();
}
- write_sysreg(val, hcr_el2);
+ write_sysreg(vcpu->arch.hcr_el2, hcr_el2);
/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
write_sysreg(1 << 15, hstr_el2);
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index cbbcd6f410a8..883a6383cd36 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -19,6 +19,7 @@
#include <linux/kvm_host.h>
#include <asm/kvm_asm.h>
+#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
/* Yes, this does nothing, on purpose */
@@ -141,7 +142,7 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
{
u64 *spsr, *sysreg;
- if (read_sysreg(hcr_el2) & HCR_RW)
+ if (!vcpu_el1_is_32bit(vcpu))
return;
spsr = vcpu->arch.ctxt.gp_regs.spsr;
@@ -166,7 +167,7 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
{
u64 *spsr, *sysreg;
- if (read_sysreg(hcr_el2) & HCR_RW)
+ if (!vcpu_el1_is_32bit(vcpu))
return;
spsr = vcpu->arch.ctxt.gp_regs.spsr;
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 2d38ede2eff0..f4d35bb551e4 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -128,7 +128,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
*/
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
{
- if (!(vcpu->arch.hcr_el2 & HCR_RW))
+ if (vcpu_el1_is_32bit(vcpu))
kvm_inject_dabt32(vcpu, addr);
else
inject_abt64(vcpu, false, addr);
@@ -144,7 +144,7 @@ void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
*/
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
{
- if (!(vcpu->arch.hcr_el2 & HCR_RW))
+ if (vcpu_el1_is_32bit(vcpu))
kvm_inject_pabt32(vcpu, addr);
else
inject_abt64(vcpu, true, addr);
@@ -158,7 +158,7 @@ void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
*/
void kvm_inject_undefined(struct kvm_vcpu *vcpu)
{
- if (!(vcpu->arch.hcr_el2 & HCR_RW))
+ if (vcpu_el1_is_32bit(vcpu))
kvm_inject_undef32(vcpu);
else
inject_undef64(vcpu);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 07/41] KVM: arm/arm64: Add kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
As we are about to move a bunch of save/restore logic for VHE kernels to
the load and put functions, we need some infrastructure to do this.
Reviewed-by: Andrew Jones <drjones@redhat.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm/include/asm/kvm_host.h | 3 +++
arch/arm64/include/asm/kvm_host.h | 3 +++
arch/arm64/kvm/hyp/sysreg-sr.c | 30 ++++++++++++++++++++++++++++++
virt/kvm/arm/arm.c | 2 ++
4 files changed, 38 insertions(+)
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 7f96b3541939..793b3bc5a56c 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -300,4 +300,7 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
/* All host FP/SIMD state is restored on guest exit, so nothing to save: */
static inline void kvm_fpsimd_flush_cpu_state(void) {}
+static inline void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) {}
+static inline void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) {}
+
#endif /* __ARM_KVM_HOST_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 59150c75f9a5..0e9e7291a7e6 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -412,4 +412,7 @@ static inline void kvm_fpsimd_flush_cpu_state(void)
sve_flush_cpu_state();
}
+void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu);
+void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu);
+
#endif /* __ARM64_KVM_HOST_H__ */
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index e19d89cabf2a..cbbcd6f410a8 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -184,6 +184,36 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
}
+/**
+ * kvm_vcpu_load_sysregs - Load guest system registers to the physical CPU
+ *
+ * @vcpu: The VCPU pointer
+ *
+ * Load system registers that do not affect the host's execution, for
+ * example EL1 system registers on a VHE system where the host kernel
+ * runs at EL2. This function is called from KVM's vcpu_load() function
+ * and loading system register state early avoids having to load them on
+ * every entry to the VM.
+ */
+void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu)
+{
+}
+
+/**
+ * kvm_vcpu_put_sysregs - Restore host system registers to the physical CPU
+ *
+ * @vcpu: The VCPU pointer
+ *
+ * Save guest system registers that do not affect the host's execution, for
+ * example EL1 system registers on a VHE system where the host kernel
+ * runs at EL2. This function is called from KVM's vcpu_put() function
+ * and deferring saving system register state until we're no longer running the
+ * VCPU avoids having to save them on every exit from the VM.
+ */
+void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu)
+{
+}
+
void __hyp_text __kvm_set_tpidr_el2(u64 tpidr_el2)
{
asm("msr tpidr_el2, %0": : "r" (tpidr_el2));
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 46c125f1da56..5b1487bd91e8 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -361,10 +361,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
kvm_arm_set_running_vcpu(vcpu);
kvm_vgic_load(vcpu);
kvm_timer_vcpu_load(vcpu);
+ kvm_vcpu_load_sysregs(vcpu);
}
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
+ kvm_vcpu_put_sysregs(vcpu);
kvm_timer_vcpu_put(vcpu);
kvm_vgic_put(vcpu);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 06/41] KVM: arm/arm64: Get rid of vcpu->arch.irq_lines
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
We currently have a separate read-modify-write of the HCR_EL2 on entry
to the guest for the sole purpose of setting the VF and VI bits, if set.
Since this is most rarely the case (only when using userspace IRQ chip
and interrupts are in flight), let's get rid of this operation and
instead modify the bits in the vcpu->arch.hcr[_el2] directly when
needed.
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Andrew Jones <drjones@redhat.com>
Reviewed-by: Julien Thierry <julien.thierry@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm/include/asm/kvm_emulate.h | 9 ++-------
arch/arm/include/asm/kvm_host.h | 3 ---
arch/arm/kvm/emulate.c | 2 +-
arch/arm/kvm/hyp/switch.c | 2 +-
arch/arm64/include/asm/kvm_emulate.h | 9 ++-------
arch/arm64/include/asm/kvm_host.h | 3 ---
arch/arm64/kvm/hyp/switch.c | 6 ------
arch/arm64/kvm/inject_fault.c | 2 +-
virt/kvm/arm/arm.c | 11 ++++++-----
virt/kvm/arm/mmu.c | 6 +++---
10 files changed, 16 insertions(+), 37 deletions(-)
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 3d22eb87f919..d5e1b8bf6422 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -92,14 +92,9 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
vcpu->arch.hcr = HCR_GUEST_MASK;
}
-static inline unsigned long vcpu_get_hcr(const struct kvm_vcpu *vcpu)
+static inline unsigned long *vcpu_hcr(const struct kvm_vcpu *vcpu)
{
- return vcpu->arch.hcr;
-}
-
-static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
-{
- vcpu->arch.hcr = hcr;
+ return (unsigned long *)&vcpu->arch.hcr;
}
static inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 6394fb99da7f..7f96b3541939 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -155,9 +155,6 @@ struct kvm_vcpu_arch {
/* HYP trapping configuration */
u32 hcr;
- /* Interrupt related fields */
- u32 irq_lines; /* IRQ and FIQ levels */
-
/* Exception Information */
struct kvm_vcpu_fault_info fault;
diff --git a/arch/arm/kvm/emulate.c b/arch/arm/kvm/emulate.c
index cdff963f133a..fa501bf437f3 100644
--- a/arch/arm/kvm/emulate.c
+++ b/arch/arm/kvm/emulate.c
@@ -174,5 +174,5 @@ unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu)
*/
void kvm_inject_vabt(struct kvm_vcpu *vcpu)
{
- vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) | HCR_VA);
+ *vcpu_hcr(vcpu) |= HCR_VA;
}
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index 330c9ce34ba5..c3b9799e2e13 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -43,7 +43,7 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host)
isb();
}
- write_sysreg(vcpu->arch.hcr | vcpu->arch.irq_lines, HCR);
+ write_sysreg(vcpu->arch.hcr, HCR);
/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
write_sysreg(HSTR_T(15), HSTR);
write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR);
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 8ff5aef44656..b36aaa1fe332 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -62,14 +62,9 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
vcpu->arch.hcr_el2 |= HCR_TID3;
}
-static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu)
+static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
{
- return vcpu->arch.hcr_el2;
-}
-
-static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
-{
- vcpu->arch.hcr_el2 = hcr;
+ return (unsigned long *)&vcpu->arch.hcr_el2;
}
static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 6ce0b428a4db..59150c75f9a5 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -269,9 +269,6 @@ struct kvm_vcpu_arch {
/* IO related fields */
struct kvm_decode mmio_decode;
- /* Interrupt related fields */
- u64 irq_lines; /* IRQ and FIQ levels */
-
/* Cache some mmu pages needed inside spinlock regions */
struct kvm_mmu_memory_cache mmu_page_cache;
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index f6189d08753e..11ec1c6f3b84 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -171,12 +171,6 @@ static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
{
- u64 val;
-
- val = read_sysreg(hcr_el2);
- val |= vcpu->arch.irq_lines;
- write_sysreg(val, hcr_el2);
-
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
__vgic_v3_restore_state(vcpu);
else
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 8ecbcb40e317..2d38ede2eff0 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -173,5 +173,5 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu)
*/
void kvm_inject_vabt(struct kvm_vcpu *vcpu)
{
- vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) | HCR_VSE);
+ *vcpu_hcr(vcpu) |= HCR_VSE;
}
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 360df72692ee..46c125f1da56 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -419,7 +419,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
*/
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
{
- return ((!!v->arch.irq_lines || kvm_vgic_vcpu_pending_irq(v))
+ bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
+ return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
&& !v->arch.power_off && !v->arch.pause);
}
@@ -806,18 +807,18 @@ static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
{
int bit_index;
bool set;
- unsigned long *ptr;
+ unsigned long *hcr;
if (number == KVM_ARM_IRQ_CPU_IRQ)
bit_index = __ffs(HCR_VI);
else /* KVM_ARM_IRQ_CPU_FIQ */
bit_index = __ffs(HCR_VF);
- ptr = (unsigned long *)&vcpu->arch.irq_lines;
+ hcr = vcpu_hcr(vcpu);
if (level)
- set = test_and_set_bit(bit_index, ptr);
+ set = test_and_set_bit(bit_index, hcr);
else
- set = test_and_clear_bit(bit_index, ptr);
+ set = test_and_clear_bit(bit_index, hcr);
/*
* If we didn't change anything, no need to wake up or kick other CPUs
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index b36945d49986..d93d56d4cc5b 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -1987,7 +1987,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
*/
void kvm_set_way_flush(struct kvm_vcpu *vcpu)
{
- unsigned long hcr = vcpu_get_hcr(vcpu);
+ unsigned long hcr = *vcpu_hcr(vcpu);
/*
* If this is the first time we do a S/W operation
@@ -2002,7 +2002,7 @@ void kvm_set_way_flush(struct kvm_vcpu *vcpu)
trace_kvm_set_way_flush(*vcpu_pc(vcpu),
vcpu_has_cache_enabled(vcpu));
stage2_flush_vm(vcpu->kvm);
- vcpu_set_hcr(vcpu, hcr | HCR_TVM);
+ *vcpu_hcr(vcpu) = hcr | HCR_TVM;
}
}
@@ -2020,7 +2020,7 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
/* Caches are now on, stop trapping VM ops (until a S/W op) */
if (now_enabled)
- vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) & ~HCR_TVM);
+ *vcpu_hcr(vcpu) &= ~HCR_TVM;
trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 05/41] KVM: arm64: Move HCR_INT_OVERRIDE to default HCR_EL2 guest flag
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
From: Shih-Wei Li <shihwei@cs.columbia.edu>
We always set the IMO and FMO bits in the HCR_EL2 when running the
guest, regardless if we use the vgic or not. By moving these flags to
HCR_GUEST_FLAGS we can avoid one of the extra save/restore operations of
HCR_EL2 in the world switch code, and we can also soon get rid of the
other one.
This is safe, because even though the IMO and FMO bits control both
taking the interrupts to EL2 and remapping ICC_*_EL1 to ICV_*_EL1
executed at EL1, as long as we ensure that these bits are clear when
running the EL1 host, as defined in the HCR_HOST_[VHE_]FLAGS, we're OK.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Shih-Wei Li <shihwei@cs.columbia.edu>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/include/asm/kvm_arm.h | 4 ++--
arch/arm64/kvm/hyp/switch.c | 3 ---
2 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 715d395ef45b..656deeb17bf2 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -79,9 +79,9 @@
*/
#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \
HCR_TVM | HCR_BSU_IS | HCR_FB | HCR_TAC | \
- HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW)
+ HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | \
+ HCR_FMO | HCR_IMO)
#define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
-#define HCR_INT_OVERRIDE (HCR_FMO | HCR_IMO)
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
/* TCR_EL2 Registers bits */
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 71700ecee308..f6189d08753e 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -167,8 +167,6 @@ static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
__vgic_v3_save_state(vcpu);
else
__vgic_v2_save_state(vcpu);
-
- write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
}
static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
@@ -176,7 +174,6 @@ static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
u64 val;
val = read_sysreg(hcr_el2);
- val |= HCR_INT_OVERRIDE;
val |= vcpu->arch.irq_lines;
write_sysreg(val, hcr_el2);
--
2.14.2
^ permalink raw reply related
* [PATCH v3 04/41] KVM: arm64: Rework hyp_panic for VHE and non-VHE
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
VHE actually doesn't rely on clearing the VTTBR when returning to the
host kernel, and that is the current key mechanism of hyp_panic to
figure out how to attempt to return to a state good enough to print a
panic statement.
Therefore, we split the hyp_panic function into two functions, a VHE and
a non-VHE, keeping the non-VHE version intact, but changing the VHE
behavior.
The vttbr_el2 check on VHE doesn't really make that much sense, because
the only situation where we can get here on VHE is when the hypervisor
assembly code actually called into hyp_panic, which only happens when
VBAR_EL2 has been set to the KVM exception vectors. On VHE, we can
always safely disable the traps and restore the host registers at this
point, so we simply do that unconditionally and call into the panic
function directly.
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/hyp/switch.c | 42 +++++++++++++++++++++++-------------------
1 file changed, 23 insertions(+), 19 deletions(-)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 6fcb37e220b5..71700ecee308 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -419,10 +419,20 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
- struct kvm_vcpu *vcpu)
+ struct kvm_cpu_context *__host_ctxt)
{
+ struct kvm_vcpu *vcpu;
unsigned long str_va;
+ vcpu = __host_ctxt->__hyp_running_vcpu;
+
+ if (read_sysreg(vttbr_el2)) {
+ __timer_disable_traps(vcpu);
+ __deactivate_traps(vcpu);
+ __deactivate_vm(vcpu);
+ __sysreg_restore_host_state(__host_ctxt);
+ }
+
/*
* Force the panic string to be loaded from the literal pool,
* making sure it is a kernel address and not a PC-relative
@@ -436,37 +446,31 @@ static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
read_sysreg(hpfar_el2), par, vcpu);
}
-static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
- struct kvm_vcpu *vcpu)
+static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
+ struct kvm_cpu_context *host_ctxt)
{
+ struct kvm_vcpu *vcpu;
+ vcpu = host_ctxt->__hyp_running_vcpu;
+
+ __deactivate_traps(vcpu);
+ __sysreg_restore_host_state(host_ctxt);
+
panic(__hyp_panic_string,
spsr, elr,
read_sysreg_el2(esr), read_sysreg_el2(far),
read_sysreg(hpfar_el2), par, vcpu);
}
-static hyp_alternate_select(__hyp_call_panic,
- __hyp_call_panic_nvhe, __hyp_call_panic_vhe,
- ARM64_HAS_VIRT_HOST_EXTN);
-
void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
{
- struct kvm_vcpu *vcpu = NULL;
-
u64 spsr = read_sysreg_el2(spsr);
u64 elr = read_sysreg_el2(elr);
u64 par = read_sysreg(par_el1);
- if (read_sysreg(vttbr_el2)) {
- vcpu = host_ctxt->__hyp_running_vcpu;
- __timer_disable_traps(vcpu);
- __deactivate_traps(vcpu);
- __deactivate_vm(vcpu);
- __sysreg_restore_host_state(host_ctxt);
- }
-
- /* Call panic for real */
- __hyp_call_panic()(spsr, elr, par, vcpu);
+ if (!has_vhe())
+ __hyp_call_panic_nvhe(spsr, elr, par, host_ctxt);
+ else
+ __hyp_call_panic_vhe(spsr, elr, par, host_ctxt);
unreachable();
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 03/41] KVM: arm64: Avoid storing the vcpu pointer on the stack
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
We already have the percpu area for the host cpu state, which points to
the VCPU, so there's no need to store the VCPU pointer on the stack on
every context switch. We can be a little more clever and just use
tpidr_el2 for the percpu offset and load the VCPU pointer from the host
context.
This does require us to calculate the percpu offset without including
the offset from the kernel mapping of the percpu array to the linear
mapping of the array (which is what we store in tpidr_el1), because a
PC-relative generated address in EL2 is already giving us the hyp alias
of the linear mapping of a kernel address. We do this in
__cpu_init_hyp_mode() by using kvm_ksym_ref().
This change also requires us to have a scratch register, so we take the
chance to rearrange some of the el1_sync code to only look at the
vttbr_el2 to determine if this is a trap from the guest or an HVC from
the host. We do add an extra check to call the panic code if the kernel
is configured with debugging enabled and we saw a trap from the host
which wasn't an HVC, indicating that we left some EL2 trap configured by
mistake.
The code that accesses ESR_EL2 was previously using an alternative to
use the _EL1 accessor on VHE systems, but this was actually unnecessary
as the _EL1 accessor aliases the ESR_EL2 register on VHE, and the _EL2
accessor does the same thing on both systems.
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/include/asm/kvm_asm.h | 14 +++++++++++++
arch/arm64/include/asm/kvm_host.h | 15 ++++++++++++++
arch/arm64/kernel/asm-offsets.c | 1 +
arch/arm64/kvm/hyp/entry.S | 6 +-----
arch/arm64/kvm/hyp/hyp-entry.S | 41 ++++++++++++++++++---------------------
arch/arm64/kvm/hyp/switch.c | 5 +----
arch/arm64/kvm/hyp/sysreg-sr.c | 5 +++++
7 files changed, 56 insertions(+), 31 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index ab4d0a926043..6c7599b5cb40 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -33,6 +33,7 @@
#define KVM_ARM64_DEBUG_DIRTY_SHIFT 0
#define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
+/* Translate a kernel address of @sym into its equivalent linear mapping */
#define kvm_ksym_ref(sym) \
({ \
void *val = &sym; \
@@ -68,6 +69,19 @@ extern u32 __kvm_get_mdcr_el2(void);
extern u32 __init_stage2_translation(void);
+#else /* __ASSEMBLY__ */
+
+.macro get_host_ctxt reg, tmp
+ adr_l \reg, kvm_host_cpu_state
+ mrs \tmp, tpidr_el2
+ add \reg, \reg, \tmp
+.endm
+
+.macro get_vcpu vcpu, ctxt
+ ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+ kern_hyp_va \vcpu
+.endm
+
#endif
#endif /* __ARM_KVM_ASM_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 048f5db120f3..6ce0b428a4db 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -350,10 +350,15 @@ int kvm_perf_teardown(void);
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
+extern void __kvm_set_tpidr_el2(u64 tpidr_el2);
+DECLARE_PER_CPU(kvm_cpu_context_t, kvm_host_cpu_state);
+
static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
unsigned long hyp_stack_ptr,
unsigned long vector_ptr)
{
+ u64 tpidr_el2;
+
/*
* Call initialization code, and switch to the full blown HYP code.
* If the cpucaps haven't been finalized yet, something has gone very
@@ -362,6 +367,16 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
*/
BUG_ON(!static_branch_likely(&arm64_const_caps_ready));
__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
+
+ /*
+ * Calculate the raw per-cpu offset without a translation from the
+ * kernel's mapping to the linear mapping, and store it in tpidr_el2
+ * so that we can use adr_l to access per-cpu variables in EL2.
+ */
+ tpidr_el2 = (u64)this_cpu_ptr(&kvm_host_cpu_state)
+ - (u64)kvm_ksym_ref(kvm_host_cpu_state);
+
+ kvm_call_hyp(__kvm_set_tpidr_el2, tpidr_el2);
}
static inline void kvm_arch_hardware_unsetup(void) {}
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 71bf088f1e4b..612021dce84f 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -135,6 +135,7 @@ int main(void)
DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));
DEFINE(VCPU_FPEXC32_EL2, offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2]));
DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));
+ DEFINE(HOST_CONTEXT_VCPU, offsetof(struct kvm_cpu_context, __hyp_running_vcpu));
#endif
#ifdef CONFIG_CPU_PM
DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 9a8ab5dddd9e..a360ac6e89e9 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -62,9 +62,6 @@ ENTRY(__guest_enter)
// Store the host regs
save_callee_saved_regs x1
- // Store host_ctxt and vcpu for use@exit time
- stp x1, x0, [sp, #-16]!
-
add x18, x0, #VCPU_CONTEXT
// Restore guest regs x0-x17
@@ -118,8 +115,7 @@ ENTRY(__guest_exit)
// Store the guest regs x19-x29, lr
save_callee_saved_regs x1
- // Restore the host_ctxt from the stack
- ldr x2, [sp], #16
+ get_host_ctxt x2, x3
// Now restore the host regs
restore_callee_saved_regs x2
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index e4f37b9dd47c..71b4cc92895e 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -56,18 +56,15 @@ ENDPROC(__vhe_hyp_call)
el1_sync: // Guest trapped into EL2
stp x0, x1, [sp, #-16]!
-alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
- mrs x1, esr_el2
-alternative_else
- mrs x1, esr_el1
-alternative_endif
- lsr x0, x1, #ESR_ELx_EC_SHIFT
+ mrs x1, vttbr_el2 // If vttbr is valid, this is a trap
+ cbnz x1, el1_trap // from the guest
- cmp x0, #ESR_ELx_EC_HVC64
- b.ne el1_trap
-
- mrs x1, vttbr_el2 // If vttbr is valid, the 64bit guest
- cbnz x1, el1_trap // called HVC
+#ifdef CONFIG_DEBUG
+ mrs x0, esr_el2
+ lsr x0, x0, #ESR_ELx_EC_SHIFT
+ cmp x0, #ESR_ELx_EC_HVC64
+ b.ne __hyp_panic
+#endif
/* Here, we're pretty sure the host called HVC. */
ldp x0, x1, [sp], #16
@@ -101,10 +98,15 @@ alternative_endif
eret
el1_trap:
+ get_host_ctxt x0, x1
+ get_vcpu x1, x0
+
+ mrs x0, esr_el2
+ lsr x0, x0, #ESR_ELx_EC_SHIFT
/*
* x0: ESR_EC
+ * x1: vcpu pointer
*/
- ldr x1, [sp, #16 + 8] // vcpu stored by __guest_enter
/*
* We trap the first access to the FP/SIMD to save the host context
@@ -122,13 +124,15 @@ alternative_else_nop_endif
el1_irq:
stp x0, x1, [sp, #-16]!
- ldr x1, [sp, #16 + 8]
+ get_host_ctxt x0, x1
+ get_vcpu x1, x0
mov x0, #ARM_EXCEPTION_IRQ
b __guest_exit
el1_error:
stp x0, x1, [sp, #-16]!
- ldr x1, [sp, #16 + 8]
+ get_host_ctxt x0, x1
+ get_vcpu x1, x0
mov x0, #ARM_EXCEPTION_EL1_SERROR
b __guest_exit
@@ -164,14 +168,7 @@ ENTRY(__hyp_do_panic)
ENDPROC(__hyp_do_panic)
ENTRY(__hyp_panic)
- /*
- * '=kvm_host_cpu_state' is a host VA from the constant pool, it may
- * not be accessible by this address from EL2, hyp_panic() converts
- * it with kern_hyp_va() before use.
- */
- ldr x0, =kvm_host_cpu_state
- mrs x1, tpidr_el2
- add x0, x0, x1
+ get_host_ctxt x0, x1
b hyp_panic
ENDPROC(__hyp_panic)
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index f7307b6b42f0..6fcb37e220b5 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -449,7 +449,7 @@ static hyp_alternate_select(__hyp_call_panic,
__hyp_call_panic_nvhe, __hyp_call_panic_vhe,
ARM64_HAS_VIRT_HOST_EXTN);
-void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *__host_ctxt)
+void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
{
struct kvm_vcpu *vcpu = NULL;
@@ -458,9 +458,6 @@ void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *__host_ctxt)
u64 par = read_sysreg(par_el1);
if (read_sysreg(vttbr_el2)) {
- struct kvm_cpu_context *host_ctxt;
-
- host_ctxt = kern_hyp_va(__host_ctxt);
vcpu = host_ctxt->__hyp_running_vcpu;
__timer_disable_traps(vcpu);
__deactivate_traps(vcpu);
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index c54cc2afb92b..e19d89cabf2a 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -183,3 +183,8 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
}
+
+void __hyp_text __kvm_set_tpidr_el2(u64 tpidr_el2)
+{
+ asm("msr tpidr_el2, %0": : "r" (tpidr_el2));
+}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 02/41] KVM: arm/arm64: Move vcpu_load call after kvm_vcpu_first_run_init
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
Moving the call to vcpu_load() in kvm_arch_vcpu_ioctl_run() to after
we've called kvm_vcpu_first_run_init() simplifies some of the vgic and
there is also no need to do vcpu_load() for things such as handling the
immediate_exit flag.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
virt/kvm/arm/arch_timer.c | 7 -------
virt/kvm/arm/arm.c | 22 ++++++++--------------
virt/kvm/arm/vgic/vgic-init.c | 11 -----------
3 files changed, 8 insertions(+), 32 deletions(-)
diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c
index cfcd0323deab..c09c701fd68e 100644
--- a/virt/kvm/arm/arch_timer.c
+++ b/virt/kvm/arm/arch_timer.c
@@ -834,14 +834,7 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu)
return ret;
no_vgic:
- preempt_disable();
timer->enabled = 1;
- if (!irqchip_in_kernel(vcpu->kvm))
- kvm_timer_vcpu_load_user(vcpu);
- else
- kvm_timer_vcpu_load_vgic(vcpu);
- preempt_enable();
-
return 0;
}
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 5e3c149a6e28..360df72692ee 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -631,27 +631,22 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
if (unlikely(!kvm_vcpu_initialized(vcpu)))
return -ENOEXEC;
- vcpu_load(vcpu);
-
ret = kvm_vcpu_first_run_init(vcpu);
if (ret)
- goto out;
+ return ret;
if (run->exit_reason == KVM_EXIT_MMIO) {
ret = kvm_handle_mmio_return(vcpu, vcpu->run);
if (ret)
- goto out;
- if (kvm_arm_handle_step_debug(vcpu, vcpu->run)) {
- ret = 0;
- goto out;
- }
-
+ return ret;
+ if (kvm_arm_handle_step_debug(vcpu, vcpu->run))
+ return 0;
}
- if (run->immediate_exit) {
- ret = -EINTR;
- goto out;
- }
+ if (run->immediate_exit)
+ return -EINTR;
+
+ vcpu_load(vcpu);
kvm_sigset_activate(vcpu);
@@ -803,7 +798,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
kvm_sigset_deactivate(vcpu);
-out:
vcpu_put(vcpu);
return ret;
}
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c
index 62310122ee78..a0688ef52ad7 100644
--- a/virt/kvm/arm/vgic/vgic-init.c
+++ b/virt/kvm/arm/vgic/vgic-init.c
@@ -300,17 +300,6 @@ int vgic_init(struct kvm *kvm)
dist->initialized = true;
- /*
- * If we're initializing GICv2 on-demand when first running the VCPU
- * then we need to load the VGIC state onto the CPU. We can detect
- * this easily by checking if we are in between vcpu_load and vcpu_put
- * when we just initialized the VGIC.
- */
- preempt_disable();
- vcpu = kvm_arm_get_running_vcpu();
- if (vcpu)
- kvm_vgic_load(vcpu);
- preempt_enable();
out:
return ret;
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 01/41] KVM: arm/arm64: Avoid vcpu_load for other vcpu ioctls than KVM_RUN
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180112120747.27999-1-christoffer.dall@linaro.org>
Calling vcpu_load() registers preempt notifiers for this vcpu and calls
kvm_arch_vcpu_load(). The latter will soon be doing a lot of heavy
lifting on arm/arm64 and will try to do things such as enabling the
virtual timer and setting us up to handle interrupts from the timer
hardware.
Loading state onto hardware registers and enabling hardware to signal
interrupts can be problematic when we're not actually about to run the
VCPU, because it makes it difficult to establish the right context when
handling interrupts from the timer, and it makes the register access
code difficult to reason about.
Luckily, now when we call vcpu_load in each ioctl implementation, we can
simply remove the call from the non-KVM_RUN vcpu ioctls, and our
kvm_arch_vcpu_load() is only used for loading vcpu content to the
physical CPU when we're actually going to run the vcpu.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
---
arch/arm64/kvm/guest.c | 3 ---
virt/kvm/arm/arm.c | 9 ---------
2 files changed, 12 deletions(-)
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index d7e3299a7734..959e50d2588c 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -363,8 +363,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
{
int ret = 0;
- vcpu_load(vcpu);
-
trace_kvm_set_guest_debug(vcpu, dbg->control);
if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
@@ -386,7 +384,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
}
out:
- vcpu_put(vcpu);
return ret;
}
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 7a6ce4830cc5..5e3c149a6e28 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -383,14 +383,11 @@ static void vcpu_power_off(struct kvm_vcpu *vcpu)
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
struct kvm_mp_state *mp_state)
{
- vcpu_load(vcpu);
-
if (vcpu->arch.power_off)
mp_state->mp_state = KVM_MP_STATE_STOPPED;
else
mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
- vcpu_put(vcpu);
return 0;
}
@@ -399,8 +396,6 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
{
int ret = 0;
- vcpu_load(vcpu);
-
switch (mp_state->mp_state) {
case KVM_MP_STATE_RUNNABLE:
vcpu->arch.power_off = false;
@@ -412,7 +407,6 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
ret = -EINVAL;
}
- vcpu_put(vcpu);
return ret;
}
@@ -1028,8 +1022,6 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
struct kvm_device_attr attr;
long r;
- vcpu_load(vcpu);
-
switch (ioctl) {
case KVM_ARM_VCPU_INIT: {
struct kvm_vcpu_init init;
@@ -1106,7 +1098,6 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
r = -EINVAL;
}
- vcpu_put(vcpu);
return r;
}
--
2.14.2
^ permalink raw reply related
* [PATCH v3 00/41] Optimize KVM/ARM for VHE systems
From: Christoffer Dall @ 2018-01-12 12:07 UTC (permalink / raw)
To: linux-arm-kernel
This series redesigns parts of KVM/ARM to optimize the performance on
VHE systems. The general approach is to try to do as little work as
possible when transitioning between the VM and the hypervisor. This has
the benefit of lower latency when waiting for interrupts and delivering
virtual interrupts, and reduces the overhead of emulating behavior and
I/O in the host kernel.
Patches 01 through 06 are not VHE specific, but rework parts of KVM/ARM
that can be generally improved. We then add infrastructure to move more
logic into vcpu_load and vcpu_put, we improve handling of VFP and debug
registers.
We then introduce a new world-switch function for VHE systems, which we
can tweak and optimize for VHE systems. To do that, we rework a lot of
the system register save/restore handling and emulation code that may
need access to system registers, so that we can defer as many system
register save/restore operations to vcpu_load and vcpu_put, and move
this logic out of the VHE world switch function.
We then optimize the configuration of traps. On non-VHE systems, both
the host and VM kernels run in EL1, but because the host kernel should
have full access to the underlying hardware, but the VM kernel should
not, we essentially make the host kernel more privileged than the VM
kernel despite them both running at the same privilege level by enabling
VE traps when entering the VM and disabling those traps when exiting the
VM. On VHE systems, the host kernel runs in EL2 and has full access to
the hardware (as much as allowed by secure side software), and is
unaffected by the trap configuration. That means we can configure the
traps for VMs running in EL1 once, and don't have to switch them on and
off for every entry/exit to/from the VM.
Finally, we improve our VGIC handling by moving all save/restore logic
out of the VHE world-switch, and we make it possible to truly only
evaluate if the AP list is empty and not do *any* VGIC work if that is
the case, and only do the minimal amount of work required in the course
of the VGIC processing when we have virtual interrupts in flight.
The patches are based on v4.15-rc3, v9 of the level-triggered mapped
interrupts support series [1], and the first five patches of James' SDEI
series [2].
I've given the patches a fair amount of testing on Thunder-X, Mustang,
Seattle, and TC2 (32-bit) for non-VHE testing, and tested VHE
functionality on the Foundation model, running both 64-bit VMs and
32-bit VMs side-by-side and using both GICv3-on-GICv3 and
GICv2-on-GICv3.
The patches are also available in the vhe-optimize-v3 branch on my
kernel.org repository [3]. The vhe-optimize-v3-base branch contains
prerequisites of this series.
Changes since v2:
- Rebased on v4.15-rc3.
- Includes two additional patches that only does vcpu_load after
kvm_vcpu_first_run_init and only for KVM_RUN.
- Addressed review comments from v2 (detailed changelogs are in the
individual patches).
Thanks,
-Christoffer
[1]: git://git.kernel.org/pub/scm/linux/kernel/git/cdall/linux.git level-mapped-v9
[2]: git://linux-arm.org/linux-jm.git sdei/v5/base
[3]: git://git.kernel.org/pub/scm/linux/kernel/git/cdall/linux.git vhe-optimize-v3
Christoffer Dall (40):
KVM: arm/arm64: Avoid vcpu_load for other vcpu ioctls than KVM_RUN
KVM: arm/arm64: Move vcpu_load call after kvm_vcpu_first_run_init
KVM: arm64: Avoid storing the vcpu pointer on the stack
KVM: arm64: Rework hyp_panic for VHE and non-VHE
KVM: arm/arm64: Get rid of vcpu->arch.irq_lines
KVM: arm/arm64: Add kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs
KVM: arm/arm64: Introduce vcpu_el1_is_32bit
KVM: arm64: Defer restoring host VFP state to vcpu_put
KVM: arm64: Move debug dirty flag calculation out of world switch
KVM: arm64: Slightly improve debug save/restore functions
KVM: arm64: Improve debug register save/restore flow
KVM: arm64: Factor out fault info population and gic workarounds
KVM: arm64: Introduce VHE-specific kvm_vcpu_run
KVM: arm64: Remove kern_hyp_va() use in VHE switch function
KVM: arm64: Don't deactivate VM on VHE systems
KVM: arm64: Remove noop calls to timer save/restore from VHE switch
KVM: arm64: Move userspace system registers into separate function
KVM: arm64: Rewrite sysreg alternatives to static keys
KVM: arm64: Introduce separate VHE/non-VHE sysreg save/restore
functions
KVM: arm/arm64: Remove leftover comment from kvm_vcpu_run_vhe
KVM: arm64: Unify non-VHE host/guest sysreg save and restore functions
KVM: arm64: Don't save the host ELR_EL2 and SPSR_EL2 on VHE systems
KVM: arm64: Change 32-bit handling of VM system registers
KVM: arm64: Rewrite system register accessors to read/write functions
KVM: arm64: Introduce framework for accessing deferred sysregs
KVM: arm/arm64: Prepare to handle deferred save/restore of SPSR_EL1
KVM: arm64: Prepare to handle deferred save/restore of ELR_EL1
KVM: arm64: Defer saving/restoring 64-bit sysregs to vcpu load/put on
VHE
KVM: arm64: Prepare to handle deferred save/restore of 32-bit
registers
KVM: arm64: Defer saving/restoring 32-bit sysregs to vcpu load/put
KVM: arm64: Move common VHE/non-VHE trap config in separate functions
KVM: arm64: Configure FPSIMD traps on vcpu load/put
KVM: arm64: Configure c15, PMU, and debug register traps on cpu
load/put for VHE
KVM: arm64: Separate activate_traps and deactive_traps for VHE and
non-VHE
KVM: arm/arm64: Get rid of vgic_elrsr
KVM: arm/arm64: Handle VGICv2 save/restore from the main VGIC code
KVM: arm/arm64: Move arm64-only vgic-v2-sr.c file to arm64
KVM: arm/arm64: Handle VGICv3 save/restore from the main VGIC code on
VHE
KVM: arm/arm64: Move VGIC APR save/restore to vgic put/load
KVM: arm/arm64: Avoid VGICv3 save/restore on VHE with no IRQs
Shih-Wei Li (1):
KVM: arm64: Move HCR_INT_OVERRIDE to default HCR_EL2 guest flag
arch/arm/include/asm/kvm_asm.h | 5 +-
arch/arm/include/asm/kvm_emulate.h | 21 +-
arch/arm/include/asm/kvm_host.h | 6 +-
arch/arm/include/asm/kvm_hyp.h | 4 +
arch/arm/kvm/emulate.c | 4 +-
arch/arm/kvm/hyp/Makefile | 1 -
arch/arm/kvm/hyp/switch.c | 16 +-
arch/arm64/include/asm/kvm_arm.h | 4 +-
arch/arm64/include/asm/kvm_asm.h | 18 +-
arch/arm64/include/asm/kvm_emulate.h | 74 +++-
arch/arm64/include/asm/kvm_host.h | 49 ++-
arch/arm64/include/asm/kvm_hyp.h | 32 +-
arch/arm64/include/asm/kvm_mmu.h | 2 +-
arch/arm64/kernel/asm-offsets.c | 2 +
arch/arm64/kvm/debug.c | 28 +-
arch/arm64/kvm/guest.c | 3 -
arch/arm64/kvm/hyp/Makefile | 2 +-
arch/arm64/kvm/hyp/debug-sr.c | 88 +++--
arch/arm64/kvm/hyp/entry.S | 9 +-
arch/arm64/kvm/hyp/hyp-entry.S | 41 +--
arch/arm64/kvm/hyp/switch.c | 404 +++++++++++++---------
arch/arm64/kvm/hyp/sysreg-sr.c | 192 ++++++++--
{virt/kvm/arm => arch/arm64/kvm}/hyp/vgic-v2-sr.c | 81 -----
arch/arm64/kvm/inject_fault.c | 24 +-
arch/arm64/kvm/regmap.c | 65 +++-
arch/arm64/kvm/sys_regs.c | 247 +++++++++++--
arch/arm64/kvm/sys_regs.h | 4 +-
arch/arm64/kvm/sys_regs_generic_v8.c | 4 +-
include/kvm/arm_vgic.h | 2 -
virt/kvm/arm/aarch32.c | 2 +-
virt/kvm/arm/arch_timer.c | 7 -
virt/kvm/arm/arm.c | 50 ++-
virt/kvm/arm/hyp/timer-sr.c | 44 +--
virt/kvm/arm/hyp/vgic-v3-sr.c | 244 +++++++------
virt/kvm/arm/mmu.c | 6 +-
virt/kvm/arm/pmu.c | 37 +-
virt/kvm/arm/vgic/vgic-init.c | 11 -
virt/kvm/arm/vgic/vgic-v2.c | 61 +++-
virt/kvm/arm/vgic/vgic-v3.c | 12 +-
virt/kvm/arm/vgic/vgic.c | 21 ++
virt/kvm/arm/vgic/vgic.h | 3 +
41 files changed, 1229 insertions(+), 701 deletions(-)
rename {virt/kvm/arm => arch/arm64/kvm}/hyp/vgic-v2-sr.c (50%)
--
2.14.2
^ permalink raw reply
* [PATCH] arm64: support do_csum with neon
From: Chen Zhou @ 2018-01-12 11:53 UTC (permalink / raw)
To: linux-arm-kernel
On arm64 little endian such as the Cortex-A57, the
neon based implementation performance increases by
about 70% when len is greater than 512.
Signed-off-by: Chen Zhou <chenzhou10@huawei.com>
---
arch/arm64/include/asm/checksum.h | 3 +
arch/arm64/lib/Makefile | 1 +
arch/arm64/lib/do_csum.S | 177 ++++++++++++++++++++++++++++++++++++++
3 files changed, 181 insertions(+)
create mode 100644 arch/arm64/lib/do_csum.S
diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h
index 09f6533..e300782 100644
--- a/arch/arm64/include/asm/checksum.h
+++ b/arch/arm64/include/asm/checksum.h
@@ -26,6 +26,9 @@ static inline __sum16 csum_fold(__wsum csum)
}
#define csum_fold csum_fold
+#define do_csum do_csum
+extern unsigned int do_csum(const unsigned char *, size_t);
+
static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
{
__uint128_t tmp;
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 1a811ec..5b6aa34 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -3,6 +3,7 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \
clear_page.o memchr.o memcpy.o memmove.o memset.o \
memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
strchr.o strrchr.o
+lib-y += do_csum.o
# Tell the compiler to treat all general purpose registers as
# callee-saved, which allows for efficient runtime patching of the bl
diff --git a/arch/arm64/lib/do_csum.S b/arch/arm64/lib/do_csum.S
new file mode 100644
index 0000000..8e7b486
--- /dev/null
+++ b/arch/arm64/lib/do_csum.S
@@ -0,0 +1,177 @@
+/*
+ * Optmized version of the standard do_csum() function
+ *
+ * Parameters:
+ * x0 - address of buffer to checksum (const unsigned char *)
+ * x1 - length of the buffer (int)
+ * Returns:
+ * x0 - the return checksum of the buffer
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ENTRY(do_csum)
+ ldr x13, =0xffff
+ eor x4, x4, x4
+ eor x5, x5, x5
+ eor v0.16b, v0.16b, v0.16b
+
+ //len is zero or negative
+ and x6, x1, #0x80000000
+ cmp x6, #0
+ b.gt out
+ cbz w1, out
+
+ tst x0, #1
+ b.eq addr_not_odd
+
+ //addr is odd
+ mov x4, #1
+ ldr x6, [x0], #1
+#ifdef __AARCH64EB__
+ and x6, x6, #0xff
+#else
+ lsl x6, x6, #8
+ and x6, x6, x13
+#endif
+ add x5, x5, x6
+ sub x1, x1, #1
+
+addr_not_odd:
+ cmp x1, #32
+ b.lt len_4
+ cmp x1, #192
+ b.ge len_than_192
+ b do_loop_16
+
+len_than_192:
+ ldp q1, q0, [x0], #32
+ ldp q3, q2, [x0], #32
+ ldp q5, q4, [x0], #32
+ sub x1, x1, #96
+
+do_loop_96:
+ ldp q7, q6, [x0], #32
+ ldp q9, q8, [x0], #32
+ ldp q11, q10, [x0], #32
+
+ uaddl v12.4s, v0.4h, v6.4h
+ uaddl2 v13.4s, v0.8h, v6.8h
+
+ uaddl v14.4s, v1.4h, v7.4h
+ uaddl2 v15.4s, v1.8h, v7.8h
+
+ uaddl v16.4s, v2.4h, v8.4h
+ uaddl2 v17.4s, v2.8h, v8.8h
+
+ uaddl v18.4s, v3.4h, v9.4h
+ uaddl2 v19.4s, v3.8h, v9.8h
+
+ uaddl v20.4s, v4.4h, v10.4h
+ uaddl2 v21.4s, v4.8h, v10.8h
+ uaddl v22.4s, v5.4h, v11.4h
+ uaddl2 v23.4s, v5.8h, v11.8h
+
+ add v0.4s, v12.4s, v13.4s
+ add v1.4s, v14.4s, v15.4s
+ add v2.4s, v16.4s, v17.4s
+ add v3.4s, v18.4s, v19.4s
+ add v4.4s, v20.4s, v21.4s
+ add v5.4s, v22.4s, v23.4s
+
+ sub x1, x1, #96
+ cmp x1, #96
+ b.ge do_loop_96
+
+ add v0.4s, v0.4s, v1.4s
+ add v2.4s, v2.4s, v3.4s
+ add v4.4s, v4.4s, v5.4s
+ add v0.4s, v0.4s, v2.4s
+ add v0.4s, v0.4s, v4.4s //get result
+
+ cmp x1, #16
+ b.lt get_64
+
+do_loop_16:
+ ldr q6, [x0], #16
+ uaddl v24.4s, v0.4h, v6.4h
+ uaddl2 v25.4s, v0.8h, v6.8h
+ add v0.4s, v24.4s, v25.4s
+ sub x1, x1, #16
+ cmp x1, #16
+ b.ge do_loop_16
+
+get_64:
+ mov x6, v0.d[0]
+ add x5, x5, x6
+ mov x6, v0.d[1]
+
+ add x5, x5, x6
+ cmp x5, x6
+ b.ge len_4
+ add x5, x5, #1
+
+len_4:
+ cmp x1, #4
+ b.lt len_2
+
+ sub x1, x1, #4
+ ldr w6, [x0], #4
+ and x6, x6, #0xffffffff
+ add x5, x5, x6
+ b len_4
+
+len_2:
+ cmp x1, #2
+ b.lt len_1
+ sub x1, x1, #2
+ ldrh w6, [x0], #2
+ and x6, x6, x13
+ add x5, x5, x6
+
+len_1:
+ cmp x1, #1
+ b.lt fold_32
+ ldr x6, [x0], #1
+#ifdef __AARCH64EB__
+ lsl x6, x6, #8
+ and x6, x6, x13
+#else
+ and x6, x6, #0xff
+#endif
+ add x5, x5, x6
+
+fold_32:
+ and x9, x5, x13 //[15:0]
+ and x10, x13, x5, lsr #16 //[31:16]
+ and x11, x13, x5, lsr #32 //[47:32]
+ and x12, x13, x5, lsr #48 //[47:32]
+
+ add x9, x9, x10
+ add x11, x11, x12
+
+ add x9, x9, x11
+
+ and x10, x9, x13
+ and x11, x13, x9, lsr #16
+
+ add x5, x10, x11
+
+ and x9, x5, x13 //add carry
+ and x10, x13, x5, lsr #16
+ add x5, x9, x10
+
+ cbz x4, out //addr isn't odd
+
+ lsr x6, x5, #8
+ and x6, x6, #0xff
+ and x7, x5, #0xff
+ lsl x7, x7, #8
+
+ orr x5, x6, x7
+
+out:
+ mov x0, x5
+ ret
+ENDPROC(do_csum)
--
1.8.3.1
I test the performance on arm64 little endian(Cortex-A57) and
run 100000 times as a benchmark. The len of test packets is
32/64/256/512/1024/1444B. The result is as follows(average):
Len get_cycles performance increase
----------------------------------------------------
The generic implementation:
----------------------------------------------------
32B: 106372.30 cycles
64B: 241362.40 cycles
256B: 791705.30 cycles
512B: 1510084.50 cycles
1024B: 2937692.60 cycles
1444B: 4113743.30 cycles
The neon implementation:
----------------------------------------------------
32B: 151483.40 cycles -42%
64B: 151069.70 cycles 59.7%
256B: 472375.10 cycles 67.6%
512B: 883657.30 cycles 70.8%
1024B: 1713357.00 cycles 71.4%
1444B: 2375959.20 cycles 73.1%
According to the result, the neon performance of small packets
such as 32B is worse than the ordinary implementation. When the
len is greater than 256, the performance increases by about 70%.
^ permalink raw reply related
* [PATCH v2 7/8] arm64: allow ID map to be extended to 52 bits
From: Kristina Martsenko @ 2018-01-12 11:48 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <0136ccc5-4055-d392-6e3a-f39080163620@arm.com>
On 09/01/18 19:29, Kristina Martsenko wrote:
> On 22/12/17 15:23, Catalin Marinas wrote:
>> From: Kristina Martsenko <kristina.martsenko@arm.com>
>>
>> Currently, when using VA_BITS < 48, if the ID map text happens to be
>> placed in physical memory above VA_BITS, we increase the VA size (up to
>> 48) and create a new table level, in order to map in the ID map text.
>> This is okay because the system always supports 48 bits of VA.
>>
>> This patch extends the code such that if the system supports 52 bits of
>> VA, and the ID map text is placed that high up, then we increase the VA
>> size accordingly, up to 52.
>>
>> One difference from the current implementation is that so far the
>> condition of VA_BITS < 48 has meant that the top level table is always
>> "full", with the maximum number of entries, and an extra table level is
>> always needed. Now, when VA_BITS = 48 (and using 64k pages), the top
>> level table is not full, and we simply need to increase the number of
>> entries in it, instead of creating a new table level.
>>
>> Tested-by: Bob Picco <bob.picco@oracle.com>
>> Reviewed-by: Bob Picco <bob.picco@oracle.com>
>> Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
>> [catalin.marinas at arm.com: reduce arguments to __create_hyp_mappings()]
>> [catalin.marinas at arm.com: reworked/renamed __cpu_uses_extended_idmap_level()]
>> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
>> ---
>> arch/arm/include/asm/kvm_mmu.h | 5 +++
>> arch/arm64/include/asm/assembler.h | 2 -
>> arch/arm64/include/asm/kvm_mmu.h | 7 +++-
>> arch/arm64/include/asm/mmu_context.h | 18 +++++++--
>> arch/arm64/kernel/head.S | 76 +++++++++++++++++++++---------------
>> arch/arm64/kvm/hyp-init.S | 17 ++++----
>> arch/arm64/mm/mmu.c | 1 +
>> virt/kvm/arm/mmu.c | 10 ++++-
>> 8 files changed, 87 insertions(+), 49 deletions(-)
>>
>> diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
>> index 8dbec683638b..8c5643e2eea4 100644
>> --- a/arch/arm/include/asm/kvm_mmu.h
>> +++ b/arch/arm/include/asm/kvm_mmu.h
>> @@ -211,6 +211,11 @@ static inline bool __kvm_cpu_uses_extended_idmap(void)
>> return false;
>> }
>>
>> +static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
>> +{
>> + return PTRS_PER_PGD;
>> +}
>> +
>> static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
>> pgd_t *hyp_pgd,
>> pgd_t *merged_hyp_pgd,
>> diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
>> index 49ea3def4bd1..942fdb5ef0ad 100644
>> --- a/arch/arm64/include/asm/assembler.h
>> +++ b/arch/arm64/include/asm/assembler.h
>> @@ -344,10 +344,8 @@ alternative_endif
>> * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map
>> */
>> .macro tcr_set_idmap_t0sz, valreg, tmpreg
>> -#ifndef CONFIG_ARM64_VA_BITS_48
>> ldr_l \tmpreg, idmap_t0sz
>> bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
>> -#endif
>> .endm
>>
>> /*
>> diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
>> index b3f7b68b042d..8d663ca0d50c 100644
>> --- a/arch/arm64/include/asm/kvm_mmu.h
>> +++ b/arch/arm64/include/asm/kvm_mmu.h
>> @@ -273,7 +273,12 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
>>
>> static inline bool __kvm_cpu_uses_extended_idmap(void)
>> {
>> - return __cpu_uses_extended_idmap();
>> + return __cpu_uses_extended_idmap_table();
>> +}
>> +
>> +static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
>> +{
>> + return idmap_ptrs_per_pgd;
>> }
>>
>> /*
>> diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
>> index accc2ff32a0e..7991718890c6 100644
>> --- a/arch/arm64/include/asm/mmu_context.h
>> +++ b/arch/arm64/include/asm/mmu_context.h
>> @@ -63,11 +63,21 @@ static inline void cpu_set_reserved_ttbr0(void)
>> * physical memory, in which case it will be smaller.
>> */
>> extern u64 idmap_t0sz;
>> +extern u64 idmap_ptrs_per_pgd;
>>
>> -static inline bool __cpu_uses_extended_idmap(void)
>> +static inline bool __cpu_uses_extended_idmap_level(void)
>> {
>> - return (!IS_ENABLED(CONFIG_ARM64_VA_BITS_48) &&
>> - unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS)));
>> + return ARM64_HW_PGTABLE_LEVELS((64 - idmap_t0sz)) > CONFIG_PGTABLE_LEVELS;
>> +}
>> +
>> +/*
>> + * True if the extended ID map requires an extra level of translation table
>> + * to be configured.
>> + */
>> +static inline bool __cpu_uses_extended_idmap_table(void)
>> +{
>> + return __cpu_uses_extended_idmap_level() &&
>> + (idmap_ptrs_per_pgd == PTRS_PER_PGD);
>> }
>>
>> /*
>> @@ -77,7 +87,7 @@ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz)
>> {
>> unsigned long tcr;
>>
>> - if (!__cpu_uses_extended_idmap())
>> + if (!__cpu_uses_extended_idmap_level())
>> return;
>>
>> tcr = read_sysreg(tcr_el1);
>> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
>> index eeec0001e204..66f01869e97c 100644
>> --- a/arch/arm64/kernel/head.S
>> +++ b/arch/arm64/kernel/head.S
>> @@ -176,7 +176,7 @@ ENDPROC(preserve_boot_args)
>> * ptrs: #imm pointers per table page
>> *
>> * Preserves: virt
>> - * Corrupts: tmp1, tmp2
>> + * Corrupts: ptrs, tmp1, tmp2
>> * Returns: tbl -> next level table page address
>> */
>> .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
>> @@ -184,7 +184,8 @@ ENDPROC(preserve_boot_args)
>> phys_to_pte \tmp1, \tmp2
>> orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type
>> lsr \tmp1, \virt, #\shift
>> - and \tmp1, \tmp1, #\ptrs - 1 // table index
>> + sub \ptrs, \ptrs, #1
>> + and \tmp1, \tmp1, \ptrs // table index
>> str \tmp2, [\tbl, \tmp1, lsl #3]
>> add \tbl, \tbl, #PAGE_SIZE // next level table page
>> .endm
>> @@ -194,15 +195,17 @@ ENDPROC(preserve_boot_args)
>> * block entry in the next level (tbl) for the given virtual address.
>> *
>> * Preserves: tbl, next, virt
>> - * Corrupts: tmp1, tmp2
>> + * Corrupts: ptrs_per_pgd, tmp1, tmp2
>> */
>> - .macro create_pgd_entry, tbl, virt, tmp1, tmp2
>> - create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
>> + .macro create_pgd_entry, tbl, virt, ptrs_per_pgd, tmp1, tmp2
>> + create_table_entry \tbl, \virt, PGDIR_SHIFT, \ptrs_per_pgd, \tmp1, \tmp2
>> #if SWAPPER_PGTABLE_LEVELS > 3
>> - create_table_entry \tbl, \virt, PUD_SHIFT, PTRS_PER_PUD, \tmp1, \tmp2
>> + mov \ptrs_per_pgd, PTRS_PER_PUD
>> + create_table_entry \tbl, \virt, PUD_SHIFT, \ptrs_per_pgd, \tmp1, \tmp2
>> #endif
>> #if SWAPPER_PGTABLE_LEVELS > 2
>> - create_table_entry \tbl, \virt, SWAPPER_TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
>> + mov \ptrs_per_pgd, PTRS_PER_PTE
>> + create_table_entry \tbl, \virt, SWAPPER_TABLE_SHIFT, \ptrs_per_pgd, \tmp1, \tmp2
>> #endif
>> .endm
>>
>> @@ -266,26 +269,13 @@ __create_page_tables:
>> adrp x0, idmap_pg_dir
>> adrp x3, __idmap_text_start // __pa(__idmap_text_start)
>>
>> -#ifndef CONFIG_ARM64_VA_BITS_48
>> -#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3)
>> -#define EXTRA_PTRS (1 << (48 - EXTRA_SHIFT))
>> -
>> - /*
>> - * If VA_BITS < 48, it may be too small to allow for an ID mapping to be
>> - * created that covers system RAM if that is located sufficiently high
>> - * in the physical address space. So for the ID map, use an extended
>> - * virtual range in that case, by configuring an additional translation
>> - * level.
>> - * First, we have to verify our assumption that the current value of
>> - * VA_BITS was chosen such that all translation levels are fully
>> - * utilised, and that lowering T0SZ will always result in an additional
>> - * translation level to be configured.
>> - */
>> -#if VA_BITS != EXTRA_SHIFT
>> -#error "Mismatch between VA_BITS and page size/number of translation levels"
>> -#endif
>> -
>> /*
>> + * VA_BITS may be too small to allow for an ID mapping to be created
>> + * that covers system RAM if that is located sufficiently high in the
>> + * physical address space. So for the ID map, use an extended virtual
>> + * range in that case, and configure an additional translation level
>> + * if needed.
>> + *
>> * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the
>> * entire ID map region can be mapped. As T0SZ == (64 - #bits used),
>> * this number conveniently equals the number of leading zeroes in
>> @@ -294,18 +284,41 @@ __create_page_tables:
>> adrp x5, __idmap_text_end
>> clz x5, x5
>> cmp x5, TCR_T0SZ(VA_BITS) // default T0SZ small enough?
>> - b.ge 1f // .. then skip additional level
>> + b.ge 1f // .. then skip VA range extension
>>
>> adr_l x6, idmap_t0sz
>> str x5, [x6]
>> dmb sy
>> dc ivac, x6 // Invalidate potentially stale cache line
>>
>> - create_table_entry x0, x3, EXTRA_SHIFT, EXTRA_PTRS, x5, x6
>> -1:
>> +#if (VA_BITS < 48)
>> +#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3)
>> +#define EXTRA_PTRS (1 << (PHYS_MASK_SHIFT - EXTRA_SHIFT))
>> +
>> + /*
>> + * If VA_BITS < 48, we have to configure an additional table level.
>> + * First, we have to verify our assumption that the current value of
>> + * VA_BITS was chosen such that all translation levels are fully
>> + * utilised, and that lowering T0SZ will always result in an additional
>> + * translation level to be configured.
>> + */
>> +#if VA_BITS != EXTRA_SHIFT
>> +#error "Mismatch between VA_BITS and page size/number of translation levels"
>> #endif
>>
>> - create_pgd_entry x0, x3, x5, x6
>> + mov x4, EXTRA_PTRS
>> + create_table_entry x0, x3, EXTRA_SHIFT, x4, x5, x6
>> +#else
>> + /*
>> + * If VA_BITS == 48, we don't have to configure an additional
>> + * translation level, but the top-level table has more entries.
>> + */
>> + mov x4, #1 << (PHYS_MASK_SHIFT - PGDIR_SHIFT)
>> + str_l x4, idmap_ptrs_per_pgd, x5
>> +#endif
>> +1:
>> + ldr_l x4, idmap_ptrs_per_pgd
>> + create_pgd_entry x0, x3, x4, x5, x6
>> mov x5, x3 // __pa(__idmap_text_start)
>> adr_l x6, __idmap_text_end // __pa(__idmap_text_end)
>> create_block_map x0, x7, x3, x5, x6, x4
>> @@ -316,7 +329,8 @@ __create_page_tables:
>> adrp x0, swapper_pg_dir
>> mov_q x5, KIMAGE_VADDR + TEXT_OFFSET // compile time __va(_text)
>> add x5, x5, x23 // add KASLR displacement
>> - create_pgd_entry x0, x5, x3, x6
>> + mov x4, PTRS_PER_PGD
>> + create_pgd_entry x0, x5, x4, x3, x6
>> adrp x6, _end // runtime __pa(_end)
>> adrp x3, _text // runtime __pa(_text)
>> sub x6, x6, x3 // _end - _text
>> diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S
>> index f9681cc00973..33c40b3eea01 100644
>> --- a/arch/arm64/kvm/hyp-init.S
>> +++ b/arch/arm64/kvm/hyp-init.S
>> @@ -72,24 +72,23 @@ __do_hyp_init:
>> mov x5, #TCR_EL2_RES1
>> orr x4, x4, x5
>>
>> -#ifndef CONFIG_ARM64_VA_BITS_48
>> /*
>> - * If we are running with VA_BITS < 48, we may be running with an extra
>> - * level of translation in the ID map. This is only the case if system
>> - * RAM is out of range for the currently configured page size and number
>> - * of translation levels, in which case we will also need the extra
>> - * level for the HYP ID map, or we won't be able to enable the EL2 MMU.
>> + * The ID map may be configured to use an extended virtual address
>> + * range. This is only the case if system RAM is out of range for the
>> + * currently configured page size and VA_BITS, in which case we will
>> + * also need the extended virtual range for the HYP ID map, or we won't
>> + * be able to enable the EL2 MMU.
>> *
>> * However, at EL2, there is only one TTBR register, and we can't switch
>> * between translation tables *and* update TCR_EL2.T0SZ at the same
>> - * time. Bottom line: we need the extra level in *both* our translation
>> - * tables.
>> + * time. Bottom line: we need to use the extended range with *both* our
>> + * translation tables.
>> *
>> * So use the same T0SZ value we use for the ID map.
>> */
>> ldr_l x5, idmap_t0sz
>> bfi x4, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
>> -#endif
>> +
>> /*
>> * Set the PS bits in TCR_EL2.
>> */
>> diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
>> index 0c631a17ae1d..baa34418c3bf 100644
>> --- a/arch/arm64/mm/mmu.c
>> +++ b/arch/arm64/mm/mmu.c
>> @@ -50,6 +50,7 @@
>> #define NO_CONT_MAPPINGS BIT(1)
>>
>> u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
>> +u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
>>
>> u64 kimage_voffset __ro_after_init;
>> EXPORT_SYMBOL(kimage_voffset);
>> diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
>> index b36945d49986..761787befd3b 100644
>> --- a/virt/kvm/arm/mmu.c
>> +++ b/virt/kvm/arm/mmu.c
>> @@ -629,14 +629,20 @@ static int __create_hyp_mappings(pgd_t *pgdp,
>> {
>> pgd_t *pgd;
>> pud_t *pud;
>> - unsigned long addr, next;
>> + unsigned long addr, next, ptrs_per_pgd = PTRS_PER_PGD;
>> int err = 0;
>>
>> + /*
>> + * If it's not the hyp_pgd, fall back to the kvm idmap layout.
>> + */
>> + if (pgdp != hyp_pgd)
>> + ptrs_per_pgd = __kvm_idmap_ptrs_per_pgd();
>
> This isn't right. When VA_BITS = 48 and the idmap is in 52-bit memory,
> we come here with hyp_pgd, but need to use __kvm_idmap_ptrs_per_pgd. (We
> don't have an extra table level, so we take the "else" case in
> kvm_mmu_init. But we have a larger number of entries in the top-level
> table, so we need __kvm_idmap_ptrs_per_pgd to mask in the extra bits.)
>
> Currently a kernel with the above configuration (and VHE disabled) fails
> to boot.
>
> I'm not sure how else to push the changes into __create_hyp_mappings, as
> Marc originally requested. One option might be to get rid of the mask
> entirely, since the top bits of all hyp addresses should be zero anyway:
> pgd = pgdp + (addr >> PGDIR_SHIFT);
I was wrong, this solution does not work for the case where we create an
extra table level, because the index will incorrectly include the bits
"above" the pgd level. Sorry about that.
I don't have any other ideas. Should I put it back as it was in v1, with
the callers passing in the mask for the table index?
Thanks,
Kristina
> but I'm not sure how nice that is. Any other ideas?
>
> If not, we should put this back as it was in v1 of the series, i.e.
> change the call sites instead.
>
> Kristina
>
>> +
>> mutex_lock(&kvm_hyp_pgd_mutex);
>> addr = start & PAGE_MASK;
>> end = PAGE_ALIGN(end);
>> do {
>> - pgd = pgdp + pgd_index(addr);
>> + pgd = pgdp + ((addr >> PGDIR_SHIFT) & (ptrs_per_pgd - 1));
>>
>> if (pgd_none(*pgd)) {
>> pud = pud_alloc_one(NULL, addr);
>>
^ permalink raw reply
* [RFC PATCH 0/2] drivers: clk: Add ZynqMP clock driver support
From: Sudeep Holla @ 2018-01-12 11:47 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <DM2PR0201MB0767E1E08918BED4CB06DB20B8160@DM2PR0201MB0767.namprd02.prod.outlook.com>
On 11/01/18 20:29, Jolly Shah wrote:
> Hi Sudeep,
>
> There are 3 things:
>
> 1> IPI mailbox transport - series is already posted
> https://patchwork.kernel.org/patch/10145795/
> 2> EEMI - Posted by me below
> https://patchwork.kernel.org/patch/10150665/
> 3> EEMI Users
> Currently posted as RFC as they are dependent on #2.
> OK, can #2 and #3 can be put together until initial review ?
> #2 doesn?t use mailbox so it is a posted as a separate series.
OK, does it use smc ? If so, the point is treat it as smc mailbox
so that smc is just a transport and EEMI can be still used with other
transport technically.
--
Regards,
Sudeep
^ permalink raw reply
* [PATCH] soc: brcmstb: Only register SoC device on STB platforms
From: Sudeep Holla @ 2018-01-12 11:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180109145409.11612-1-thierry.reding@gmail.com>
On 09/01/18 14:54, Thierry Reding wrote:
> From: Thierry Reding <treding@nvidia.com>
>
> After moving the SoC device initialization to an early initcall in
> commit f780429adfbc ("soc: brcmstb: biuctrl: Move to early_initcall"),
> the Broadcom STB SoC device is registered on all platforms if support
> for the device is enabled in the kernel configuration.
>
> This causes an additional SoC device to appear on platforms that already
> register a native one. In case of Tegra the STB SoC device is registered
> as soc0 (with totally meaningless content in the sysfs attributes) and
> causes various scripts and programs to fail because they don't know how
> to parse that data.
>
> To fix this, duplicate the check from brcmstb_soc_device_early_init()
> that already prevents the code from doing anything nonsensical on non-
> STB platforms.
>
> Fixes: f780429adfbc ("soc: brcmstb: biuctrl: Move to early_initcall")
> Signed-off-by: Thierry Reding <treding@nvidia.com>
> ---
> drivers/soc/bcm/brcmstb/common.c | 5 +++++
> 1 file changed, 5 insertions(+)
>
> diff --git a/drivers/soc/bcm/brcmstb/common.c b/drivers/soc/bcm/brcmstb/common.c
> index 781ada62d0a3..4fe1cb73b39a 100644
> --- a/drivers/soc/bcm/brcmstb/common.c
> +++ b/drivers/soc/bcm/brcmstb/common.c
> @@ -89,8 +89,13 @@ early_initcall(brcmstb_soc_device_early_init);
> static int __init brcmstb_soc_device_init(void)
> {
> struct soc_device_attribute *soc_dev_attr;
> + struct device_node *sun_top_ctrl;
> struct soc_device *soc_dev;
>
> + sun_top_ctrl = of_find_matching_node(NULL, sun_top_ctrl_match);
> + if (!sun_top_ctrl)
> + return -ENODEV;
> +
missing of_node_put(sun_top_ctrl) ? or am I missing to see that elsewhere ?
--
Regards,
Sudeep
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox