From: Marc Zyngier <maz@kernel.org>
To: kvmarm@lists.linux.dev, linux-arm-kernel@lists.infradead.org,
kvm@vger.kernel.org
Cc: Joey Gouly <joey.gouly@arm.com>,
Suzuki K Poulose <suzuki.poulose@arm.com>,
Oliver Upton <oliver.upton@linux.dev>,
Zenghui Yu <yuzenghui@huawei.com>,
Alexandru Elisei <alexandru.elisei@arm.com>,
Mark Brown <broonie@kernel.org>
Subject: [PATCH v4 07/36] KVM: arm64: nv: Save/Restore vEL2 sysregs
Date: Wed, 9 Oct 2024 19:59:50 +0100 [thread overview]
Message-ID: <20241009190019.3222687-8-maz@kernel.org> (raw)
In-Reply-To: <20241009190019.3222687-1-maz@kernel.org>
Whenever we need to restore the guest's system registers to the CPU, we
now need to take care of the EL2 system registers as well. Most of them
are accessed via traps only, but some have an immediate effect and also
a guest running in VHE mode would expect them to be accessible via their
EL1 encoding, which we do not trap.
For vEL2 we write the virtual EL2 registers with an identical format directly
into their EL1 counterpart, and translate the few registers that have a
different format for the same effect on the execution when running a
non-VHE guest guest hypervisor.
Based on an initial patch from Andre Przywara, rewritten many times
since.
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
---
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 5 +-
arch/arm64/kvm/hyp/nvhe/sysreg-sr.c | 2 +-
arch/arm64/kvm/hyp/vhe/sysreg-sr.c | 137 ++++++++++++++++++++-
3 files changed, 139 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
index 1579a3c08a36b..d67628d01bf5e 100644
--- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
@@ -152,9 +152,10 @@ static inline void __sysreg_restore_user_state(struct kvm_cpu_context *ctxt)
write_sysreg(ctxt_sys_reg(ctxt, TPIDRRO_EL0), tpidrro_el0);
}
-static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
+static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt,
+ u64 mpidr)
{
- write_sysreg(ctxt_sys_reg(ctxt, MPIDR_EL1), vmpidr_el2);
+ write_sysreg(mpidr, vmpidr_el2);
if (has_vhe() ||
!cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
diff --git a/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c b/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c
index 29305022bc048..dba101565de36 100644
--- a/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c
@@ -28,7 +28,7 @@ void __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt)
void __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt)
{
- __sysreg_restore_el1_state(ctxt);
+ __sysreg_restore_el1_state(ctxt, ctxt_sys_reg(ctxt, MPIDR_EL1));
__sysreg_restore_common_state(ctxt);
__sysreg_restore_user_state(ctxt);
__sysreg_restore_el2_return_state(ctxt);
diff --git a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c
index e12bd7d6d2dce..e0df14ead2657 100644
--- a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c
@@ -15,6 +15,108 @@
#include <asm/kvm_hyp.h>
#include <asm/kvm_nested.h>
+static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
+{
+ /* These registers are common with EL1 */
+ __vcpu_sys_reg(vcpu, PAR_EL1) = read_sysreg(par_el1);
+ __vcpu_sys_reg(vcpu, TPIDR_EL1) = read_sysreg(tpidr_el1);
+
+ __vcpu_sys_reg(vcpu, ESR_EL2) = read_sysreg_el1(SYS_ESR);
+ __vcpu_sys_reg(vcpu, AFSR0_EL2) = read_sysreg_el1(SYS_AFSR0);
+ __vcpu_sys_reg(vcpu, AFSR1_EL2) = read_sysreg_el1(SYS_AFSR1);
+ __vcpu_sys_reg(vcpu, FAR_EL2) = read_sysreg_el1(SYS_FAR);
+ __vcpu_sys_reg(vcpu, MAIR_EL2) = read_sysreg_el1(SYS_MAIR);
+ __vcpu_sys_reg(vcpu, VBAR_EL2) = read_sysreg_el1(SYS_VBAR);
+ __vcpu_sys_reg(vcpu, CONTEXTIDR_EL2) = read_sysreg_el1(SYS_CONTEXTIDR);
+ __vcpu_sys_reg(vcpu, AMAIR_EL2) = read_sysreg_el1(SYS_AMAIR);
+
+ /*
+ * In VHE mode those registers are compatible between EL1 and EL2,
+ * and the guest uses the _EL1 versions on the CPU naturally.
+ * So we save them into their _EL2 versions here.
+ * For nVHE mode we trap accesses to those registers, so our
+ * _EL2 copy in sys_regs[] is always up-to-date and we don't need
+ * to save anything here.
+ */
+ if (vcpu_el2_e2h_is_set(vcpu)) {
+ u64 val;
+
+ /*
+ * We don't save CPTR_EL2, as accesses to CPACR_EL1
+ * are always trapped, ensuring that the in-memory
+ * copy is always up-to-date. A small blessing...
+ */
+ __vcpu_sys_reg(vcpu, SCTLR_EL2) = read_sysreg_el1(SYS_SCTLR);
+ __vcpu_sys_reg(vcpu, TTBR0_EL2) = read_sysreg_el1(SYS_TTBR0);
+ __vcpu_sys_reg(vcpu, TTBR1_EL2) = read_sysreg_el1(SYS_TTBR1);
+ __vcpu_sys_reg(vcpu, TCR_EL2) = read_sysreg_el1(SYS_TCR);
+
+ /*
+ * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where
+ * the interesting CNTHCTL_EL2 bits live. So preserve these
+ * bits when reading back the guest-visible value.
+ */
+ val = read_sysreg_el1(SYS_CNTKCTL);
+ val &= CNTKCTL_VALID_BITS;
+ __vcpu_sys_reg(vcpu, CNTHCTL_EL2) &= ~CNTKCTL_VALID_BITS;
+ __vcpu_sys_reg(vcpu, CNTHCTL_EL2) |= val;
+ }
+
+ __vcpu_sys_reg(vcpu, SP_EL2) = read_sysreg(sp_el1);
+ __vcpu_sys_reg(vcpu, ELR_EL2) = read_sysreg_el1(SYS_ELR);
+ __vcpu_sys_reg(vcpu, SPSR_EL2) = read_sysreg_el1(SYS_SPSR);
+}
+
+static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
+{
+ u64 val;
+
+ /* These registers are common with EL1 */
+ write_sysreg(__vcpu_sys_reg(vcpu, PAR_EL1), par_el1);
+ write_sysreg(__vcpu_sys_reg(vcpu, TPIDR_EL1), tpidr_el1);
+
+ write_sysreg(read_cpuid_id(), vpidr_el2);
+ write_sysreg(__vcpu_sys_reg(vcpu, MPIDR_EL1), vmpidr_el2);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, MAIR_EL2), SYS_MAIR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, VBAR_EL2), SYS_VBAR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, CONTEXTIDR_EL2), SYS_CONTEXTIDR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, AMAIR_EL2), SYS_AMAIR);
+
+ if (vcpu_el2_e2h_is_set(vcpu)) {
+ /*
+ * In VHE mode those registers are compatible between
+ * EL1 and EL2.
+ */
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2), SYS_SCTLR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, CPTR_EL2), SYS_CPACR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2), SYS_TTBR0);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR1_EL2), SYS_TTBR1);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR_EL2), SYS_TCR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, CNTHCTL_EL2), SYS_CNTKCTL);
+ } else {
+ /*
+ * CNTHCTL_EL2 only affects EL1 when running nVHE, so
+ * no need to restore it.
+ */
+ val = translate_sctlr_el2_to_sctlr_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2));
+ write_sysreg_el1(val, SYS_SCTLR);
+ val = translate_cptr_el2_to_cpacr_el1(__vcpu_sys_reg(vcpu, CPTR_EL2));
+ write_sysreg_el1(val, SYS_CPACR);
+ val = translate_ttbr0_el2_to_ttbr0_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2));
+ write_sysreg_el1(val, SYS_TTBR0);
+ val = translate_tcr_el2_to_tcr_el1(__vcpu_sys_reg(vcpu, TCR_EL2));
+ write_sysreg_el1(val, SYS_TCR);
+ }
+
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, ESR_EL2), SYS_ESR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR0_EL2), SYS_AFSR0);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR1_EL2), SYS_AFSR1);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, FAR_EL2), SYS_FAR);
+ write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR);
+}
+
/*
* VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
* pstate, which are handled as part of the el2 return state) on every
@@ -66,6 +168,7 @@ void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
struct kvm_cpu_context *host_ctxt;
+ u64 mpidr;
host_ctxt = host_data_ptr(host_ctxt);
__sysreg_save_user_state(host_ctxt);
@@ -89,7 +192,29 @@ void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu)
*/
__sysreg32_restore_state(vcpu);
__sysreg_restore_user_state(guest_ctxt);
- __sysreg_restore_el1_state(guest_ctxt);
+
+ if (unlikely(__is_hyp_ctxt(guest_ctxt))) {
+ __sysreg_restore_vel2_state(vcpu);
+ } else {
+ if (vcpu_has_nv(vcpu)) {
+ /*
+ * Only set VPIDR_EL2 for nested VMs, as this is the
+ * only time it changes. We'll restore the MIDR_EL1
+ * view on put.
+ */
+ write_sysreg(ctxt_sys_reg(guest_ctxt, VPIDR_EL2), vpidr_el2);
+
+ /*
+ * As we're restoring a nested guest, set the value
+ * provided by the guest hypervisor.
+ */
+ mpidr = ctxt_sys_reg(guest_ctxt, VMPIDR_EL2);
+ } else {
+ mpidr = ctxt_sys_reg(guest_ctxt, MPIDR_EL1);
+ }
+
+ __sysreg_restore_el1_state(guest_ctxt, mpidr);
+ }
vcpu_set_flag(vcpu, SYSREGS_ON_CPU);
}
@@ -112,12 +237,20 @@ void __vcpu_put_switch_sysregs(struct kvm_vcpu *vcpu)
host_ctxt = host_data_ptr(host_ctxt);
- __sysreg_save_el1_state(guest_ctxt);
+ if (unlikely(__is_hyp_ctxt(guest_ctxt)))
+ __sysreg_save_vel2_state(vcpu);
+ else
+ __sysreg_save_el1_state(guest_ctxt);
+
__sysreg_save_user_state(guest_ctxt);
__sysreg32_save_state(vcpu);
/* Restore host user state */
__sysreg_restore_user_state(host_ctxt);
+ /* If leaving a nesting guest, restore MPIDR_EL1 default view */
+ if (vcpu_has_nv(vcpu))
+ write_sysreg(read_cpuid_id(), vpidr_el2);
+
vcpu_clear_flag(vcpu, SYSREGS_ON_CPU);
}
--
2.39.2
next prev parent reply other threads:[~2024-10-09 20:10 UTC|newest]
Thread overview: 53+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-10-09 18:59 [PATCH v4 00/36] KVM: arm64: Add EL2 support to FEAT_S1PIE/S1POE Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 01/36] arm64: Drop SKL0/SKL1 from TCR2_EL2 Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 02/36] arm64: Remove VNCR definition for PIRE0_EL2 Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 03/36] arm64: Add encoding " Marc Zyngier
2024-10-10 10:46 ` Mark Brown
2024-10-09 18:59 ` [PATCH v4 04/36] KVM: arm64: Drop useless struct s2_mmu in __kvm_at_s1e2() Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 05/36] KVM: arm64: nv: Add missing EL2->EL1 mappings in get_el2_to_el1_mapping() Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 06/36] KVM: arm64: nv: Handle CNTHCTL_EL2 specially Marc Zyngier
2024-10-16 9:37 ` Alexandru Elisei
2024-10-16 11:29 ` Marc Zyngier
2024-10-16 13:19 ` Alexandru Elisei
2024-10-16 13:41 ` Marc Zyngier
2024-10-09 18:59 ` Marc Zyngier [this message]
2024-10-09 19:55 ` [PATCH v4 07/36] KVM: arm64: nv: Save/Restore vEL2 sysregs Oliver Upton
2024-10-16 13:12 ` Alexandru Elisei
2024-10-16 13:57 ` Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 08/36] KVM: arm64: Correctly access TCR2_EL1, PIR_EL1, PIRE0_EL1 with VHE Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 09/36] KVM: arm64: Extend masking facility to arbitrary registers Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 10/36] arm64: Define ID_AA64MMFR1_EL1.HAFDBS advertising FEAT_HAFT Marc Zyngier
2024-10-10 16:20 ` Mark Brown
2024-10-09 18:59 ` [PATCH v4 11/36] KVM: arm64: Add TCR2_EL2 to the sysreg arrays Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 12/36] KVM: arm64: Sanitise TCR2_EL2 Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 13/36] KVM: arm64: Add save/restore for TCR2_EL2 Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 14/36] KVM: arm64: Add PIR{,E0}_EL2 to the sysreg arrays Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 15/36] KVM: arm64: Add save/restore for PIR{,E0}_EL2 Marc Zyngier
2024-10-09 18:59 ` [PATCH v4 16/36] KVM: arm64: Handle PIR{,E0}_EL2 traps Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 17/36] KVM: arm64: Sanitise ID_AA64MMFR3_EL1 Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 18/36] KVM: arm64: Add AT fast-path support for S1PIE Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 19/36] KVM: arm64: Split S1 permission evaluation into direct and hierarchical parts Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 20/36] KVM: arm64: Disable hierarchical permissions when S1PIE is enabled Marc Zyngier
2024-10-10 7:33 ` Oliver Upton
2024-10-10 8:04 ` Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 21/36] KVM: arm64: Implement AT S1PIE support Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 22/36] KVM: arm64: Define helper for EL2 registers with custom visibility Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 23/36] KVM: arm64: Hide TCR2_EL1 from userspace when disabled for guests Marc Zyngier
2024-10-10 7:50 ` Oliver Upton
2024-10-09 19:00 ` [PATCH v4 24/36] KVM: arm64: Hide S1PIE registers " Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 25/36] KVM: arm64: Rely on visibility to let PIR*_ELx/TCR2_ELx UNDEF Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 26/36] arm64: Add encoding for POR_EL2 Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 27/36] KVM: arm64: Add a composite EL2 visibility helper Marc Zyngier
2024-10-10 7:52 ` Oliver Upton
2024-10-09 19:00 ` [PATCH v4 28/36] KVM: arm64: Drop bogus CPTR_EL2.E0POE trap routing Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 29/36] KVM: arm64: Subject S1PIE/S1POE registers to HCR_EL2.{TVM,TRVM} Marc Zyngier
2024-10-10 7:53 ` Oliver Upton
2024-10-09 19:00 ` [PATCH v4 30/36] KVM: arm64: Add basic support for POR_EL2 Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 31/36] KVM: arm64: Add save/retore " Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 32/36] KVM: arm64: Add POE save/restore for AT emulation fast-path Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 33/36] KVM: arm64: Disable hierarchical permissions when POE is enabled Marc Zyngier
2024-10-10 8:08 ` Oliver Upton
2024-10-13 14:27 ` Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 34/36] KVM: arm64: Make PAN conditions part of the S1 walk context Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 35/36] KVM: arm64: Handle stage-1 permission overlays Marc Zyngier
2024-10-09 19:00 ` [PATCH v4 36/36] KVM: arm64: Handle WXN attribute Marc Zyngier
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20241009190019.3222687-8-maz@kernel.org \
--to=maz@kernel.org \
--cc=alexandru.elisei@arm.com \
--cc=broonie@kernel.org \
--cc=joey.gouly@arm.com \
--cc=kvm@vger.kernel.org \
--cc=kvmarm@lists.linux.dev \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=oliver.upton@linux.dev \
--cc=suzuki.poulose@arm.com \
--cc=yuzenghui@huawei.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).