[PATCH 0/3] KVM: arm64: Support for Arm v8.8 memcpy instructions in KVM guests

[PATCH 0/3] KVM: arm64: Support for Arm v8.8 memcpy instructions in KVM guests

[Kristina Martsenko]

Hi,

This series adds support for using the new Arm memory copy instructions in KVM
guests.  A previous series [1] added support for using the instructions in
userspace applications.  Similarly to that series, the main thing to note here
is the handling for when a memcpy is stopped and continued on a different type
of CPU. See patch #2 for details.

As background, the Armv8.8 extension adds new instructions to perform memcpy(),
memset() and memmove() operations in hardware (FEAT_MOPS).  The aim is to avoid
having many different performance-optimal memcpy implementations in software
(tailored to CPU model and copy size) and the overhead of selecting between
them.  The new instructions are intended to be at least as fast as any
alternative instruction sequence.

The series has been tested on the Arm FVP.

Thanks,
Kristina

[1] https://lore.kernel.org/all/20230509142235.3284028-1-kristina.martsenko@arm.com/


Kristina Martsenko (3):
  KVM: arm64: Configure HCRX_EL2 dynamically
  KVM: arm64: Add handler for MOPS exceptions
  KVM: arm64: Expose MOPS instructions to guests

 arch/arm64/include/asm/kvm_emulate.h          | 10 ++++
 arch/arm64/include/asm/kvm_host.h             |  1 +
 arch/arm64/include/asm/traps.h                | 54 ++++++++++++++++++-
 arch/arm64/kernel/traps.c                     | 48 +----------------
 arch/arm64/kvm/arm.c                          |  1 +
 arch/arm64/kvm/hyp/include/hyp/switch.h       | 19 ++++++-
 .../arm64/kvm/hyp/include/nvhe/fixed_config.h |  3 +-
 arch/arm64/kvm/hyp/nvhe/hyp-main.c            |  1 +
 arch/arm64/kvm/hyp/nvhe/pkvm.c                |  1 +
 arch/arm64/kvm/hyp/nvhe/switch.c              |  2 +
 arch/arm64/kvm/hyp/vhe/switch.c               |  1 +
 arch/arm64/kvm/sys_regs.c                     |  1 -
 12 files changed, 90 insertions(+), 52 deletions(-)


base-commit: 0bb80ecc33a8fb5a682236443c1e740d5c917d1d
-- 
2.25.1

[PATCH 0/3] KVM: arm64: Support for Arm v8.8 memcpy instructions in KVM guests

[Kristina Martsenko]

Hi,

This series adds support for using the new Arm memory copy instructions in KVM
guests.  A previous series [1] added support for using the instructions in
userspace applications.  Similarly to that series, the main thing to note here
is the handling for when a memcpy is stopped and continued on a different type
of CPU. See patch #2 for details.

As background, the Armv8.8 extension adds new instructions to perform memcpy(),
memset() and memmove() operations in hardware (FEAT_MOPS).  The aim is to avoid
having many different performance-optimal memcpy implementations in software
(tailored to CPU model and copy size) and the overhead of selecting between
them.  The new instructions are intended to be at least as fast as any
alternative instruction sequence.

The series has been tested on the Arm FVP.

Thanks,
Kristina

[1] https://lore.kernel.org/all/20230509142235.3284028-1-kristina.martsenko@arm.com/


Kristina Martsenko (3):
  KVM: arm64: Configure HCRX_EL2 dynamically
  KVM: arm64: Add handler for MOPS exceptions
  KVM: arm64: Expose MOPS instructions to guests

 arch/arm64/include/asm/kvm_emulate.h          | 10 ++++
 arch/arm64/include/asm/kvm_host.h             |  1 +
 arch/arm64/include/asm/traps.h                | 54 ++++++++++++++++++-
 arch/arm64/kernel/traps.c                     | 48 +----------------
 arch/arm64/kvm/arm.c                          |  1 +
 arch/arm64/kvm/hyp/include/hyp/switch.h       | 19 ++++++-
 .../arm64/kvm/hyp/include/nvhe/fixed_config.h |  3 +-
 arch/arm64/kvm/hyp/nvhe/hyp-main.c            |  1 +
 arch/arm64/kvm/hyp/nvhe/pkvm.c                |  1 +
 arch/arm64/kvm/hyp/nvhe/switch.c              |  2 +
 arch/arm64/kvm/hyp/vhe/switch.c               |  1 +
 arch/arm64/kvm/sys_regs.c                     |  1 -
 12 files changed, 90 insertions(+), 52 deletions(-)


base-commit: 0bb80ecc33a8fb5a682236443c1e740d5c917d1d
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 1/3] KVM: arm64: Configure HCRX_EL2 dynamically

[Kristina Martsenko]

At the moment the HCRX_EL2 system register is always initialized to
HCRX_GUEST_FLAGS when running a guest. Instead, choose the configuration
at vcpu reset time and save it in the vcpu struct, similarly to how
HCR_EL2 is set up. This will be needed in a subsequent change to
configure the register based on CPU features detected at runtime.

Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
---
 arch/arm64/include/asm/kvm_emulate.h    | 5 +++++
 arch/arm64/include/asm/kvm_host.h       | 1 +
 arch/arm64/kvm/arm.c                    | 1 +
 arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
 arch/arm64/kvm/hyp/nvhe/hyp-main.c      | 1 +
 arch/arm64/kvm/hyp/nvhe/pkvm.c          | 1 +
 6 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 3d6725ff0bf6..64ea27e6deb1 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -134,6 +134,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
 	vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
 }
 
+static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
+}
+
 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.vsesr_el2;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index af06ccb7ee34..2764748756a7 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -487,6 +487,7 @@ struct kvm_vcpu_arch {
 
 	/* Values of trap registers for the guest. */
 	u64 hcr_el2;
+	u64 hcrx_el2;
 	u64 mdcr_el2;
 	u64 cptr_el2;
 
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 4866b3f7b4ea..fa359fd6fca9 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1318,6 +1318,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
 	}
 
 	vcpu_reset_hcr(vcpu);
+	vcpu_reset_hcrx(vcpu);
 	vcpu->arch.cptr_el2 = kvm_get_reset_cptr_el2(vcpu);
 
 	/*
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 9cfe6bd1dbe4..119b75344505 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -198,7 +198,7 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
 	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
 
 	if (cpus_have_final_cap(ARM64_HAS_HCX)) {
-		u64 hcrx = HCRX_GUEST_FLAGS;
+		u64 hcrx = vcpu->arch.hcrx_el2;
 		if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
 			u64 clr = 0, set = 0;
 
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 857d9bc04fd4..26692083b80b 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -35,6 +35,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 	hyp_vcpu->vcpu.arch.hw_mmu	= host_vcpu->arch.hw_mmu;
 
 	hyp_vcpu->vcpu.arch.hcr_el2	= host_vcpu->arch.hcr_el2;
+	hyp_vcpu->vcpu.arch.hcrx_el2	= host_vcpu->arch.hcrx_el2;
 	hyp_vcpu->vcpu.arch.mdcr_el2	= host_vcpu->arch.mdcr_el2;
 	hyp_vcpu->vcpu.arch.cptr_el2	= host_vcpu->arch.cptr_el2;
 
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 8033ef353a5d..8c8b37525dce 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -188,6 +188,7 @@ static void pvm_init_trap_regs(struct kvm_vcpu *vcpu)
 
 	/* Clear res0 and set res1 bits to trap potential new features. */
 	vcpu->arch.hcr_el2 &= ~(HCR_RES0);
+	vcpu->arch.hcrx_el2 &= ~(HCRX_EL2_RES0);
 	vcpu->arch.mdcr_el2 &= ~(MDCR_EL2_RES0);
 	if (!has_hvhe()) {
 		vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1;
-- 
2.25.1

[PATCH 1/3] KVM: arm64: Configure HCRX_EL2 dynamically

[Kristina Martsenko]

At the moment the HCRX_EL2 system register is always initialized to
HCRX_GUEST_FLAGS when running a guest. Instead, choose the configuration
at vcpu reset time and save it in the vcpu struct, similarly to how
HCR_EL2 is set up. This will be needed in a subsequent change to
configure the register based on CPU features detected at runtime.

Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
---
 arch/arm64/include/asm/kvm_emulate.h    | 5 +++++
 arch/arm64/include/asm/kvm_host.h       | 1 +
 arch/arm64/kvm/arm.c                    | 1 +
 arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
 arch/arm64/kvm/hyp/nvhe/hyp-main.c      | 1 +
 arch/arm64/kvm/hyp/nvhe/pkvm.c          | 1 +
 6 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 3d6725ff0bf6..64ea27e6deb1 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -134,6 +134,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
 	vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
 }
 
+static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
+}
+
 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.vsesr_el2;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index af06ccb7ee34..2764748756a7 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -487,6 +487,7 @@ struct kvm_vcpu_arch {
 
 	/* Values of trap registers for the guest. */
 	u64 hcr_el2;
+	u64 hcrx_el2;
 	u64 mdcr_el2;
 	u64 cptr_el2;
 
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 4866b3f7b4ea..fa359fd6fca9 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1318,6 +1318,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
 	}
 
 	vcpu_reset_hcr(vcpu);
+	vcpu_reset_hcrx(vcpu);
 	vcpu->arch.cptr_el2 = kvm_get_reset_cptr_el2(vcpu);
 
 	/*
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 9cfe6bd1dbe4..119b75344505 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -198,7 +198,7 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
 	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
 
 	if (cpus_have_final_cap(ARM64_HAS_HCX)) {
-		u64 hcrx = HCRX_GUEST_FLAGS;
+		u64 hcrx = vcpu->arch.hcrx_el2;
 		if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
 			u64 clr = 0, set = 0;
 
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 857d9bc04fd4..26692083b80b 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -35,6 +35,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 	hyp_vcpu->vcpu.arch.hw_mmu	= host_vcpu->arch.hw_mmu;
 
 	hyp_vcpu->vcpu.arch.hcr_el2	= host_vcpu->arch.hcr_el2;
+	hyp_vcpu->vcpu.arch.hcrx_el2	= host_vcpu->arch.hcrx_el2;
 	hyp_vcpu->vcpu.arch.mdcr_el2	= host_vcpu->arch.mdcr_el2;
 	hyp_vcpu->vcpu.arch.cptr_el2	= host_vcpu->arch.cptr_el2;
 
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 8033ef353a5d..8c8b37525dce 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -188,6 +188,7 @@ static void pvm_init_trap_regs(struct kvm_vcpu *vcpu)
 
 	/* Clear res0 and set res1 bits to trap potential new features. */
 	vcpu->arch.hcr_el2 &= ~(HCR_RES0);
+	vcpu->arch.hcrx_el2 &= ~(HCRX_EL2_RES0);
 	vcpu->arch.mdcr_el2 &= ~(MDCR_EL2_RES0);
 	if (!has_hvhe()) {
 		vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1;
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 1/3] KVM: arm64: Configure HCRX_EL2 dynamically

[Marc Zyngier]

Hi Kristina,

On Fri, 15 Sep 2023 13:48:38 +0100,
Kristina Martsenko <kristina.martsenko@arm.com> wrote:
> 
> At the moment the HCRX_EL2 system register is always initialized to
> HCRX_GUEST_FLAGS when running a guest. Instead, choose the configuration
> at vcpu reset time and save it in the vcpu struct, similarly to how
> HCR_EL2 is set up. This will be needed in a subsequent change to
> configure the register based on CPU features detected at runtime.
> 
> Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
> ---
>  arch/arm64/include/asm/kvm_emulate.h    | 5 +++++
>  arch/arm64/include/asm/kvm_host.h       | 1 +
>  arch/arm64/kvm/arm.c                    | 1 +
>  arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
>  arch/arm64/kvm/hyp/nvhe/hyp-main.c      | 1 +
>  arch/arm64/kvm/hyp/nvhe/pkvm.c          | 1 +
>  6 files changed, 10 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
> index 3d6725ff0bf6..64ea27e6deb1 100644
> --- a/arch/arm64/include/asm/kvm_emulate.h
> +++ b/arch/arm64/include/asm/kvm_emulate.h
> @@ -134,6 +134,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
>  	vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
>  }
>  
> +static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
> +{
> +	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
> +}
> +
>  static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
>  {
>  	return vcpu->arch.vsesr_el2;
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index af06ccb7ee34..2764748756a7 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -487,6 +487,7 @@ struct kvm_vcpu_arch {
>  
>  	/* Values of trap registers for the guest. */
>  	u64 hcr_el2;
> +	u64 hcrx_el2;

Do we really need this extra field? Yes, this is only an extra 64bit,
but they tend to accumulate...

Looking at patch #3, the change is related to this:

 	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
+
+	if (cpus_have_final_cap(ARM64_HAS_MOPS)) {
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MSCEn;
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MCE2;
+	}

meaning that this is a constant value for a given boot of the host.

At this stage, I'd rather you define HCRX_GUEST_FLAGS as:

#define HCRX_GUEST_FLAGS \
	(HCRX_EL2_SMPME | HCRX_EL2_TCR2En | \
	 cpus_have_final_cap(ARM64_HAS_MOPS) ? \
	 (HCRX_EL2_MSCEn | HCRX_EL2_MCE2) : 0)

and drop the new field altogether, until we have something that
requires dynamic flipping of an HCRX_EL2 field.

Thanks,

	M.

-- 
Without deviation from the norm, progress is not possible.

Re: [PATCH 1/3] KVM: arm64: Configure HCRX_EL2 dynamically

[Marc Zyngier]

Hi Kristina,

On Fri, 15 Sep 2023 13:48:38 +0100,
Kristina Martsenko <kristina.martsenko@arm.com> wrote:
> 
> At the moment the HCRX_EL2 system register is always initialized to
> HCRX_GUEST_FLAGS when running a guest. Instead, choose the configuration
> at vcpu reset time and save it in the vcpu struct, similarly to how
> HCR_EL2 is set up. This will be needed in a subsequent change to
> configure the register based on CPU features detected at runtime.
> 
> Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
> ---
>  arch/arm64/include/asm/kvm_emulate.h    | 5 +++++
>  arch/arm64/include/asm/kvm_host.h       | 1 +
>  arch/arm64/kvm/arm.c                    | 1 +
>  arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
>  arch/arm64/kvm/hyp/nvhe/hyp-main.c      | 1 +
>  arch/arm64/kvm/hyp/nvhe/pkvm.c          | 1 +
>  6 files changed, 10 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
> index 3d6725ff0bf6..64ea27e6deb1 100644
> --- a/arch/arm64/include/asm/kvm_emulate.h
> +++ b/arch/arm64/include/asm/kvm_emulate.h
> @@ -134,6 +134,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
>  	vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
>  }
>  
> +static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
> +{
> +	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
> +}
> +
>  static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
>  {
>  	return vcpu->arch.vsesr_el2;
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index af06ccb7ee34..2764748756a7 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -487,6 +487,7 @@ struct kvm_vcpu_arch {
>  
>  	/* Values of trap registers for the guest. */
>  	u64 hcr_el2;
> +	u64 hcrx_el2;

Do we really need this extra field? Yes, this is only an extra 64bit,
but they tend to accumulate...

Looking at patch #3, the change is related to this:

 	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
+
+	if (cpus_have_final_cap(ARM64_HAS_MOPS)) {
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MSCEn;
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MCE2;
+	}

meaning that this is a constant value for a given boot of the host.

At this stage, I'd rather you define HCRX_GUEST_FLAGS as:

#define HCRX_GUEST_FLAGS \
	(HCRX_EL2_SMPME | HCRX_EL2_TCR2En | \
	 cpus_have_final_cap(ARM64_HAS_MOPS) ? \
	 (HCRX_EL2_MSCEn | HCRX_EL2_MCE2) : 0)

and drop the new field altogether, until we have something that
requires dynamic flipping of an HCRX_EL2 field.

Thanks,

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 1/3] KVM: arm64: Configure HCRX_EL2 dynamically

[Kristina Martsenko]

On 18/09/2023 12:10, Marc Zyngier wrote:
> Hi Kristina,

Hi Marc,

> On Fri, 15 Sep 2023 13:48:38 +0100,
> Kristina Martsenko <kristina.martsenko@arm.com> wrote:
>>
>> At the moment the HCRX_EL2 system register is always initialized to
>> HCRX_GUEST_FLAGS when running a guest. Instead, choose the configuration
>> at vcpu reset time and save it in the vcpu struct, similarly to how
>> HCR_EL2 is set up. This will be needed in a subsequent change to
>> configure the register based on CPU features detected at runtime.
>>
>> Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
>> ---
>>  arch/arm64/include/asm/kvm_emulate.h    | 5 +++++
>>  arch/arm64/include/asm/kvm_host.h       | 1 +
>>  arch/arm64/kvm/arm.c                    | 1 +
>>  arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
>>  arch/arm64/kvm/hyp/nvhe/hyp-main.c      | 1 +
>>  arch/arm64/kvm/hyp/nvhe/pkvm.c          | 1 +
>>  6 files changed, 10 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
>> index 3d6725ff0bf6..64ea27e6deb1 100644
>> --- a/arch/arm64/include/asm/kvm_emulate.h
>> +++ b/arch/arm64/include/asm/kvm_emulate.h
>> @@ -134,6 +134,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
>>  	vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
>>  }
>>  
>> +static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
>> +{
>> +	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
>> +}
>> +
>>  static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
>>  {
>>  	return vcpu->arch.vsesr_el2;
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index af06ccb7ee34..2764748756a7 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -487,6 +487,7 @@ struct kvm_vcpu_arch {
>>  
>>  	/* Values of trap registers for the guest. */
>>  	u64 hcr_el2;
>> +	u64 hcrx_el2;
> 
> Do we really need this extra field? Yes, this is only an extra 64bit,
> but they tend to accumulate...
> 
> Looking at patch #3, the change is related to this:
> 
>  	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
> +
> +	if (cpus_have_final_cap(ARM64_HAS_MOPS)) {
> +		vcpu->arch.hcrx_el2 |= HCRX_EL2_MSCEn;
> +		vcpu->arch.hcrx_el2 |= HCRX_EL2_MCE2;
> +	}
> 
> meaning that this is a constant value for a given boot of the host.
> 
> At this stage, I'd rather you define HCRX_GUEST_FLAGS as:
> 
> #define HCRX_GUEST_FLAGS \
> 	(HCRX_EL2_SMPME | HCRX_EL2_TCR2En | \
> 	 cpus_have_final_cap(ARM64_HAS_MOPS) ? \
> 	 (HCRX_EL2_MSCEn | HCRX_EL2_MCE2) : 0)
> 
> and drop the new field altogether, until we have something that
> requires dynamic flipping of an HCRX_EL2 field.

Makes sense, the field isn't strictly required yet, I'll drop it in v2.

Thanks,
Kristina

Re: [PATCH 1/3] KVM: arm64: Configure HCRX_EL2 dynamically

[Kristina Martsenko]

On 18/09/2023 12:10, Marc Zyngier wrote:
> Hi Kristina,

Hi Marc,

> On Fri, 15 Sep 2023 13:48:38 +0100,
> Kristina Martsenko <kristina.martsenko@arm.com> wrote:
>>
>> At the moment the HCRX_EL2 system register is always initialized to
>> HCRX_GUEST_FLAGS when running a guest. Instead, choose the configuration
>> at vcpu reset time and save it in the vcpu struct, similarly to how
>> HCR_EL2 is set up. This will be needed in a subsequent change to
>> configure the register based on CPU features detected at runtime.
>>
>> Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
>> ---
>>  arch/arm64/include/asm/kvm_emulate.h    | 5 +++++
>>  arch/arm64/include/asm/kvm_host.h       | 1 +
>>  arch/arm64/kvm/arm.c                    | 1 +
>>  arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +-
>>  arch/arm64/kvm/hyp/nvhe/hyp-main.c      | 1 +
>>  arch/arm64/kvm/hyp/nvhe/pkvm.c          | 1 +
>>  6 files changed, 10 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
>> index 3d6725ff0bf6..64ea27e6deb1 100644
>> --- a/arch/arm64/include/asm/kvm_emulate.h
>> +++ b/arch/arm64/include/asm/kvm_emulate.h
>> @@ -134,6 +134,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
>>  	vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
>>  }
>>  
>> +static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
>> +{
>> +	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
>> +}
>> +
>>  static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
>>  {
>>  	return vcpu->arch.vsesr_el2;
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index af06ccb7ee34..2764748756a7 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -487,6 +487,7 @@ struct kvm_vcpu_arch {
>>  
>>  	/* Values of trap registers for the guest. */
>>  	u64 hcr_el2;
>> +	u64 hcrx_el2;
> 
> Do we really need this extra field? Yes, this is only an extra 64bit,
> but they tend to accumulate...
> 
> Looking at patch #3, the change is related to this:
> 
>  	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
> +
> +	if (cpus_have_final_cap(ARM64_HAS_MOPS)) {
> +		vcpu->arch.hcrx_el2 |= HCRX_EL2_MSCEn;
> +		vcpu->arch.hcrx_el2 |= HCRX_EL2_MCE2;
> +	}
> 
> meaning that this is a constant value for a given boot of the host.
> 
> At this stage, I'd rather you define HCRX_GUEST_FLAGS as:
> 
> #define HCRX_GUEST_FLAGS \
> 	(HCRX_EL2_SMPME | HCRX_EL2_TCR2En | \
> 	 cpus_have_final_cap(ARM64_HAS_MOPS) ? \
> 	 (HCRX_EL2_MSCEn | HCRX_EL2_MCE2) : 0)
> 
> and drop the new field altogether, until we have something that
> requires dynamic flipping of an HCRX_EL2 field.

Makes sense, the field isn't strictly required yet, I'll drop it in v2.

Thanks,
Kristina


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 2/3] KVM: arm64: Add handler for MOPS exceptions

[Kristina Martsenko]

An Armv8.8 FEAT_MOPS main or epilogue instruction will take an exception
if executed on a CPU with a different MOPS implementation option (A or
B) than the CPU where the preceding prologue instruction ran. In this
case the OS exception handler is expected to reset the registers and
restart execution from the prologue instruction.

A KVM guest may use the instructions at EL1 at times when the guest is
not able to handle the exception, expecting that the instructions will
only run on one CPU (e.g. when running UEFI boot services in the guest).
As KVM may reschedule the guest between different types of CPUs at any
time (on an asymmetric system), it needs to also handle the resulting
exception itself in case the guest is not able to. A similar situation
will also occur in the future when live migrating a guest from one type
of CPU to another.

Add handling for the MOPS exception to KVM. The handling can be shared
with the EL0 exception handler, as the logic and register layouts are
the same. The exception can be handled right after exiting a guest,
which avoids the cost of returning to the host exit handler.

Similarly to the EL0 exception handler, in case the main or epilogue
instruction is being single stepped, it makes sense to finish the step
before executing the prologue instruction, so advance the single step
state machine.

Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
---
 arch/arm64/include/asm/traps.h          | 54 ++++++++++++++++++++++++-
 arch/arm64/kernel/traps.c               | 48 +---------------------
 arch/arm64/kvm/hyp/include/hyp/switch.h | 17 ++++++++
 arch/arm64/kvm/hyp/nvhe/switch.c        |  2 +
 arch/arm64/kvm/hyp/vhe/switch.c         |  1 +
 5 files changed, 73 insertions(+), 49 deletions(-)

diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h
index d66dfb3a72dd..eefe766d6161 100644
--- a/arch/arm64/include/asm/traps.h
+++ b/arch/arm64/include/asm/traps.h
@@ -9,10 +9,9 @@
 
 #include <linux/list.h>
 #include <asm/esr.h>
+#include <asm/ptrace.h>
 #include <asm/sections.h>
 
-struct pt_regs;
-
 #ifdef CONFIG_ARMV8_DEPRECATED
 bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn);
 #else
@@ -101,4 +100,55 @@ static inline unsigned long arm64_ras_serror_get_severity(unsigned long esr)
 
 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr);
 void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr);
+
+static inline void arm64_mops_reset_regs(struct user_pt_regs *regs, unsigned long esr)
+{
+	bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION;
+	bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A;
+	int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr);
+	int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr);
+	int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr);
+	unsigned long dst, src, size;
+
+	dst = regs->regs[dstreg];
+	src = regs->regs[srcreg];
+	size = regs->regs[sizereg];
+
+	/*
+	 * Put the registers back in the original format suitable for a
+	 * prologue instruction, using the generic return routine from the
+	 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
+	 */
+	if (esr & ESR_ELx_MOPS_ISS_MEM_INST) {
+		/* SET* instruction */
+		if (option_a ^ wrong_option) {
+			/* Format is from Option A; forward set */
+			regs->regs[dstreg] = dst + size;
+			regs->regs[sizereg] = -size;
+		}
+	} else {
+		/* CPY* instruction */
+		if (!(option_a ^ wrong_option)) {
+			/* Format is from Option B */
+			if (regs->pstate & PSR_N_BIT) {
+				/* Backward copy */
+				regs->regs[dstreg] = dst - size;
+				regs->regs[srcreg] = src - size;
+			}
+		} else {
+			/* Format is from Option A */
+			if (size & BIT(63)) {
+				/* Forward copy */
+				regs->regs[dstreg] = dst + size;
+				regs->regs[srcreg] = src + size;
+				regs->regs[sizereg] = -size;
+			}
+		}
+	}
+
+	if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE)
+		regs->pc -= 8;
+	else
+		regs->pc -= 4;
+}
 #endif
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 8b70759cdbb9..ede65a20e7dc 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -516,53 +516,7 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
 
 void do_el0_mops(struct pt_regs *regs, unsigned long esr)
 {
-	bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION;
-	bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A;
-	int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr);
-	int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr);
-	int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr);
-	unsigned long dst, src, size;
-
-	dst = pt_regs_read_reg(regs, dstreg);
-	src = pt_regs_read_reg(regs, srcreg);
-	size = pt_regs_read_reg(regs, sizereg);
-
-	/*
-	 * Put the registers back in the original format suitable for a
-	 * prologue instruction, using the generic return routine from the
-	 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
-	 */
-	if (esr & ESR_ELx_MOPS_ISS_MEM_INST) {
-		/* SET* instruction */
-		if (option_a ^ wrong_option) {
-			/* Format is from Option A; forward set */
-			pt_regs_write_reg(regs, dstreg, dst + size);
-			pt_regs_write_reg(regs, sizereg, -size);
-		}
-	} else {
-		/* CPY* instruction */
-		if (!(option_a ^ wrong_option)) {
-			/* Format is from Option B */
-			if (regs->pstate & PSR_N_BIT) {
-				/* Backward copy */
-				pt_regs_write_reg(regs, dstreg, dst - size);
-				pt_regs_write_reg(regs, srcreg, src - size);
-			}
-		} else {
-			/* Format is from Option A */
-			if (size & BIT(63)) {
-				/* Forward copy */
-				pt_regs_write_reg(regs, dstreg, dst + size);
-				pt_regs_write_reg(regs, srcreg, src + size);
-				pt_regs_write_reg(regs, sizereg, -size);
-			}
-		}
-	}
-
-	if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE)
-		regs->pc -= 8;
-	else
-		regs->pc -= 4;
+	arm64_mops_reset_regs(&regs->user_regs, esr);
 
 	/*
 	 * If single stepping then finish the step before executing the
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 119b75344505..c1db9ce0db07 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -30,6 +30,7 @@
 #include <asm/fpsimd.h>
 #include <asm/debug-monitors.h>
 #include <asm/processor.h>
+#include <asm/traps.h>
 
 struct kvm_exception_table_entry {
 	int insn, fixup;
@@ -265,6 +266,22 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
 	return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);
 }
 
+static bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
+{
+	*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
+	arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2);
+	write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
+
+	/*
+	 * Finish potential single step before executing the prologue
+	 * instruction.
+	 */
+	*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
+	write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
+
+	return true;
+}
+
 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
 {
 	sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index c353a06ee7e6..c50f8459e4fc 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -192,6 +192,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
 	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,
 	[ESR_ELx_EC_WATCHPT_LOW]	= kvm_hyp_handle_watchpt_low,
 	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,
+	[ESR_ELx_EC_MOPS]		= kvm_hyp_handle_mops,
 };
 
 static const exit_handler_fn pvm_exit_handlers[] = {
@@ -203,6 +204,7 @@ static const exit_handler_fn pvm_exit_handlers[] = {
 	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,
 	[ESR_ELx_EC_WATCHPT_LOW]	= kvm_hyp_handle_watchpt_low,
 	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,
+	[ESR_ELx_EC_MOPS]		= kvm_hyp_handle_mops,
 };
 
 static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 6537f58b1a8c..796202f2e08f 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -126,6 +126,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
 	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,
 	[ESR_ELx_EC_WATCHPT_LOW]	= kvm_hyp_handle_watchpt_low,
 	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,
+	[ESR_ELx_EC_MOPS]		= kvm_hyp_handle_mops,
 };
 
 static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
-- 
2.25.1

[PATCH 2/3] KVM: arm64: Add handler for MOPS exceptions

[Kristina Martsenko]

An Armv8.8 FEAT_MOPS main or epilogue instruction will take an exception
if executed on a CPU with a different MOPS implementation option (A or
B) than the CPU where the preceding prologue instruction ran. In this
case the OS exception handler is expected to reset the registers and
restart execution from the prologue instruction.

A KVM guest may use the instructions at EL1 at times when the guest is
not able to handle the exception, expecting that the instructions will
only run on one CPU (e.g. when running UEFI boot services in the guest).
As KVM may reschedule the guest between different types of CPUs at any
time (on an asymmetric system), it needs to also handle the resulting
exception itself in case the guest is not able to. A similar situation
will also occur in the future when live migrating a guest from one type
of CPU to another.

Add handling for the MOPS exception to KVM. The handling can be shared
with the EL0 exception handler, as the logic and register layouts are
the same. The exception can be handled right after exiting a guest,
which avoids the cost of returning to the host exit handler.

Similarly to the EL0 exception handler, in case the main or epilogue
instruction is being single stepped, it makes sense to finish the step
before executing the prologue instruction, so advance the single step
state machine.

Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
---
 arch/arm64/include/asm/traps.h          | 54 ++++++++++++++++++++++++-
 arch/arm64/kernel/traps.c               | 48 +---------------------
 arch/arm64/kvm/hyp/include/hyp/switch.h | 17 ++++++++
 arch/arm64/kvm/hyp/nvhe/switch.c        |  2 +
 arch/arm64/kvm/hyp/vhe/switch.c         |  1 +
 5 files changed, 73 insertions(+), 49 deletions(-)

diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h
index d66dfb3a72dd..eefe766d6161 100644
--- a/arch/arm64/include/asm/traps.h
+++ b/arch/arm64/include/asm/traps.h
@@ -9,10 +9,9 @@
 
 #include <linux/list.h>
 #include <asm/esr.h>
+#include <asm/ptrace.h>
 #include <asm/sections.h>
 
-struct pt_regs;
-
 #ifdef CONFIG_ARMV8_DEPRECATED
 bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn);
 #else
@@ -101,4 +100,55 @@ static inline unsigned long arm64_ras_serror_get_severity(unsigned long esr)
 
 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr);
 void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr);
+
+static inline void arm64_mops_reset_regs(struct user_pt_regs *regs, unsigned long esr)
+{
+	bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION;
+	bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A;
+	int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr);
+	int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr);
+	int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr);
+	unsigned long dst, src, size;
+
+	dst = regs->regs[dstreg];
+	src = regs->regs[srcreg];
+	size = regs->regs[sizereg];
+
+	/*
+	 * Put the registers back in the original format suitable for a
+	 * prologue instruction, using the generic return routine from the
+	 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
+	 */
+	if (esr & ESR_ELx_MOPS_ISS_MEM_INST) {
+		/* SET* instruction */
+		if (option_a ^ wrong_option) {
+			/* Format is from Option A; forward set */
+			regs->regs[dstreg] = dst + size;
+			regs->regs[sizereg] = -size;
+		}
+	} else {
+		/* CPY* instruction */
+		if (!(option_a ^ wrong_option)) {
+			/* Format is from Option B */
+			if (regs->pstate & PSR_N_BIT) {
+				/* Backward copy */
+				regs->regs[dstreg] = dst - size;
+				regs->regs[srcreg] = src - size;
+			}
+		} else {
+			/* Format is from Option A */
+			if (size & BIT(63)) {
+				/* Forward copy */
+				regs->regs[dstreg] = dst + size;
+				regs->regs[srcreg] = src + size;
+				regs->regs[sizereg] = -size;
+			}
+		}
+	}
+
+	if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE)
+		regs->pc -= 8;
+	else
+		regs->pc -= 4;
+}
 #endif
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 8b70759cdbb9..ede65a20e7dc 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -516,53 +516,7 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
 
 void do_el0_mops(struct pt_regs *regs, unsigned long esr)
 {
-	bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION;
-	bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A;
-	int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr);
-	int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr);
-	int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr);
-	unsigned long dst, src, size;
-
-	dst = pt_regs_read_reg(regs, dstreg);
-	src = pt_regs_read_reg(regs, srcreg);
-	size = pt_regs_read_reg(regs, sizereg);
-
-	/*
-	 * Put the registers back in the original format suitable for a
-	 * prologue instruction, using the generic return routine from the
-	 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
-	 */
-	if (esr & ESR_ELx_MOPS_ISS_MEM_INST) {
-		/* SET* instruction */
-		if (option_a ^ wrong_option) {
-			/* Format is from Option A; forward set */
-			pt_regs_write_reg(regs, dstreg, dst + size);
-			pt_regs_write_reg(regs, sizereg, -size);
-		}
-	} else {
-		/* CPY* instruction */
-		if (!(option_a ^ wrong_option)) {
-			/* Format is from Option B */
-			if (regs->pstate & PSR_N_BIT) {
-				/* Backward copy */
-				pt_regs_write_reg(regs, dstreg, dst - size);
-				pt_regs_write_reg(regs, srcreg, src - size);
-			}
-		} else {
-			/* Format is from Option A */
-			if (size & BIT(63)) {
-				/* Forward copy */
-				pt_regs_write_reg(regs, dstreg, dst + size);
-				pt_regs_write_reg(regs, srcreg, src + size);
-				pt_regs_write_reg(regs, sizereg, -size);
-			}
-		}
-	}
-
-	if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE)
-		regs->pc -= 8;
-	else
-		regs->pc -= 4;
+	arm64_mops_reset_regs(&regs->user_regs, esr);
 
 	/*
 	 * If single stepping then finish the step before executing the
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 119b75344505..c1db9ce0db07 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -30,6 +30,7 @@
 #include <asm/fpsimd.h>
 #include <asm/debug-monitors.h>
 #include <asm/processor.h>
+#include <asm/traps.h>
 
 struct kvm_exception_table_entry {
 	int insn, fixup;
@@ -265,6 +266,22 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
 	return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);
 }
 
+static bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
+{
+	*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
+	arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2);
+	write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
+
+	/*
+	 * Finish potential single step before executing the prologue
+	 * instruction.
+	 */
+	*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
+	write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
+
+	return true;
+}
+
 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
 {
 	sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index c353a06ee7e6..c50f8459e4fc 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -192,6 +192,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
 	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,
 	[ESR_ELx_EC_WATCHPT_LOW]	= kvm_hyp_handle_watchpt_low,
 	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,
+	[ESR_ELx_EC_MOPS]		= kvm_hyp_handle_mops,
 };
 
 static const exit_handler_fn pvm_exit_handlers[] = {
@@ -203,6 +204,7 @@ static const exit_handler_fn pvm_exit_handlers[] = {
 	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,
 	[ESR_ELx_EC_WATCHPT_LOW]	= kvm_hyp_handle_watchpt_low,
 	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,
+	[ESR_ELx_EC_MOPS]		= kvm_hyp_handle_mops,
 };
 
 static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 6537f58b1a8c..796202f2e08f 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -126,6 +126,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
 	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,
 	[ESR_ELx_EC_WATCHPT_LOW]	= kvm_hyp_handle_watchpt_low,
 	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,
+	[ESR_ELx_EC_MOPS]		= kvm_hyp_handle_mops,
 };
 
 static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 3/3] KVM: arm64: Expose MOPS instructions to guests

[Kristina Martsenko]

Expose the Armv8.8 FEAT_MOPS feature to guests in the ID register and
allow the MOPS instructions to be run in a guest. Only expose MOPS if
the whole system supports it.

Note, it is expected that guests do not use these instructions on MMIO,
similarly to other instructions where ESR_EL2.ISV==0 such as LDP/STP.

Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
---
 arch/arm64/include/asm/kvm_emulate.h           | 5 +++++
 arch/arm64/kvm/hyp/include/nvhe/fixed_config.h | 3 ++-
 arch/arm64/kvm/sys_regs.c                      | 1 -
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 64ea27e6deb1..ba46004be0e8 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -137,6 +137,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
 static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
+
+	if (cpus_have_final_cap(ARM64_HAS_MOPS)) {
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MSCEn;
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MCE2;
+	}
 }
 
 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
index 37440e1dda93..e91922daa8ca 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
@@ -197,7 +197,8 @@
 
 #define PVM_ID_AA64ISAR2_ALLOW (\
 	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3) | \
-	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) \
+	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) | \
+	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS) \
 	)
 
 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id);
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index e92ec810d449..153baf2f72cb 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1338,7 +1338,6 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
 				 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
 		if (!cpus_have_final_cap(ARM64_HAS_WFXT))
 			val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
-		val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS);
 		break;
 	case SYS_ID_AA64MMFR2_EL1:
 		val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
-- 
2.25.1

[PATCH 3/3] KVM: arm64: Expose MOPS instructions to guests

[Kristina Martsenko]

Expose the Armv8.8 FEAT_MOPS feature to guests in the ID register and
allow the MOPS instructions to be run in a guest. Only expose MOPS if
the whole system supports it.

Note, it is expected that guests do not use these instructions on MMIO,
similarly to other instructions where ESR_EL2.ISV==0 such as LDP/STP.

Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
---
 arch/arm64/include/asm/kvm_emulate.h           | 5 +++++
 arch/arm64/kvm/hyp/include/nvhe/fixed_config.h | 3 ++-
 arch/arm64/kvm/sys_regs.c                      | 1 -
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 64ea27e6deb1..ba46004be0e8 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -137,6 +137,11 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
 static inline void vcpu_reset_hcrx(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
+
+	if (cpus_have_final_cap(ARM64_HAS_MOPS)) {
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MSCEn;
+		vcpu->arch.hcrx_el2 |= HCRX_EL2_MCE2;
+	}
 }
 
 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
index 37440e1dda93..e91922daa8ca 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
@@ -197,7 +197,8 @@
 
 #define PVM_ID_AA64ISAR2_ALLOW (\
 	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3) | \
-	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) \
+	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) | \
+	ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS) \
 	)
 
 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id);
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index e92ec810d449..153baf2f72cb 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1338,7 +1338,6 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
 				 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
 		if (!cpus_have_final_cap(ARM64_HAS_WFXT))
 			val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
-		val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS);
 		break;
 	case SYS_ID_AA64MMFR2_EL1:
 		val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel