[PATCH 1/8] KVM: arm64: nv: Disallow KVM_CAP_ARM_MTE + FEAT_EL2

[PATCH 1/8] KVM: arm64: nv: Disallow KVM_CAP_ARM_MTE + FEAT_EL2

[Oliver Upton]

KVM's nested implementation already hides FEAT_MTE and its laundry list
of derivatives from an NV guest. While good, this is insonsistent with
the UAPI which does not explicitly prevent the combination of MTE and
NV.

Bite the bullet and refuse to create a nested VM if KVM_CAP_ARM_MTE has
been enabled.

Cc: stable@vger.kernel.org
Fixes: a7484c80e5ca ("KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2*")
Signed-off-by: Oliver Upton <oupton@kernel.org>
---
 arch/arm64/kvm/arm.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 157bf4b81202..a750413c684b 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1793,13 +1793,15 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
 	    !test_bit(KVM_ARM_VCPU_PMU_V3, &features))
 		return -EINVAL;
 
+	/* MTE is incompatible with AArch32 and nested virt */
+	if (kvm_has_mte(vcpu->kvm) &&
+	    (test_bit(KVM_ARM_VCPU_EL1_32BIT, &features) ||
+	     test_bit(KVM_ARM_VCPU_HAS_EL2, &features)))
+		return -EINVAL;
+
 	if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
 		return 0;
 
-	/* MTE is incompatible with AArch32 */
-	if (kvm_has_mte(vcpu->kvm))
-		return -EINVAL;
-
 	/* NV is incompatible with AArch32 */
 	if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
 		return -EINVAL;
-- 
2.47.3

[PATCH 7/8] KVM: arm64: nv: Enforce sysreg visibility before evaulating guest traps

[Oliver Upton]

Not all sysregs / system instructions can be described by FGUs. Of
these, there are some which may be not implemented for the VM (e.g. MTE
cache maintenance instructions) yet intersect with valid trap bits for
the VM (HCR_EL2.TPCP).

This is annoying to deal with since most of our UNDEF infrastructure is
built around traps being RESx for unimplemented features. Test the
visibility of the sys_reg_desc to avoid injecting traps for non-existent
system registers into the guest hypervisor.

Cc: stable@vger.kernel.org
Fixes: a7484c80e5ca ("KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2*")
Signed-off-by: Oliver Upton <oupton@kernel.org>
---
 arch/arm64/kvm/emulate-nested.c | 24 +++++++++++++++++++++---
 arch/arm64/kvm/sys_regs.c       | 18 ++++++++++--------
 arch/arm64/kvm/sys_regs.h       |  4 +++-
 3 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index d04ac5203454..46dc658b041b 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -2546,12 +2546,13 @@ static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr,
 	return !(kvm_get_sysreg_res0(kvm, sr) & BIT(tc.bit));
 }
 
-bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
+bool triage_sysreg_trap(struct kvm_vcpu *vcpu, const struct sys_reg_desc **r)
 {
 	enum vcpu_sysreg fgtreg;
 	union trap_config tc;
 	enum trap_behaviour b;
 	bool is_read;
+	int sr_index;
 	u32 sysreg;
 	u64 esr;
 
@@ -2559,6 +2560,7 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
 	sysreg = esr_sys64_to_sysreg(esr);
 	is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
 
+	*r = NULL;
 	tc = get_trap_config(sysreg);
 
 	/*
@@ -2569,6 +2571,11 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
 	if (!tc.val)
 		goto local;
 
+	if (tc.sri) {
+		sr_index = tc.sri - 1;
+		*r = kvm_sys_reg_idx_to_desc(esr_sys64_to_sysreg(esr), sr_index);
+	}
+
 	/*
 	 * If a sysreg can be trapped using a FGT, first check whether we
 	 * trap for the purpose of forbidding the feature. In that case,
@@ -2595,6 +2602,18 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
 	if (is_hyp_ctxt(vcpu) && !vcpu_is_host_el0(vcpu))
 		goto local;
 
+	/*
+	 * Certain sysregs and system instructions could be UNDEF for the VM
+	 * but lack a corresponding FGU description and intersect with a valid
+	 * trap bit for the VM. Test the visibility the sysreg (if we have a
+	 * sys_reg_desc) before determining the effects of the guest
+	 * hypervisor's traps.
+	 */
+	if (*r && sysreg_guest_hidden(vcpu, *r)) {
+		kvm_inject_undefined(vcpu);
+		return true;
+	}
+
 	switch ((enum fgt_group_id)tc.fgt) {
 	case __NO_FGT_GROUP__:
 		break;
@@ -2666,7 +2685,7 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
 		goto inject;
 
 local:
-	if (!tc.sri) {
+	if (!(*r)) {
 		struct sys_reg_params params;
 
 		params = esr_sys64_to_params(esr);
@@ -2697,7 +2716,6 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
 		return true;
 	}
 
-	*sr_index = tc.sri - 1;
 	return false;
 
 inject:
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 7f7a197a9ad6..7c8277de0494 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -5407,6 +5407,15 @@ void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
 		kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu);
 }
 
+const struct sys_reg_desc *kvm_sys_reg_idx_to_desc(u32 encoding, int sr_idx)
+{
+	/* System registers have Op0=={2,3}, as per DDI487 J.a C5.1.2 */
+	if (sys_reg_Op0(encoding) == 2 || sys_reg_Op0(encoding) == 3)
+		return &sys_reg_descs[sr_idx];
+	else
+		return &sys_insn_descs[sr_idx];
+}
+
 /**
  * kvm_handle_sys_reg -- handles a system instruction or mrs/msr instruction
  *			 trap on a guest execution
@@ -5418,22 +5427,15 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
 	struct sys_reg_params params;
 	unsigned long esr = kvm_vcpu_get_esr(vcpu);
 	int Rt = kvm_vcpu_sys_get_rt(vcpu);
-	int sr_idx;
 
 	trace_kvm_handle_sys_reg(esr);
 
-	if (triage_sysreg_trap(vcpu, &sr_idx))
+	if (triage_sysreg_trap(vcpu, &desc))
 		return 1;
 
 	params = esr_sys64_to_params(esr);
 	params.regval = vcpu_get_reg(vcpu, Rt);
 
-	/* System registers have Op0=={2,3}, as per DDI487 J.a C5.1.2 */
-	if (params.Op0 == 2 || params.Op0 == 3)
-		desc = &sys_reg_descs[sr_idx];
-	else
-		desc = &sys_insn_descs[sr_idx];
-
 	perform_access(vcpu, &params, desc);
 
 	/* Read from system register? */
diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h
index 50214271b24a..d8ae21578665 100644
--- a/arch/arm64/kvm/sys_regs.h
+++ b/arch/arm64/kvm/sys_regs.h
@@ -263,10 +263,12 @@ int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
 int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
 			 const struct sys_reg_desc table[], unsigned int num);
 
-bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index);
+bool triage_sysreg_trap(struct kvm_vcpu *vcpu, const struct sys_reg_desc **r);
 
 int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu);
 
+const struct sys_reg_desc *kvm_sys_reg_idx_to_desc(u32 encoding, int sr_idx);
+
 #define AA32(_x)	.aarch32_map = AA32_##_x
 #define Op0(_x) 	.Op0 = _x
 #define Op1(_x) 	.Op1 = _x
-- 
2.47.3

Re: [PATCH 7/8] KVM: arm64: nv: Enforce sysreg visibility before evaulating guest traps

[Leonardo Bras]

On Wed, Nov 04, 2026 at 11:21:23AM -0800, Oliver Upton wrote:
> Not all sysregs / system instructions can be described by FGUs. Of
> these, there are some which may be not implemented for the VM (e.g. MTE
> cache maintenance instructions) yet intersect with valid trap bits for
> the VM (HCR_EL2.TPCP).
> 
> This is annoying to deal with since most of our UNDEF infrastructure is
> built around traps being RESx for unimplemented features. Test the
> visibility of the sys_reg_desc to avoid injecting traps for non-existent
> system registers into the guest hypervisor.
> 
> Cc: stable@vger.kernel.org
> Fixes: a7484c80e5ca ("KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2*")
> Signed-off-by: Oliver Upton <oupton@kernel.org>
> ---
>  arch/arm64/kvm/emulate-nested.c | 24 +++++++++++++++++++++---
>  arch/arm64/kvm/sys_regs.c       | 18 ++++++++++--------
>  arch/arm64/kvm/sys_regs.h       |  4 +++-
>  3 files changed, 34 insertions(+), 12 deletions(-)
> 
> diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
> index d04ac5203454..46dc658b041b 100644
> --- a/arch/arm64/kvm/emulate-nested.c
> +++ b/arch/arm64/kvm/emulate-nested.c
> @@ -2546,12 +2546,13 @@ static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr,
>  	return !(kvm_get_sysreg_res0(kvm, sr) & BIT(tc.bit));
>  }
>  
> -bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
> +bool triage_sysreg_trap(struct kvm_vcpu *vcpu, const struct sys_reg_desc **r)
>  {
>  	enum vcpu_sysreg fgtreg;
>  	union trap_config tc;
>  	enum trap_behaviour b;
>  	bool is_read;
> +	int sr_index;
>  	u32 sysreg;
>  	u64 esr;
>  
> @@ -2559,6 +2560,7 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
>  	sysreg = esr_sys64_to_sysreg(esr);
>  	is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
>  
> +	*r = NULL;

In the caller, desc is already NULL, and here we are overwriting as NULL 
again. Is it needed here?

(For some reason it looks weird to me that the first thing we do with a 
parameter is to clean it's contents here)

>  	tc = get_trap_config(sysreg);
>  
>  	/*
> @@ -2569,6 +2571,11 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
>  	if (!tc.val)
>  		goto local;
>  
> +	if (tc.sri) {
> +		sr_index = tc.sri - 1;
> +		*r = kvm_sys_reg_idx_to_desc(esr_sys64_to_sysreg(esr), sr_index);

IIUC sr_index is used only in this block, maybe we could make it local to 
it?

> +	}
> +
>  	/*
>  	 * If a sysreg can be trapped using a FGT, first check whether we
>  	 * trap for the purpose of forbidding the feature. In that case,
> @@ -2595,6 +2602,18 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
>  	if (is_hyp_ctxt(vcpu) && !vcpu_is_host_el0(vcpu))
>  		goto local;
>  
> +	/*
> +	 * Certain sysregs and system instructions could be UNDEF for the VM
> +	 * but lack a corresponding FGU description and intersect with a valid
> +	 * trap bit for the VM. Test the visibility the sysreg (if we have a
> +	 * sys_reg_desc) before determining the effects of the guest
> +	 * hypervisor's traps.
> +	 */
> +	if (*r && sysreg_guest_hidden(vcpu, *r)) {
> +		kvm_inject_undefined(vcpu);
> +		return true;
> +	}
> +
>  	switch ((enum fgt_group_id)tc.fgt) {
>  	case __NO_FGT_GROUP__:
>  		break;
> @@ -2666,7 +2685,7 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
>  		goto inject;
>  
>  local:
> -	if (!tc.sri) {
> +	if (!(*r)) {
>  		struct sys_reg_params params;
>  
>  		params = esr_sys64_to_params(esr);
> @@ -2697,7 +2716,6 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
>  		return true;
>  	}
>  
> -	*sr_index = tc.sri - 1;
>  	return false;
>  
>  inject:
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 7f7a197a9ad6..7c8277de0494 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -5407,6 +5407,15 @@ void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
>  		kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu);
>  }
>  
> +const struct sys_reg_desc *kvm_sys_reg_idx_to_desc(u32 encoding, int sr_idx)
> +{
> +	/* System registers have Op0=={2,3}, as per DDI487 J.a C5.1.2 */
> +	if (sys_reg_Op0(encoding) == 2 || sys_reg_Op0(encoding) == 3)
> +		return &sys_reg_descs[sr_idx];
> +	else
> +		return &sys_insn_descs[sr_idx];
> +}
> +
>  /**
>   * kvm_handle_sys_reg -- handles a system instruction or mrs/msr instruction
>   *			 trap on a guest execution
> @@ -5418,22 +5427,15 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
>  	struct sys_reg_params params;
>  	unsigned long esr = kvm_vcpu_get_esr(vcpu);
>  	int Rt = kvm_vcpu_sys_get_rt(vcpu);
> -	int sr_idx;
>  
>  	trace_kvm_handle_sys_reg(esr);
>  
> -	if (triage_sysreg_trap(vcpu, &sr_idx))
> +	if (triage_sysreg_trap(vcpu, &desc))
>  		return 1;
>  
>  	params = esr_sys64_to_params(esr);
>  	params.regval = vcpu_get_reg(vcpu, Rt);
>  
> -	/* System registers have Op0=={2,3}, as per DDI487 J.a C5.1.2 */
> -	if (params.Op0 == 2 || params.Op0 == 3)
> -		desc = &sys_reg_descs[sr_idx];
> -	else
> -		desc = &sys_insn_descs[sr_idx];
> -
>  	perform_access(vcpu, &params, desc);
>  
>  	/* Read from system register? */
> diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h
> index 50214271b24a..d8ae21578665 100644
> --- a/arch/arm64/kvm/sys_regs.h
> +++ b/arch/arm64/kvm/sys_regs.h
> @@ -263,10 +263,12 @@ int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
>  int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
>  			 const struct sys_reg_desc table[], unsigned int num);
>  
> -bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index);
> +bool triage_sysreg_trap(struct kvm_vcpu *vcpu, const struct sys_reg_desc **r);
>  
>  int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu);
>  
> +const struct sys_reg_desc *kvm_sys_reg_idx_to_desc(u32 encoding, int sr_idx);
> +
>  #define AA32(_x)	.aarch32_map = AA32_##_x
>  #define Op0(_x) 	.Op0 = _x
>  #define Op1(_x) 	.Op1 = _x
> -- 
> 2.47.3
> 

In general terms, I like that the change makes it easier to understand the 
function.

Thanks!
Leo

[PATCH 8/8] KVM: arm64: Stop leaning on __kvm_inject_sea() for abort injection

[Oliver Upton]

KVM's handling of local abort injection is, politely stated, a giant
fscking mess. At surface level, things look somewhat innocuous: inject
an SEA and directly update ESR_EL1 with the correct FSC afterwards. But
this is broken for a couple reasons.

For one, on implementations with FEAT_DoubleFault2 KVM may inject the
SEA as an SError depending on the falue of SCTLR2_EL1.NMEA, meaning that
the eventual non-SEA abort is incorrectly taken to the SError vector.

AArch32 is also quite bitrotted. No surprise. The FSR value is written
to IFSR not DFSR, when a prefetch abort is injected. So while DFSR is
aliased to ESR_EL1 architecturally, fixing up ESR_EL1.FSC after
injection has no effect on IFSR.

Finally, injecting aborts into non-LPAE relics may use an invalid value
for xFSC.FS, as KVM unconditionally uses the LPAE/VMSAv8-64 format.

Fix the first two by tearing the whole mess, adding __kvm_inject_abort()
for injecting non-SEA aborts which takes the desired FSC from the
caller. Fix the third one by selecting a fault status value based on the
LPAE-ness of the guest.

Cc: stable@vger.kernel.org
Fixes: fff97df2a0bd ("KVM: arm64: Route SEAs to the SError vector when EASE is set")
Fixes: 2937aeec9dc5 ("KVM: arm64: Handle DABT caused by LS64* instructions on unsupported memory")
Signed-off-by: Oliver Upton <oupton@kernel.org>
---
 arch/arm64/kvm/inject_fault.c | 133 +++++++++++++++++++---------------
 1 file changed, 76 insertions(+), 57 deletions(-)

diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index aa2011cc0d5a..89c7890b8743 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -102,12 +102,32 @@ static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu)
 	return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT);
 }
 
-static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)
+static void __inject_abt64(struct kvm_vcpu *vcpu, bool iabt, u32 iss, unsigned long addr)
 {
-	unsigned long cpsr = *vcpu_cpsr(vcpu);
-	bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
-	u64 esr = 0, fsc;
-	int level;
+	u64 esr;
+	u8 ec;
+
+	if (vcpu_mode_priv(vcpu))
+		ec = iabt ? ESR_ELx_EC_IABT_CUR : ESR_ELx_EC_DABT_CUR;
+	else
+		ec = iabt ? ESR_ELx_EC_IABT_LOW : ESR_ELx_EC_DABT_LOW;
+
+	esr = FIELD_PREP(ESR_ELx_EC_MASK, ec) | FIELD_PREP(ESR_ELx_ISS_MASK, iss) |
+	      ESR_ELx_IL;
+
+	vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
+	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
+}
+
+static void inject_abt64(struct kvm_vcpu *vcpu, bool iabt, u32 iss, u64 addr)
+{
+	pend_sync_exception(vcpu);
+	__inject_abt64(vcpu, iabt, iss, addr);
+}
+
+static void inject_sea64(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
+{
+	u8 fsc;
 
 	/*
 	 * If injecting an abort from a failed S1PTW, rewalk the S1 PTs to
@@ -116,7 +136,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
 	 */
 	if (kvm_vcpu_abt_iss1tw(vcpu)) {
 		u64 hpfar = kvm_vcpu_get_fault_ipa(vcpu);
-		int ret;
+		int ret, level;
 
 		if (hpfar == INVALID_GPA)
 			return;
@@ -137,28 +157,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
 	else
 		pend_sync_exception(vcpu);
 
-	/*
-	 * Build an {i,d}abort, depending on the level.
-	 * Report an external synchronous abort.
-	 */
-	esr |= ESR_ELx_IL;
-
-	/*
-	 * Here, the guest runs in AArch64 mode when in EL1. If we get
-	 * an AArch32 fault, it means we managed to trap an EL0 fault.
-	 */
-	if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
-		esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
-	else
-		esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
-
-	if (!is_iabt)
-		esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT;
-
-	esr |= fsc;
-
-	vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
-	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
+	__inject_abt64(vcpu, iabt, fsc, addr);
 }
 
 void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr)
@@ -174,31 +173,41 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
 	kvm_inject_sync(vcpu, esr);
 }
 
-#define DFSR_FSC_EXTABT_LPAE	0x10
-#define DFSR_FSC_EXTABT_nLPAE	0x08
-#define DFSR_LPAE		BIT(9)
-#define TTBCR_EAE		BIT(31)
+#define DFSR_FS_4			BIT(10)
+#define DFSR_FS_3_0			GENMASK(3, 0)
+#define DFSR_FS_EXTABT_nLPAE		0x08
+#define DFSR_FS_EXCL_ATOMIC_nLPAE	0x15
+#define DFSR_FSC			GENMASK(5, 0)
+#define DFSR_FSC_EXTABT_LPAE		0x10
+#define DFSR_LPAE			BIT(9)
+#define TTBCR_EAE			BIT(31)
 
 static void inject_undef32(struct kvm_vcpu *vcpu)
 {
 	kvm_pend_exception(vcpu, EXCEPT_AA32_UND);
 }
 
+static bool vcpu_trans_32bit_non_lpae(struct kvm_vcpu *vcpu)
+{
+	return vcpu_el1_is_32bit(vcpu) && !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE);
+}
+
 /*
  * Modelled after TakeDataAbortException() and TakePrefetchAbortException
  * pseudocode.
  */
-static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
+static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u8 status, u32 addr)
 {
 	u64 far;
 	u32 fsr;
 
-	/* Give the guest an IMPLEMENTATION DEFINED exception */
-	if (vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE) {
-		fsr = DFSR_LPAE | DFSR_FSC_EXTABT_LPAE;
+	if (vcpu_trans_32bit_non_lpae(vcpu)) {
+		/* Ensure status actually fits in DFSR.FS */
+		KVM_BUG_ON(status > 0x1F, vcpu->kvm);
+		fsr = FIELD_PREP(DFSR_FS_4, FIELD_GET(BIT(4), status)) |
+		      FIELD_PREP(DFSR_FS_3_0, FIELD_GET(GENMASK(3, 0), status));
 	} else {
-		/* no need to shuffle FS[4] into DFSR[10] as it's 0 */
-		fsr = DFSR_FSC_EXTABT_nLPAE;
+		fsr = FIELD_PREP(DFSR_FSC, status) | DFSR_LPAE;
 	}
 
 	far = vcpu_read_sys_reg(vcpu, FAR_EL1);
@@ -218,12 +227,28 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
 	vcpu_write_sys_reg(vcpu, far, FAR_EL1);
 }
 
+static void inject_sea32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
+{
+	if (vcpu_trans_32bit_non_lpae(vcpu))
+		inject_abt32(vcpu, is_pabt, DFSR_FS_EXTABT_nLPAE, addr);
+	else
+		inject_abt32(vcpu, is_pabt, DFSR_FSC_EXTABT_LPAE, addr);
+}
+
 static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
 {
 	if (vcpu_el1_is_32bit(vcpu))
-		inject_abt32(vcpu, iabt, addr);
+		inject_sea32(vcpu, iabt, addr);
 	else
-		inject_abt64(vcpu, iabt, addr);
+		inject_sea64(vcpu, iabt, addr);
+}
+
+static void __kvm_inject_abort(struct kvm_vcpu *vcpu, bool iabt, u8 fsc, u64 addr)
+{
+	if (vcpu_el1_is_32bit(vcpu))
+		inject_abt32(vcpu, iabt, fsc, addr);
+	else
+		inject_abt64(vcpu, iabt, fsc, addr);
 }
 
 static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
@@ -270,28 +295,25 @@ static int kvm_inject_nested_excl_atomic(struct kvm_vcpu *vcpu, u64 addr)
  */
 int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr)
 {
-	u64 esr;
-
 	if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM))
 		return kvm_inject_nested_excl_atomic(vcpu, addr);
 
-	__kvm_inject_sea(vcpu, false, addr);
-	esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
-	esr &= ~ESR_ELx_FSC;
-	esr |= ESR_ELx_FSC_EXCL_ATOMIC;
-	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
+	if (vcpu_trans_32bit_non_lpae(vcpu))
+		__kvm_inject_abort(vcpu, false, DFSR_FS_EXCL_ATOMIC_nLPAE, addr);
+	else
+		__kvm_inject_abort(vcpu, false, ESR_ELx_FSC_EXCL_ATOMIC, addr);
+
 	return 1;
 }
 
 void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
 {
-	unsigned long addr, esr;
+	bool iabt = kvm_vcpu_trap_is_iabt(vcpu);
+	unsigned long addr;
 
 	addr  = kvm_vcpu_get_fault_ipa(vcpu);
 	addr |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu));
 
-	__kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr);
-
 	/*
 	 * If AArch64 or LPAE, set FSC to 0 to indicate an Address
 	 * Size Fault at level 0, as if exceeding PARange.
@@ -299,13 +321,10 @@ void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
 	 * Non-LPAE guests will only get the external abort, as there
 	 * is no way to describe the ASF.
 	 */
-	if (vcpu_el1_is_32bit(vcpu) &&
-	    !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))
-		return;
-
-	esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
-	esr &= ~GENMASK_ULL(5, 0);
-	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
+	if (vcpu_trans_32bit_non_lpae(vcpu))
+		__kvm_inject_sea(vcpu, iabt, addr);
+	else
+		__kvm_inject_abort(vcpu, iabt, ESR_ELx_FSC_ADDRSZ, addr);
 }
 
 /**
-- 
2.47.3

Re: [PATCH 8/8] KVM: arm64: Stop leaning on __kvm_inject_sea() for abort injection

[Leonardo Bras]

On Wed, Nov 04, 2026 at 11:21:24AM -0800, Oliver Upton wrote:
> KVM's handling of local abort injection is, politely stated, a giant
> fscking mess. At surface level, things look somewhat innocuous: inject
> an SEA and directly update ESR_EL1 with the correct FSC afterwards. But
> this is broken for a couple reasons.
> 
> For one, on implementations with FEAT_DoubleFault2 KVM may inject the
> SEA as an SError depending on the falue of SCTLR2_EL1.NMEA, meaning that
> the eventual non-SEA abort is incorrectly taken to the SError vector.
> 
> AArch32 is also quite bitrotted. No surprise. The FSR value is written
> to IFSR not DFSR, when a prefetch abort is injected. So while DFSR is
> aliased to ESR_EL1 architecturally, fixing up ESR_EL1.FSC after
> injection has no effect on IFSR.
> 
> Finally, injecting aborts into non-LPAE relics may use an invalid value
> for xFSC.FS, as KVM unconditionally uses the LPAE/VMSAv8-64 format.
> 
> Fix the first two by tearing the whole mess, adding __kvm_inject_abort()
> for injecting non-SEA aborts which takes the desired FSC from the
> caller. Fix the third one by selecting a fault status value based on the
> LPAE-ness of the guest.
> 
> Cc: stable@vger.kernel.org
> Fixes: fff97df2a0bd ("KVM: arm64: Route SEAs to the SError vector when EASE is set")
> Fixes: 2937aeec9dc5 ("KVM: arm64: Handle DABT caused by LS64* instructions on unsupported memory")
> Signed-off-by: Oliver Upton <oupton@kernel.org>
> ---
>  arch/arm64/kvm/inject_fault.c | 133 +++++++++++++++++++---------------
>  1 file changed, 76 insertions(+), 57 deletions(-)
> 
> diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
> index aa2011cc0d5a..89c7890b8743 100644
> --- a/arch/arm64/kvm/inject_fault.c
> +++ b/arch/arm64/kvm/inject_fault.c
> @@ -102,12 +102,32 @@ static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu)
>  	return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT);
>  }
>  
> -static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)
> +static void __inject_abt64(struct kvm_vcpu *vcpu, bool iabt, u32 iss, unsigned long addr)
>  {
> -	unsigned long cpsr = *vcpu_cpsr(vcpu);
> -	bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
> -	u64 esr = 0, fsc;
> -	int level;
> +	u64 esr;
> +	u8 ec;
> +
> +	if (vcpu_mode_priv(vcpu))

Maybe I am getting this wrong, but this line was previously:
-	if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)

but vcpu_mode_priv() does not test exactly the same thing, it would test:
vcpu_mode_is_32bit(), tested in is_aarch32, &&
cpsr_mode > PSR_AA32_MODE_USR, not tested above, ||
*vcpu_cpsr(vcpu) & PSR_MODE_MASK != PSR_MODE_EL0t

This last line is testing the exact opposite of what was here. 
Is that another fix?

> +		ec = iabt ? ESR_ELx_EC_IABT_CUR : ESR_ELx_EC_DABT_CUR;
> +	else
> +		ec = iabt ? ESR_ELx_EC_IABT_LOW : ESR_ELx_EC_DABT_LOW;
> +
> +	esr = FIELD_PREP(ESR_ELx_EC_MASK, ec) | FIELD_PREP(ESR_ELx_ISS_MASK, iss) |
> +	      ESR_ELx_IL;
> +
> +	vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
> +	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
> +}
> +
> +static void inject_abt64(struct kvm_vcpu *vcpu, bool iabt, u32 iss, u64 addr)
> +{
> +	pend_sync_exception(vcpu);
> +	__inject_abt64(vcpu, iabt, iss, addr);
> +}
> +
> +static void inject_sea64(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
> +{
> +	u8 fsc;
>  
>  	/*
>  	 * If injecting an abort from a failed S1PTW, rewalk the S1 PTs to
> @@ -116,7 +136,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
>  	 */
>  	if (kvm_vcpu_abt_iss1tw(vcpu)) {
>  		u64 hpfar = kvm_vcpu_get_fault_ipa(vcpu);
> -		int ret;
> +		int ret, level;
>  
>  		if (hpfar == INVALID_GPA)
>  			return;
> @@ -137,28 +157,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
>  	else
>  		pend_sync_exception(vcpu);
>  
> -	/*
> -	 * Build an {i,d}abort, depending on the level.
> -	 * Report an external synchronous abort.
> -	 */
> -	esr |= ESR_ELx_IL;
> -
> -	/*
> -	 * Here, the guest runs in AArch64 mode when in EL1. If we get
> -	 * an AArch32 fault, it means we managed to trap an EL0 fault.
> -	 */
> -	if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
> -		esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
> -	else
> -		esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
> -
> -	if (!is_iabt)
> -		esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT;
> -
> -	esr |= fsc;
> -
> -	vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
> -	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
> +	__inject_abt64(vcpu, iabt, fsc, addr);
>  }
>  
>  void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr)
> @@ -174,31 +173,41 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
>  	kvm_inject_sync(vcpu, esr);
>  }
>  
> -#define DFSR_FSC_EXTABT_LPAE	0x10
> -#define DFSR_FSC_EXTABT_nLPAE	0x08
> -#define DFSR_LPAE		BIT(9)
> -#define TTBCR_EAE		BIT(31)
> +#define DFSR_FS_4			BIT(10)
> +#define DFSR_FS_3_0			GENMASK(3, 0)
> +#define DFSR_FS_EXTABT_nLPAE		0x08
> +#define DFSR_FS_EXCL_ATOMIC_nLPAE	0x15

Maybe I am reading the wrong place, but in M.d, I see:
DFSR.FS = 0b10101
IMPLEMENTATION DEFINED fault (Unsupported Exclusive access fault).

> +#define DFSR_FSC			GENMASK(5, 0)
> +#define DFSR_FSC_EXTABT_LPAE		0x10

Wouldn't this be STATUS bits instead of FSC?
(Maybe FSC got renamed STATUS in the meantime)


> +#define DFSR_LPAE			BIT(9)
> +#define TTBCR_EAE			BIT(31)
>  
>  static void inject_undef32(struct kvm_vcpu *vcpu)
>  {
>  	kvm_pend_exception(vcpu, EXCEPT_AA32_UND);
>  }
>  
> +static bool vcpu_trans_32bit_non_lpae(struct kvm_vcpu *vcpu)
> +{
> +	return vcpu_el1_is_32bit(vcpu) && !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE);
> +}
> +
>  /*
>   * Modelled after TakeDataAbortException() and TakePrefetchAbortException
>   * pseudocode.
>   */
> -static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
> +static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u8 status, u32 addr)
>  {
>  	u64 far;
>  	u32 fsr;
>  
> -	/* Give the guest an IMPLEMENTATION DEFINED exception */
> -	if (vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE) {
> -		fsr = DFSR_LPAE | DFSR_FSC_EXTABT_LPAE;
> +	if (vcpu_trans_32bit_non_lpae(vcpu)) {
> +		/* Ensure status actually fits in DFSR.FS */
> +		KVM_BUG_ON(status > 0x1F, vcpu->kvm);
> +		fsr = FIELD_PREP(DFSR_FS_4, FIELD_GET(BIT(4), status)) |
> +		      FIELD_PREP(DFSR_FS_3_0, FIELD_GET(GENMASK(3, 0), status));
>  	} else {
> -		/* no need to shuffle FS[4] into DFSR[10] as it's 0 */
> -		fsr = DFSR_FSC_EXTABT_nLPAE;
> +		fsr = FIELD_PREP(DFSR_FSC, status) | DFSR_LPAE;
>  	}
>  
>  	far = vcpu_read_sys_reg(vcpu, FAR_EL1);
> @@ -218,12 +227,28 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
>  	vcpu_write_sys_reg(vcpu, far, FAR_EL1);
>  }
>  
> +static void inject_sea32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
> +{
> +	if (vcpu_trans_32bit_non_lpae(vcpu))
> +		inject_abt32(vcpu, is_pabt, DFSR_FS_EXTABT_nLPAE, addr);
> +	else
> +		inject_abt32(vcpu, is_pabt, DFSR_FSC_EXTABT_LPAE, addr);
> +}
> +
>  static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
>  {
>  	if (vcpu_el1_is_32bit(vcpu))
> -		inject_abt32(vcpu, iabt, addr);
> +		inject_sea32(vcpu, iabt, addr);
>  	else
> -		inject_abt64(vcpu, iabt, addr);
> +		inject_sea64(vcpu, iabt, addr);
> +}
> +
> +static void __kvm_inject_abort(struct kvm_vcpu *vcpu, bool iabt, u8 fsc, u64 addr)
> +{
> +	if (vcpu_el1_is_32bit(vcpu))
> +		inject_abt32(vcpu, iabt, fsc, addr);
> +	else
> +		inject_abt64(vcpu, iabt, fsc, addr);
>  }
>  
>  static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
> @@ -270,28 +295,25 @@ static int kvm_inject_nested_excl_atomic(struct kvm_vcpu *vcpu, u64 addr)
>   */
>  int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr)
>  {
> -	u64 esr;
> -
>  	if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM))
>  		return kvm_inject_nested_excl_atomic(vcpu, addr);
>  
> -	__kvm_inject_sea(vcpu, false, addr);
> -	esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
> -	esr &= ~ESR_ELx_FSC;
> -	esr |= ESR_ELx_FSC_EXCL_ATOMIC;
> -	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
> +	if (vcpu_trans_32bit_non_lpae(vcpu))
> +		__kvm_inject_abort(vcpu, false, DFSR_FS_EXCL_ATOMIC_nLPAE, addr);
> +	else
> +		__kvm_inject_abort(vcpu, false, ESR_ELx_FSC_EXCL_ATOMIC, addr);
> +
>  	return 1;
>  }
>  
>  void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
>  {
> -	unsigned long addr, esr;
> +	bool iabt = kvm_vcpu_trap_is_iabt(vcpu);
> +	unsigned long addr;
>  
>  	addr  = kvm_vcpu_get_fault_ipa(vcpu);
>  	addr |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu));
>  
> -	__kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr);
> -
>  	/*
>  	 * If AArch64 or LPAE, set FSC to 0 to indicate an Address
>  	 * Size Fault at level 0, as if exceeding PARange.
> @@ -299,13 +321,10 @@ void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
>  	 * Non-LPAE guests will only get the external abort, as there
>  	 * is no way to describe the ASF.
>  	 */
> -	if (vcpu_el1_is_32bit(vcpu) &&
> -	    !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))
> -		return;
> -
> -	esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
> -	esr &= ~GENMASK_ULL(5, 0);
> -	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
> +	if (vcpu_trans_32bit_non_lpae(vcpu))
> +		__kvm_inject_sea(vcpu, iabt, addr);
> +	else
> +		__kvm_inject_abort(vcpu, iabt, ESR_ELx_FSC_ADDRSZ, addr);

The rest seems correct, and make easier to comprehend. :)

Thanks!
Leo