* [PATCH v13 15/32] KVM: arm64: Factor SVE code out of fpsimd_lazy_switch_to_host()
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
Since the function will grow as a result of adding SME support move the
SVE code out of fpsimd_lazy_switch_to_host(). No functional change, just
code motion.
Reviewed-by: Jean-Philippe Brucker <jpb@kernel.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kvm/hyp/include/hyp/switch.h | 48 ++++++++++++++++++---------------
1 file changed, 27 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 35e8d952c7b6..ce586110c226 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -519,11 +519,11 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
}
}
-static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
+static inline void sve_lazy_switch_to_host(struct kvm_vcpu *vcpu)
{
u64 zcr_el1, zcr_el2;
- if (!guest_owns_fp_regs())
+ if (!vcpu_has_sve(vcpu))
return;
/*
@@ -534,29 +534,35 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
* synchronization event, we don't need an ISB here to avoid taking
* traps for anything that was exposed to the guest.
*/
- if (vcpu_has_sve(vcpu)) {
- zcr_el1 = read_sysreg_el1(SYS_ZCR);
- __vcpu_assign_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu), zcr_el1);
+ zcr_el1 = read_sysreg_el1(SYS_ZCR);
+ __vcpu_assign_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu), zcr_el1);
- /*
- * The guest's state is always saved using the guest's max VL.
- * Ensure that the host has the guest's max VL active such that
- * the host can save the guest's state lazily, but don't
- * artificially restrict the host to the guest's max VL.
- */
- if (has_vhe()) {
- zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
- write_sysreg_el2(zcr_el2, SYS_ZCR);
- } else {
- zcr_el2 = sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1;
- write_sysreg_el2(zcr_el2, SYS_ZCR);
-
- zcr_el1 = vcpu_sve_max_vq(vcpu) - 1;
- write_sysreg_el1(zcr_el1, SYS_ZCR);
- }
+ /*
+ * The guest's state is always saved using the guest's max VL.
+ * Ensure that the host has the guest's max VL active such
+ * that the host can save the guest's state lazily, but don't
+ * artificially restrict the host to the guest's max VL.
+ */
+ if (has_vhe()) {
+ zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+ write_sysreg_el2(zcr_el2, SYS_ZCR);
+ } else {
+ zcr_el2 = sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1;
+ write_sysreg_el2(zcr_el2, SYS_ZCR);
+
+ zcr_el1 = vcpu_sve_max_vq(vcpu) - 1;
+ write_sysreg_el1(zcr_el1, SYS_ZCR);
}
}
+static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
+{
+ if (!guest_owns_fp_regs())
+ return;
+
+ sve_lazy_switch_to_host(vcpu);
+}
+
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
--
2.47.3
^ permalink raw reply related
* [PATCH v13 14/32] KVM: arm64: Store vector lengths in an array
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
SME adds a second vector length configured in a very similar way to the
SVE vector length, in order to facilitate future code sharing for SME
refactor our storage of vector lengths to use an array like the host does.
We do not yet take much advantage of this so the intermediate code is not
as clean as might be.
No functional change.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/kvm_host.h | 11 ++++++-----
arch/arm64/include/asm/kvm_hyp.h | 2 +-
arch/arm64/include/asm/kvm_pkvm.h | 2 +-
arch/arm64/kvm/guest.c | 6 +++---
arch/arm64/kvm/hyp/include/hyp/switch.h | 4 ++--
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 4 ++--
arch/arm64/kvm/hyp/nvhe/pkvm.c | 7 ++++---
arch/arm64/kvm/reset.c | 22 +++++++++++-----------
8 files changed, 30 insertions(+), 28 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 8e185e43fbff..5e071381ae5b 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -77,8 +77,9 @@ enum kvm_mode kvm_get_mode(void);
static inline enum kvm_mode kvm_get_mode(void) { return KVM_MODE_NONE; };
#endif
-extern unsigned int __ro_after_init kvm_sve_max_vl;
-extern unsigned int __ro_after_init kvm_host_sve_max_vl;
+extern unsigned int __ro_after_init kvm_max_vl[ARM64_VEC_MAX];
+extern unsigned int __ro_after_init kvm_host_max_vl[ARM64_VEC_MAX];
+
int __init kvm_arm_init_sve(void);
u32 __attribute_const__ kvm_target_cpu(void);
@@ -857,7 +858,7 @@ struct kvm_vcpu_arch {
*/
struct arm64_sve_state *sve_state;
enum fp_type fp_type;
- unsigned int sve_max_vl;
+ unsigned int max_vl[ARM64_VEC_MAX];
/* Stage 2 paging state used by the hardware on next switch */
struct kvm_s2_mmu *hw_mmu;
@@ -1100,7 +1101,7 @@ struct kvm_vcpu_arch {
/* KVM is currently emulating an L2 to L1 exception */
#define IN_NESTED_EXCEPTION __vcpu_single_flag(sflags, BIT(9))
-#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
+#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.max_vl[ARM64_VEC_SVE])
#define vcpu_sve_zcr_elx(vcpu) \
(unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1)
@@ -1119,7 +1120,7 @@ struct kvm_vcpu_arch {
__size_ret; \
})
-#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.sve_max_vl)
+#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.max_vl[ARM64_VEC_SVE])
#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
KVM_GUESTDBG_USE_SW_BP | \
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 4974492744cc..3d05533a0f67 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -150,7 +150,7 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val);
extern unsigned long kvm_nvhe_sym(__icache_flags);
extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
-extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
+extern unsigned int kvm_nvhe_sym(kvm_host_max_vl[ARM64_VEC_MAX]);
extern unsigned long kvm_nvhe_sym(hyp_nr_cpus);
extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr);
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index 74fedd9c5ff0..d4d22acf2fe7 100644
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -188,7 +188,7 @@ static inline size_t pkvm_host_sve_state_size(void)
if (!system_supports_sve())
return 0;
- return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl));
+ return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]));
}
struct pkvm_mapping {
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index 3ae751e72c95..2370bb0ad94e 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -318,7 +318,7 @@ static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (!vcpu_has_sve(vcpu))
return -ENOENT;
- if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl)))
+ if (WARN_ON(!sve_vl_valid(vcpu->arch.max_vl[ARM64_VEC_SVE])))
return -EINVAL;
memset(vqs, 0, sizeof(vqs));
@@ -356,7 +356,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (vq_present(vqs, vq))
max_vq = vq;
- if (max_vq > sve_vq_from_vl(kvm_sve_max_vl))
+ if (max_vq > sve_vq_from_vl(kvm_max_vl[ARM64_VEC_SVE]))
return -EINVAL;
/*
@@ -375,7 +375,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
return -EINVAL;
/* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_vec() */
- vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq);
+ vcpu->arch.max_vl[ARM64_VEC_SVE] = sve_vl_from_vq(max_vq);
return 0;
}
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index b4680baffc6e..35e8d952c7b6 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -493,7 +493,7 @@ static inline void __hyp_sve_save_host(void)
struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs);
ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
- write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
+ write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2);
sve_save_state(sve_regs, true);
fpsimd_save_common(&hctxt->fp_regs);
}
@@ -548,7 +548,7 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
write_sysreg_el2(zcr_el2, SYS_ZCR);
} else {
- zcr_el2 = sve_vq_from_vl(kvm_host_sve_max_vl) - 1;
+ zcr_el2 = sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1;
write_sysreg_el2(zcr_el2, SYS_ZCR);
zcr_el1 = vcpu_sve_max_vq(vcpu) - 1;
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index d3c69de698f4..14e24e257dcc 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -40,7 +40,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true);
fpsimd_save_common(&vcpu->arch.ctxt.fp_regs);
- write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
+ write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2);
}
static void __hyp_sve_restore_host(void)
@@ -57,7 +57,7 @@ static void __hyp_sve_restore_host(void)
* that was discovered, if we wish to use larger VLs this will
* need to be revisited.
*/
- write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
+ write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2);
sve_load_state(sve_regs, true);
fpsimd_load_common(&hctxt->fp_regs);
write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR);
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 9d7f632f01f8..d49f7f327adf 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -22,7 +22,7 @@ unsigned long __icache_flags;
/* Used by kvm_get_vttbr(). */
unsigned int kvm_arm_vmid_bits;
-unsigned int kvm_host_sve_max_vl;
+unsigned int kvm_host_max_vl[ARM64_VEC_MAX];
/*
* The currently loaded hyp vCPU for each physical CPU. Used in protected mode
@@ -459,7 +459,8 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h
}
/* Limit guest vector length to the maximum supported by the host. */
- sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl);
+ sve_max_vl = min(READ_ONCE(host_vcpu->arch.max_vl[ARM64_VEC_SVE]),
+ kvm_host_max_vl[ARM64_VEC_SVE]);
sve_state_size = sve_state_size_from_vl(sve_max_vl);
sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state));
@@ -473,7 +474,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h
goto err;
vcpu->arch.sve_state = sve_state;
- vcpu->arch.sve_max_vl = sve_max_vl;
+ vcpu->arch.max_vl[ARM64_VEC_SVE] = sve_max_vl;
return 0;
err:
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 0fae62a9eaef..fee01c38fa13 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -32,7 +32,7 @@
/* Maximum phys_shift supported for any VM on this host */
static u32 __ro_after_init kvm_ipa_limit;
-unsigned int __ro_after_init kvm_host_sve_max_vl;
+unsigned int __ro_after_init kvm_host_max_vl[ARM64_VEC_MAX];
/*
* ARMv8 Reset Values
@@ -46,14 +46,14 @@ unsigned int __ro_after_init kvm_host_sve_max_vl;
#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \
PSR_AA32_I_BIT | PSR_AA32_F_BIT)
-unsigned int __ro_after_init kvm_sve_max_vl;
+unsigned int __ro_after_init kvm_max_vl[ARM64_VEC_MAX];
int __init kvm_arm_init_sve(void)
{
if (system_supports_sve()) {
- kvm_sve_max_vl = sve_max_virtualisable_vl();
- kvm_host_sve_max_vl = sve_max_vl();
- kvm_nvhe_sym(kvm_host_sve_max_vl) = kvm_host_sve_max_vl;
+ kvm_max_vl[ARM64_VEC_SVE] = sve_max_virtualisable_vl();
+ kvm_host_max_vl[ARM64_VEC_SVE] = sve_max_vl();
+ kvm_nvhe_sym(kvm_host_max_vl[ARM64_VEC_SVE]) = kvm_host_max_vl[ARM64_VEC_SVE];
/*
* The get_sve_reg()/set_sve_reg() ioctl interface will need
@@ -61,16 +61,16 @@ int __init kvm_arm_init_sve(void)
* order to support vector lengths greater than
* VL_ARCH_MAX:
*/
- if (WARN_ON(kvm_sve_max_vl > VL_ARCH_MAX))
- kvm_sve_max_vl = VL_ARCH_MAX;
+ if (WARN_ON(kvm_max_vl[ARM64_VEC_SVE] > VL_ARCH_MAX))
+ kvm_max_vl[ARM64_VEC_SVE] = VL_ARCH_MAX;
/*
* Don't even try to make use of vector lengths that
* aren't available on all CPUs, for now:
*/
- if (kvm_sve_max_vl < sve_max_vl())
+ if (kvm_max_vl[ARM64_VEC_SVE] < sve_max_vl())
pr_warn("KVM: SVE vector length for guests limited to %u bytes\n",
- kvm_sve_max_vl);
+ kvm_max_vl[ARM64_VEC_SVE]);
}
return 0;
@@ -78,7 +78,7 @@ int __init kvm_arm_init_sve(void)
static void kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu)
{
- vcpu->arch.sve_max_vl = kvm_sve_max_vl;
+ vcpu->arch.max_vl[ARM64_VEC_SVE] = kvm_max_vl[ARM64_VEC_SVE];
/*
* Userspace can still customize the vector lengths by writing
@@ -99,7 +99,7 @@ static int kvm_vcpu_finalize_vec(struct kvm_vcpu *vcpu)
size_t reg_sz;
int ret;
- vl = vcpu->arch.sve_max_vl;
+ vl = vcpu->arch.max_vl[ARM64_VEC_SVE];
/*
* Responsibility for these properties is shared between
--
2.47.3
^ permalink raw reply related
* [PATCH v13 13/32] KVM: arm64: Rename sve_state_reg_region
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
As for SVE we will need to pull parts of dynamically sized registers out of
a block of memory for SME so we will use a similar code pattern for this.
Rename the current struct sve_state_reg_region in preparation for this.
No functional change.
Reviewed-by: Fuad Tabba <tabba@google.com>
Reviewed-by: Jean-Philippe Brucker <jpb@kernel.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kvm/guest.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index 0b789f73bf7c..3ae751e72c95 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -404,9 +404,9 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
*/
#define vcpu_sve_slices(vcpu) 1
-/* Bounds of a single SVE register slice within vcpu->arch.sve_state */
-struct sve_state_reg_region {
- unsigned int koffset; /* offset into sve_state in kernel memory */
+/* Bounds of a single register slice within vcpu->arch.s[mv]e_state */
+struct vec_state_reg_region {
+ unsigned int koffset; /* offset into s[mv]e_state in kernel memory */
unsigned int klen; /* length in kernel memory */
unsigned int upad; /* extra trailing padding in user memory */
};
@@ -415,7 +415,7 @@ struct sve_state_reg_region {
* Validate SVE register ID and get sanitised bounds for user/kernel SVE
* register copy
*/
-static int sve_reg_to_region(struct sve_state_reg_region *region,
+static int sve_reg_to_region(struct vec_state_reg_region *region,
struct kvm_vcpu *vcpu,
const struct kvm_one_reg *reg)
{
@@ -485,7 +485,7 @@ static int sve_reg_to_region(struct sve_state_reg_region *region,
static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
{
int ret;
- struct sve_state_reg_region region;
+ struct vec_state_reg_region region;
char __user *uptr = (char __user *)reg->addr;
/* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
@@ -511,7 +511,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
{
int ret;
- struct sve_state_reg_region region;
+ struct vec_state_reg_region region;
const char __user *uptr = (const char __user *)reg->addr;
/* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
--
2.47.3
^ permalink raw reply related
* [PATCH v13 12/32] KVM: arm64: Define internal features for SME
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
In order to simplify interdependencies in the rest of the series define
the feature detection for SME and its subfeatures. Due to the need for
vector length configuration we define a flag for SME like for SVE. We
also have two subfeatures which add architectural state, FA64 and SME2,
which are configured via the normal ID register scheme.
Also provide helpers which check if the vCPU is in streaming mode or has
ZA enabled.
Reviewed-by: Fuad Tabba <tabba@google.com>
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/kvm_host.h | 35 ++++++++++++++++++++++++++++++++++-
arch/arm64/kvm/sys_regs.c | 2 +-
2 files changed, 35 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 8b746b1a1e53..8e185e43fbff 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -367,6 +367,8 @@ struct kvm_arch {
#define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10
/* Unhandled SEAs are taken to userspace */
#define KVM_ARCH_FLAG_EXIT_SEA 11
+ /* SME exposed to guest */
+#define KVM_ARCH_FLAG_GUEST_HAS_SME 12
unsigned long flags;
/* VM-wide vCPU feature set */
@@ -1133,7 +1135,16 @@ struct kvm_vcpu_arch {
#define vcpu_has_sve(vcpu) kvm_has_sve((vcpu)->kvm)
#endif
-#define vcpu_has_vec(vcpu) vcpu_has_sve(vcpu)
+#define kvm_has_sme(kvm) (system_supports_sme() && \
+ test_bit(KVM_ARCH_FLAG_GUEST_HAS_SME, &(kvm)->arch.flags))
+
+#ifdef __KVM_NVHE_HYPERVISOR__
+#define vcpu_has_sme(vcpu) kvm_has_sme(kern_hyp_va((vcpu)->kvm))
+#else
+#define vcpu_has_sme(vcpu) kvm_has_sme((vcpu)->kvm)
+#endif
+
+#define vcpu_has_vec(vcpu) (vcpu_has_sve(vcpu) || vcpu_has_sme(vcpu))
#ifdef CONFIG_ARM64_PTR_AUTH
#define vcpu_has_ptrauth(vcpu) \
@@ -1650,6 +1661,28 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val);
#define kvm_has_sctlr2(k) \
(kvm_has_feat((k), ID_AA64MMFR3_EL1, SCTLRX, IMP))
+#define kvm_has_fa64(k) \
+ (system_supports_fa64() && \
+ kvm_has_feat((k), ID_AA64SMFR0_EL1, FA64, IMP))
+
+#define kvm_has_sme2(k) \
+ (system_supports_sme2() && \
+ kvm_has_feat((k), ID_AA64PFR1_EL1, SME, SME2))
+
+#ifdef __KVM_NVHE_HYPERVISOR__
+#define vcpu_has_sme2(vcpu) kvm_has_sme2(kern_hyp_va((vcpu)->kvm))
+#define vcpu_has_fa64(vcpu) kvm_has_fa64(kern_hyp_va((vcpu)->kvm))
+#else
+#define vcpu_has_sme2(vcpu) kvm_has_sme2((vcpu)->kvm)
+#define vcpu_has_fa64(vcpu) kvm_has_fa64((vcpu)->kvm)
+#endif
+
+#define vcpu_in_streaming_mode(vcpu) \
+ (__vcpu_sys_reg(vcpu, SVCR) & SVCR_SM_MASK)
+
+#define vcpu_za_enabled(vcpu) \
+ (__vcpu_sys_reg(vcpu, SVCR) & SVCR_ZA_MASK)
+
static inline bool kvm_arch_has_irq_bypass(void)
{
return true;
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index b352cd323e30..ba8a3ed8f5ff 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -2028,7 +2028,7 @@ static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
static unsigned int sme_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
- if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SME, IMP))
+ if (vcpu_has_sme(vcpu))
return 0;
return REG_HIDDEN;
--
2.47.3
^ permalink raw reply related
* [PATCH v13 11/32] KVM: arm64: Remove special case for FP state loading from ZCR_EL2 traps
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
As part of adding the nested virtualisation support commit 0cfc85b8f5cf3
("KVM: arm64: nv: Load guest FP state for ZCR_EL2 trap") added code
which handles access to ZCR_EL2 from a guest as a SVE access trap.
While the reasoning for this was not specifically articulated in the
commit log it was part of the lazy switching mechanism that we used to
have for ZCR_ELx. Since commit 59419f10045bc ("KVM: arm64: Eagerly
switch ZCR_EL{1,2}") this mechanism has been removed. We now switch
ZCR_EL2 in the hypervisor when transitioning between guest and host,
meaning that we no longer need the special casing.
Remove the redundant special casing, this simplifies the code and will
make the implementation of SME support (which architecturally follows a
similar pattern to SVE) easier. Accesses to ZCR_EL2 from a nested EL2
will use the standard system register access handling path which works
with the in memory copy, and if the FP state is not yet loaded any
floating point operations will cause an access trap which we handle by
loading the state.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kvm/hyp/include/hyp/switch.h | 4 ----
arch/arm64/kvm/hyp/vhe/switch.c | 27 ---------------------------
2 files changed, 31 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 18131e395e24..b4680baffc6e 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -601,10 +601,6 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
if (guest_hyp_fpsimd_traps_enabled(vcpu))
return false;
break;
- case ESR_ELx_EC_SYS64:
- if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu)))
- return false;
- fallthrough;
case ESR_ELx_EC_SVE:
if (!sve_guest)
return false;
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index bbe9cebd3d9d..625fe81a20b1 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -458,30 +458,6 @@ static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code)
return true;
}
-static bool kvm_hyp_handle_zcr_el2(struct kvm_vcpu *vcpu, u64 *exit_code)
-{
- u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));
-
- if (!vcpu_has_nv(vcpu))
- return false;
-
- if (sysreg != SYS_ZCR_EL2)
- return false;
-
- if (guest_owns_fp_regs())
- return false;
-
- /*
- * ZCR_EL2 traps are handled in the slow path, with the expectation
- * that the guest's FP context has already been loaded onto the CPU.
- *
- * Load the guest's FP context and unconditionally forward to the
- * slow path for handling (i.e. return false).
- */
- kvm_hyp_handle_fpsimd(vcpu, exit_code);
- return false;
-}
-
static bool kvm_hyp_handle_sysreg_vhe(struct kvm_vcpu *vcpu, u64 *exit_code)
{
if (kvm_hyp_handle_tlbi_el2(vcpu, exit_code))
@@ -493,9 +469,6 @@ static bool kvm_hyp_handle_sysreg_vhe(struct kvm_vcpu *vcpu, u64 *exit_code)
if (kvm_hyp_handle_cpacr_el1(vcpu, exit_code))
return true;
- if (kvm_hyp_handle_zcr_el2(vcpu, exit_code))
- return true;
-
return kvm_hyp_handle_sysreg(vcpu, exit_code);
}
--
2.47.3
^ permalink raw reply related
* [PATCH v13 10/32] KVM: arm64: Rename SVE finalization constants to be more general
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
Due to the overlap between SVE and SME vector length configuration
created by streaming mode SVE we will finalize both at once. Rename the
existing finalization to use _VEC (vector) for the naming to avoid
confusion.
Since this includes the userspace API we create an alias
KVM_ARM_VCPU_VEC for the existing KVM_ARM_VCPU_SVE capability, existing
code which does not enable SME will be unaffected and any SME only code
will not need to use SVE constants.
No functional change.
Reviewed-by: Fuad Tabba <tabba@google.com>
Reviewed-by: Jean-Philippe Brucker <jpb@kernel.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/kvm_host.h | 8 +++++---
arch/arm64/include/uapi/asm/kvm.h | 6 ++++++
arch/arm64/kvm/guest.c | 12 ++++++------
arch/arm64/kvm/hyp/nvhe/pkvm.c | 2 +-
arch/arm64/kvm/reset.c | 20 ++++++++++----------
5 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index bae2c4f92ef5..8b746b1a1e53 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -1037,8 +1037,8 @@ struct kvm_vcpu_arch {
/* KVM_ARM_VCPU_INIT completed */
#define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0))
-/* SVE config completed */
-#define VCPU_SVE_FINALIZED __vcpu_single_flag(cflags, BIT(1))
+/* Vector config completed */
+#define VCPU_VEC_FINALIZED __vcpu_single_flag(cflags, BIT(1))
/* pKVM VCPU setup completed */
#define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2))
@@ -1133,6 +1133,8 @@ struct kvm_vcpu_arch {
#define vcpu_has_sve(vcpu) kvm_has_sve((vcpu)->kvm)
#endif
+#define vcpu_has_vec(vcpu) vcpu_has_sve(vcpu)
+
#ifdef CONFIG_ARM64_PTR_AUTH
#define vcpu_has_ptrauth(vcpu) \
((cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH) || \
@@ -1505,7 +1507,7 @@ struct kvm *kvm_arch_alloc_vm(void);
int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature);
bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);
-#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED)
+#define kvm_arm_vcpu_vec_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_VEC_FINALIZED)
#define kvm_has_mte(kvm) \
(system_supports_mte() && \
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 1c13bfa2d38a..83af99ca4e1b 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -107,6 +107,12 @@ struct kvm_regs {
#define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */
#define KVM_ARM_VCPU_HAS_EL2_E2H0 8 /* Limit NV support to E2H RES0 */
+/*
+ * An alias for _SVE since we finalize VL configuration for both SVE and SME
+ * simultaneously.
+ */
+#define KVM_ARM_VCPU_VEC KVM_ARM_VCPU_SVE
+
struct kvm_vcpu_init {
__u32 target;
__u32 features[7];
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index b01d6622b872..0b789f73bf7c 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -342,7 +342,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (!vcpu_has_sve(vcpu))
return -ENOENT;
- if (kvm_arm_vcpu_sve_finalized(vcpu))
+ if (kvm_arm_vcpu_vec_finalized(vcpu))
return -EPERM; /* too late! */
if (WARN_ON(vcpu->arch.sve_state))
@@ -374,7 +374,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (max_vq < SVE_VQ_MIN)
return -EINVAL;
- /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */
+ /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_vec() */
vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq);
return 0;
@@ -497,7 +497,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (ret)
return ret;
- if (!kvm_arm_vcpu_sve_finalized(vcpu))
+ if (!kvm_arm_vcpu_vec_finalized(vcpu))
return -EPERM;
if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset,
@@ -523,7 +523,7 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (ret)
return ret;
- if (!kvm_arm_vcpu_sve_finalized(vcpu))
+ if (!kvm_arm_vcpu_vec_finalized(vcpu))
return -EPERM;
if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr,
@@ -599,7 +599,7 @@ static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
return 0;
/* Policed by KVM_GET_REG_LIST: */
- WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
+ WARN_ON(!kvm_arm_vcpu_vec_finalized(vcpu));
return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */)
+ 1; /* KVM_REG_ARM64_SVE_VLS */
@@ -617,7 +617,7 @@ static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu,
return 0;
/* Policed by KVM_GET_REG_LIST: */
- WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
+ WARN_ON(!kvm_arm_vcpu_vec_finalized(vcpu));
/*
* Enumerate this first, so that userspace can save/restore in
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 3b2c4fbc34d8..9d7f632f01f8 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -454,7 +454,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h
int ret = 0;
if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) {
- vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED);
+ vcpu_clear_flag(vcpu, VCPU_VEC_FINALIZED);
return 0;
}
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index b963fd975aac..0fae62a9eaef 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -92,7 +92,7 @@ static void kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu)
* Finalize vcpu's maximum SVE vector length, allocating
* vcpu->arch.sve_state as necessary.
*/
-static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu)
+static int kvm_vcpu_finalize_vec(struct kvm_vcpu *vcpu)
{
void *buf;
unsigned int vl;
@@ -122,21 +122,21 @@ static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu)
}
vcpu->arch.sve_state = buf;
- vcpu_set_flag(vcpu, VCPU_SVE_FINALIZED);
+ vcpu_set_flag(vcpu, VCPU_VEC_FINALIZED);
return 0;
}
int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)
{
switch (feature) {
- case KVM_ARM_VCPU_SVE:
- if (!vcpu_has_sve(vcpu))
+ case KVM_ARM_VCPU_VEC:
+ if (!vcpu_has_vec(vcpu))
return -EINVAL;
- if (kvm_arm_vcpu_sve_finalized(vcpu))
+ if (kvm_arm_vcpu_vec_finalized(vcpu))
return -EPERM;
- return kvm_vcpu_finalize_sve(vcpu);
+ return kvm_vcpu_finalize_vec(vcpu);
}
return -EINVAL;
@@ -144,7 +144,7 @@ int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)
bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)
{
- if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu))
+ if (vcpu_has_vec(vcpu) && !kvm_arm_vcpu_vec_finalized(vcpu))
return false;
return true;
@@ -163,7 +163,7 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu)
kfree(vcpu->arch.ccsidr);
}
-static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu)
+static void kvm_vcpu_reset_vec(struct kvm_vcpu *vcpu)
{
if (vcpu_has_sve(vcpu))
memset(vcpu->arch.sve_state, 0, vcpu_sve_state_size(vcpu));
@@ -203,11 +203,11 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu)
if (loaded)
kvm_arch_vcpu_put(vcpu);
- if (!kvm_arm_vcpu_sve_finalized(vcpu)) {
+ if (!kvm_arm_vcpu_vec_finalized(vcpu)) {
if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE))
kvm_vcpu_enable_sve(vcpu);
} else {
- kvm_vcpu_reset_sve(vcpu);
+ kvm_vcpu_reset_vec(vcpu);
}
if (vcpu_el1_is_32bit(vcpu))
--
2.47.3
^ permalink raw reply related
* [PATCH v13 09/32] KVM: arm64: Pull ctxt_has_ helpers to start of sysreg-sr.h
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
Rather than add earlier prototypes of specific ctxt_has_ helpers let's just
pull all their definitions to the top of sysreg-sr.h so they're all
available to all the individual save/restore functions.
Reviewed-by: Fuad Tabba <tabba@google.com>
Reviewed-by: Jean-Philippe Brucker <jpb@kernel.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 84 +++++++++++++++---------------
1 file changed, 41 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
index a17cbe7582de..5624fd705ae3 100644
--- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
@@ -16,8 +16,6 @@
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
-static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt);
-
static inline struct kvm_vcpu *ctxt_to_vcpu(struct kvm_cpu_context *ctxt)
{
struct kvm_vcpu *vcpu = ctxt->__hyp_running_vcpu;
@@ -28,47 +26,6 @@ static inline struct kvm_vcpu *ctxt_to_vcpu(struct kvm_cpu_context *ctxt)
return vcpu;
}
-static inline bool ctxt_is_guest(struct kvm_cpu_context *ctxt)
-{
- return host_data_ptr(host_ctxt) != ctxt;
-}
-
-static inline u64 *ctxt_mdscr_el1(struct kvm_cpu_context *ctxt)
-{
- struct kvm_vcpu *vcpu = ctxt_to_vcpu(ctxt);
-
- if (ctxt_is_guest(ctxt) && kvm_host_owns_debug_regs(vcpu))
- return &vcpu->arch.external_mdscr_el1;
-
- return &ctxt_sys_reg(ctxt, MDSCR_EL1);
-}
-
-static inline u64 ctxt_midr_el1(struct kvm_cpu_context *ctxt)
-{
- struct kvm *kvm = kern_hyp_va(ctxt_to_vcpu(ctxt)->kvm);
-
- if (!(ctxt_is_guest(ctxt) &&
- test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags)))
- return read_cpuid_id();
-
- return kvm_read_vm_id_reg(kvm, SYS_MIDR_EL1);
-}
-
-static inline void __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
-{
- *ctxt_mdscr_el1(ctxt) = read_sysreg(mdscr_el1);
-
- // POR_EL0 can affect uaccess, so must be saved/restored early.
- if (ctxt_has_s1poe(ctxt))
- ctxt_sys_reg(ctxt, POR_EL0) = read_sysreg_s(SYS_POR_EL0);
-}
-
-static inline void __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
-{
- ctxt_sys_reg(ctxt, TPIDR_EL0) = read_sysreg(tpidr_el0);
- ctxt_sys_reg(ctxt, TPIDRRO_EL0) = read_sysreg(tpidrro_el0);
-}
-
static inline bool ctxt_has_mte(struct kvm_cpu_context *ctxt)
{
struct kvm_vcpu *vcpu = ctxt_to_vcpu(ctxt);
@@ -131,6 +88,47 @@ static inline bool ctxt_has_sctlr2(struct kvm_cpu_context *ctxt)
return kvm_has_sctlr2(kern_hyp_va(vcpu->kvm));
}
+static inline bool ctxt_is_guest(struct kvm_cpu_context *ctxt)
+{
+ return host_data_ptr(host_ctxt) != ctxt;
+}
+
+static inline u64 *ctxt_mdscr_el1(struct kvm_cpu_context *ctxt)
+{
+ struct kvm_vcpu *vcpu = ctxt_to_vcpu(ctxt);
+
+ if (ctxt_is_guest(ctxt) && kvm_host_owns_debug_regs(vcpu))
+ return &vcpu->arch.external_mdscr_el1;
+
+ return &ctxt_sys_reg(ctxt, MDSCR_EL1);
+}
+
+static inline u64 ctxt_midr_el1(struct kvm_cpu_context *ctxt)
+{
+ struct kvm *kvm = kern_hyp_va(ctxt_to_vcpu(ctxt)->kvm);
+
+ if (!(ctxt_is_guest(ctxt) &&
+ test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags)))
+ return read_cpuid_id();
+
+ return kvm_read_vm_id_reg(kvm, SYS_MIDR_EL1);
+}
+
+static inline void __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
+{
+ *ctxt_mdscr_el1(ctxt) = read_sysreg(mdscr_el1);
+
+ // POR_EL0 can affect uaccess, so must be saved/restored early.
+ if (ctxt_has_s1poe(ctxt))
+ ctxt_sys_reg(ctxt, POR_EL0) = read_sysreg_s(SYS_POR_EL0);
+}
+
+static inline void __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
+{
+ ctxt_sys_reg(ctxt, TPIDR_EL0) = read_sysreg(tpidr_el0);
+ ctxt_sys_reg(ctxt, TPIDRRO_EL0) = read_sysreg(tpidrro_el0);
+}
+
static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
{
ctxt_sys_reg(ctxt, SCTLR_EL1) = read_sysreg_el1(SYS_SCTLR);
--
2.47.3
^ permalink raw reply related
* [PATCH v13 08/32] KVM: arm64: Handle FEAT_IDST for guest accesses to hidden registers
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
In preparation for adding support for SMIDR_EL1 which is only available in
systems with SME factor out the FEAT_IDST injection from emulate-nested.c
into a helper and use it when handling hidden ID registers, ensuring that
we provide FEAT_IDST behaviour for them.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/kvm_emulate.h | 1 +
arch/arm64/kvm/emulate-nested.c | 6 +-----
arch/arm64/kvm/inject_fault.c | 15 +++++++++++++++
arch/arm64/kvm/sys_regs.c | 6 +++++-
4 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 5bf3d7e1d92c..994afbf479fc 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -45,6 +45,7 @@ bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
void kvm_skip_instr32(struct kvm_vcpu *vcpu);
void kvm_inject_undefined(struct kvm_vcpu *vcpu);
+void kvm_inject_undefined_idreg(struct kvm_vcpu *vcpu);
void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr);
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index e688bc5139c1..f49b7b311d09 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -2669,11 +2669,7 @@ bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
* helper for the purpose of dealing with FEAT_IDST.
*/
if (in_feat_id_space(¶ms)) {
- if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, IDS, IMP))
- kvm_inject_sync(vcpu, kvm_vcpu_get_esr(vcpu));
- else
- kvm_inject_undefined(vcpu);
-
+ kvm_inject_undefined_idreg(vcpu);
return true;
}
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 89982bd3345f..e2f519ca3045 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -331,6 +331,21 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu)
inject_undef64(vcpu);
}
+/**
+ * kvm_inject_undefined_idreg - emulate UnimplementedIDRegister() pseudocode
+ * @vcpu: The vCPU in which to inject the exception
+ *
+ * It is assumed that this code is called from the VCPU thread and that the
+ * VCPU therefore is not currently executing guest code.
+ */
+void kvm_inject_undefined_idreg(struct kvm_vcpu *vcpu)
+{
+ if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, IDS, IMP))
+ kvm_inject_sync(vcpu, kvm_vcpu_get_esr(vcpu));
+ else
+ kvm_inject_undefined(vcpu);
+}
+
static bool serror_is_masked(struct kvm_vcpu *vcpu)
{
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && !effective_sctlr2_nmea(vcpu);
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 5d5c579d4579..b352cd323e30 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -4759,7 +4759,11 @@ static void perform_access(struct kvm_vcpu *vcpu,
/* Check for regs disabled by runtime config */
if (sysreg_hidden(vcpu, r)) {
- kvm_inject_undefined(vcpu);
+ if (in_feat_id_space(params)) {
+ kvm_inject_undefined_idreg(vcpu);
+ } else {
+ kvm_inject_undefined(vcpu);
+ }
return;
}
--
2.47.3
^ permalink raw reply related
* [PATCH v13 07/32] KVM: arm64: Remove bitrotted comment on handle_sve()
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
The comment for handle_sve() says that we only call it if the guest does
not support SVE but it is now also used for nested guests which have
enabled traps at EL2. Just remove the comment.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kvm/handle_exit.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 54aedf93c78b..f674cd25d894 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -224,10 +224,6 @@ static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu)
return 1;
}
-/*
- * Guest access to SVE registers should be routed to this handler only
- * when the system doesn't support SVE.
- */
static int handle_sve(struct kvm_vcpu *vcpu)
{
if (guest_hyp_sve_traps_enabled(vcpu))
--
2.47.3
^ permalink raw reply related
* [PATCH v13 06/32] arm64/fpsimd: Determine maximum virtualisable SME vector length
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
As with SVE we can only virtualise SME vector lengths that are supported by
all CPUs in the system, implement similar checks to those for SVE. Since
unlike SVE there are no specific vector lengths that are architecturally
required the handling is subtly different, we report a system where this
happens with a maximum vector length of 0.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 314f5d61be11..7f2c44bd9fd5 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1281,6 +1281,7 @@ void __init sme_setup(void)
* minimum available VL will be used.
*/
set_sme_default_vl(find_supported_vector_length(ARM64_VEC_SME, 32));
+ info->max_virtualisable_vl = vec_virtualisable_vl(info);
pr_info("SME: minimum available vector length %u bytes per vector\n",
info->min_vl);
@@ -1288,6 +1289,10 @@ void __init sme_setup(void)
info->max_vl);
pr_info("SME: default vector length %u bytes per vector\n",
get_sme_default_vl());
+
+ /* KVM decides whether to support mismatched systems. Just warn here: */
+ if (info->max_virtualisable_vl < info->max_vl)
+ pr_warn("SME: unvirtualisable vector lengths present\n");
}
#endif /* CONFIG_ARM64_SME */
--
2.47.3
^ permalink raw reply related
* [PATCH v13 05/32] arm64/sve: Factor virtualizable VL discovery out of SVE specific code
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
In preparation for reuising it for SME pull the code for discovering the
maximum virtualizable vector length out of sve_setup() into a separate
function, updating to return 0 without a WARN_ON() for the case where no
virtualisable VL exists. For SVE this case is architecturally invalid
but for SME it is valid if ill advised.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 41 ++++++++++++++++++++++++-----------------
1 file changed, 24 insertions(+), 17 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index b5535f8c9bd9..314f5d61be11 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1118,6 +1118,29 @@ int vec_verify_vq_map(enum vec_type type)
return 0;
}
+static int __init vec_virtualisable_vl(struct vl_info *info)
+{
+ DECLARE_BITMAP(partial_only_map, SVE_VQ_MAX);
+ unsigned long b_min_partial, b_max_virt;
+
+ bitmap_andnot(partial_only_map, info->vq_partial_map, info->vq_map,
+ SVE_VQ_MAX);
+ b_min_partial = find_last_bit(partial_only_map, SVE_VQ_MAX);
+
+ /* All implemented VLs are virtualisable */
+ if (b_min_partial >= SVE_VQ_MAX)
+ return info->max_vl;
+
+ b_max_virt = find_next_bit(info->vq_map, SVE_VQ_MAX, b_min_partial);
+
+ /* No implemented VLs are virtualisable */
+ if (b_max_virt >= SVE_VQ_MAX)
+ return 0;
+
+ /* At least one virtualisable VL exists */
+ return sve_vl_from_vq(__bit_to_vq(b_max_virt));
+}
+
void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p)
{
write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
@@ -1129,8 +1152,6 @@ void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p)
void __init sve_setup(void)
{
struct vl_info *info = &vl_info[ARM64_VEC_SVE];
- DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
- unsigned long b;
int max_bit;
if (!system_supports_sve())
@@ -1153,21 +1174,7 @@ void __init sve_setup(void)
*/
set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64));
- bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map,
- SVE_VQ_MAX);
-
- b = find_last_bit(tmp_map, SVE_VQ_MAX);
- if (b >= SVE_VQ_MAX)
- /* No non-virtualisable VLs found */
- info->max_virtualisable_vl = SVE_VQ_MAX;
- else if (WARN_ON(b == SVE_VQ_MAX - 1))
- /* No virtualisable VLs? This is architecturally forbidden. */
- info->max_virtualisable_vl = SVE_VQ_MIN;
- else /* b + 1 < SVE_VQ_MAX */
- info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
-
- if (info->max_virtualisable_vl > info->max_vl)
- info->max_virtualisable_vl = info->max_vl;
+ info->max_virtualisable_vl = vec_virtualisable_vl(info);
pr_info("%s: maximum available vector length %u bytes per vector\n",
info->name, info->max_vl);
--
2.47.3
^ permalink raw reply related
* [PATCH v13 04/32] arm64/fpsimd: Decide to save ZT0 and streaming mode FFR at bind time
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
Some parts of the SME state are optional, enabled by additional features
on top of the base FEAT_SME and controlled with enable bits in SMCR_ELx. We
unconditionally enable these for the host but for KVM we will allow the
feature set exposed to guests to be restricted by the VMM. These are the
FFR register (FEAT_SME_FA64) and ZT0 (FEAT_SME2).
We defer saving of guest floating point state for non-protected guests to
the host kernel. We also want to avoid having to reconfigure the guest
floating point state if nothing used the floating point state while running
the host. If the guest was running with the optional features disabled then
traps will be enabled for them so the host kernel will need to skip
accessing that state when saving state for the guest.
Support this by moving the decision about saving this state to the point
where we bind floating point state to the CPU, instead of only storing
the SME VL to use we store the SMCR value. This includes all the enable
controls for the subfeatures along the vector length.
In order to keep the code paths for the vector extensions consistent
also adjust the SVE path to store a ZCR value instead of the VL, since
no fields other than LEN are currently defined for ZCR this is much less
of a meaningful change.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 4 ++--
arch/arm64/kernel/fpsimd.c | 16 ++++++++--------
arch/arm64/kvm/fpsimd.c | 3 ++-
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 59f3c4a9e390..395606e01e18 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -166,8 +166,8 @@ struct cpu_fp_state {
struct arm64_sme_state *sme_state;
u64 *svcr;
u64 *fpmr;
- unsigned int sve_vl;
- unsigned int sme_vl;
+ u64 smcr;
+ u64 zcr;
enum fp_type *fp_type;
enum fp_type to_save;
};
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index b868db2982ee..b5535f8c9bd9 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -474,7 +474,7 @@ static void fpsimd_save_user_state(void)
/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
bool save_sve_regs = false;
bool save_ffr;
- unsigned int vl;
+ unsigned int vq;
WARN_ON(!system_supports_fpsimd());
WARN_ON(preemptible());
@@ -498,7 +498,7 @@ static void fpsimd_save_user_state(void)
last->to_save == FP_STATE_SVE) {
save_sve_regs = true;
save_ffr = true;
- vl = last->sve_vl;
+ vq = SYS_FIELD_GET(ZCR_ELx, LEN, last->zcr) + 1;
}
if (system_supports_sme()) {
@@ -508,19 +508,19 @@ static void fpsimd_save_user_state(void)
if (*svcr & SVCR_ZA_MASK)
sme_save_state(last->sme_state,
- system_supports_sme2());
+ last->smcr & SMCR_ELx_EZT0);
/* If we are in streaming mode override regular SVE. */
if (*svcr & SVCR_SM_MASK) {
save_sve_regs = true;
- save_ffr = system_supports_fa64();
- vl = last->sme_vl;
+ save_ffr = last->smcr & SMCR_ELx_FA64;
+ vq = SYS_FIELD_GET(SMCR_ELx, LEN, last->smcr) + 1;
}
}
if (IS_ENABLED(CONFIG_ARM64_SVE) && save_sve_regs) {
/* Get the configured VL from RDVL, will account for SM */
- if (WARN_ON(sve_get_vl() != vl)) {
+ if (WARN_ON(sve_get_vl() != sve_vl_from_vq(vq))) {
/*
* Can't save the user regs, so current would
* re-enter user with corrupt state.
@@ -1712,8 +1712,8 @@ static void fpsimd_bind_task_to_cpu(void)
last->st = ¤t->thread.uw.fpsimd_state;
last->sve_state = current->thread.sve_state;
last->sme_state = current->thread.sme_state;
- last->sve_vl = task_get_sve_vl(current);
- last->sme_vl = task_get_sme_vl(current);
+ last->zcr = task_zcr(current);
+ last->smcr = task_smcr(current);
last->svcr = ¤t->thread.svcr;
last->fpmr = ¤t->thread.uw.fpmr;
last->fp_type = ¤t->thread.fp_type;
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index 3f6b1e29cd6b..567dd43970c5 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -89,7 +89,8 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
*/
fp_state.st = &vcpu->arch.ctxt.fp_regs;
fp_state.sve_state = vcpu->arch.sve_state;
- fp_state.sve_vl = vcpu->arch.sve_max_vl;
+ fp_state.zcr = vcpu_sve_max_vq(vcpu) - 1;
+ fp_state.smcr = 0;
fp_state.sme_state = NULL;
fp_state.svcr = __ctxt_sys_reg(&vcpu->arch.ctxt, SVCR);
fp_state.fpmr = __ctxt_sys_reg(&vcpu->arch.ctxt, FPMR);
--
2.47.3
^ permalink raw reply related
* [PATCH v13 03/32] arm64/fpsimd: Configure all ZCR/SMCR bits when loading task state
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
Currently, when loading a task's SVE and/or SME state, we configure
ZCR_ELx.LEN and/or SMCR_ELx.LEN with read-modify-write sequences,
preserving all other bits of the registers (e.g. SMCR_ELx.{FA64,ZT0}).
This relies on all the other bits already holding the expected value
for the task.
It would be simpler and more robust to configure all the bits without
a read-modify-write sequence:
- Doing so will remove the need to configure these registers during
feature detection and when returning from idle, simplifying the code
and removing some redundant writes to the registers.
- Doing so will permit KVM to clobber other bits of the registers when
no task state is bound. Along with other changes (e.g. when saving
state), this will make it possible for KVM to expose a different
configuration to guests (e.g. disabling ZT0 for vCPUs, even if ZT0
is exposed to userspace).
While SMCR_ELx.LEN and ZCR_ELx.LEN are self synchronizing SMCR_ELx.FA64
and SMCR_ELx.EZT0 are not so we add an isb() if those have changed.
This could be further optimised by considering if the SME configuration
of the loaded state means we will rely on the newly configured values
but for clarity this is not done here, it can be improved in future.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 2 -
arch/arm64/kernel/cpufeature.c | 2 -
arch/arm64/kernel/fpsimd.c | 84 +++++++++++++++++++----------------------
3 files changed, 38 insertions(+), 50 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index e13a85320ab6..59f3c4a9e390 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -360,8 +360,6 @@ struct arm64_cpu_capabilities;
extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
-extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
-extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
extern void fpsimd_suspend_exit(void);
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9a22df0c5120..0609dce1989e 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2992,7 +2992,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.capability = ARM64_SME_FA64,
.matches = has_cpuid_feature,
- .cpu_enable = cpu_enable_fa64,
ARM64_CPUID_FIELDS(ID_AA64SMFR0_EL1, FA64, IMP)
},
{
@@ -3000,7 +2999,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.capability = ARM64_SME2,
.matches = has_cpuid_feature,
- .cpu_enable = cpu_enable_sme2,
ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, SME2)
},
#endif /* CONFIG_ARM64_SME */
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 78c9d8dab545..b868db2982ee 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -277,6 +277,27 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
task->thread.vl_onexec[type] = vl;
}
+static unsigned long task_zcr(const struct task_struct *task)
+{
+ unsigned long vq = sve_vq_from_vl(task_get_sve_vl(task));
+ unsigned long zcr = vq - 1;
+
+ return zcr;
+}
+
+static unsigned long task_smcr(const struct task_struct *task)
+{
+ unsigned long vq = sve_vq_from_vl(task_get_sme_vl(task));
+ unsigned long smcr = vq - 1;
+
+ if (system_supports_fa64())
+ smcr |= SMCR_ELx_FA64;
+ if (system_supports_sme2())
+ smcr |= SMCR_ELx_EZT0;
+
+ return smcr;
+}
+
/*
* TIF_SME controls whether a task can use SME without trapping while
* in userspace, when TIF_SME is set then we must have storage
@@ -377,10 +398,8 @@ static void task_fpsimd_load(void)
if (!thread_sm_enabled(¤t->thread))
WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE));
- if (test_thread_flag(TIF_SVE)) {
- unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
- sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
- }
+ if (system_supports_sve())
+ sysreg_cond_update_s(SYS_ZCR_EL1, task_zcr(current));
restore_sve_regs = true;
restore_ffr = true;
@@ -402,12 +421,17 @@ static void task_fpsimd_load(void)
/* Restore SME, override SVE register configuration if needed */
if (system_supports_sme()) {
- unsigned long sme_vl = task_get_sme_vl(current);
-
- /* Ensure VL is set up for restoring data */
if (test_thread_flag(TIF_SME)) {
- unsigned long vq = sve_vq_from_vl(sme_vl);
- sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1);
+ u64 old_smcr = read_sysreg_s(SYS_SMCR_EL1);
+ u64 new_smcr = task_smcr(current);
+
+ sysreg_cond_update_s(SYS_SMCR_EL1, new_smcr);
+
+ /* FA64 and EZT0 are not self synchronising */
+ old_smcr &= SMCR_ELx_FA64 | SMCR_ELx_EZT0;
+ new_smcr &= SMCR_ELx_FA64 | SMCR_ELx_EZT0;
+ if (old_smcr != new_smcr)
+ isb();
}
write_sysreg_s(current->thread.svcr, SYS_SVCR);
@@ -1217,26 +1241,6 @@ void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p)
isb();
}
-void cpu_enable_sme2(const struct arm64_cpu_capabilities *__always_unused p)
-{
- /* This must be enabled after SME */
- BUILD_BUG_ON(ARM64_SME2 <= ARM64_SME);
-
- /* Allow use of ZT0 */
- write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_EZT0_MASK,
- SYS_SMCR_EL1);
-}
-
-void cpu_enable_fa64(const struct arm64_cpu_capabilities *__always_unused p)
-{
- /* This must be enabled after SME */
- BUILD_BUG_ON(ARM64_SME_FA64 <= ARM64_SME);
-
- /* Allow use of FA64 */
- write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_FA64_MASK,
- SYS_SMCR_EL1);
-}
-
void __init sme_setup(void)
{
struct vl_info *info = &vl_info[ARM64_VEC_SME];
@@ -1283,20 +1287,10 @@ void __init sme_setup(void)
void fpsimd_suspend_exit(void)
{
- u64 smcr = 0;
-
- if (system_supports_sve())
- write_sysreg_s(0, SYS_ZCR_EL1);
-
- if (system_supports_sme()) {
- if (system_supports_fa64())
- smcr |= SMCR_ELx_FA64;
- if (system_supports_sme2())
- smcr |= SMCR_ELx_EZT0;
+ if (!system_supports_sme())
+ return;
- write_sysreg_s(smcr, SYS_SMCR_EL1);
- write_sysreg_s(0, SYS_SMPRI_EL1);
- }
+ write_sysreg_s(0, SYS_SMPRI_EL1);
}
/*
@@ -1338,8 +1332,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs)
* any effective streaming mode SVE state.
*/
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
- unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
- sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
+ sysreg_cond_update_s(SYS_ZCR_EL1, task_zcr(current));
sve_flush_live();
fpsimd_bind_task_to_cpu();
} else {
@@ -1474,8 +1467,7 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs)
WARN_ON(1);
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
- unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current));
- sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1);
+ sysreg_cond_update_s(SYS_SMCR_EL1, task_smcr(current));
fpsimd_bind_task_to_cpu();
} else {
--
2.47.3
^ permalink raw reply related
* [PATCH v13 02/32] arm64/fpsimd: Ensure all of ZCR_EL1 is initialised from idle
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
At present when exiting from idle we do not fully reinitialise ZCR_EL1,
we update ZCR_EL1.LEN with a read/modify/write cycle when loading task
state but never set any of the other bits to an explicit value. Since
currently they are all architecturally RES0 or RAZ/WI this is not a
practical issue but it may become one if further fields are defined in
the register so we should explicitly configure the whole register.
Rename the existing sme_suspend_exit() (which handles this for SME) to
fpsimd_suspend_exit() and add set ZCR_EL1 to 0 there, if needed LEN will
be updated when loading task state.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 4 ++--
arch/arm64/kernel/fpsimd.c | 24 +++++++++++++-----------
arch/arm64/kernel/suspend.c | 2 +-
3 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index a67d5774e672..e13a85320ab6 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -364,6 +364,8 @@ extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
+extern void fpsimd_suspend_exit(void);
+
/*
* Helpers to translate bit indices in sve_vq_map to VQ values (and
* vice versa). This allows find_next_bit() to be used to find the
@@ -623,7 +625,6 @@ static inline unsigned int sme_get_vl(void)
extern void sme_alloc(struct task_struct *task, bool flush);
extern int sme_set_current_vl(unsigned long arg);
extern int sme_get_current_vl(void);
-extern void sme_suspend_exit(void);
static inline size_t __sme_state_size(unsigned int sme_vl)
{
@@ -779,7 +780,6 @@ static inline int sme_max_vl(void) { return 0; }
static inline int sme_max_virtualisable_vl(void) { return 0; }
static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
static inline int sme_get_current_vl(void) { return -EINVAL; }
-static inline void sme_suspend_exit(void) { }
static inline size_t __sme_state_size(unsigned int sme_vl)
{
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index e7f1682a3059..78c9d8dab545 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1279,24 +1279,26 @@ void __init sme_setup(void)
get_sme_default_vl());
}
-void sme_suspend_exit(void)
+#endif /* CONFIG_ARM64_SME */
+
+void fpsimd_suspend_exit(void)
{
u64 smcr = 0;
- if (!system_supports_sme())
- return;
+ if (system_supports_sve())
+ write_sysreg_s(0, SYS_ZCR_EL1);
- if (system_supports_fa64())
- smcr |= SMCR_ELx_FA64;
- if (system_supports_sme2())
- smcr |= SMCR_ELx_EZT0;
+ if (system_supports_sme()) {
+ if (system_supports_fa64())
+ smcr |= SMCR_ELx_FA64;
+ if (system_supports_sme2())
+ smcr |= SMCR_ELx_EZT0;
- write_sysreg_s(smcr, SYS_SMCR_EL1);
- write_sysreg_s(0, SYS_SMPRI_EL1);
+ write_sysreg_s(smcr, SYS_SMCR_EL1);
+ write_sysreg_s(0, SYS_SMPRI_EL1);
+ }
}
-#endif /* CONFIG_ARM64_SME */
-
/*
* Trapped SVE access
*
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index eaaff94329cd..e02728d14e0d 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -81,7 +81,7 @@ void notrace __cpu_suspend_exit(void)
*/
spectre_v4_enable_mitigation(NULL);
- sme_suspend_exit();
+ fpsimd_suspend_exit();
/* Restore additional feature-specific configuration */
ptrauth_suspend_exit();
--
2.47.3
^ permalink raw reply related
* [PATCH v13 01/32] arm64/sysreg: Define full value read/modify/write helpers
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
In-Reply-To: <20260720-kvm-arm64-sme-v13-0-d9abd3ffa245@kernel.org>
We have read/modify write helpers for updating bitfields in sysregs if they
have changed but we do not have them for updating the whole register.
Define sysreg_cond_update() for that.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7aa08d59d494..4b96449e0ffa 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -1235,6 +1235,12 @@
write_sysreg_s(__scs_new, sysreg); \
} while (0)
+#define sysreg_cond_update(sysreg, val) \
+ sysreg_clear_set(sysreg, ~0UL, val)
+
+#define sysreg_cond_update_s(sysreg, val) \
+ sysreg_clear_set_s(sysreg, ~0UL, val)
+
#define write_sysreg_hcr(__val) do { \
if (IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23) && \
(!system_capabilities_finalized() || \
--
2.47.3
^ permalink raw reply related
* [PATCH v13 00/32] KVM: arm64: Implement support for SME
From: Mark Brown @ 2026-07-19 23:07 UTC (permalink / raw)
To: Marc Zyngier, Joey Gouly, Catalin Marinas, Suzuki K Poulose,
Will Deacon, Paolo Bonzini, Jonathan Corbet, Shuah Khan,
Oliver Upton
Cc: Dave Martin, Fuad Tabba, Mark Rutland, Ben Horgan,
Jean-Philippe Brucker, linux-arm-kernel, kvmarm, linux-kernel,
kvm, linux-doc, linux-kselftest, Peter Maydell, Eric Auger,
Mark Brown
[ Note: I've spoken with both Mark Rutland and James Morse both of whom
might potentially have some overlapping changes, they've both agreed
that they'll deal with any rebasing if it's a question of this getting
in or something else. ]
I've removed the RFC tag from this version of the series, but the items
that I'm looking for feedback on remains the same:
- The userspace ABI, in particular:
- The vector length used for the SVE registers, access to the SVE
registers and access to ZA and (if available) ZT0 depending on
the current state of PSTATE.{SM,ZA}.
- The use of a single finalisation for both SVE and SME.
- There was some review from the qemu people who seemed OK with the
structure, and there's some initial patches from them at:
https://patchew.org/QEMU/20260216034432.23912-1-richard.henderson@linaro.org/
This series implements support for SME use in non-protected KVM guests.
Much of this is very similar to SVE, the main additional challenge that
SME presents is that it introduces a new vector length similar to the
SVE vector length and two new controls which change the registers seen
by guests:
- PSTATE.ZA enables the ZA matrix register and, if SME2 is supported,
the ZT0 LUT register.
- PSTATE.SM enables streaming mode, a new floating point mode which
uses the SVE register set with the separately configured SME vector
length. In streaming mode implementation of the FFR register is
optional.
It is also permitted to build systems which support SME without SVE, in
this case when not in streaming mode no SVE registers or instructions
are available. Further, there is no requirement that there be any
overlap in the set of vector lengths supported by SVE and SME in a
system, this is expected to be a common situation in practical systems.
Since there is a new vector length to configure we introduce a new
feature parallel to the existing SVE one with a new pseudo register for
the streaming mode vector length. Due to the overlap with SVE caused by
streaming mode rather than finalising SME as a separate feature we use
the existing SVE finalisation to also finalise SME, a new define
KVM_ARM_VCPU_VEC is provided to help make user code clearer. Finalising
SVE and SME separately would introduce complication with register access
since finalising SVE makes the SVE registers writeable by userspace and
doing multiple finalisations results in an error being reported.
Dealing with a state where the SVE registers are writeable due to one of
SVE or SME being finalised but may have their VL changed by the other
being finalised seems like needless complexity with minimal practical
utility, it seems clearer to just express directly that only one
finalisation can be done in the ABI.
Access to the floating point registers follows the architecture:
- When both SVE and SME are present:
- If PSTATE.SM == 0 the vector length used for the Z and P registers
is the SVE vector length.
- If PSTATE.SM == 1 the vector length used for the Z and P registers
is the SME vector length.
- If only SME is present:
- If PSTATE.SM == 0 the Z and P registers are inaccessible and the
floating point state accessed via the encodings for the V registers.
- If PSTATE.SM == 1 the vector length used for the Z and P registers
- The SME specific ZA and ZT0 registers are only accessible if SVCR.ZA is 1.
The VMM must understand this, in particular when loading state SVCR
should be configured before other state. It should be noted that while
the architecture refers to PSTATE.SM and PSTATE.ZA these PSTATE bits are
not preserved in SPSR_ELx, they are only accessible via SVCR.
There are a large number of subfeatures for SME, most of which only
offer additional instructions but some of which (SME2 and FA64) add
architectural state. These are configured via the ID registers as per
usual.
Protected KVM is supported, with the implementation maintaining the
existing restriction that the hypervisor will refuse to run if streaming
mode or ZA is enabled. This both simplfies the code and avoids the need
to allocate storage for host ZA and ZT0 state, there seems to be little
practical use case for supporting this and the memory usage would be
non-trivial. SME is not made available to protected guests, only normal
guests.
The new KVM_ARM_VCPU_VEC feature and ZA and ZT0 registers have not been
added to the get-reg-list selftest, the idea of supporting additional
features there without restructuring the program to generate all
possible feature combinations has been rejected. I will post a separate
series which does that restructuring.
There is some room for optimisation here, for example we could do more
work to suppress isb()s on SMCR changes (we don't really try in the
hypervisor), but this is already a very large series.
Signed-off-by: Mark Brown <broonie@kernel.org>
---
Changes in v13:
- Rebase onto v7.2-rc3.
- Remove existing special casing of ZCR_EL2 as an access trap and the
corresponding newly added SMCR_EL2 handling.
- Fix handling of nested EZT0/FA64, factoring out shared initialisation
for both SMCR_EL2 and nested EL2 traps.
- Document why we isb() after updating SMCR, and suppress redundant
isb()s for the host kernel.
- Add missing isb()s in the hypervisor.
- Use generic RES0 handling for SMPRIMAP_EL2.
- Fix comment on SMPRI_EL1.
- Emulate SCMR_EL2.EZT0 traps for nested guests.
- Remove a bitrotted comment on the SVE exit handler.
- Don't restore SMCR_EL1 after running non-SME guests with pKVM.
- Add an explicit reset of ZCR_EL1.
- Add a missing __init annotation to vec_virtualisable_vl().
- Add SMIDR_EL1 to the ABI documentation for KVM_CAP_ARM_WRITABLE_IMP_ID_REGS.
- Block attempts to set ID_AA64PFR1_EL1.SME to 0 when SME is enabled.
- Process vCPU resets before enumerating registers.
- Changelog updates.
- Link to v12: https://patch.msgid.link/20260709-kvm-arm64-sme-v12-0-d0301d79ef58@kernel.org
Changes in v12:
- Resend with commit rather than tag object specified as the base.
- Link to v11: https://patch.msgid.link/20260709-kvm-arm64-sme-v11-0-32799f66db9d@kernel.org
Changes in v11:
- Rebase onto v7.2-rc2.
- Rework VL enumeration.
- Refactor ZCR and SMCR value generation for the host into helper
functions.
- Add handling of FEAT_IDST for hidden system registers.
- Ensure we manage traps correctly for emulated EL2.
- Always allocate space for ZT0 if the hardware supports it.
- Tighten the register enumeration ABI to reflect current VM state.
- Sanitise SMIDR_EL1.
- Fix generation of invalid values in set_id_regs.
- Various smaller fixups, including from Jean-Phillipe's review.
- Link to v10: https://patch.msgid.link/20260306-kvm-arm64-sme-v10-0-43f7683a0fb7@kernel.org
Changes in v10:
- Define and use a SME_VQ_INVALID for the case where there is no
virtuablisable SME VL.
- Fix handling of SMCR_EL2 accesses.
- Correct VNCR constant for SMPRI_EL2.
- Correct trapping for SMPRI_EL1.
- Reject userspace access to FFR when in streaming mode without FA64.
- Constrain the VL set by sme_cond_update_smcr() to fit within LEN.
- Reject userspace access to ZA and ZT0 when SVCR.SM is 0.
- Use -EACCESS for inaccessible SME registers.
- Remove some unused functions.
- Further bugfixes from review.
- Commit log typo fixes.
- Link to v9: https://patch.msgid.link/20251223-kvm-arm64-sme-v9-0-8be3867cb883@kernel.org
Changes in v9:
- Rebase onto v6.19-rc1.
- ABI document clarifications.
- Add changes dropping asserts on single bit wide bitfields in set_id_regs.
- Link to v8: https://lore.kernel.org/r/20250902-kvm-arm64-sme-v8-0-2cb2199c656c@kernel.org
Changes in v8:
- Small fixes in ABI documentation.
- Link to v7: https://lore.kernel.org/r/20250822-kvm-arm64-sme-v7-0-7a65d82b8b10@kernel.org
Changes in v7:
- Rebase onto v6.17-rc1.
- Handle SMIDR_EL1 as a VM wide ID register and use this in feat_sme_smps().
- Expose affinity fields in SMIDR_EL1.
- Remove SMPRI_EL1 from vcpu_sysreg, the value is always 0 currently.
- Prevent userspace writes to SMPRIMAP_EL2.
- Link to v6: https://lore.kernel.org/r/20250625-kvm-arm64-sme-v6-0-114cff4ffe04@kernel.org
Changes in v6:
- Rebase onto v6.16-rc3.
- Link to v5: https://lore.kernel.org/r/20250417-kvm-arm64-sme-v5-0-f469a2d5f574@kernel.org
Changes in v5:
- Rebase onto v6.15-rc2.
- Add pKVM guest support.
- Always restore SVCR.
- Link to v4: https://lore.kernel.org/r/20250214-kvm-arm64-sme-v4-0-d64a681adcc2@kernel.org
Changes in v4:
- Rebase onto v6.14-rc2 and Mark Rutland's fixes.
- Expose SME to nested guests.
- Additional cleanups and test fixes following on from the rebase.
- Flush register state on VMM PSTATE.{SM,ZA}.
- Link to v3: https://lore.kernel.org/r/20241220-kvm-arm64-sme-v3-0-05b018c1ffeb@kernel.org
Changes in v3:
- Rebase onto v6.12-rc2.
- Link to v2: https://lore.kernel.org/r/20231222-kvm-arm64-sme-v2-0-da226cb180bb@kernel.org
Changes in v2:
- Rebase onto v6.7-rc3.
- Configure subfeatures based on host system only.
- Complete nVHE support.
- There was some snafu with sending v1 out, it didn't make it to the
lists but in case it hit people's inboxes I'm sending as v2.
---
Mark Brown (32):
arm64/sysreg: Define full value read/modify/write helpers
arm64/fpsimd: Ensure all of ZCR_EL1 is initialised from idle
arm64/fpsimd: Configure all ZCR/SMCR bits when loading task state
arm64/fpsimd: Decide to save ZT0 and streaming mode FFR at bind time
arm64/sve: Factor virtualizable VL discovery out of SVE specific code
arm64/fpsimd: Determine maximum virtualisable SME vector length
KVM: arm64: Remove bitrotted comment on handle_sve()
KVM: arm64: Handle FEAT_IDST for guest accesses to hidden registers
KVM: arm64: Pull ctxt_has_ helpers to start of sysreg-sr.h
KVM: arm64: Rename SVE finalization constants to be more general
KVM: arm64: Remove special case for FP state loading from ZCR_EL2 traps
KVM: arm64: Define internal features for SME
KVM: arm64: Rename sve_state_reg_region
KVM: arm64: Store vector lengths in an array
KVM: arm64: Factor SVE code out of fpsimd_lazy_switch_to_host()
KVM: arm64: Document the KVM ABI for SME
KVM: arm64: Implement SME vector length configuration
KVM: arm64: Support SME control registers
KVM: arm64: Support TPIDR2_EL0
KVM: arm64: Support SME identification registers for guests
KVM: arm64: Support SME priority registers
KVM: arm64: Support userspace access to streaming mode Z and P registers
KVM: arm64: Flush register state on writes to SVCR.SM and SVCR.ZA
KVM: arm64: Expose SME specific state to userspace
KVM: arm64: Context switch SME state for guests
KVM: arm64: Handle SME exceptions
KVM: arm64: Expose SME to nested guests
KVM: arm64: Provide interface for configuring and enabling SME for guests
KVM: arm64: selftests: Remove spurious check for single bit safe values
KVM: arm64: selftests: Skip impossible invalid value tests
KVM: arm64: selftests: Add SME system registers to get-reg-list
KVM: arm64: selftests: Add SME to set_id_regs test
Documentation/virt/kvm/api.rst | 127 +++++---
arch/arm64/include/asm/fpsimd.h | 11 +-
arch/arm64/include/asm/kvm_emulate.h | 16 +
arch/arm64/include/asm/kvm_host.h | 117 ++++++-
arch/arm64/include/asm/kvm_hyp.h | 2 +-
arch/arm64/include/asm/kvm_nested.h | 2 +
arch/arm64/include/asm/kvm_pkvm.h | 2 +-
arch/arm64/include/asm/sysreg.h | 8 +
arch/arm64/include/asm/vncr_mapping.h | 2 +
arch/arm64/include/uapi/asm/kvm.h | 34 +++
arch/arm64/kernel/cpufeature.c | 2 -
arch/arm64/kernel/fpsimd.c | 144 ++++-----
arch/arm64/kernel/suspend.c | 2 +-
arch/arm64/kvm/arm.c | 19 ++
arch/arm64/kvm/config.c | 13 +-
arch/arm64/kvm/emulate-nested.c | 6 +-
arch/arm64/kvm/fpsimd.c | 27 +-
arch/arm64/kvm/guest.c | 369 ++++++++++++++++++++---
arch/arm64/kvm/handle_exit.c | 23 +-
arch/arm64/kvm/hyp/include/hyp/switch.h | 262 +++++++++++++---
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 96 +++---
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 133 ++++++--
arch/arm64/kvm/hyp/nvhe/pkvm.c | 90 ++++--
arch/arm64/kvm/hyp/nvhe/switch.c | 2 +
arch/arm64/kvm/hyp/nvhe/sys_regs.c | 6 +
arch/arm64/kvm/hyp/vhe/switch.c | 28 +-
arch/arm64/kvm/hyp/vhe/sysreg-sr.c | 7 +
arch/arm64/kvm/inject_fault.c | 15 +
arch/arm64/kvm/nested.c | 15 +-
arch/arm64/kvm/reset.c | 159 +++++++---
arch/arm64/kvm/sys_regs.c | 155 +++++++++-
include/uapi/linux/kvm.h | 1 +
tools/testing/selftests/kvm/arm64/get-reg-list.c | 15 +-
tools/testing/selftests/kvm/arm64/set_id_regs.c | 96 +++++-
34 files changed, 1602 insertions(+), 404 deletions(-)
---
base-commit: a13c140cc289c0b7b3770bce5b3ad42ab35074aa
change-id: 20230301-kvm-arm64-sme-06a1246d3636
Best regards,
--
Mark Brown <broonie@kernel.org>
^ permalink raw reply
* Re: [PATCH] Documentation: wmi: lenovo-wmi-other: Document intentional BIOS misspelling
From: Weijie Yuan @ 2026-07-19 22:54 UTC (permalink / raw)
To: Yahya Toubali
Cc: Jonathan Corbet, mpearson-lenovo, linux-doc, platform-driver-x86,
linux-kernel
In-Reply-To: <20260719224317.2450320-1-yahya@yahyatoubali.me>
On Sun, Jul 19, 2026 at 11:42:46PM +0100, Yahya Toubali wrote:
> Hi Jon,
>
> Yes, I used an LLM to help me format the email text because I am still
> learning how the mailing list process works.
I suspect it's not just the format.
> I will work on my next patches to make sure they are well-structured
> and follow the proper git mailing list standards.
>
> Thanks for the patience and the explanation,
> Yahya
Friendly reminder ;-)
You may get started by wrapping your email to 72 columns first.
Thanks.
^ permalink raw reply
* Re: [PATCH] Documentation: wmi: lenovo-wmi-other: Document intentional BIOS misspelling
From: Yahya Toubali @ 2026-07-19 22:42 UTC (permalink / raw)
To: Jonathan Corbet
Cc: Yahya Toubali, Weijie Yuan, mpearson-lenovo, linux-doc,
platform-driver-x86, linux-kernel
In-Reply-To: <87o6g250ya.fsf@trenco.lwn.net>
Hi Jon,
Yes, I used an LLM to help me format the email text because I am still learning how the mailing list process works.
I will work on my next patches to make sure they are well-structured and follow the proper git mailing list standards.
Thanks for the patience and the explanation,
Yahya
^ permalink raw reply
* Re: [RFC PATCH 0/7] mm: defer address-space teardown of large exiting processes to a kthread
From: Liam R . Howlett @ 2026-07-19 22:28 UTC (permalink / raw)
To: Aditya Sharma
Cc: Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Vlastimil Babka, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
David Rientjes, Shakeel Butt, Jonathan Corbet, Shuah Khan,
Steven Rostedt, Masami Hiramatsu, Mathieu Desnoyers, Kees Cook,
Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Ben Segall, Mel Gorman, Valentin Schneider,
K Prateek Nayak, linux-mm, linux-doc, linux-trace-kernel,
linux-kernel, imbrenda
In-Reply-To: <20260719185409.409685-1-adi.sharma@zohomail.in>
On 26/07/20 12:24AM, Aditya Sharma wrote:
> Address-space teardown on process exit runs synchronously in the dying
> task's context: exit_mm() -> mmput() -> __mmput() -> exit_mmap() walks
> page tables, updates rmap, frees the RSS and drops file references, all
> on the exiting CPU. For a multi-GB process that is hundreds of
> milliseconds of exit-path latency, paid by whoever is waiting on the
> death: a supervisor's kill-and-respawn cycle, a shell's waitpid(), an
> orchestrator reaping a fleet of workers. On the test box below, killing
> a 16GB process costs ~280ms before the parent's waitpid() returns.
>
> This series adds CONFIG_ASYNC_MM_TEARDOWN: an opt-in, default-off path
> that defers __mmput() of large exiting processes to a dedicated kernel
> thread (mm_reaper), so the exiting CPU is released as soon as the task
> is reaped and the teardown runs off to the side.
This reads like it was written by an AI, but there is no assisted-by
tags. Which AI did you use to generate these patches and/or cover
letter?
I do not see a benefit to running the task cleanup 'off the side'. In
modern computers, that may mean switching core types that run at
different speeds.
Regardless of speed, the CPU that will be doing the work is remote to
the workload by design, (or may be, depending on the scheduling?). That
is, you are tasking another CPU to do cleanup of local CPU-aware
information.. The ideal CPU to do a task exit is the one that's doing it
already.
There are hidden costs to your approach and I don't really understand
how this could possibly speed things up - even parallelized tasks will
take more CPU time in total, even if locking issues are avoided.
Sure, the task waiting for cleanup to complete may think it's done and
continue to do the Next Thing - but if it really needs the cleanup to be
completed then you've just made it impossible to know when it has
happened.
Also, you've just made this extremely hard to debug if something is
missed.
The work that is done in teardown is necessary. Your time (and tokens?)
would be better spent trying to improve what we have to do instead of
shifting the work around.
Thanks,
Liam
^ permalink raw reply
* Re: [PATCH v7 2/3] iommu/arm-smmu-v3: Introduce CFGI/TLBI-repeat workaround infrastructure
From: Nicolin Chen @ 2026-07-19 20:37 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: Ashish Mhetre, catalin.marinas, will, corbet, skhan, robin.murphy,
joro, linux-arm-kernel, linux-doc, linux-kernel, iommu,
linux-tegra
In-Reply-To: <20260718164548.GI701389@ziepe.ca>
On Sat, Jul 18, 2026 at 01:45:48PM -0300, Jason Gunthorpe wrote:
> On Fri, Jul 17, 2026 at 02:16:53PM -0700, Nicolin Chen wrote:
> > Given the use cases on Tegra264, instead of patching the iommufd
> > path as this patch does, perhaps we should simply spit a WARN in
> > arm_vsmmu_init():
>
> Nope, in this case you need to make the errata discoverable through
> the viommu info so the VMM can validate the right DT is used, or apply
> the WAR itself before submitting commands to the kernel.
OK.
> Some kind of new flag in the viommu struct I guess
struct iommu_hw_info_arm_smmuv3 has an unused "flags" field:
/**
* enum iommu_hw_info_smmuv3_flags - Flags for ARM SMMUv3 hw_info
* @IOMMU_HW_INFO_SMMUV3_ERRATA_REPEAT_TLBI_CFGI: If set, user space must issue
* TLBI/CFGI+SYNC commands twice
* due to a hardware erratum:
* <link-to-erratum>
*/
enum iommu_hw_info_smmuv3_flags {
IOMMU_HW_INFO_SMMUV3_ERRATA_REPEAT_TLBI_CFGI = 1 << 0,
};
Thanks
Nicolin
^ permalink raw reply
* [groeck-staging:hwmon-next 51/58] htmldocs: Documentation/hwmon/mpq82d00.rst:4: WARNING: Title underline too short.
From: kernel test robot @ 2026-07-19 20:14 UTC (permalink / raw)
To: Wensheng Wang; +Cc: oe-kbuild-all, Guenter Roeck, linux-doc
tree: https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-next
head: 428f63b8f73dd48efa8fa17330d8769ad765afd9
commit: cac7a93ef369c067263a2a7090b68177336c8fad [51/58] hwmon: add MPQ82D00 driver
compiler: clang version 22.1.8 (https://github.com/llvm/llvm-project ca7933e47d3a3451d81e72ac174dcb5aa28b59d1)
docutils: docutils (Docutils 0.21.2, Python 3.13.5, on linux)
reproduce: (https://download.01.org/0day-ci/archive/20260719/202607192203.4qtHWNRY-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202607192203.4qtHWNRY-lkp@intel.com/
All warnings (new ones prefixed by >>):
Runtime Survivability
===================== [docutils]
>> Documentation/hwmon/mpq82d00.rst:4: WARNING: Title underline too short.
vim +4 Documentation/hwmon/mpq82d00.rst
2
3 Kernel driver mpq82d00
> 4 ====================
5
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* [RFC PATCH 3/7] mm/oom_kill: mark an OOM victim's mm with MMF_OOM_TARGETED
From: Aditya Sharma @ 2026-07-19 18:54 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko
Cc: David Rientjes, Shakeel Butt, Jonathan Corbet, Shuah Khan,
Steven Rostedt, Masami Hiramatsu, Mathieu Desnoyers, Kees Cook,
Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Ben Segall, Mel Gorman, Valentin Schneider,
K Prateek Nayak, linux-mm, linux-doc, linux-trace-kernel,
linux-kernel, imbrenda, Aditya Sharma
In-Reply-To: <20260719185409.409685-1-adi.sharma@zohomail.in>
A later part of this series defers the address-space teardown of large
exiting processes to a kernel thread (mm_reaper) instead of running it
inline on the exiting CPU. That deferral must never be applied to an
OOM victim: its memory has to be released promptly to relieve the
pressure that triggered the kill, and queuing it behind the
low-priority, housekeeping-CPU-confined mm_reaper would work directly
against the oom_reaper. The teardown path therefore needs a cheap
"is this mm an OOM victim?" test.
The eventual consumer runs from mmput_exit() on the exit path, after
exit_mm() has already cleared current->mm, so the only handle it has
is the mm pointer itself. Existing OOM state does not answer the
question well from an mm alone:
- signal->oom_mm and TIF_MEMDIE are per-task / per-signal and are
set only on the single task passed to mark_oom_victim(). A sibling
thread, or a CLONE_VM-sharing process in another thread group
(which __oom_kill_process() SIGKILLs but never marks), can be the
task that drops the last mm_users reference and runs the teardown.
A per-task check misses those.
- MMF_OOM_SKIP is mm-scoped but does not mean "OOM victim".
exit_mmap() sets it on every exiting mm once the memory is freed,
and dup_mmap() sets it on the fork-failure cleanup path, so it
produces false positives on ordinary exits. It is also not reliably
early: on a real victim it is normally set only once exit_mmap() or
oom_reap_task_mm() has already released the memory -- the exception
being the mm-pinned-by-init case in __oom_kill_process(), which sets
it at kill time with RSS fully intact. Wrong in both directions, so
it cannot serve as the predicate this path needs.
Add MMF_OOM_TARGETED, an mm-scoped flag, and set it in two places:
- in mark_oom_victim(), covering every caller uniformly (this is
the only marking for the two task_will_free_mem() fast paths in
oom_kill_process() and out_of_memory());
- early in __oom_kill_process(), before any SIGKILL for the kill
event is dispatched, to victim's own thread group or to other
thread groups sharing the mm.
Observing this flag is not load-bearing for correctness. Should it
ever be missed, the consequence is bounded: the victim's mm is queued
to mm_reaper instead of being torn down inline, i.e. a latency and
priority-inversion regression against the oom_reaper -- not
corruption, use-after-free or a leak. The mm is still fully torn down
and the oom_reaper still reaps the victim independently. The
visibility argument below is therefore belt-and-suspenders.
Ordering and coverage: this flag is reliably visible to whichever
thread ends up dropping the last reference in mmput_exit(), because
marking and queuing for async teardown can never overlap in time.
Every mark site runs while some task's ->mm still points at this mm --
either tsk is current, marking itself in program order (the
task_will_free_mem() fast paths), or tsk is task_lock()ed by the
caller (oom_kill_process()'s fast path, and __oom_kill_process() via
find_lock_task_mm()) -- while mmput_exit(), the only path that queues
for teardown, is reached only after mm_users hits 0, which requires
that same task's own exit_mm() (or exec_mmap(), the other path that
sheds ->mm) to have already cleared ->mm under that same task_lock().
Program order, or task_lock() release/acquire, therefore orders the
flag store before that task's own eventual mmput_exit(). This is stated
for userspace tasks: any userspace task's ->mm holds an mm_users
reference, whereas kthread_use_mm() borrows an mm under mmgrab() alone
-- kthreads are not signal-killable and are never OOM victims, so the
borrowed-mm case does not arise here.
If a different CLONE_VM sharer in another thread group performs the
actual final decrement instead -- a task mark_oom_victim() never marks
directly -- the ordering still carries, though not from the flag store
itself. mm_flags_set() is set_bit(), a non-value-returning RMW, so it
is unordered and contributes nothing on its own. The ordering comes
from the marking task's own atomic_dec_and_test(), a value-returning
RMW and therefore fully ordered (Documentation/atomic_t.txt: an
smp_mb() before and after). Decrements are totally ordered in
mm_users' modification order with the zeroing one last, so the general
barriers on both sides make the flag store visible to whichever thread
performs it.
The redundant set in mark_oom_victim() on the __oom_kill_process()
path is harmless and keeps the invariant "mark_oom_victim() implies
MMF_OOM_TARGETED" independent of future code motion.
The flag is intentionally sticky: it is never cleared. That is fine
because a marked mm is dying: __oom_kill_process() SIGKILLs every
process sharing the victim's mm -- only global init (whose mm also
gets MMF_OOM_SKIP set on the spot, keeping it off the async path
anyway) and kthreads are exempt from the sweep, and a kthread's
kthread_unuse_mm() drop is a plain mmput(), never mmput_exit(). So
no task that could reach the async path survives owning a marked mm.
Should some future path leave a survivor, the consequence is only
conservative: that mm keeps tearing down synchronously. The flag is
structurally outside legacy-mask inheritance: mm_init() clears the
whole flags bitmap and re-seeds only word 0 from
MMF_INIT_LEGACY_MASK, so flags above bit 31 are never copied on
fork(). NUM_MM_FLAG_BITS is a fixed 64 on all architectures, so bit 32
exists on 32-bit builds as well.
This commit only defines and sets the flag; nothing reads it yet. The
write is gated by IS_ENABLED(CONFIG_ASYNC_MM_TEARDOWN) and compiles
out entirely on kernels that will not consume it. The reader
(async_mm_teardown_eligible()) is added in the next patch, and the
deferred teardown is not routed onto the exit path until later in the
series, so this change is inert on its own.
Note: a bit named MMF_OOM_VICTIM existed until 2022 and was removed
as dead state once exit_mmap()'s special-case reap-before-teardown
was replaced by mmap_lock/MMF_OOM_SKIP-based concurrency between
exit_mmap() and oom_reap_task_mm(). MMF_OOM_TARGETED is a
deliberately distinct name for a new consumer -- deferred exit-path
teardown -- and does not revive the old exit_mmap() behaviour.
Signed-off-by: Aditya Sharma <adi.sharma@zohomail.in>
---
include/linux/mm_types.h | 10 +++++++
mm/oom_kill.c | 61 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 71 insertions(+)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index fffa285ef..3240c029f 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1989,6 +1989,16 @@ enum {
#define MMF_TOPDOWN 31 /* mm searches top down by default */
#define MMF_TOPDOWN_MASK BIT(MMF_TOPDOWN)
+/*
+ * mm is the target of an in-flight OOM kill. Only written under
+ * CONFIG_ASYNC_MM_TEARDOWN today. Sticky: never cleared -- the mm is
+ * dying (__oom_kill_process() kills every process sharing it), and a
+ * hypothetical surviving sharer would merely keep tearing down
+ * synchronously. Above bit 31, so it is outside MMF_INIT_LEGACY_MASK's
+ * domain entirely and is never inherited on fork.
+ */
+#define MMF_OOM_TARGETED 32
+
#define MMF_INIT_LEGACY_MASK (MMF_DUMP_FILTER_MASK |\
MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\
MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK)
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 5f372f6e2..b5490d1b1 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -754,6 +754,40 @@ static void mark_oom_victim(struct task_struct *tsk)
struct mm_struct *mm = tsk->mm;
WARN_ON(oom_killer_disabled);
+
+ /*
+ * Mark the mm itself, not just this task/signal, as an OOM target,
+ * so the deferred-teardown path can test it from the mm alone. Set
+ * ahead of the TIF_MEMDIE test-and-set below so that every call
+ * marks the mm, including a repeat mark on a task that is already
+ * TIF_MEMDIE.
+ *
+ * This is reliably visible to whichever thread ends up dropping the
+ * last reference in mmput_exit(): marking and queuing for async
+ * teardown can never overlap in time. Every caller reaches this with
+ * tsk->mm still set -- either tsk is current, marking itself in
+ * program order, or tsk is task_lock()ed by the caller (the
+ * task_will_free_mem() fast path in oom_kill_process()) -- while
+ * mmput_exit(), the only path that queues for teardown, is reached
+ * only after mm_users hits 0, which requires tsk's own exit_mm() (or
+ * exec_mmap(), the other path that sheds ->mm) to have already
+ * cleared ->mm under that same task_lock(). Program order or
+ * task_lock() release/acquire therefore orders this store before
+ * tsk's own eventual mmput_exit().
+ *
+ * If a different CLONE_VM sharer performs the actual final decrement
+ * instead, the ordering does not come from this store:
+ * mm_flags_set() is set_bit(), a non-value-returning RMW, so it is
+ * unordered and contributes nothing on its own. It comes from the
+ * marking task's own atomic_dec_and_test(), a value-returning RMW
+ * and therefore fully ordered (an smp_mb() before and after).
+ * Decrements are totally ordered in mm_users' modification order
+ * with the zeroing one last, so the general barriers on both sides
+ * make this store visible to whichever thread performs it.
+ */
+ if (IS_ENABLED(CONFIG_ASYNC_MM_TEARDOWN))
+ mm_flags_set(MMF_OOM_TARGETED, mm);
+
/* OOM killer might race with memcg OOM */
if (test_and_set_tsk_thread_flag(tsk, TIF_MEMDIE))
return;
@@ -931,6 +965,33 @@ static void __oom_kill_process(struct task_struct *victim, const char *message)
mm = victim->mm;
mmgrab(mm);
+ /*
+ * Set this before any SIGKILL for this kill event goes out below,
+ * while task_lock(victim) (held since find_lock_task_mm() above) is
+ * still held. This is reliably visible to whichever thread ends up
+ * running mmput_exit() and dropping the last reference to this mm --
+ * victim itself, a sibling, or a CLONE_VM sharer in another thread
+ * group (which mark_oom_victim() never marks, but which still
+ * observes this store on the shared mm). Marking and queuing for
+ * async teardown can never overlap: mmput_exit() is reached only
+ * after mm_users hits 0, and that requires victim's own exit_mm()
+ * (or exec_mmap(), the other path that sheds ->mm) to have cleared
+ * ->mm under this same task_lock() first, which release/acquire
+ * orders after the store above.
+ *
+ * If some other sharer performs the actual final decrement instead,
+ * the ordering does not come from this store: mm_flags_set() is
+ * set_bit(), a non-value-returning RMW, so it is unordered and
+ * contributes nothing on its own. It comes from the marking task's
+ * own atomic_dec_and_test(), a value-returning RMW and therefore
+ * fully ordered (an smp_mb() before and after). Decrements are
+ * totally ordered in mm_users' modification order with the zeroing
+ * one last, so the general barriers on both sides make this store
+ * visible to whichever thread performs it.
+ */
+ if (IS_ENABLED(CONFIG_ASYNC_MM_TEARDOWN))
+ mm_flags_set(MMF_OOM_TARGETED, mm);
+
/* Raise event before sending signal: task reaper must see this */
count_vm_event(OOM_KILL);
memcg_memory_event_mm(mm, MEMCG_OOM_KILL);
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 0/7] mm: defer address-space teardown of large exiting processes to a kthread
From: Aditya Sharma @ 2026-07-19 18:54 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko
Cc: David Rientjes, Shakeel Butt, Jonathan Corbet, Shuah Khan,
Steven Rostedt, Masami Hiramatsu, Mathieu Desnoyers, Kees Cook,
Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Ben Segall, Mel Gorman, Valentin Schneider,
K Prateek Nayak, linux-mm, linux-doc, linux-trace-kernel,
linux-kernel, imbrenda, Aditya Sharma
Address-space teardown on process exit runs synchronously in the dying
task's context: exit_mm() -> mmput() -> __mmput() -> exit_mmap() walks
page tables, updates rmap, frees the RSS and drops file references, all
on the exiting CPU. For a multi-GB process that is hundreds of
milliseconds of exit-path latency, paid by whoever is waiting on the
death: a supervisor's kill-and-respawn cycle, a shell's waitpid(), an
orchestrator reaping a fleet of workers. On the test box below, killing
a 16GB process costs ~280ms before the parent's waitpid() returns.
This series adds CONFIG_ASYNC_MM_TEARDOWN: an opt-in, default-off path
that defers __mmput() of large exiting processes to a dedicated kernel
thread (mm_reaper), so the exiting CPU is released as soon as the task
is reaped and the teardown runs off to the side.
Design
======
- A single freezable kthread, mm_reaper, modeled on oom_reaper, fed
through a lock-free llist + waitqueue. llist rather than oom_reaper's
spinlock+list so many-core parallel exits do not serialize on a
global lock at enqueue. The kthread runs at nice 19; its affinity is
a preference (kthread_affine_preferred()) for the HK_TYPE_KTHREAD
housekeeping mask, so it keeps off isolcpus= and cpuset-isolated
CPUs without the hard bind of kthread_bind_mask() --
PF_NO_SETAFFINITY would leave admins unable to re-affine it.
- mmput_exit(), called only from exit_mm()'s final reference drop
(patch 7). Every other mmput() caller (get_task_mm() users such as
ptrace and /proc, kthread_unuse_mm(), ...) is unchanged and still
tears down inline if it happens to hold the last reference.
- Deferral only when ALL of the following hold, else inline __mmput():
* vm.async_mm_teardown=1 (static key, default off)
* RSS >= vm.async_mm_teardown_thresh_pages (default 64MB)
* the mm is not an OOM-kill target (MMF_OOM_TARGETED, patch 3) and
not already reaped (MMF_OOM_SKIP)
* charging the RSS to a pending-pages budget stays under
vm.async_mm_teardown_max_pending_pages (add-then-check: can
over-reject under concurrency, never over-admit)
- exit_aio() is pulled forward: mmput_exit() runs it eagerly on the
exiting task before reserving/queuing. exit_aio() can block
indefinitely waiting for in-flight AIO to drain; run eagerly, a
stuck AIO context stalls only that task -- exactly what a
synchronous mmput() would do -- instead of blocking the single
reaper and stranding every teardown queued behind it. It is
idempotent (a no-op once ioctx_table is gone), so __mmput() calling
it again later is harmless.
- OOM exclusion: an OOM victim's memory must come back promptly, not
sit behind a nice-19 kthread working against the oom_reaper.
MMF_OOM_TARGETED is mm-scoped (not per-task) because the final
mm_users drop can come from a CLONE_VM sharer in another thread
group that mark_oom_victim() never sees; it is set before any
SIGKILL for the kill event is dispatched, and the marking and the
eligibility check cannot overlap in time (see patch 3 for the
ordering argument). That argument is belt-and-suspenders, not a
correctness dependency: a missed flag would only defer the
victim's teardown (a latency/priority-inversion regression), never
corrupt anything, and the oom_reaper reaps the victim
independently either way. An LKMM litmus test for the
CLONE_VM-sharer case is available on request.
- No extra mm refcounting: deferring __mmput() keeps the mm_count
reference that mm_users > 0 stands for, exactly as mmput_async()
does; the mm and its embedded llist node stay alive until the
kthread runs.
- Observability: three vmstat counters (queued / sync / rejected) and
two tracepoints. queue carries pid/comm/rss/node of the exiting
task; reap carries the charged RSS vs a fresh RSS read plus the
reaper's node, so the gap measures how much reclaim ate out of the
queue, and queue-vs-reap node mismatches expose NUMA-remote
teardown.
The work is moved, not eliminated. Measured checks (same box
as below):
- Freeing latency: with async on, a killed 16GB region is freed on
the same timescale as inline teardown -- nr_free_pages recovers
within ~300ms of the kill in both configurations (single-rep
watcher readings of 240ms async vs 283ms inline, within run-to-run
variance; not a claim that async frees faster). Deferral does not
delay the return of memory; it makes the exit path stop waiting
for it.
- CPU conservation: 5 consecutive 4GB inline teardowns cost 372ms of
exit-path CPU; mm_reaper spends 440ms of kthread CPU draining the
same five, read from its schedstat over a fully drained window
(ratio 1.18, nice-19 kthread, cache-cold on another CPU).
- Pages queued for teardown are invisible to reclaim until the reaper
frees them. The pending cap bounds that exposure coarsely; it is not
a reclaim mechanism. Reclaim integration (unmap-first draining, a
shrinker) is future work, and is also an answer to the cap
sizing question below.
Scope of the claims: what this series claims today is the common-case
exit-latency win under a modest threshold and a conservative cap. It
deliberately does not claim the multi-GB-teardown-under-memory-pressure
regime: until the queue is visible to reclaim, that regime is exactly
where the cap must stay conservative, and the 16GB+ numbers below are
capability demonstrations, run with the cap deliberately raised on an
otherwise idle machine -- not a recommended production configuration.
Open question 1 is where this either gets fixed (reclaim integration)
or stays scoped.
Numbers
=======
Bare metal, 32 CPUs / 32GB RAM (31 GiB usable), single NUMA node,
performance governor, medians, THP off unless noted. reap = SIGKILL -> waitpid()
returns. For all async-on runs vm.async_mm_teardown_max_pending_pages
was raised to 3/4 of RAM so the measurements exercise the reaper, not
the backpressure fallback: with the RAM/4 default, a single 16GB mm on
this box (8GB cap) -- or the 64GB guest case -- exceeds the cap and
falls back to sync teardown by design. See open question 1.
kill, async off -> on:
1GB 18.01 ms -> 0.14 ms
4GB 73.71 ms -> 0.14 ms
16GB 282.16 ms -> 0.15 ms
16GB 27.68 ms -> 0.13 ms (THP on)
voluntary exit (_exit() -> reap), 16GB:
271.27 ms -> 0.12 ms
M simultaneous 0.5GB exits, tail until ALL reaped:
M=8 40.64 ms -> 0.15 ms
M=16 79.99 ms -> 0.17 ms
M=32 157.42 ms -> 0.32 ms
below-threshold 30MB process, feature ON vs OFF:
0.56 ms -> 0.60 ms (sync path, within noise)
file-backed (tmpfs) 4GB mapping, kill:
35.34 ms -> 0.12 ms
real workload -- redis-server with a 16GB populated dataset
(DEBUG POPULATE, VmRSS ~16.2GB), kill:
292.97 ms -> 0.24 ms
The redis point (5 reps/pass, sync spread 288.5-309.2ms, async
0.12-0.32ms) lands within ~3% of the synthetic 16GB per-GB rate, so
the benchmark's memset region is a fair proxy for a real heap, and
the async side carries over unchanged.
CONFIG=n control: the same driver on a =n build of the same tree
(interface verified fully absent: no sysctls, no vmstat counters, no
kthread, no tracepoints) reproduces the =y async-off column within
noise -- 18.01 vs 18.01 ms at 1GB kill, 282.14 vs 282.16 ms at 16GB
kill -- so with the key off, the config costs nothing measurable.
Single-reaper drain throughput: 16.4GB backlog (32 x 0.5GB) returned in
650ms, ~25GB/s on this box. For scale, a 32-vCPU/88GB QEMU guest,
pinned to one NUMA node of its host (vCPUs and preallocated guest RAM
both bound node-local; guest numbers, same kernel/config): 16GB kill
426.15ms -> 0.13ms, a 64GB backlog (32 x 2GB) drained in ~2.1s
(~31GB/s), a 16GB shmem inode evicted in reaper context, and the
pm_test freezer cycle passed with the full 64GB backlog queued.
Robustness testing (driver script, all sections pass, dmesg clean)
==================================================================
- Backpressure: with the cap forced to 1GB, 8 parallel 0.5GB exits
produce queued += 4, sync fallbacks += 12; the cap is respected and
fallback is graceful.
- OOM killer firing while 8GB of teardowns are queued: 3/3 reps clean,
backlog drains in ~650ms alongside the kill.
- Targeted MMF_OOM_TARGETED probes, verified per-pid against the
tracepoints (a positive control first proves a plain large exit of
the same binary IS seen as queued): a ~512MB memcg OOM victim is
never queued; a CLONE_VM pair sharing one mm, swept by the same
kill, produces no queue event from either thread group. Two claims
are documented as untestable from userspace rather than tested:
task_will_free_mem() fast-path marking, and flag stickiness/fork
non-inheritance (no userspace process can survive owning a targeted
mm: oom_score_adj writes propagate across mm sharers, and the
__oom_kill_process() sweep kills every sharer unconditionally).
- unlink-while-mapped tmpfs file: the VMA's fput at teardown is the
last reference, so the final iput -> shmem_evict_inode of a 4GB
shmem inode runs in reaper context; tmpfs is empty 212ms after the
reap. Note: a kthread's fput cannot use task_work, so the evict
actually executes in the (bound) delayed_fput workqueue -- see open
question 4.
- Toggling: disabling mid-drain still drains the backlog (8.0GB in
670ms) while new exits take the sync path; 50 rapid static-key
flip cycles under concurrent exits, clean.
- pm_test freezer cycle with a 16GB backlog queued: completes, and
the async path works after thaw. Caveat in open question 6.
NOT tested: NUMA. I have no bare-metal access to a multi-node machine;
all bare-metal numbers above are single-node, and the QEMU guests
present a single node too. The driver has a ready-to-run NUMA section
(reaper pinned to node 0, victim on node 0 vs node 1, kthread_tail
compared, tracepoints capture queue-vs-reap node) that skips on
single-node systems. Numbers from a multi-node box would be very
welcome and directly feed open question 2.
Open questions, input wanted
============================
1. The default for vm.async_mm_teardown_max_pending_pages. It is
currently totalram_pages()/4, an explicitly marked placeholder.
A static fraction of RAM is arbitrary; but making it a dynamic
function of available RAM is wrong in a different way: high
occupancy is not load. A box full of page cache or running a
legitimately large in-memory workload is not necessarily under
pressure, and those are exactly the boxes that want async teardown.
Under real pressure the cap is the wrong tool anyway, because the
queued pages are invisible to reclaim regardless of the cap value;
the correct fix there is reclaim integration (above).
The default also cuts against the feature: RAM/4 rejects any
single mm larger than a quarter of RAM from deferral entirely --
the exact processes with the worst exit latency are the first ones
sent back to the sync path. Every async number above needed the
cap raised to 3/4 of RAM to be measurable at all.
And the loose end of the range bites in practice: in a large
guest with the cap at that same 3/4 of RAM, a stress run that
started a new round of large exits before the previous round had
drained ran the box into OOM kills. That is what the design
predicts: queued mms are invisible to reclaim and unreachable by
the OOM killer -- their tasks are already dead, so they appear in
no task dump and the oom_reaper has no task to reap; only
mm_reaper can free them. The kills do resolve the pressure (OOM
victims are MMF_OOM_TARGETED, so their teardown is sync), but
tasks die on a box that is not really out of memory -- the memory
is in the queue. With the RAM/4 default the same pattern
self-limits: reservation fails and exits fall back to sync
teardown, which is its own backpressure.
Note the motivating workload (kill-and-respawn) is exactly the
pattern that re-faults immediately after reap, so transient demand
approaches old+new RSS; the cap bounds the old half. So: is RAM/4
acceptable as a coarse safety bound for now? Should the cap react
to watermarks or PSI instead? Should out_of_memory() drain or
expedite the pending queue before picking a victim, the way it
already coordinates with the oom_reaper? That last one is
attractive because every queued mm is already dead: draining is
guaranteed-progress reclaim with no victim to choose. But an
unbounded synchronous drain inside the allocation path is its own
stall (seconds for a large backlog), so it would need a budget.
Or stay dumb until the shrinker exists?
The shape I have in mind for the real fix: a shrinker whose
count_objects() reports the pending pages and whose scan path
kicks the reaper and switches it to unmap-first draining, so
memory returns ahead of the slow metadata teardown. Running full
__mmput()s inside direct reclaim would trade one latency cliff for
another, so the shrinker's job is to accelerate and prioritize the
drain, not to execute it in the caller's context. If the consensus
is that reclaim visibility must land before exit_mm() is routed
(patch 7), I would make that the centerpiece of v2.
2. One reaper vs per-node reapers vs an unbound workqueue. One thread
sustains ~25GB/s on this box and a 64GB backlog still drains in
~2.1s in the big guest, but all of that is node-local. The tracepoint
node fields exist precisely to evaluate this once multi-node
numbers exist.
3. CPU accounting. Teardown CPU moves from the exiting task (charged
to its cgroup) to a root-cgroup kthread. That is the same class of
escape as kswapd or the oom_reaper, but this extends it to routine
process exit. Patch 5's admin docs state the escape explicitly so
nobody enables this blind; the open question is whether that
suffices for an opt-in feature or whether cgroup-aware charging of
the reaper's CPU is a prerequisite.
4. The delayed_fput hop. Because mm_reaper is a kthread, the final
fput() it drops cannot use task_work and is punted to the bound
delayed_fput workqueue -- so a multi-GB shmem inode evict runs on a
bound system_wq worker (the workqueue watchdog already complains:
"delayed_fput hogged CPU for >13333us, consider switching to
WQ_UNBOUND"). Should the reaper drain its own fput backlog inline,
or is this an argument for making delayed_fput unbound generally?
5. memcg zombies. Deferring teardown also defers the final uncharge,
so a dying memcg can be pinned a little longer by a queued mm
(bounded by the pending cap and measured drain times, i.e.
normally milliseconds). I believe this is acceptable.
6. Freezing. The drain loop calls try_to_freeze() after every entry,
so a freeze request is honoured between mms mid-batch, and the
pm_test freezer cycle passes with a 16GB backlog queued. What
remains is that a single in-flight __mmput() cannot freeze
mid-teardown, so one sufficiently large mm could still exceed the
freeze timeout -- but that is equally true of the same exit_mmap()
running synchronously in the exiting task today. Is per-mm
granularity enough?
Related work
============
- "Two alternatives for mm async teardown" (Claudio Imbrenda, 2021,
https://lore.kernel.org/all/20211111095008.264412-1-imbrenda@linux.ibm.com/
): an earlier RFC with the same
goal, motivated by huge s390 KVM guest address spaces. It offered
either an arch hook in exit_mm() or a process_mmput_async syscall
that runs the teardown in another process's context, plus an OOM
notifier to shield the teardown from the OOM killer. Neither
variant was merged; the s390 protected-guest case was later
addressed with KVM-specific asynchronous destruction. This series
is arch-independent, needs no syscall or userspace agent (kernel
kthread plus an eligibility gate), and integrates with the OOM
killer by exclusion (MMF_OOM_TARGETED keeps victims synchronous)
rather than by blocking it.
- oom_reaper: same kthread pattern, but it strips memory from a
victim that cannot exit; this series is about not making healthy
exits pay the teardown latency.
- process_mrelease(2): userspace-driven reaping of a killed process.
It needs a daemon to issue it per-kill, only helps the kill case
(not voluntary exit), and does not remove the exit-path work from
the dying task -- it races it. This series needs no userspace
agent and covers both exit flavors.
- mmput_async(): existing deferral of __mmput() to a workqueue for
contexts that cannot sleep; this series reuses its lifetime
argument but adds gating, backpressure, OOM exclusion and a
dedicated low-priority thread, none of which mmput_async() has.
Patches
=======
1 mm: add CONFIG_ASYNC_MM_TEARDOWN scaffolding for off-CPU exit
teardown -- config, mm_struct::async_reap_node, mmput_exit()
fallback; no functional change.
2 mm: dispatch __mmput() to an mm_reaper kthread via llist -- the
kthread and queue; built but unreferenced.
3 mm/oom_kill: mark an OOM victim's mm with MMF_OOM_TARGETED -- the
mm-scoped victim flag and its visibility/ordering argument;
nothing reads it yet.
4 mm: gate async teardown on RSS with pending-pages backpressure --
eligibility predicate + budget reservation; charge memoized at
enqueue because RSS keeps shrinking after mm_users hits zero;
exit_aio() pulled forward onto the exiting task.
5 mm: add runtime toggle and sysctls for async teardown -- static
key, three vm.* sysctls, admin-guide docs.
6 mm: add tracepoints and vmstat counters for async teardown.
7 exit: route exit_mm()'s final mmput() through mmput_exit() -- the
switch that makes 1-6 operative.
Patches 1-6 are individually inert (each is either scaffolding or
dead code until patch 7), so the series bisects cleanly with the
feature dark until the last commit, and the runtime default is off
even then.
The benchmark (exit_latency.c) and the driver that produced every
number above (sanity/headline/wait-free/CPU-conservation/reuse/
parallel/backpressure/numa/smallproc/oom/oom-targeted/file-backed/
toggle/freezer sections, ~2k lines total), plus the standalone redis
driver behind the real-workload number, are available on request; if
there is interest I am happy to rework them into
tools/testing/selftests/mm for a later revision.
Based on mm-unstable, commit 890f8c4e827c
("mm/secretmem: don't allow highmem folios").
Aditya Sharma (7):
mm: add CONFIG_ASYNC_MM_TEARDOWN scaffolding for off-CPU exit teardown
mm: dispatch __mmput() to an mm_reaper kthread via llist
mm/oom_kill: mark an OOM victim's mm with MMF_OOM_TARGETED
mm: gate async teardown on RSS with pending-pages backpressure
mm: add runtime toggle and sysctls for async teardown
mm: add tracepoints and vmstat counters for async teardown
exit: route exit_mm()'s final mmput() through mmput_exit()
Documentation/admin-guide/sysctl/vm.rst | 38 +++++
include/linux/mm_types.h | 15 ++
include/linux/sched/mm.h | 6 +
include/linux/vm_event_item.h | 5 +
include/trace/events/mm_reaper.h | 73 +++++++++
kernel/exit.c | 2 +-
kernel/fork.c | 199 ++++++++++++++++++++++++
mm/Kconfig | 12 ++
mm/oom_kill.c | 61 ++++++++
mm/vmstat.c | 5 +
10 files changed, 415 insertions(+), 1 deletion(-)
create mode 100644 include/trace/events/mm_reaper.h
base-commit: 890f8c4e827c918dac668a12eaf63180ba8a9e6d
--
2.34.1
^ permalink raw reply
* [RFC PATCH 4/7] mm: gate async teardown on RSS with pending-pages backpressure
From: Aditya Sharma @ 2026-07-19 18:54 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko
Cc: David Rientjes, Shakeel Butt, Jonathan Corbet, Shuah Khan,
Steven Rostedt, Masami Hiramatsu, Mathieu Desnoyers, Kees Cook,
Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Ben Segall, Mel Gorman, Valentin Schneider,
K Prateek Nayak, linux-mm, linux-doc, linux-trace-kernel,
linux-kernel, imbrenda, Aditya Sharma
In-Reply-To: <20260719185409.409685-1-adi.sharma@zohomail.in>
Deferring every exit would churn small processes for no benefit and let
unreaped memory grow without bound. Split the defer decision into a
pure predicate and a budget reservation:
- async_mm_teardown_eligible() (side-effect free): defer only when
RSS >= async_mm_teardown_thresh_pages and both MMF_OOM_SKIP and
MMF_OOM_TARGETED are clear:
* MMF_OOM_SKIP -- the mm is hidden from the OOM killer and
reaper. Usually that means it has already been reaped or run
through exit_mmap() and has no RSS left worth deferring; in the
mm-pinned-by-init case (__oom_kill_process()) the flag is set at
kill time with RSS intact, and keeping such an mm synchronous is
equally what we want;
* MMF_OOM_TARGETED -- the mm is the target of an in-flight OOM
kill (added in the preceding patch). Such a victim is torn
down synchronously so its memory is released promptly rather
than queued behind the low-priority, housekeeping-confined
mm_reaper, which would work against the oom_reaper during the
exact pressure that triggered the kill. The flag is set while
some task still holds a live reference to the mm, and this
check is reached only after mm_users has dropped to 0 -- which
requires that task's own exit_mm() (or exec_mmap(), the other
path that sheds ->mm) to have already cleared ->mm under that
same task_lock() -- so marking and checking can never overlap in
time. Whichever thread performs the final decrement is therefore
guaranteed to observe the flag already set, per the preceding
patch.
- async_mm_teardown_reserve(): charge RSS against a pending-pages
budget with atomic_long_add_return(); if the result exceeds
async_mm_teardown_max_pending_pages it subtracts back and returns
false, so the caller falls back to synchronous __mmput().
mmput_exit() reads RSS once and checks eligibility before reservation,
so the budget is only touched for processes that pass the cheap
checks. On success it stores the charged amount in mm->async_reap_node's
companion field async_reap_rss before enqueueing, and the reaper
releases exactly that stored charge after __mmput(). The charge cannot
be re-derived at reap time: RSS keeps shrinking after mm_users reaches
zero -- rmap-based reclaim can still swap out anon pages and truncation
can still unmap file pages, neither of which requires mm_users -- and
get_mm_rss() is an approximate per-CPU counter read besides. Two
independent reads at enqueue and reap would systematically
under-release, drifting the pending counter upward until the cap
permanently forces synchronous fallback.
An eligible mm would otherwise have its entire __mmput() -- including
exit_aio() -- deferred onto the single mm_reaper kthread.
exit_aio() can block indefinitely waiting for in-flight AIO to drain,
so mmput_exit() calls it eagerly, on the exiting task's own context,
before reservation: a stuck AIO context then only stalls this task,
same as a synchronous mmput() would, instead of blocking mm_reaper and
stranding every other mm already queued behind it. exit_aio() is
idempotent -- it is a no-op once mm->ioctx_table has been torn down --
so __mmput() calling it again unconditionally is harmless, whether that
second call lands on the reaper or -- when the reservation fails --
back on the exiting task itself.
Because reservation is add-then-check rather than read-then-add,
committed pending never stays above the cap: an exit that would exceed
it tears down synchronously instead. Under concurrent exits a failing
reservation can transiently inflate the counter between its add and
roll-back, which may cause another exiter to fall back to sync
conservatively -- it can over-reject but never over-admit.
The cap is a coarse safety bound on unreaped memory, not a reclaim
mechanism: pages queued here are invisible to reclaim until the reaper
frees them. Integrating the queue with reclaim (unmap-first draining,
a shrinker) is future work; see the cover letter.
The RSS threshold defaults to a fixed 64MB at init; the pending cap
defaults to totalram_pages() / 4 as a placeholder pending tuning. Both
are static module variables here and not yet user-tunable. The static
key that gates the whole path, and sysctl registration, follow in the
next patch.
This patch is dormant: nothing calls mmput_exit() yet (exit_mm() is
routed to it later in the series), so no teardown is deferred.
Signed-off-by: Aditya Sharma <adi.sharma@zohomail.in>
---
include/linux/mm_types.h | 1 +
kernel/fork.c | 79 +++++++++++++++++++++++++++++++++++++++-
2 files changed, 79 insertions(+), 1 deletion(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 3240c029f..ee6c0ac84 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1370,6 +1370,7 @@ struct mm_struct {
struct work_struct async_put_work;
#ifdef CONFIG_ASYNC_MM_TEARDOWN
struct llist_node async_reap_node;
+ unsigned long async_reap_rss;
#endif /* CONFIG_ASYNC_MM_TEARDOWN */
#ifdef CONFIG_IOMMU_MM_DATA
diff --git a/kernel/fork.c b/kernel/fork.c
index 884e4e767..d45418e66 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -113,6 +113,7 @@
#include <linux/pgalloc.h>
#include <linux/uaccess.h>
#include <linux/sched/isolation.h>
+#include <linux/sizes.h>
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
@@ -3412,6 +3413,48 @@ subsys_initcall(init_fork_sysctl);
#ifdef CONFIG_ASYNC_MM_TEARDOWN
static LLIST_HEAD(mm_reaper_list);
static DECLARE_WAIT_QUEUE_HEAD(mm_reaper_wait);
+static atomic_long_t mm_reaper_pending_pages;
+static unsigned long sysctl_async_mm_teardown_thresh_pages;
+static unsigned long sysctl_async_mm_teardown_max_pending_pages;
+
+static bool async_mm_teardown_eligible(struct mm_struct *mm, unsigned long rss)
+{
+ if (rss < READ_ONCE(sysctl_async_mm_teardown_thresh_pages))
+ return false;
+ if (mm_flags_test(MMF_OOM_SKIP, mm)) /* reaped, or hidden from the reaper */
+ return false;
+ /*
+ * MMF_OOM_TARGETED is set by the OOM killer while some task still
+ * holds a live reference to this mm (see mark_oom_victim() and
+ * __oom_kill_process()), and this eligibility check is reached only
+ * from mmput_exit(), after mm_users has dropped to 0 -- so marking
+ * and this check can never overlap in time. Whichever thread
+ * performs the final decrement is therefore guaranteed to observe
+ * the flag already set, regardless of whether that thread was
+ * itself ever passed to mark_oom_victim() (a CLONE_VM-sharing
+ * process in another thread group never is, but still observes
+ * this flag on the shared mm).
+ */
+ if (mm_flags_test(MMF_OOM_TARGETED, mm))
+ return false;
+ return true;
+}
+
+/*
+ * Charge rss against the pending-teardown budget. Returns true if it fits
+ * under the cap, in which case the caller enqueues and the reaper releases
+ * the same rss after __mmput(). On overshoot nothing stays charged and the
+ * caller must tear down synchronously.
+ */
+static bool async_mm_teardown_reserve(unsigned long rss)
+{
+ if (atomic_long_add_return(rss, &mm_reaper_pending_pages) >
+ READ_ONCE(sysctl_async_mm_teardown_max_pending_pages)) {
+ atomic_long_sub(rss, &mm_reaper_pending_pages);
+ return false;
+ }
+ return true;
+}
static void async_mm_teardown_queue(struct mm_struct *mm)
{
@@ -3429,7 +3472,10 @@ static int mm_reaper(void *unused)
wait_event_freezable(mm_reaper_wait, !llist_empty(&mm_reaper_list));
batch = llist_del_all(&mm_reaper_list);
llist_for_each_entry_safe(mm, n, batch, async_reap_node) {
+ unsigned long pages = mm->async_reap_rss;
+
__mmput(mm); /* may free mm via mmdrop */
+ atomic_long_sub(pages, &mm_reaper_pending_pages);
cond_resched();
try_to_freeze();
}
@@ -3439,10 +3485,38 @@ static int mm_reaper(void *unused)
void mmput_exit(struct mm_struct *mm)
{
+ unsigned long rss;
+
might_sleep();
+ /*
+ * Fully ordered RMW: combined with exit_mm() (or exec_mmap(), the
+ * other path that sheds ->mm) clearing ->mm under task_lock() before
+ * this call, it guarantees MMF_OOM_TARGETED set at any mark site is
+ * visible here -- see mark_oom_victim(). Any new caller of
+ * mmput_exit() outside exit_mm() must re-verify that argument.
+ */
if (!atomic_dec_and_test(&mm->mm_users))
return;
- async_mm_teardown_queue(mm);
+
+ rss = get_mm_rss(mm);
+
+ if (async_mm_teardown_eligible(mm, rss)) {
+ /*
+ * exit_aio() can block indefinitely. Run it here so a stuck
+ * AIO only hangs this task (same as how it would happen in
+ * case of synchronous mmput()) instead of stranding every
+ * teardown queued behind this mm
+ */
+ exit_aio(mm);
+
+ if (async_mm_teardown_reserve(rss)) {
+ mm->async_reap_rss = rss;
+ async_mm_teardown_queue(mm);
+ return;
+ }
+ }
+
+ __mmput(mm);
}
static int __init mm_reaper_init(void)
@@ -3456,7 +3530,10 @@ static int __init mm_reaper_init(void)
}
set_user_nice(th, 19); /* minimize competition with other fair class tasks */
kthread_affine_preferred(th, housekeeping_cpumask(HK_TYPE_KTHREAD));
+ sysctl_async_mm_teardown_thresh_pages = SZ_64M >> PAGE_SHIFT;
+ sysctl_async_mm_teardown_max_pending_pages = totalram_pages() / 4; /* TODO: placeholder */
wake_up_process(th);
+
return 0;
}
subsys_initcall(mm_reaper_init);
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 5/7] mm: add runtime toggle and sysctls for async teardown
From: Aditya Sharma @ 2026-07-19 18:54 UTC (permalink / raw)
To: Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko
Cc: David Rientjes, Shakeel Butt, Jonathan Corbet, Shuah Khan,
Steven Rostedt, Masami Hiramatsu, Mathieu Desnoyers, Kees Cook,
Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Ben Segall, Mel Gorman, Valentin Schneider,
K Prateek Nayak, linux-mm, linux-doc, linux-trace-kernel,
linux-kernel, imbrenda, Aditya Sharma
In-Reply-To: <20260719185409.409685-1-adi.sharma@zohomail.in>
Add the userspace interface that arms and tunes off-CPU exit teardown,
and the static key that gates it.
Introduce async_mm_teardown_key (static key, default false) and gate the
deferral path in mmput_exit() on it via static_branch_unlikely(); when
off, mmput_exit() runs __mmput() inline. The get_mm_rss() read moves
inside the static_branch_unlikely() block, alongside eligibility and
exit_aio(), so the default (key off) path costs nothing beyond the
branch itself -- no RSS scan for every exiting process on kernels or
boots that never enable the feature. Under CONFIG_SYSCTL, register
three knobs under vm. with register_sysctl_init():
- async_mm_teardown: write 1/0 to enable/disable. A custom handler
flips the static key, serialized by a mutex so the key state always
matches the last value written.
- async_mm_teardown_thresh_pages: RSS threshold (default ~64MB).
- async_mm_teardown_max_pending_pages: backpressure cap.
The threshold defaults are set in the initcall outside any CONFIG_SYSCTL
guard, since eligible()/reserve() read them whenever the key is on; only
the toggle variable, its lock, the handler and the table depend on
CONFIG_SYSCTL. With CONFIG_SYSCTL=n there is no path to enable the key,
so the feature stays off and the thresholds are inert.
This patch is still dormant: nothing calls mmput_exit() yet, so flipping
async_mm_teardown on has no observable effect. exit_mm() is routed
through mmput_exit() later in the series; only then does enabling the key
defer teardown. The toggle and gate are introduced here so that, once
routing lands, the feature defaults off and an explicit opt-in is
required.
Signed-off-by: Aditya Sharma <adi.sharma@zohomail.in>
---
Documentation/admin-guide/sysctl/vm.rst | 38 +++++++++++
kernel/fork.c | 90 ++++++++++++++++++++-----
mm/Kconfig | 3 +-
3 files changed, 114 insertions(+), 17 deletions(-)
diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst
index 5b318d17a..ccda6c75f 100644
--- a/Documentation/admin-guide/sysctl/vm.rst
+++ b/Documentation/admin-guide/sysctl/vm.rst
@@ -22,6 +22,9 @@ the writeout of dirty data to disk.
Currently, these files are in /proc/sys/vm:
- admin_reserve_kbytes
+- async_mm_teardown
+- async_mm_teardown_max_pending_pages
+- async_mm_teardown_thresh_pages
- compact_memory
- compaction_proactiveness
- compact_unevictable_allowed
@@ -108,6 +111,41 @@ On x86_64 this is about 128MB.
Changing this takes effect whenever an application requests memory.
+async_mm_teardown
+=================
+
+Available only when CONFIG_ASYNC_MM_TEARDOWN is set. Controls whether the
+address-space teardown (exit_mmap()) of large exiting processes is deferred
+to the mm_reaper kernel thread instead of running on the exiting CPU.
+
+Writing 1 enables deferral, 0 disables it. Disabling makes mmput_exit() tear
+down inline again; it does not drain mms already queued. Defaults to 0.
+
+Deferred teardown consumes CPU in the mm_reaper kernel thread, which runs in
+the root cgroup: the teardown work of a task in a CPU-limited cgroup is not
+charged to that cgroup, similar to kswapd and the oom_reaper. Consider this
+before enabling the feature on systems that rely on strict per-cgroup CPU
+accounting.
+
+
+async_mm_teardown_max_pending_pages
+===================================
+
+Available only when CONFIG_ASYNC_MM_TEARDOWN is set. Backpressure cap, in
+pages, on the total RSS of mms queued for the mm_reaper but not yet torn
+down. An exit that would push the total over this cap tears down
+synchronously instead of queuing. Defaults to totalram_pages() / 4.
+
+
+async_mm_teardown_thresh_pages
+==============================
+
+Available only when CONFIG_ASYNC_MM_TEARDOWN is set. Minimum RSS, in pages,
+for an exiting mm to be eligible for deferral to the mm_reaper. Exiting mms
+below this threshold are always torn down synchronously. Defaults to 64MB
+worth of pages.
+
+
compact_memory
==============
diff --git a/kernel/fork.c b/kernel/fork.c
index d45418e66..0976770db 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -3414,8 +3414,13 @@ subsys_initcall(init_fork_sysctl);
static LLIST_HEAD(mm_reaper_list);
static DECLARE_WAIT_QUEUE_HEAD(mm_reaper_wait);
static atomic_long_t mm_reaper_pending_pages;
+static DEFINE_STATIC_KEY_FALSE(async_mm_teardown_key);
static unsigned long sysctl_async_mm_teardown_thresh_pages;
static unsigned long sysctl_async_mm_teardown_max_pending_pages;
+#ifdef CONFIG_SYSCTL
+static u8 sysctl_async_mm_teardown_enabled;
+static DEFINE_MUTEX(async_mm_teardown_lock);
+#endif
static bool async_mm_teardown_eligible(struct mm_struct *mm, unsigned long rss)
{
@@ -3485,8 +3490,6 @@ static int mm_reaper(void *unused)
void mmput_exit(struct mm_struct *mm)
{
- unsigned long rss;
-
might_sleep();
/*
* Fully ordered RMW: combined with exit_mm() (or exec_mmap(), the
@@ -3498,27 +3501,80 @@ void mmput_exit(struct mm_struct *mm)
if (!atomic_dec_and_test(&mm->mm_users))
return;
- rss = get_mm_rss(mm);
+ if (static_branch_unlikely(&async_mm_teardown_key)) {
+ unsigned long rss = get_mm_rss(mm);
- if (async_mm_teardown_eligible(mm, rss)) {
- /*
- * exit_aio() can block indefinitely. Run it here so a stuck
- * AIO only hangs this task (same as how it would happen in
- * case of synchronous mmput()) instead of stranding every
- * teardown queued behind this mm
- */
- exit_aio(mm);
+ if (async_mm_teardown_eligible(mm, rss)) {
+ /*
+ * exit_aio() can block indefinitely. Run it here so a stuck
+ * AIO only hangs this task (same as how it would happen in
+ * case of synchronous mmput()) instead of stranding every
+ * teardown queued behind this mm
+ */
+ exit_aio(mm);
- if (async_mm_teardown_reserve(rss)) {
- mm->async_reap_rss = rss;
- async_mm_teardown_queue(mm);
- return;
+ if (async_mm_teardown_reserve(rss)) {
+ mm->async_reap_rss = rss;
+ async_mm_teardown_queue(mm);
+ return;
+ }
}
}
__mmput(mm);
}
+#ifdef CONFIG_SYSCTL
+static int async_mm_teardown_enabled_handler(const struct ctl_table *table,
+ int write, void *buffer,
+ size_t *lenp, loff_t *ppos)
+{
+ int ret;
+
+ /*
+ * Serialize concurrent writers so the static key state always matches
+ * the last value written to the variable.
+ */
+ guard(mutex)(&async_mm_teardown_lock);
+
+ ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos);
+ if (ret || !write)
+ return ret;
+
+ if (sysctl_async_mm_teardown_enabled)
+ static_branch_enable(&async_mm_teardown_key);
+ else
+ static_branch_disable(&async_mm_teardown_key);
+ return 0;
+}
+
+static const struct ctl_table async_mm_teardown_table[] = {
+ {
+ .procname = "async_mm_teardown",
+ .data = &sysctl_async_mm_teardown_enabled,
+ .maxlen = sizeof(u8),
+ .mode = 0644,
+ .proc_handler = async_mm_teardown_enabled_handler,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
+ },
+ {
+ .procname = "async_mm_teardown_thresh_pages",
+ .data = &sysctl_async_mm_teardown_thresh_pages,
+ .maxlen = sizeof(unsigned long),
+ .mode = 0644,
+ .proc_handler = proc_doulongvec_minmax,
+ },
+ {
+ .procname = "async_mm_teardown_max_pending_pages",
+ .data = &sysctl_async_mm_teardown_max_pending_pages,
+ .maxlen = sizeof(unsigned long),
+ .mode = 0644,
+ .proc_handler = proc_doulongvec_minmax,
+ },
+};
+#endif /* CONFIG_SYSCTL */
+
static int __init mm_reaper_init(void)
{
struct task_struct *th;
@@ -3533,7 +3589,9 @@ static int __init mm_reaper_init(void)
sysctl_async_mm_teardown_thresh_pages = SZ_64M >> PAGE_SHIFT;
sysctl_async_mm_teardown_max_pending_pages = totalram_pages() / 4; /* TODO: placeholder */
wake_up_process(th);
-
+#ifdef CONFIG_SYSCTL
+ register_sysctl_init("vm", async_mm_teardown_table);
+#endif
return 0;
}
subsys_initcall(mm_reaper_init);
diff --git a/mm/Kconfig b/mm/Kconfig
index 296b16ad4..a9d925406 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1507,7 +1507,8 @@ config ASYNC_MM_TEARDOWN
help
Defer the address space teardown (exit_mmap()) of large exiting processes
to a kernel thread instead of running it on the exiting CPU. The feature
- is off by default.
+ is off by default and is enabled at runtime via the vm.async_mm_teardown
+ sysctl.
If unsure, say N.
--
2.34.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox