[PATCH v2 18/20] KVM: s390: arm64: Implement SVE for arm guests

[Steffen Eiden <seiden@linux.ibm.com>]

Add support for Scalable Vector Extension (SVE) in ARM64 guests running
on s390 hardware, providing 128-bit vector operations backed by s390's
Vector Extension (VX).

The implementation is limited to 128-bit vectors (VQ=1) matching s390 VX
register width. Z registers reuse existing FP/SIMD register space (vregs),
while P registers and FFR are stored in the SAE block.

This enables ARM64 guests to use basic SVE instructions while maintaining
compatibility with s390 hardware capabilities.

Co-developed-by: Friedrich Welter <fritz@linux.ibm.com>
Signed-off-by: Friedrich Welter <fritz@linux.ibm.com>
Signed-off-by: Steffen Eiden <seiden@linux.ibm.com>
---
 arch/s390/include/asm/kvm_host_arm64.h |   6 +-
 arch/s390/kvm/arm64/arm.c              |   7 +
 arch/s390/kvm/arm64/feature.c          |   9 +-
 arch/s390/kvm/arm64/feature.h          |  10 ++
 arch/s390/kvm/arm64/guest.c            | 204 +++++++++++++++++++++++++
 arch/s390/kvm/arm64/reset.c            |  22 ++-
 6 files changed, 253 insertions(+), 5 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host_arm64.h b/arch/s390/include/asm/kvm_host_arm64.h
index c0e8fbddc9dc..73eb09e98848 100644
--- a/arch/s390/include/asm/kvm_host_arm64.h
+++ b/arch/s390/include/asm/kvm_host_arm64.h
@@ -31,8 +31,9 @@ bool cpus_have_final_cap(unsigned int num);
 
 #define KVM_HAVE_MMU_RWLOCK
 #define KVM_MAX_VCPUS 1
-#define KVM_S390_ARM64_IMPL_FEATURES \
-	(BIT(KVM_ARM_VCPU_PTRAUTH_ADDRESS) | BIT(KVM_ARM_VCPU_PTRAUTH_GENERIC))
+#define KVM_S390_ARM64_IMPL_FEATURES (		\
+	 BIT(KVM_ARM_VCPU_SVE)			\
+)
 
 #define KVM_HALT_POLL_NS_DEFAULT 50000
 
@@ -132,7 +133,6 @@ struct kvm_vcpu_stat {
 };
 
 #define kvm_has_mte(_kvm) false
-#define vcpu_has_sve(_vcpu) false
 #define vcpu_has_ptrauth(_vcpu) false
 
 struct kvm_arch_memory_slot {
diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c
index d2ff69422217..a46c9c62a5f9 100644
--- a/arch/s390/kvm/arm64/arm.c
+++ b/arch/s390/kvm/arm64/arm.c
@@ -22,6 +22,7 @@
 #include <kvm_mmu.h>
 
 #include "arm.h"
+#include "feature.h"
 #include "handle_exit.h"
 #include "qaaf.h"
 #include "feature.h"
@@ -53,6 +54,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_IOEVENTFD:
 		ret = 1;
 		break;
+	case KVM_CAP_ARM_SVE:
+		ret = system_supports_sve();
+		break;
 	default:
 		ret = 0;
 	}
@@ -258,6 +262,9 @@ static unsigned long system_supported_vcpu_features(void)
 {
 	unsigned long features = KVM_S390_ARM64_IMPL_FEATURES;
 
+	if (!system_supports_sve())
+		clear_bit(KVM_ARM_VCPU_SVE, &features);
+
 	return features;
 }
 
diff --git a/arch/s390/kvm/arm64/feature.c b/arch/s390/kvm/arm64/feature.c
index 39c8ae7e951c..3847cdd2f92b 100644
--- a/arch/s390/kvm/arm64/feature.c
+++ b/arch/s390/kvm/arm64/feature.c
@@ -56,7 +56,6 @@ int __init kvm_arm_host_sanitize_features(struct qaaf_qmc_block *qaaf_qmc)
 	MASK_RESERVED(qaaf_qmc, ID_AA64MMFR4_EL1);
 
 	MODIFY(qaaf_qmc, ID_AA64PFR0_EL1, SEL2, NI);
-	MODIFY(qaaf_qmc, ID_AA64PFR0_EL1, SVE, NI);
 	MASK_RESERVED(qaaf_qmc, ID_AA64PFR0_EL1);
 	MASK_RESERVED(qaaf_qmc, ID_AA64PFR1_EL1);
 	MASK_RESERVED(qaaf_qmc, ID_AA64PFR2_EL1);
@@ -163,3 +162,11 @@ bool cpus_have_final_cap(unsigned int num)
 		return false;
 	}
 }
+
+bool system_supports_sve(void)
+{
+	return cpu_has_vx() &&
+	       SYS_FIELD_GET(ID_AA64PFR0_EL1, SVE,
+			     read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) ==
+		       ID_AA64PFR0_EL1_SVE_IMP;
+}
diff --git a/arch/s390/kvm/arm64/feature.h b/arch/s390/kvm/arm64/feature.h
index 9c902df1f26c..892cf8bcf3a1 100644
--- a/arch/s390/kvm/arm64/feature.h
+++ b/arch/s390/kvm/arm64/feature.h
@@ -5,9 +5,13 @@
 
 #include <linux/types.h>
 #include <linux/bitfield.h>
+#include <linux/cpufeature.h>
 
 #include <asm/sae.h>
 
+#include <arm64/sysreg.h>
+#include <arm64/sys_regs.h>
+
 #include "qaaf.h"
 
 int __init kvm_arm_host_sanitize_features(struct qaaf_qmc_block *qaaf_qmc);
@@ -84,4 +88,10 @@ static inline u64 kvm_sae_irptc(void)
 
 #define kvm_vcpu_has_pmu(_v) false
 
+#define vcpu_has_sve(_vcpu)							\
+	(system_supports_sve() &&						\
+	 test_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &((_vcpu)->kvm)->arch.flags))
+
+bool system_supports_sve(void);
+
 #endif /* ARCH_S390_KVM_FEATURE_H */
diff --git a/arch/s390/kvm/arm64/guest.c b/arch/s390/kvm/arm64/guest.c
index 047460975936..63fc4a64ab48 100644
--- a/arch/s390/kvm/arm64/guest.c
+++ b/arch/s390/kvm/arm64/guest.c
@@ -5,6 +5,9 @@
 #include <arm64/kvm_emulate.h>
 #include <arm64/kvm_nested.h>
 #include <arm64/sys_regs.h>
+#include <arm64/sve_context.h>
+
+#include "feature.h"
 
 #define SVE_VQ_MIN	__SVE_VQ_MIN
 #define SVE_NUM_ZREGS	KVM_ARM64_SVE_NUM_ZREGS
@@ -52,6 +55,197 @@ const struct kvm_stats_header kvm_vcpu_stats_header = {
 		       sizeof(kvm_vcpu_stats_desc),
 };
 
+static int sve_zreg_index(__u64 id, unsigned int *regnum)
+{
+	/* Currently only one slice is supported on arm, and our zreg is only 128 bit */
+	const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(KVM_ARM64_SVE_NUM_ZREGS - 1,
+						       KVM_ARM64_SVE_MAX_SLICES - 1);
+	const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0);
+
+	if (id < zreg_id_min || id > zreg_id_max)
+		return -EINVAL;
+	if ((id & SVE_REG_SLICE_MASK) > 0)
+		return -ENOENT;
+
+	*regnum = (id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
+
+	return 0;
+}
+
+static int sve_preg_index(u64 id, unsigned int *regnum)
+{
+	const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(KVM_ARM64_SVE_MAX_SLICES - 1);
+	const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0);
+
+	if (id < preg_id_min || id > preg_id_max)
+		return -EINVAL;
+	if ((id & SVE_REG_SLICE_MASK) > 0)
+		return -ENOENT;
+
+	*regnum = (id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
+
+	return 0;
+}
+
+static int sve_ffr_index(u64 id, unsigned int *regnum)
+{
+	if (id != KVM_REG_ARM64_SVE_FFR(0))
+		return -EINVAL;
+	if ((id & SVE_REG_SLICE_MASK) > 0)
+		return -ENOENT;
+
+	*regnum = (id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
+
+	return 0;
+}
+
+static inline int get_sve_ffr_reg(struct kvm_vcpu *vcpu, unsigned int regnum,
+				  u16 __user *uptr)
+{
+	/* ffr is pregmax + 1 */
+	if (regnum != KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (put_user(vcpu->arch.sae_block.sve_ffr, uptr))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int get_sve_preg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       u16 __user *uptr)
+{
+	if (regnum < 0 || regnum >= KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (put_user(vcpu->arch.sae_block.sve_pregs[regnum], uptr))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int get_sve_zreg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       __vector128 __user *uptr)
+{
+	if (regnum < 0 || regnum >= KVM_ARM64_SVE_NUM_ZREGS)
+		return -EINVAL;
+
+	/* vreg and svreg overlap and zreg is also just 128 bit so we reuse the vreg space */
+	if (copy_to_user(uptr, &vcpu->arch.ctxt.vregs[regnum],
+			 sizeof(vcpu->arch.ctxt.vregs[regnum])))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int set_sve_ffr_reg(struct kvm_vcpu *vcpu, unsigned int regnum,
+				  const u16 __user *uptr)
+{
+	/* ffr is pregmax + 1*/
+	if (regnum != KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (get_user(vcpu->arch.sae_block.sve_ffr, uptr))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int set_sve_preg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       const u16 __user *uptr)
+{
+	if (regnum < 0 || regnum >= KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (get_user(vcpu->arch.sae_block.sve_pregs[regnum], uptr))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int set_sve_zreg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       const __vector128 __user *uptr)
+{
+	if (regnum < 0 || regnum >= KVM_ARM64_SVE_NUM_ZREGS)
+		return -EINVAL;
+
+	/*vreg and svreg overlap and zreg is also just 128 bit so we reuse the vreg space*/
+	if (copy_from_user(&vcpu->arch.ctxt.vregs[regnum], uptr,
+			   sizeof(vcpu->arch.ctxt.vregs[regnum])))
+		return -EFAULT;
+	return 0;
+}
+
+static int set_sve_vls(struct kvm_vcpu *vcpu, const void __user *uptr)
+{
+	u64 vqs[KVM_ARM64_SVE_VLS_WORDS] = { 0 };
+	unsigned int vq;
+
+	if (!vcpu_has_sve(vcpu))
+		return -ENOENT;
+
+	if (kvm_arm_vcpu_sve_finalized(vcpu))
+		return -EPERM;
+
+	if (copy_from_user(vqs, uptr, sizeof(vqs)))
+		return -EFAULT;
+
+	/* only 128 bit and 1 VQ are supported , nothing saved just check validity */
+	for (vq = KVM_ARM64_SVE_VQ_MIN + 1; vq <= KVM_ARM64_SVE_VQ_MAX; ++vq)
+		if (vq_present(vqs, vq))
+			return -EINVAL;
+
+	/* run with a vl of 0 not valid */
+	if (!vq_present(vqs, KVM_ARM64_SVE_VQ_MIN))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int get_sve_vls(struct kvm_vcpu *vcpu, void __user *uptr)
+{
+	u64 vqs[KVM_ARM64_SVE_VLS_WORDS] = { 0 };
+
+	/* currently only 128 bit are supported so we only set bit 0  hardcoded */
+	vqs[0] |= vq_mask(KVM_ARM64_SVE_VQ_MIN);
+
+	if (copy_to_user(uptr, vqs, sizeof(vqs)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+	void __user *uptr = (void __user *)reg->addr;
+	unsigned int regnum;
+	int ret = -EFAULT;
+
+	if (reg->id == KVM_REG_ARM64_SVE_VLS)
+		ret = get_sve_vls(vcpu, uptr);
+	else if (sve_ffr_index(reg->id, &regnum) >= 0)
+		ret = get_sve_ffr_reg(vcpu, regnum, uptr);
+	else if (sve_preg_index(reg->id, &regnum) >= 0)
+		ret = get_sve_preg(vcpu, regnum, uptr);
+	else if (sve_zreg_index(reg->id, &regnum) >= 0)
+		ret = get_sve_zreg(vcpu, regnum, uptr);
+
+	return ret;
+}
+
+static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+	const void __user *uptr = (const void __user *)reg->addr;
+	int ret = -EFAULT;
+	unsigned int regnum;
+
+	if (reg->id == KVM_REG_ARM64_SVE_VLS)
+		ret = set_sve_vls(vcpu, uptr);
+	else if (sve_ffr_index(reg->id, &regnum) >= 0)
+		ret = set_sve_ffr_reg(vcpu, regnum, uptr);
+	else if (sve_preg_index(reg->id, &regnum) >= 0)
+		ret = set_sve_preg(vcpu, regnum, uptr);
+	else if (sve_zreg_index(reg->id, &regnum) >= 0)
+		ret = set_sve_zreg(vcpu, regnum, uptr);
+
+	return ret;
+}
+
 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 {
 	int ret;
@@ -61,6 +255,11 @@ int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 		return ret;
 	uindices += ret;
 
+	ret = copy_sve_reg_indices(vcpu, uindices);
+	if (ret < 0)
+		return ret;
+	uindices += ret;
+
 	return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
 }
 
@@ -68,6 +267,7 @@ unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
 {
 	unsigned long num = num_core_regs(vcpu);
 
+	num += num_sve_regs(vcpu);
 	num += kvm_arm_num_sys_reg_descs(vcpu);
 	return num;
 }
@@ -81,6 +281,8 @@ int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 	switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
 	case KVM_REG_ARM_CORE:
 		return get_core_reg(vcpu, reg);
+	case KVM_REG_ARM64_SVE:
+		return get_sve_reg(vcpu, reg);
 	default:
 		return kvm_arm_sys_reg_get_reg(vcpu, reg);
 	}
@@ -95,6 +297,8 @@ int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 	switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
 	case KVM_REG_ARM_CORE:
 		return set_core_reg(vcpu, reg);
+	case KVM_REG_ARM64_SVE:
+		return set_sve_reg(vcpu, reg);
 	default:
 		return kvm_arm_sys_reg_set_reg(vcpu, reg);
 	}
diff --git a/arch/s390/kvm/arm64/reset.c b/arch/s390/kvm/arm64/reset.c
index bc7bcee75779..754c4d4a70b6 100644
--- a/arch/s390/kvm/arm64/reset.c
+++ b/arch/s390/kvm/arm64/reset.c
@@ -3,6 +3,8 @@
 #include <linux/kvm_host.h>
 #include <linux/fpu.h>
 
+#include "asm/cpufeature.h"
+
 #include <arm64/kvm_emulate.h>
 #include <arm64/kvm_nested.h>
 #include <arm64/sys_regs.h>
@@ -11,9 +13,13 @@
 #include <clocksource/arm_arch_timer.h>
 
 #include "qaaf.h"
+#include "feature.h"
 
 bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)
 {
+	if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu))
+		return false;
+
 	return true;
 }
 
@@ -76,6 +82,10 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 	 */
 	preempt_disable();
 
+	if (!vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) &&
+	    vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE))
+		set_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &vcpu->kvm->arch.flags);
+
 	/* The reset must run with an unloaded save area */
 	loaded = vcpu_is_loaded(vcpu);
 	if (loaded)
@@ -114,5 +124,15 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 
 int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)
 {
-	return 0;
+	switch (feature) {
+	case KVM_ARM_VCPU_SVE:
+		if (!cpu_has_vx())
+			return -EINVAL;
+		if (vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED))
+			return -EPERM;
+		vcpu_set_flag(vcpu, VCPU_SVE_FINALIZED);
+		return 0;
+	}
+
+	return -EINVAL;
 }
-- 
2.53.0