[PATCH v3 21/23] KVM: s390: arm64: Implement SVE for arm guests

[Steffen Eiden <seiden@linux.ibm.com>]

Add support for Scalable Vector Extension (SVE) in ARM64 guests running
on s390 hardware, providing 128-bit vector operations backed by s390's
Vector Extension (VX).

The implementation is limited to 128-bit vectors (VQ=1) matching s390 VX
register width. Z registers reuse existing FP/SIMD register space (vregs),
while P registers and FFR are stored in the SAE block.

Co-developed-by: Friedrich Welter <fritz@linux.ibm.com>
Signed-off-by: Friedrich Welter <fritz@linux.ibm.com>
Co-developed-by: Andreas Grapentin <gra@linux.ibm.com>
Signed-off-by: Andreas Grapentin <gra@linux.ibm.com>
Signed-off-by: Steffen Eiden <seiden@linux.ibm.com>
---
 arch/s390/include/asm/kvm_host_arm64.h |   5 +-
 arch/s390/kvm/arm64/arm.c              |   6 +
 arch/s390/kvm/arm64/feature.c          |   9 +-
 arch/s390/kvm/arm64/feature.h          |   6 +
 arch/s390/kvm/arm64/guest.c            | 250 +++++++++++++++++++++++++++++++++
 arch/s390/kvm/arm64/reset.c            |  24 ++++
 6 files changed, 297 insertions(+), 3 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host_arm64.h b/arch/s390/include/asm/kvm_host_arm64.h
index 20f51cf9ce37..f096faceb19d 100644
--- a/arch/s390/include/asm/kvm_host_arm64.h
+++ b/arch/s390/include/asm/kvm_host_arm64.h
@@ -34,7 +34,9 @@ bool cpus_have_final_cap(unsigned int num);
 #define KVM_MAX_VCPUS 1
 #define __KVM_HAVE_VCPU_RUN_PID_CHANGE
 
-#define KVM_S390_ARM64_IMPL_FEATURES 0
+#define KVM_S390_ARM64_IMPL_FEATURES (		\
+	 BIT(KVM_ARM_VCPU_SVE)			\
+)
 #define KVM_HALT_POLL_NS_DEFAULT 50000
 
 #define KVM_S390_MANAGES_S390_GUEST 0
@@ -130,7 +132,6 @@ struct kvm_vcpu_stat {
 };
 
 #define kvm_has_mte(_kvm) false
-#define vcpu_has_sve(_vcpu) false
 #define vcpu_has_ptrauth(_vcpu) false
 
 #define kvm_has_fpmr(kvm) \
diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c
index b5a0f8c1e9a4..8c2a07a96ee3 100644
--- a/arch/s390/kvm/arm64/arm.c
+++ b/arch/s390/kvm/arm64/arm.c
@@ -54,6 +54,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_ONE_REG:
 		ret = 1;
 		break;
+	case KVM_CAP_ARM_SVE:
+		ret = system_supports_sve();
+		break;
 	default:
 		ret = 0;
 	}
@@ -264,6 +267,9 @@ static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu)
 {
 	unsigned long features = KVM_S390_ARM64_IMPL_FEATURES;
 
+	if (!system_supports_sve())
+		clear_bit(KVM_ARM_VCPU_SVE, &features);
+
 	return features;
 }
 
diff --git a/arch/s390/kvm/arm64/feature.c b/arch/s390/kvm/arm64/feature.c
index f8b23ba85f9e..0979e0cadc66 100644
--- a/arch/s390/kvm/arm64/feature.c
+++ b/arch/s390/kvm/arm64/feature.c
@@ -56,7 +56,6 @@ void __init kvm_arm_host_sanitize_features(struct qaaf_qmc_block *qaaf_qmc)
 	MASK_RESERVED(qaaf_qmc, ID_AA64MMFR4_EL1);
 
 	MODIFY(qaaf_qmc, ID_AA64PFR0_EL1, SEL2, NI);
-	MODIFY(qaaf_qmc, ID_AA64PFR0_EL1, SVE, NI);
 	MASK_RESERVED(qaaf_qmc, ID_AA64PFR0_EL1);
 	MASK_RESERVED(qaaf_qmc, ID_AA64PFR1_EL1);
 	MASK_RESERVED(qaaf_qmc, ID_AA64PFR2_EL1);
@@ -161,3 +160,11 @@ bool cpus_have_final_cap(unsigned int num)
 		return false;
 	}
 }
+
+bool system_supports_sve(void)
+{
+	return cpu_has_vx() &&
+	       SYS_FIELD_GET(ID_AA64PFR0_EL1, SVE,
+			     read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) ==
+		       ID_AA64PFR0_EL1_SVE_IMP;
+}
diff --git a/arch/s390/kvm/arm64/feature.h b/arch/s390/kvm/arm64/feature.h
index 0a59047fda6e..9af6173f6b11 100644
--- a/arch/s390/kvm/arm64/feature.h
+++ b/arch/s390/kvm/arm64/feature.h
@@ -83,4 +83,10 @@ static inline u64 kvm_sae_irptc(void)
 
 #define kvm_vcpu_has_pmu(_v) false
 
+#define vcpu_has_sve(_vcpu)							\
+	(system_supports_sve() &&						\
+	 test_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &((_vcpu)->kvm)->arch.flags))
+
+bool system_supports_sve(void);
+
 #endif /* ARCH_S390_KVM_FEATURE_H */
diff --git a/arch/s390/kvm/arm64/guest.c b/arch/s390/kvm/arm64/guest.c
index 047460975936..9e9d0034ddfc 100644
--- a/arch/s390/kvm/arm64/guest.c
+++ b/arch/s390/kvm/arm64/guest.c
@@ -5,6 +5,9 @@
 #include <arm64/kvm_emulate.h>
 #include <arm64/kvm_nested.h>
 #include <arm64/sys_regs.h>
+#include <arm64/sve_context.h>
+
+#include "feature.h"
 
 #define SVE_VQ_MIN	__SVE_VQ_MIN
 #define SVE_NUM_ZREGS	KVM_ARM64_SVE_NUM_ZREGS
@@ -52,6 +55,243 @@ const struct kvm_stats_header kvm_vcpu_stats_header = {
 		       sizeof(kvm_vcpu_stats_desc),
 };
 
+static int sve_zreg_index(__u64 id, unsigned int *regnum)
+{
+	/* Currently only one slice is supported on arm, and our zreg is only 128 bit */
+	const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(KVM_ARM64_SVE_NUM_ZREGS - 1,
+						       KVM_ARM64_SVE_MAX_SLICES - 1);
+	const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0);
+
+	if (id < zreg_id_min || id > zreg_id_max)
+		return -EINVAL;
+	if ((id & SVE_REG_SLICE_MASK) > 0)
+		return -ENOENT;
+
+	*regnum = (id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
+
+	return 0;
+}
+
+static int sve_preg_index(u64 id, unsigned int *regnum)
+{
+	const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(KVM_ARM64_SVE_MAX_SLICES - 1);
+	const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0);
+
+	if (id < preg_id_min || id > preg_id_max)
+		return -EINVAL;
+	if ((id & SVE_REG_SLICE_MASK) > 0)
+		return -ENOENT;
+
+	*regnum = (id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
+
+	return 0;
+}
+
+static int sve_ffr_index(u64 id, unsigned int *regnum)
+{
+	if (id != KVM_REG_ARM64_SVE_FFR(0))
+		return -EINVAL;
+	if ((id & SVE_REG_SLICE_MASK) > 0)
+		return -ENOENT;
+
+	*regnum = (id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
+
+	return 0;
+}
+
+static inline int get_sve_ffr_reg(struct kvm_vcpu *vcpu, unsigned int regnum,
+				  u16 __user *uptr)
+{
+	/* ffr is pregmax + 1 */
+	if (regnum != KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (put_user(vcpu->arch.sae_block.sve_ffr, uptr))
+		return -EFAULT;
+	return sizeof(vcpu->arch.sae_block.sve_ffr);
+}
+
+static inline int get_sve_preg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       u16 __user *uptr)
+{
+	if (regnum >= KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (put_user(vcpu->arch.sae_block.sve_pregs[regnum], uptr))
+		return -EFAULT;
+	return sizeof(vcpu->arch.sae_block.sve_pregs[regnum]);
+}
+
+static inline int get_sve_zreg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       __vector128 __user *uptr)
+{
+	if (regnum >= KVM_ARM64_SVE_NUM_ZREGS)
+		return -EINVAL;
+
+	/* vreg and svreg overlap and zreg is also just 128 bit so we reuse the vreg space */
+	if (copy_to_user(uptr, &vcpu->arch.ctxt.vregs[regnum],
+			 sizeof(vcpu->arch.ctxt.vregs[regnum])))
+		return -EFAULT;
+	return sizeof(vcpu->arch.ctxt.vregs[regnum]);
+}
+
+static inline int set_sve_ffr_reg(struct kvm_vcpu *vcpu, unsigned int regnum,
+				  const u16 __user *uptr)
+{
+	/* ffr is pregmax + 1*/
+	if (regnum != KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (get_user(vcpu->arch.sae_block.sve_ffr, uptr))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int set_sve_preg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       const u16 __user *uptr)
+{
+	if (regnum >= KVM_ARM64_SVE_NUM_PREGS)
+		return -EINVAL;
+
+	if (get_user(vcpu->arch.sae_block.sve_pregs[regnum], uptr))
+		return -EFAULT;
+	return 0;
+}
+
+static inline int set_sve_zreg(struct kvm_vcpu *vcpu, unsigned int regnum,
+			       const __vector128 __user *uptr)
+{
+	if (regnum >= KVM_ARM64_SVE_NUM_ZREGS)
+		return -EINVAL;
+
+	/*vreg and svreg overlap and zreg is also just 128 bit so we reuse the vreg space*/
+	if (copy_from_user(&vcpu->arch.ctxt.vregs[regnum], uptr,
+			   sizeof(vcpu->arch.ctxt.vregs[regnum])))
+		return -EFAULT;
+	return 0;
+}
+
+static int set_sve_vls(struct kvm_vcpu *vcpu, const void __user *uptr)
+{
+	u64 vqs[KVM_ARM64_SVE_VLS_WORDS] = { 0 };
+	unsigned int vq;
+
+	if (!vcpu_has_sve(vcpu))
+		return -ENOENT;
+
+	if (kvm_arm_vcpu_sve_finalized(vcpu))
+		return -EPERM;
+
+	if (copy_from_user(vqs, uptr, sizeof(vqs)))
+		return -EFAULT;
+
+	/* only 128 bit and 1 VQ are supported , nothing saved just check validity */
+	for (vq = KVM_ARM64_SVE_VQ_MIN + 1; vq <= KVM_ARM64_SVE_VQ_MAX; ++vq)
+		if (vq_present(vqs, vq))
+			return -EINVAL;
+
+	/* run with a vl of 0 not valid */
+	if (!vq_present(vqs, KVM_ARM64_SVE_VQ_MIN))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int get_sve_vls(struct kvm_vcpu *vcpu, void __user *uptr)
+{
+	u64 vqs[KVM_ARM64_SVE_VLS_WORDS] = { 0 };
+
+	if (!vcpu_has_sve(vcpu))
+		return -ENOENT;
+
+	/* currently only 128 bit are supported so we only set bit 0  hardcoded */
+	vqs[0] |= vq_mask(KVM_ARM64_SVE_VQ_MIN);
+
+	if (copy_to_user(uptr, vqs, sizeof(vqs)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int sve_reg_index(u64 id, unsigned int *regnum)
+{
+	int ret;
+
+	ret = sve_ffr_index(id, regnum);
+	if (ret == -EINVAL)
+		ret = sve_preg_index(id, regnum);
+	if (ret == -EINVAL)
+		ret = sve_zreg_index(id, regnum);
+
+	return ret;
+}
+
+static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+	void __user *uptr = (void __user *)reg->addr;
+	unsigned int regnum;
+	int size;
+	int ret;
+
+	if (reg->id == KVM_REG_ARM64_SVE_VLS)
+		return get_sve_vls(vcpu, uptr);
+
+	ret = sve_reg_index(reg->id, &regnum);
+	if (ret)
+		return ret;
+
+	if (!vcpu_has_sve(vcpu))
+		return -ENOENT;
+
+	if (!kvm_arm_vcpu_sve_finalized(vcpu))
+		return -EPERM;
+
+	if (sve_ffr_index(reg->id, &regnum) == 0)
+		size = get_sve_ffr_reg(vcpu, regnum, uptr);
+	else if (sve_preg_index(reg->id, &regnum) == 0)
+		size = get_sve_preg(vcpu, regnum, uptr);
+	else
+		size = get_sve_zreg(vcpu, regnum, uptr);
+
+	if (size < 0)
+		return size;
+
+	/*
+	 * Only the low 128 bits of a Z register and the low 16 bits of a
+	 * P register or the FFR are live: zero-pad the rest.
+	 */
+	if (clear_user((u8 __user *)uptr + size, KVM_REG_SIZE(reg->id) - size))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+	const void __user *uptr = (const void __user *)reg->addr;
+	unsigned int regnum;
+	int ret;
+
+	if (reg->id == KVM_REG_ARM64_SVE_VLS)
+		return set_sve_vls(vcpu, uptr);
+
+	ret = sve_reg_index(reg->id, &regnum);
+	if (ret)
+		return ret;
+
+	if (!vcpu_has_sve(vcpu))
+		return -ENOENT;
+
+	if (!kvm_arm_vcpu_sve_finalized(vcpu))
+		return -EPERM;
+
+	if (sve_ffr_index(reg->id, &regnum) == 0)
+		return set_sve_ffr_reg(vcpu, regnum, uptr);
+	if (sve_preg_index(reg->id, &regnum) == 0)
+		return set_sve_preg(vcpu, regnum, uptr);
+	return set_sve_zreg(vcpu, regnum, uptr);
+}
+
 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 {
 	int ret;
@@ -61,6 +301,11 @@ int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 		return ret;
 	uindices += ret;
 
+	ret = copy_sve_reg_indices(vcpu, uindices);
+	if (ret < 0)
+		return ret;
+	uindices += ret;
+
 	return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
 }
 
@@ -68,6 +313,7 @@ unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
 {
 	unsigned long num = num_core_regs(vcpu);
 
+	num += num_sve_regs(vcpu);
 	num += kvm_arm_num_sys_reg_descs(vcpu);
 	return num;
 }
@@ -81,6 +327,8 @@ int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 	switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
 	case KVM_REG_ARM_CORE:
 		return get_core_reg(vcpu, reg);
+	case KVM_REG_ARM64_SVE:
+		return get_sve_reg(vcpu, reg);
 	default:
 		return kvm_arm_sys_reg_get_reg(vcpu, reg);
 	}
@@ -95,6 +343,8 @@ int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 	switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
 	case KVM_REG_ARM_CORE:
 		return set_core_reg(vcpu, reg);
+	case KVM_REG_ARM64_SVE:
+		return set_sve_reg(vcpu, reg);
 	default:
 		return kvm_arm_sys_reg_set_reg(vcpu, reg);
 	}
diff --git a/arch/s390/kvm/arm64/reset.c b/arch/s390/kvm/arm64/reset.c
index dc00bf398b15..1ce05a95689f 100644
--- a/arch/s390/kvm/arm64/reset.c
+++ b/arch/s390/kvm/arm64/reset.c
@@ -2,6 +2,7 @@
 
 #include <linux/kvm_host.h>
 #include <linux/fpu.h>
+#include <linux/cpufeature.h>
 
 #include <asm/aef.h>
 
@@ -12,8 +13,13 @@
 
 #include <clocksource/arm_arch_timer.h>
 
+#include "feature.h"
+
 bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)
 {
+	if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu))
+		return false;
+
 	return true;
 }
 
@@ -76,6 +82,14 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 	 */
 	preempt_disable();
 
+	if (!vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED)) {
+		if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE))
+			set_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &vcpu->kvm->arch.flags);
+	} else if (vcpu_has_sve(vcpu)) {
+		memset(vcpu->arch.sae_block.sve_pregs, 0, sizeof(vcpu->arch.sae_block.sve_pregs));
+		vcpu->arch.sae_block.sve_ffr = 0;
+	}
+
 	/* The reset must run with an unloaded save area */
 	loaded = vcpu_is_loaded(vcpu);
 	if (loaded)
@@ -114,5 +128,15 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 
 int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)
 {
+	switch (feature) {
+	case KVM_ARM_VCPU_SVE:
+		if (!vcpu_has_sve(vcpu))
+			return -EINVAL;
+		if (vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED))
+			return -EPERM;
+		vcpu_set_flag(vcpu, VCPU_SVE_FINALIZED);
+		return 0;
+	}
+
 	return -EINVAL;
 }

-- 
2.56.0