Re: [PATCH v10 2/3] KVM: s390: guest support for topology function

[Janosch Frank <frankja@linux.ibm.com>]

On 6/20/22 14:54, Pierre Morel wrote:
> We report a topology change to the guest for any CPU hotplug.
> 
> The reporting to the guest is done using the Multiprocessor
> Topology-Change-Report (MTCR) bit of the utility entry in the guest's
> SCA which will be cleared during the interpretation of PTF.
> 
> On every vCPU creation we set the MCTR bit to let the guest know the
> next time he uses the PTF with command 2 instruction that the
> topology changed and that he should use the STSI(15.1.x) instruction
> to get the topology details.
> 
> STSI(15.1.x) gives information on the CPU configuration topology.
> Let's accept the interception of STSI with the function code 15 and
> let the userland part of the hypervisor handle it when userland
> support the CPU Topology facility.
> 
> Signed-off-by: Pierre Morel <pmorel@linux.ibm.com>
> ---
>   arch/s390/include/asm/kvm_host.h | 11 ++++++++---
>   arch/s390/kvm/kvm-s390.c         | 27 ++++++++++++++++++++++++++-
>   arch/s390/kvm/priv.c             | 15 +++++++++++----
>   arch/s390/kvm/vsie.c             |  3 +++
>   4 files changed, 48 insertions(+), 8 deletions(-)
> 
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 766028d54a3e..bb54196d4ed6 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -97,15 +97,19 @@ struct bsca_block {
>   	union ipte_control ipte_control;
>   	__u64	reserved[5];
>   	__u64	mcn;
> -	__u64	reserved2;
> +#define SCA_UTILITY_MTCR	0x8000

I'm not too happy having this in the bsca but not in the esca. I'd 
suggest putting it outside the structs or to go with my next suggestion:

Just make it a bit field struct and make that a member in bsca/esca.
No messing about with ANDing, ORing etc.

It's unfortunate that we only use one bit in that field but I'd still 
find it easier to read.

> +	__u16	utility;
> +	__u8	reserved2[6];
>   	struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS];
>   };
>   
>   struct esca_block {
>   	union ipte_control ipte_control;
> -	__u64   reserved1[7];
> +	__u64   reserved1[6];
> +	__u16	utility;
> +	__u8	reserved2[6];
>   	__u64   mcn[4];
> -	__u64   reserved2[20];
> +	__u64   reserved3[20];
>   	struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS];
>   };
>   
> @@ -249,6 +253,7 @@ struct kvm_s390_sie_block {
>   #define ECB_SPECI	0x08
>   #define ECB_SRSI	0x04
>   #define ECB_HOSTPROTINT	0x02
> +#define ECB_PTF		0x01
>   	__u8	ecb;			/* 0x0061 */
>   #define ECB2_CMMA	0x80
>   #define ECB2_IEP	0x20
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 8fcb56141689..95b96019ca8e 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -1691,6 +1691,25 @@ static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
>   	return ret;
>   }
>   
> +/**
> + * kvm_s390_sca_set_mtcr
> + * @kvm: guest KVM description
> + *
> + * Is only relevant if the topology facility is present,
> + * the caller should check KVM facility 11

I'm not sure that this statement make sense since you set the mctr in 
kvm_s390_vcpu_setup() unconditionally and don't check stfle 11.

I think we can remove the second line from this.

> + *
> + * Updates the Multiprocessor Topology-Change-Report to signal
> + * the guest with a topology change.

Please swap those two comments

> + */
> +static void kvm_s390_sca_set_mtcr(struct kvm *kvm)
> +{
> +	struct bsca_block *sca = kvm->arch.sca; /* SCA version doesn't matter */

Please put the comment above the statement and maybe extend it a bit:
SCA version doesn't matter, the utility field always has the same offset.

> +
> +	ipte_lock(kvm);
> +	sca->utility |= SCA_UTILITY_MTCR;
> +	ipte_unlock(kvm);
> +}
> +
>   static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
>   {
>   	int ret;
> @@ -3143,7 +3162,6 @@ __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
>   
>   void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
>   {
> -

Please remove that change

>   	gmap_enable(vcpu->arch.enabled_gmap);
>   	kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
>   	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
> @@ -3272,6 +3290,11 @@ static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
>   		vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
>   	if (test_kvm_facility(vcpu->kvm, 9))
>   		vcpu->arch.sie_block->ecb |= ECB_SRSI;
> +
> +	/* PTF needs guest facilities to enable interpretation */
> +	if (test_kvm_facility(vcpu->kvm, 11))
> +		vcpu->arch.sie_block->ecb |= ECB_PTF;
> +
>   	if (test_kvm_facility(vcpu->kvm, 73))
>   		vcpu->arch.sie_block->ecb |= ECB_TE;
>   	if (!kvm_is_ucontrol(vcpu->kvm))
> @@ -3403,6 +3426,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
>   	rc = kvm_s390_vcpu_setup(vcpu);
>   	if (rc)
>   		goto out_ucontrol_uninit;
> +
> +	kvm_s390_sca_set_mtcr(vcpu->kvm);
>   	return 0;
>   
>   out_ucontrol_uninit:
> diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
> index 12c464c7cddf..77a692238585 100644
> --- a/arch/s390/kvm/priv.c
> +++ b/arch/s390/kvm/priv.c
> @@ -873,10 +873,13 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
>   	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
>   		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
>   
> -	if (fc > 3) {
> -		kvm_s390_set_psw_cc(vcpu, 3);
> -		return 0;
> -	}
> +	/* Bailout forbidden function codes */
> +	if (fc > 3 && fc != 15)
> +		goto out_no_data;
> +
> +	/* fc 15 is provided with PTF/CPU topology support */
> +	if (fc == 15 && !test_kvm_facility(vcpu->kvm, 11))
> +		goto out_no_data;
>   
>   	if (vcpu->run->s.regs.gprs[0] & 0x0fffff00
>   	    || vcpu->run->s.regs.gprs[1] & 0xffff0000)
> @@ -910,6 +913,10 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
>   			goto out_no_data;
>   		handle_stsi_3_2_2(vcpu, (void *) mem);
>   		break;
> +	case 15:
> +		trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
> +		insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2);
> +		return -EREMOTE;
>   	}
>   	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
>   		memcpy((void *)sida_origin(vcpu->arch.sie_block), (void *)mem,
> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> index dada78b92691..4f4fee697550 100644
> --- a/arch/s390/kvm/vsie.c
> +++ b/arch/s390/kvm/vsie.c
> @@ -503,6 +503,9 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
>   	/* Host-protection-interruption introduced with ESOP */
>   	if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ESOP))
>   		scb_s->ecb |= scb_o->ecb & ECB_HOSTPROTINT;
> +	/* CPU Topology */

Maybe also add:
This facility only uses the utility field of the SCA and none of the cpu 
entries that are problematic with the other interpretation facilities so 
we can pass it through.

> +	if (test_kvm_facility(vcpu->kvm, 11))
> +		scb_s->ecb |= scb_o->ecb & ECB_PTF;
>   	/* transactional execution */
>   	if (test_kvm_facility(vcpu->kvm, 73) && wants_tx) {
>   		/* remap the prefix is tx is toggled on */