Re: [kvm-unit-tests PATCH v3 4/6] lib: s390x: Use a new asce for each PV guest

[Claudio Imbrenda <imbrenda@linux.ibm.com>]

On Fri, 21 Oct 2022 06:39:00 +0000
Janosch Frank <frankja@linux.ibm.com> wrote:

> Every PV guest needs its own ASCE so let's copy the topmost table
> designated by CR1 to create a new ASCE for the PV guest. Before and
> after SIE we now need to switch ASCEs to and from the PV guest / test
> ASCE. The SIE assembly function does that automatically.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

> ---
>  lib/s390x/asm-offsets.c |  2 ++
>  lib/s390x/sie.c         |  2 ++
>  lib/s390x/sie.h         |  2 ++
>  lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
>  lib/s390x/uv.h          |  5 ++---
>  s390x/cpu.S             |  6 ++++++
>  6 files changed, 37 insertions(+), 4 deletions(-)
> 
> diff --git a/lib/s390x/asm-offsets.c b/lib/s390x/asm-offsets.c
> index fbea3278..f612f327 100644
> --- a/lib/s390x/asm-offsets.c
> +++ b/lib/s390x/asm-offsets.c
> @@ -75,9 +75,11 @@ int main(void)
>  	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
>  	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
>  	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
> +	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
>  	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
>  	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
>  	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
> +	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
>  	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
>  	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
>  	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
> diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
> index 3fee3def..6efad965 100644
> --- a/lib/s390x/sie.c
> +++ b/lib/s390x/sie.c
> @@ -85,6 +85,8 @@ void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
>  
>  	/* Guest memory chunks are always 1MB */
>  	assert(!(guest_mem_len & ~HPAGE_MASK));
> +	/* For non-PV guests we re-use the host's ASCE for ease of use */
> +	vm->save_area.guest.asce = stctg(1);
>  	/* Currently MSO/MSL is the easiest option */
>  	vm->sblk->mso = (uint64_t)guest_mem;
>  	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
> diff --git a/lib/s390x/sie.h b/lib/s390x/sie.h
> index 320c4218..3e3605c9 100644
> --- a/lib/s390x/sie.h
> +++ b/lib/s390x/sie.h
> @@ -205,12 +205,14 @@ union {
>  struct vm_uv {
>  	uint64_t vm_handle;
>  	uint64_t vcpu_handle;
> +	uint64_t asce;
>  	void *conf_base_stor;
>  	void *conf_var_stor;
>  	void *cpu_stor;
>  };
>  
>  struct vm_save_regs {
> +	uint64_t asce;
>  	uint64_t grs[16];
>  	uint64_t fprs[16];
>  	uint32_t fpc;
> diff --git a/lib/s390x/uv.c b/lib/s390x/uv.c
> index 3b4cafa9..b2a43424 100644
> --- a/lib/s390x/uv.c
> +++ b/lib/s390x/uv.c
> @@ -90,6 +90,25 @@ void uv_init(void)
>  	initialized = true;
>  }
>  
> +/*
> + * Create a new ASCE for the UV config because they can't be shared
> + * for security reasons. We just simply copy the top most table into a
> + * fresh set of allocated pages and use those pages as the asce.
> + */
> +static uint64_t create_asce(void)
> +{
> +	void *pgd_new, *pgd_old;
> +	uint64_t asce = stctg(1);
> +
> +	pgd_new = memalign_pages(PAGE_SIZE, PAGE_SIZE * 4);
> +	pgd_old = (void *)(asce & PAGE_MASK);
> +
> +	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
> +
> +	asce = __pa(pgd_new) | ASCE_P | (asce & (ASCE_DT | ASCE_TL));
> +	return asce;
> +}
> +
>  void uv_create_guest(struct vm *vm)
>  {
>  	struct uv_cb_cgc uvcb_cgc = {
> @@ -125,7 +144,8 @@ void uv_create_guest(struct vm *vm)
>  	vm->uv.cpu_stor = memalign_pages_flags(PAGE_SIZE, uvcb_qui.cpu_stor_len, 0);
>  	uvcb_csc.stor_origin = (uint64_t)vm->uv.cpu_stor;
>  
> -	uvcb_cgc.guest_asce = (uint64_t)stctg(1);
> +	uvcb_cgc.guest_asce = create_asce();
> +	vm->save_area.guest.asce = uvcb_cgc.guest_asce;
>  	uvcb_cgc.guest_sca = (uint64_t)vm->sca;
>  
>  	cc = uv_call(0, (uint64_t)&uvcb_cgc);
> @@ -166,6 +186,8 @@ void uv_destroy_guest(struct vm *vm)
>  	assert(cc == 0);
>  	free_pages(vm->uv.conf_base_stor);
>  	free_pages(vm->uv.conf_var_stor);
> +
> +	free_pages((void *)(vm->uv.asce & PAGE_MASK));
>  }
>  
>  int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
> diff --git a/lib/s390x/uv.h b/lib/s390x/uv.h
> index 44264861..5fe29bda 100644
> --- a/lib/s390x/uv.h
> +++ b/lib/s390x/uv.h
> @@ -28,9 +28,8 @@ static inline void uv_setup_asces(void)
>  	/* We need to have a valid primary ASCE to run guests. */
>  	setup_vm();
>  
> -	/* Set P bit in ASCE as it is required for PV guests */
> -	asce = stctg(1) | ASCE_P;
> -	lctlg(1, asce);
> +	/* Grab the ASCE which setup_vm() just set up */
> +	asce = stctg(1);
>  
>  	/* Copy ASCE into home space CR */
>  	lctlg(13, asce);
> diff --git a/s390x/cpu.S b/s390x/cpu.S
> index 82b5e25d..45bd551a 100644
> --- a/s390x/cpu.S
> +++ b/s390x/cpu.S
> @@ -76,6 +76,9 @@ sie64a:
>  	.endr
>  	stfpc	SIE_SAVEAREA_HOST_FPC(%r3)
>  
> +	stctg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r3)
> +	lctlg	%c1, %c1, SIE_SAVEAREA_GUEST_ASCE(%r3)
> +
>  	# Store scb and save_area pointer into stack frame
>  	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
>  	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
> @@ -102,6 +105,9 @@ sie_exit:
>  	# Load guest register save area
>  	lg	%r14,__SF_SIE_SAVEAREA(%r15)
>  
> +	# Restore the host asce
> +	lctlg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r14)
> +
>  	# Store guest's gprs, fprs and fpc
>  	stmg	%r0,%r13,SIE_SAVEAREA_GUEST_GRS(%r14)	# save guest gprs 0-13
>  	.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15