Re: [kvm-unit-tests PATCH v2 3/3] s390x: pv: Add IPL reset tests

[Claudio Imbrenda <imbrenda@linux.ibm.com>]

On Fri, 24 Mar 2023 12:04:31 +0000
Janosch Frank <frankja@linux.ibm.com> wrote:

> The diag308 requires extensive cooperation between the hypervisor and
> the Ultravisor so the Ultravisor can make sure all necessary reset
> steps have been done.
> 
> Let's check if we get the correct validity errors.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  s390x/Makefile                           |   2 +
>  s390x/pv-ipl.c                           | 246 +++++++++++++++++++++++
>  s390x/snippets/asm/snippet-pv-diag-308.S |  67 ++++++
>  3 files changed, 315 insertions(+)
>  create mode 100644 s390x/pv-ipl.c
>  create mode 100644 s390x/snippets/asm/snippet-pv-diag-308.S
> 
> diff --git a/s390x/Makefile b/s390x/Makefile
> index 858f5af4..e8559a4e 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -42,6 +42,7 @@ tests += $(TEST_DIR)/exittime.elf
>  
>  pv-tests += $(TEST_DIR)/pv-diags.elf
>  pv-tests += $(TEST_DIR)/pv-icptcode.elf
> +pv-tests += $(TEST_DIR)/pv-ipl.elf
>  
>  ifneq ($(HOST_KEY_DOCUMENT),)
>  ifneq ($(GEN_SE_HEADER),)
> @@ -124,6 +125,7 @@ $(TEST_DIR)/pv-icptcode.elf: pv-snippets += $(SNIPPET_DIR)/asm/snippet-pv-icpt-1
>  $(TEST_DIR)/pv-icptcode.elf: pv-snippets += $(SNIPPET_DIR)/asm/snippet-pv-icpt-loop.gbin
>  $(TEST_DIR)/pv-icptcode.elf: pv-snippets += $(SNIPPET_DIR)/asm/snippet-loop.gbin
>  $(TEST_DIR)/pv-icptcode.elf: pv-snippets += $(SNIPPET_DIR)/asm/snippet-pv-icpt-vir-timing.gbin
> +$(TEST_DIR)/pv-ipl.elf: pv-snippets += $(SNIPPET_DIR)/asm/snippet-pv-diag-308.gbin
>  
>  ifneq ($(GEN_SE_HEADER),)
>  snippets += $(pv-snippets)
> diff --git a/s390x/pv-ipl.c b/s390x/pv-ipl.c
> new file mode 100644
> index 00000000..d17cf59d
> --- /dev/null
> +++ b/s390x/pv-ipl.c
> @@ -0,0 +1,246 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * PV diagnose 308 (IPL) tests
> + *
> + * Copyright (c) 2023 IBM Corp
> + *
> + * Authors:
> + *  Janosch Frank <frankja@linux.ibm.com>
> + */
> +#include <libcflat.h>
> +#include <sie.h>
> +#include <sclp.h>
> +#include <snippet.h>
> +#include <asm/facility.h>
> +#include <asm/uv.h>
> +
> +static struct vm vm;
> +
> +static void setup_gbin(void)
> +{
> +	extern const char SNIPPET_NAME_START(asm, snippet_pv_diag_308)[];
> +	extern const char SNIPPET_NAME_END(asm, snippet_pv_diag_308)[];
> +	extern const char SNIPPET_HDR_START(asm, snippet_pv_diag_308)[];
> +	extern const char SNIPPET_HDR_END(asm, snippet_pv_diag_308)[];
> +	int size_hdr = SNIPPET_HDR_LEN(asm, snippet_pv_diag_308);
> +	int size_gbin = SNIPPET_LEN(asm, snippet_pv_diag_308);
> +
> +	snippet_pv_init(&vm, SNIPPET_NAME_START(asm, snippet_pv_diag_308),
> +			SNIPPET_HDR_START(asm, snippet_pv_diag_308),
> +			size_gbin, size_hdr, SNIPPET_UNPACK_OFF);
> +}
> +
> +static void test_diag_308_1(void)
> +{
> +	uint16_t rc, rrc;
> +	int cc;
> +
> +	report_prefix_push("subcode 1");
> +	setup_gbin();
> +
> +	sie(&vm);
> +	report(vm.sblk->icptcode == ICPT_PV_INSTR && vm.sblk->ipa == 0x8302 &&
> +	       vm.sblk->ipb == 0x50000000 && vm.save_area.guest.grs[5] == 0x500,
> +	       "intercept values diag 500");
> +	/* The snippet asked us for the subcode and we answer with 1 in gr2 */
> +	vm.save_area.guest.grs[2] = 1;
> +
> +	/* Continue after diag 0x500, next icpt should be the 0x308 */
> +	sie(&vm);
> +	report(vm.sblk->icptcode == ICPT_PV_NOTIFY && vm.sblk->ipa == 0x8302 &&
> +	       vm.sblk->ipb == 0x50000000 && vm.save_area.guest.grs[5] == 0x308,
> +	       "intercept values diag 0x308");
> +	report(vm.save_area.guest.grs[2] == 1,
> +	       "subcode 1");
> +
> +	/*
> +	 * We need to perform several UV calls to emulate the subcode
> +	 * 1. Failing to do that should result in a validity.
> +	 *
> +	 * - Mark all cpus as stopped
> +	 * - Unshare all
> +	 * - Prepare for reset
> +	 * - Reset the cpus, calling one gets an initial reset
> +	 * - Load the reset PSW
> +	 */
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity no UVCs");

didn't you introduce a new function in patch 1 to check for PV validity?

> +
> +	/* Mark the CPU as stopped so we can unshare and reset */
> +	cc = uv_set_cpu_state(vm.sblk->pv_handle_cpu, PV_CPU_STATE_STP);
> +	report(!cc, "Set cpu stopped");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity stopped");
> +
> +	/* Unshare all memory */
> +	cc = uv_cmd_nodata(vm.sblk->pv_handle_config,
> +			   UVC_CMD_SET_UNSHARED_ALL, &rc, &rrc);
> +	report(cc == 0 && rc == 1, "Unshare all");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000,
> +	       "validity stopped, unshared");
> +
> +	/* Prepare the CPU reset */
> +	cc = uv_cmd_nodata(vm.sblk->pv_handle_config,
> +			   UVC_CMD_PREPARE_RESET, &rc, &rrc);
> +	report(cc == 0 && rc == 1, "Prepare reset call");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000,
> +	       "validity stopped, unshared, prepare");
> +
> +	/* Do the reset */
> +	cc = uv_cmd_nodata(vm.sblk->pv_handle_cpu,
> +			   UVC_CMD_CPU_RESET_INITIAL, &rc, &rrc);
> +	report(cc == 0 && rc == 1, "Initial reset cpu");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000,
> +	       "validity stopped, unshared, prepare, reset");
> +
> +	/* Load the PSW from 0x0 */
> +	cc = uv_set_cpu_state(vm.sblk->pv_handle_cpu, PV_CPU_STATE_OPR_LOAD);
> +	report(!cc, "Set cpu load");
> +
> +	/*
> +	 * Check if we executed the iaddr of the reset PSW, we should
> +	 * see a diagnose 0x9c PV instruction notification.
> +	 */
> +	sie(&vm);
> +	report(vm.sblk->icptcode == ICPT_PV_NOTIFY && vm.sblk->ipa == 0x8302 &&
> +	       vm.sblk->ipb == 0x50000000 && vm.save_area.guest.grs[5] == 0x9c &&
> +	       vm.save_area.guest.grs[0] == 42,
> +	       "intercept values after diag 0x308");
> +
> +
> +	uv_destroy_guest(&vm);
> +	report_prefix_pop();
> +}
> +
> +static void test_diag_308_0(void)
> +{

this function seems a clone of the previous, with very minimal changes,
can't you merge them?

> +	uint16_t rc, rrc;
> +	int cc;
> +
> +	report_prefix_push("subcode 0");
> +	setup_gbin();
> +
> +	sie(&vm);
> +	report(vm.sblk->icptcode == ICPT_PV_INSTR && vm.sblk->ipa == 0x8302 &&
> +	       vm.sblk->ipb == 0x50000000 && vm.save_area.guest.grs[5] == 0x500,
> +	       "intercept values diag 500");
> +	/* The snippet asked us for the subcode and we answer with 0 in gr2 */
> +	vm.save_area.guest.grs[2] = 0;
> +
> +	/* Continue after diag 0x500, next icpt should be the 0x308 */
> +	sie(&vm);
> +	report(vm.sblk->icptcode == ICPT_PV_NOTIFY && vm.sblk->ipa == 0x8302 &&
> +	       vm.sblk->ipb == 0x50000000 && vm.save_area.guest.grs[5] == 0x308,
> +	       "intercept values");
> +	report(vm.save_area.guest.grs[2] == 0,
> +	       "subcode 0");
> +
> +	/*
> +	 * We need to perform several UV calls to emulate the subcode
> +	 * 0. Failing to do that should result in a validity.
> +	 *
> +	 * - Mark all cpus as stopped
> +	 * - Unshare all memory
> +	 * - Prepare the reset
> +	 * - Reset the cpus
> +	 * - Load the reset PSW
> +	 */
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity, no action");
> +
> +	/* Mark the CPU as stopped so we can unshare and reset */
> +	cc = uv_set_cpu_state(vm.sblk->pv_handle_cpu, PV_CPU_STATE_STP);
> +	report(!cc, "Set cpu stopped");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity, stopped");
> +
> +	/* Unshare all memory */
> +	cc = uv_cmd_nodata(vm.sblk->pv_handle_config,
> +			   UVC_CMD_SET_UNSHARED_ALL, &rc, &rrc);
> +	report(cc == 0 && rc == 1, "Unshare all");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity stopped, unshared");
> +
> +	/* Prepare the CPU reset */
> +	cc = uv_cmd_nodata(vm.sblk->pv_handle_config,
> +			   UVC_CMD_PREPARE_RESET, &rc, &rrc);
> +	report(cc == 0 && rc == 1, "Prepare reset call");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity stopped, unshared, prep reset");
> +
> +	/* Do the reset */
> +	cc = uv_cmd_nodata(vm.sblk->pv_handle_cpu,
> +			   UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
> +	report(cc == 0 && rc == 1, "Clear reset cpu");
> +
> +	sie_expect_validity(&vm);
> +	sie(&vm);
> +	report((sie_get_validity(&vm) & 0xff00) == 0x2000, "validity stopped, unshared, prep reset, cpu reset");
> +
> +	/* Load the PSW from 0x0 */
> +	cc = uv_set_cpu_state(vm.sblk->pv_handle_cpu, PV_CPU_STATE_OPR_LOAD);
> +	report(!cc, "Set cpu load");
> +
> +	/*
> +	 * Check if we executed the iaddr of the reset PSW, we should
> +	 * see a diagnose 0x9c PV instruction notification.
> +	 */
> +	sie(&vm);
> +	report(vm.sblk->icptcode == ICPT_PV_NOTIFY && vm.sblk->ipa == 0x8302 &&
> +	       vm.sblk->ipb == 0x50000000 && vm.save_area.guest.grs[5] == 0x9c &&
> +	       vm.save_area.guest.grs[0] == 42,

you are checking for DIAGs a lot, maybe it's worth adding a helper
function like for validity

something like:
report(uv_diag_check(&vm, 0x9c) && ... 

> +	       "intercept values");
> +
> +	uv_destroy_guest(&vm);
> +	report_prefix_pop();
> +}
> +
> +int main(void)
> +{
> +	report_prefix_push("uv-sie");
> +	if (!test_facility(158)) {
> +		report_skip("UV Call facility unavailable");
> +		goto done;
> +	}
> +	if (!sclp_facilities.has_sief2) {
> +		report_skip("SIEF2 facility unavailable");
> +		goto done;
> +	}
> +	/*
> +	 * Some of the UV memory needs to be allocated with >31 bit
> +	 * addresses which means we need a lot more memory than other
> +	 * tests.
> +	 */
> +	if (get_ram_size() < (SZ_1M * 2200UL)) {

I think it makes sense to put this in a macro. first, so that it isn't
a magic value, and second so that you have a single point where you can
change it if it ever needs to be changed in the future 

> +		report_skip("Not enough memory. This test needs about 2200MB of memory");

then you can do "%luMB" and use the macro here ^

> +		goto done;
> +	}
> +
> +	snippet_setup_guest(&vm, true);
> +	test_diag_308_0();
> +	test_diag_308_1();
> +	sie_guest_destroy(&vm);
> +
> +done:
> +	report_prefix_pop();
> +	return report_summary();
> +}
> diff --git a/s390x/snippets/asm/snippet-pv-diag-308.S b/s390x/snippets/asm/snippet-pv-diag-308.S
> new file mode 100644
> index 00000000..58c96173
> --- /dev/null
> +++ b/s390x/snippets/asm/snippet-pv-diag-308.S
> @@ -0,0 +1,67 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Diagnose 0x308 snippet used for PV IPL and reset testing
> + *
> + * Copyright (c) 2023 IBM Corp
> + *
> + * Authors:
> + *  Janosch Frank <frankja@linux.ibm.com>
> + */
> +#include <asm/asm-offsets.h>
> +.section .text
> +
> +/* Sets a reset PSW with the given PSW address */
> +.macro SET_RESET_PSW_ADDR label
> +lgrl	%r5, reset_psw
> +larl	%r6, \label
> +ogr	%r5, %r6
> +stg	%r5, 0
> +.endm
> +
> +/* Does a diagnose 308 with the given subcode */

but it seems you are not actually using this macro?

> +.macro DIAG308 subcode
> +xgr	%r3, %r3
> +lghi	%r3, \subcode
> +diag	1, 3, 0x308
> +.endm
> +
> +sam64
> +
> +/* Execute the diag500 which will set the subcode we execute in gr2 */
> +diag	0, 0, 0x500
> +
> +/*
> + * A valid PGM new PSW can be a real problem since we never fall out
> + * of SIE and therefore effectively loop forever. 0 is a valid PSW
> + * therefore we re-use the reset_psw as this has the short PSW
> + * bit set which is invalid for a long PSW like the exception new
> + * PSWs.
> + *
> + * For subcode 0/1 there are no PGMs to consider.
> + */
> +lgrl   %r5, reset_psw
> +stg    %r5, GEN_LC_PGM_NEW_PSW
> +
> +/* Clean registers that are used */
> +xgr	%r0, %r0
> +xgr	%r1, %r1
> +xgr	%r3, %r3
> +xgr	%r4, %r4
> +xgr	%r5, %r5
> +xgr	%r6, %r6
> +
> +/* Subcode 0 - Modified Clear */
> +SET_RESET_PSW_ADDR done
> +diag	%r0, %r2, 0x308
> +
> +/* Should never be executed because of the reset PSW */
> +diag	0, 0, 0x44
> +
> +done:
> +lghi	%r1, 42
> +diag	%r1, 0, 0x9c
> +
> +
> +	.align	8
> +reset_psw:
> +	.quad	0x0008000180000000