[kvm-unit-tests PATCH v1 0/4] s390x: add tests for SIGP call orders in enabled wait

[kvm-unit-tests PATCH v1 0/4] s390x: add tests for SIGP call orders in enabled wait

[Nico Boehr]

When a CPU is in enabled wait, it can still receive SIGP calls from
other CPUs.

Since this requires some special handling in KVM, we should have tests
for it. This has already revealed a KVM bug with ecall under PV, which
is why this test currently fails there.

Some refactoring is done as part of this series to reduce code
duplication.

Nico Boehr (4):
  lib: s390x: add cleanup function for external interrupts
  s390x: smp: move sigp calls with invalid cpu address to array
  s390x: smp: use an array for sigp calls
  s390x: smp: add tests for calls in wait state

 lib/s390x/asm/interrupt.h |   1 +
 lib/s390x/interrupt.c     |   9 ++
 s390x/smp.c               | 190 +++++++++++++++++++++-----------------
 3 files changed, 115 insertions(+), 85 deletions(-)

-- 
2.35.3

[kvm-unit-tests PATCH v1 1/4] lib: s390x: add cleanup function for external interrupts

[Nico Boehr]

Upcoming changes require a way to clear the wait bit in the PSW when
returning from an interrupt handler.

Similar to pgm ints, add a way to register a cleanup function which runs
in the interrupt handler.

Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
---
 lib/s390x/asm/interrupt.h | 1 +
 lib/s390x/interrupt.c     | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/lib/s390x/asm/interrupt.h b/lib/s390x/asm/interrupt.h
index d9ab0bd781c9..e662f0f0a190 100644
--- a/lib/s390x/asm/interrupt.h
+++ b/lib/s390x/asm/interrupt.h
@@ -38,6 +38,7 @@ union teid {
 };
 
 void register_pgm_cleanup_func(void (*f)(void));
+void register_ext_cleanup_func(void (*f)(struct stack_frame_int *));
 void handle_pgm_int(struct stack_frame_int *stack);
 void handle_ext_int(struct stack_frame_int *stack);
 void handle_mcck_int(void);
diff --git a/lib/s390x/interrupt.c b/lib/s390x/interrupt.c
index 6da20c4494ad..bb12ddf2d734 100644
--- a/lib/s390x/interrupt.c
+++ b/lib/s390x/interrupt.c
@@ -18,6 +18,7 @@
 static bool pgm_int_expected;
 static bool ext_int_expected;
 static void (*pgm_cleanup_func)(void);
+static void (*ext_cleanup_func)(struct stack_frame_int *stack);
 
 void expect_pgm_int(void)
 {
@@ -197,6 +198,11 @@ void handle_pgm_int(struct stack_frame_int *stack)
 		fixup_pgm_int(stack);
 }
 
+void register_ext_cleanup_func(void (*f)(struct stack_frame_int *))
+{
+	ext_cleanup_func = f;
+}
+
 void handle_ext_int(struct stack_frame_int *stack)
 {
 	if (!ext_int_expected &&
@@ -215,6 +221,9 @@ void handle_ext_int(struct stack_frame_int *stack)
 
 	if (!(stack->crs[0] & CR0_EXTM_MASK))
 		lowcore.ext_old_psw.mask &= ~PSW_MASK_EXT;
+
+	if (ext_cleanup_func)
+		(*ext_cleanup_func)(stack);
 }
 
 void handle_mcck_int(void)
-- 
2.35.3

[kvm-unit-tests PATCH v1 2/4] s390x: smp: move sigp calls with invalid cpu address to array

[Nico Boehr]

We have the nice array to test SIGP calls with invalid CPU addresses.
Move the SIGP cases there to eliminate some of the duplicated code in
test_emcall and test_cond_emcall.

Since adding coverage for invalid CPU addresses in the ecall case is now
trivial, do that as well.

Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
---
 s390x/smp.c | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/s390x/smp.c b/s390x/smp.c
index 6d474d0d4f99..ea811087587e 100644
--- a/s390x/smp.c
+++ b/s390x/smp.c
@@ -30,6 +30,9 @@ static const struct sigp_invalid_cases cases_invalid_cpu_addr[] = {
 	{ SIGP_STOP,                  "stop with invalid CPU address" },
 	{ SIGP_START,                 "start with invalid CPU address" },
 	{ SIGP_CPU_RESET,             "reset with invalid CPU address" },
+	{ SIGP_COND_EMERGENCY_SIGNAL, "conditional emcall with invalid CPU address" },
+	{ SIGP_EMERGENCY_SIGNAL,      "emcall with invalid CPU address" },
+	{ SIGP_EXTERNAL_CALL,         "ecall with invalid CPU address" },
 	{ INVALID_ORDER_CODE,         "invalid order code and CPU address" },
 	{ SIGP_SENSE,                 "sense with invalid CPU address" },
 	{ SIGP_STOP_AND_STORE_STATUS, "stop and store status with invalid CPU address" },
@@ -337,7 +340,6 @@ static void emcall(void)
 static void test_emcall(void)
 {
 	struct psw psw;
-	int cc;
 	psw.mask = extract_psw_mask();
 	psw.addr = (unsigned long)emcall;
 
@@ -351,13 +353,6 @@ static void test_emcall(void)
 	wait_for_flag();
 	smp_cpu_stop(1);
 
-	report_prefix_push("invalid CPU address");
-
-	cc = sigp(INVALID_CPU_ADDRESS, SIGP_EMERGENCY_SIGNAL, 0, NULL);
-	report(cc == 3, "CC = 3");
-
-	report_prefix_pop();
-
 	report_prefix_pop();
 }
 
@@ -376,13 +371,6 @@ static void test_cond_emcall(void)
 		goto out;
 	}
 
-	report_prefix_push("invalid CPU address");
-
-	cc = sigp(INVALID_CPU_ADDRESS, SIGP_COND_EMERGENCY_SIGNAL, 0, NULL);
-	report(cc == 3, "CC = 3");
-
-	report_prefix_pop();
-
 	report_prefix_push("success");
 	set_flag(0);
 
-- 
2.35.3

[kvm-unit-tests PATCH v1 3/4] s390x: smp: use an array for sigp calls

[Nico Boehr]

Tests for the SIGP calls are quite similar, so we have a lot of code
duplication right now. Since upcoming changes will add more cases,
refactor the code to iterate over an array, similarily as we already do
for test_invalid().

The receiving CPU is disabled for IO interrupts. This makes sure the
conditional emergency signal is accepted and doesn't hurt the other
orders.

Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
---
 s390x/smp.c | 130 ++++++++++++++++++----------------------------------
 1 file changed, 44 insertions(+), 86 deletions(-)

diff --git a/s390x/smp.c b/s390x/smp.c
index ea811087587e..857eae206daa 100644
--- a/s390x/smp.c
+++ b/s390x/smp.c
@@ -43,6 +43,20 @@ static const struct sigp_invalid_cases cases_valid_cpu_addr[] = {
 
 static uint32_t cpu1_prefix;
 
+struct sigp_call_cases {
+	char name[20];
+	int call;
+	uint16_t ext_int_expected_type;
+	uint32_t cr0_bit;
+	bool supports_pv;
+};
+static const struct sigp_call_cases cases_sigp_call[] = {
+	{ "emcall",      SIGP_EMERGENCY_SIGNAL,      0x1201, CTL0_EMERGENCY_SIGNAL, true },
+	{ "cond emcall", SIGP_COND_EMERGENCY_SIGNAL, 0x1201, CTL0_EMERGENCY_SIGNAL, false },
+	{ "ecall",       SIGP_EXTERNAL_CALL,         0x1202, CTL0_EXTERNAL_CALL,    true },
+};
+static const struct sigp_call_cases *current_sigp_call_case;
+
 static void test_invalid(void)
 {
 	const struct sigp_invalid_cases *c;
@@ -289,105 +303,51 @@ static void test_set_prefix(void)
 
 }
 
-static void ecall(void)
-{
-	unsigned long mask;
-
-	expect_ext_int();
-	ctl_set_bit(0, CTL0_EXTERNAL_CALL);
-	mask = extract_psw_mask();
-	mask |= PSW_MASK_EXT;
-	load_psw_mask(mask);
-	set_flag(1);
-	while (lowcore.ext_int_code != 0x1202) { mb(); }
-	report_pass("received");
-	set_flag(1);
-}
-
-static void test_ecall(void)
-{
-	struct psw psw;
-	psw.mask = extract_psw_mask();
-	psw.addr = (unsigned long)ecall;
-
-	report_prefix_push("ecall");
-	set_flag(0);
-
-	smp_cpu_start(1, psw);
-	wait_for_flag();
-	set_flag(0);
-	smp_sigp(1, SIGP_EXTERNAL_CALL, 0, NULL);
-	wait_for_flag();
-	smp_cpu_stop(1);
-	report_prefix_pop();
-}
-
-static void emcall(void)
+static void call_received(void)
 {
 	unsigned long mask;
 
 	expect_ext_int();
-	ctl_set_bit(0, CTL0_EMERGENCY_SIGNAL);
+	ctl_set_bit(0, current_sigp_call_case->cr0_bit);
 	mask = extract_psw_mask();
 	mask |= PSW_MASK_EXT;
+	/* make sure conditional emergency is accepted by disabling IO interrupts */
+	mask &= ~PSW_MASK_IO;
 	load_psw_mask(mask);
 	set_flag(1);
-	while (lowcore.ext_int_code != 0x1201) { mb(); }
+	while (lowcore.ext_int_code != current_sigp_call_case->ext_int_expected_type)
+		mb();
 	report_pass("received");
 	set_flag(1);
 }
 
-static void test_emcall(void)
+static void test_calls(void)
 {
+	int i;
 	struct psw psw;
-	psw.mask = extract_psw_mask();
-	psw.addr = (unsigned long)emcall;
-
-	report_prefix_push("emcall");
-	set_flag(0);
-
-	smp_cpu_start(1, psw);
-	wait_for_flag();
-	set_flag(0);
-	smp_sigp(1, SIGP_EMERGENCY_SIGNAL, 0, NULL);
-	wait_for_flag();
-	smp_cpu_stop(1);
-
-	report_prefix_pop();
-}
-
-static void test_cond_emcall(void)
-{
-	uint32_t status = 0;
-	struct psw psw;
-	int cc;
-	psw.mask = extract_psw_mask() & ~PSW_MASK_IO;
-	psw.addr = (unsigned long)emcall;
-
-	report_prefix_push("conditional emergency call");
 
-	if (uv_os_is_guest()) {
-		report_skip("unsupported under PV");
-		goto out;
+	for (i = 0; i < ARRAY_SIZE(cases_sigp_call); i++) {
+		current_sigp_call_case = &cases_sigp_call[i];
+
+		report_prefix_push(current_sigp_call_case->name);
+		if (!current_sigp_call_case->supports_pv && uv_os_is_guest()) {
+			report_skip("Not supported under PV");
+			report_prefix_pop();
+			continue;
+		}
+
+		set_flag(0);
+		psw.mask = extract_psw_mask();
+		psw.addr = (unsigned long)call_received;
+
+		smp_cpu_start(1, psw);
+		wait_for_flag();
+		set_flag(0);
+		smp_sigp(1, current_sigp_call_case->call, 0, NULL);
+		wait_for_flag();
+		smp_cpu_stop(1);
+		report_prefix_pop();
 	}
-
-	report_prefix_push("success");
-	set_flag(0);
-
-	smp_cpu_start(1, psw);
-	wait_for_flag();
-	set_flag(0);
-	cc = smp_sigp(1, SIGP_COND_EMERGENCY_SIGNAL, 0, &status);
-	report(!cc, "CC = 0");
-
-	wait_for_flag();
-	smp_cpu_stop(1);
-
-	report_prefix_pop();
-
-out:
-	report_prefix_pop();
-
 }
 
 static void test_sense_running(void)
@@ -511,9 +471,7 @@ int main(void)
 	test_stop_store_status();
 	test_store_status();
 	test_set_prefix();
-	test_ecall();
-	test_emcall();
-	test_cond_emcall();
+	test_calls();
 	test_sense_running();
 	test_reset();
 	test_reset_initial();
-- 
2.35.3

[kvm-unit-tests PATCH v1 4/4] s390x: smp: add tests for calls in wait state

[Nico Boehr]

Under PV, SIGP ecall requires some special handling by the hypervisor
when the receiving CPU is in enabled wait. Hence, we should have
coverage for the various SIGP call orders when the receiving CPU is in
enabled wait.

The ecall test currently fails under PV due to a KVM bug under
investigation.

Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
---
 s390x/smp.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/s390x/smp.c b/s390x/smp.c
index 857eae206daa..2b48f83d2b6d 100644
--- a/s390x/smp.c
+++ b/s390x/smp.c
@@ -350,6 +350,79 @@ static void test_calls(void)
 	}
 }
 
+static void call_in_wait_setup(void)
+{
+	expect_ext_int();
+	ctl_set_bit(0, current_sigp_call_case->cr0_bit);
+	set_flag(1);
+}
+
+static void call_in_wait_received(void)
+{
+	report(lowcore.ext_int_code == current_sigp_call_case->ext_int_expected_type, "received");
+	set_flag(1);
+}
+
+static void call_in_wait_ext_int_fixup(struct stack_frame_int *stack)
+{
+	/* leave wait after returning */
+	lowcore.ext_old_psw.mask &= ~PSW_MASK_WAIT;
+
+	stack->crs[0] &= ~current_sigp_call_case->cr0_bit;
+}
+
+static void test_calls_in_wait(void)
+{
+	int i;
+	struct psw psw;
+
+	report_prefix_push("psw wait");
+	for (i = 0; i < ARRAY_SIZE(cases_sigp_call); i++) {
+		current_sigp_call_case = &cases_sigp_call[i];
+
+		report_prefix_push(current_sigp_call_case->name);
+		if (!current_sigp_call_case->supports_pv && uv_os_is_guest()) {
+			report_skip("Not supported under PV");
+			report_prefix_pop();
+			continue;
+		}
+
+		set_flag(0);
+		psw.mask = extract_psw_mask();
+		psw.addr = (unsigned long)call_in_wait_setup;
+		smp_cpu_start(1, psw);
+		wait_for_flag();
+		set_flag(0);
+
+		register_ext_cleanup_func(call_in_wait_ext_int_fixup);
+
+		/*
+		 * To avoid races, we need to know that the secondary CPU has entered wait,
+		 * but the architecture provides no way to check whether the secondary CPU
+		 * is in wait.
+		 *
+		 * But since a waiting CPU is considered operating, simply stop the CPU, set
+		 * up the restart new PSW mask in wait, send the restart interrupt and then
+		 * wait until the CPU becomes operating (done by smp_cpu_start).
+		 */
+		smp_cpu_stop(1);
+		expect_ext_int();
+		psw.mask = extract_psw_mask() | PSW_MASK_EXT | PSW_MASK_WAIT;
+		psw.addr = (unsigned long)call_in_wait_received;
+		smp_cpu_start(1, psw);
+
+		smp_sigp(1, current_sigp_call_case->call, 0, NULL);
+
+		wait_for_flag();
+		smp_cpu_stop(1);
+
+		register_ext_cleanup_func(NULL);
+
+		report_prefix_pop();
+	}
+	report_prefix_pop();
+}
+
 static void test_sense_running(void)
 {
 	report_prefix_push("sense_running");
@@ -472,6 +545,7 @@ int main(void)
 	test_store_status();
 	test_set_prefix();
 	test_calls();
+	test_calls_in_wait();
 	test_sense_running();
 	test_reset();
 	test_reset_initial();
-- 
2.35.3