[kvm-unit-tests 0/5] s390x: Cleanup virtualization library

[kvm-unit-tests 0/5] s390x: Cleanup virtualization library

[Janosch Frank]

In order to support sae virtualization we need to cleanup the library
a bit and make it more flexible when it comes to guest creation and
allocation.

Janosch Frank (5):
  lib: s390x: Add function to get page root
  lib: s390x: sie: Allocate physical guest memory via memalign
  lib: s390x: sie: Free guest memory on destroy
  lib: s390x: snippet: Add function to create a guest of specific length
  lib: s390x: Remove kvm s390 prefix from sie control block

 lib/s390x/asm/sie-arch.h |  2 +-
 lib/s390x/interrupt.c    |  4 ++--
 lib/s390x/mmu.h          |  7 +++++++
 lib/s390x/sie-icpt.c     |  2 +-
 lib/s390x/sie-icpt.h     |  2 +-
 lib/s390x/sie.c          |  7 +++++--
 lib/s390x/sie.h          |  4 ++--
 lib/s390x/snippet.h      | 18 ++++++++++++++----
 s390x/diag258.c          |  4 ++--
 s390x/edat.c             |  5 +++--
 s390x/mvpg-sie.c         |  2 +-
 s390x/pv-edat1.c         |  4 ++--
 s390x/sie-dat.c          |  2 +-
 s390x/skey.c             |  7 +++----
 14 files changed, 45 insertions(+), 25 deletions(-)

-- 
2.51.0

[kvm-unit-tests 1/5] lib: s390x: Add function to get page root

[Janosch Frank]

It's time to hide the access to cr1 behind a function and add typing
to the tests that used void* instead of pgdt_t*.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/mmu.h  | 7 +++++++
 lib/s390x/sie.c  | 3 ++-
 s390x/diag258.c  | 4 ++--
 s390x/edat.c     | 5 +++--
 s390x/mvpg-sie.c | 2 +-
 s390x/pv-edat1.c | 4 ++--
 s390x/sie-dat.c  | 2 +-
 s390x/skey.c     | 7 +++----
 8 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/lib/s390x/mmu.h b/lib/s390x/mmu.h
index dadc2e60..6ba19d28 100644
--- a/lib/s390x/mmu.h
+++ b/lib/s390x/mmu.h
@@ -10,6 +10,8 @@
 #ifndef _S390X_MMU_H_
 #define _S390X_MMU_H_
 
+#include <asm/arch_def.h>
+
 enum pgt_level {
 	pgtable_level_pgd = 1,
 	pgtable_level_p4d,
@@ -95,4 +97,9 @@ static inline void unprotect_page(void *vaddr, unsigned long prot)
 
 void *get_dat_entry(pgd_t *pgtable, void *vaddr, enum pgt_level level);
 
+static inline pgd_t *get_page_root(void)
+{
+	return (pgd_t *)(stctg(1) & PAGE_MASK);
+}
+
 #endif /* _ASMS390X_MMU_H_ */
diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
index 0fa915cf..a397400b 100644
--- a/lib/s390x/sie.c
+++ b/lib/s390x/sie.c
@@ -17,6 +17,7 @@
 #include <alloc_page.h>
 #include <vmalloc.h>
 #include <sclp.h>
+#include <mmu.h>
 
 void sie_expect_validity(struct vm *vm)
 {
@@ -156,7 +157,7 @@ uint8_t *sie_guest_alloc(uint64_t guest_size)
 	pgd_t *root;
 
 	setup_vm();
-	root = (pgd_t *)(stctg(1) & PAGE_MASK);
+	root = get_page_root();
 
 	/*
 	 * Start of guest memory in host virtual space needs to be aligned to
diff --git a/s390x/diag258.c b/s390x/diag258.c
index 8ba75a72..bbcb92db 100644
--- a/s390x/diag258.c
+++ b/s390x/diag258.c
@@ -82,7 +82,7 @@ static void test_priv(void)
 
 static void *page_map_outside_real_space(phys_addr_t page_real)
 {
-	pgd_t *root = (pgd_t *)(stctg(1) & PAGE_MASK);
+	pgd_t *root = get_page_root();
 	void *vaddr = alloc_vpage();
 
 	install_page(root, page_real, vaddr);
@@ -109,7 +109,7 @@ static void test_refbk_real(void)
 	refbk_page = alloc_page();
 
 	/* Map refblk page outside of physical memory identity mapping */
-	root = (pgd_t *)(stctg(1) & PAGE_MASK);
+	root = get_page_root();
 	refbk = page_map_outside_real_space(virt_to_pte_phys(root, refbk_page));
 
 	/* Assert the mapping really is outside identity mapping */
diff --git a/s390x/edat.c b/s390x/edat.c
index 89b9c2d3..c6a8021a 100644
--- a/s390x/edat.c
+++ b/s390x/edat.c
@@ -21,7 +21,8 @@
 
 static uint8_t prefix_buf[LC_SIZE] __attribute__((aligned(LC_SIZE)));
 static unsigned int tmp[1024] __attribute__((aligned(PAGE_SIZE)));
-static void *root, *mem, *m;
+static void *mem, *m;
+static pgd_t *root;
 volatile unsigned int *p;
 
 /*
@@ -237,7 +238,7 @@ static unsigned int setup(void)
 
 	/* Setup DAT 1:1 mapping and memory management */
 	setup_vm();
-	root = (void *)(stctg(1) & PAGE_MASK);
+	root = get_page_root();
 
 	/*
 	 * Get a pgd worth of virtual memory, so we can test things later
diff --git a/s390x/mvpg-sie.c b/s390x/mvpg-sie.c
index 893de2cf..8666d554 100644
--- a/s390x/mvpg-sie.c
+++ b/s390x/mvpg-sie.c
@@ -89,7 +89,7 @@ static void setup_guest(void)
 	pgd_t *root;
 
 	setup_vm();
-	root = (pgd_t *)(stctg(1) & PAGE_MASK);
+	root = get_page_root();
 
 	snippet_setup_guest(&vm, false);
 	snippet_init(&vm, SNIPPET_NAME_START(c, mvpg_snippet),
diff --git a/s390x/pv-edat1.c b/s390x/pv-edat1.c
index 6acfe6c4..43b3bd4c 100644
--- a/s390x/pv-edat1.c
+++ b/s390x/pv-edat1.c
@@ -34,7 +34,7 @@
 #define PARAM(n, step)	(((unsigned long)(n) << 32) | (step))
 
 static struct vm vm;
-static void *root;
+static pgd_t *root;
 
 extern const char SNIPPET_NAME_START(c, pv_memhog)[];
 extern const char SNIPPET_NAME_END(c, pv_memhog)[];
@@ -441,7 +441,7 @@ static void init(void)
 
 	setup_vm();
 
-	root = (void *)(stctg(1) & PAGE_MASK);
+	root = get_page_root();
 	ctl_set_bit(0, CTL0_EDAT);
 
 	guest_memory = alloc_pages(GUEST_ORDER - PAGE_SHIFT);
diff --git a/s390x/sie-dat.c b/s390x/sie-dat.c
index 44bf29fe..18b536d1 100644
--- a/s390x/sie-dat.c
+++ b/s390x/sie-dat.c
@@ -69,7 +69,7 @@ static void setup_guest(void)
 	pgd_t *root;
 
 	setup_vm();
-	root = (pgd_t *)(stctg(1) & PAGE_MASK);
+	root = get_page_root();
 
 	snippet_setup_guest(&vm, false);
 
diff --git a/s390x/skey.c b/s390x/skey.c
index 7c7a8090..0aa4b6c6 100644
--- a/s390x/skey.c
+++ b/s390x/skey.c
@@ -13,6 +13,7 @@
 #include <asm/interrupt.h>
 #include <vmalloc.h>
 #include <css.h>
+#include <mmu.h>
 #include <asm/page.h>
 #include <asm/facility.h>
 #include <asm/mem.h>
@@ -465,10 +466,9 @@ static void test_set_prefix(void)
 	uint32_t *prefix_ptr = (uint32_t *)pagebuf;
 	uint32_t *no_override_prefix_ptr;
 	uint32_t old_prefix;
-	pgd_t *root;
+	pgd_t *root = get_page_root();
 
 	report_prefix_push("SET PREFIX");
-	root = (pgd_t *)(stctg(1) & PAGE_MASK);
 	old_prefix = get_prefix();
 	memcpy(lowcore_tmp, 0, sizeof(lowcore_tmp));
 	assert(((uint64_t)&lowcore_tmp >> 31) == 0);
@@ -583,11 +583,10 @@ static void test_msch(void)
 	struct schib *schib = (struct schib *)pagebuf;
 	struct schib *no_override_schib;
 	int test_device_sid;
-	pgd_t *root;
+	pgd_t *root = get_page_root();
 	int cc;
 
 	report_prefix_push("MSCH");
-	root = (pgd_t *)(stctg(1) & PAGE_MASK);
 	test_device_sid = css_enumerate();
 
 	if (!(test_device_sid & SCHID_ONE)) {
-- 
2.51.0

Re: [kvm-unit-tests 1/5] lib: s390x: Add function to get page root

[Nico Boehr]

On Wed Apr 15, 2026 at 10:45 AM CEST, Janosch Frank wrote:
[...]
> diff --git a/lib/s390x/mmu.h b/lib/s390x/mmu.h
> index dadc2e60..6ba19d28 100644
> --- a/lib/s390x/mmu.h
> +++ b/lib/s390x/mmu.h
> @@ -10,6 +10,8 @@
>  #ifndef _S390X_MMU_H_
>  #define _S390X_MMU_H_
>  
> +#include <asm/arch_def.h>
> +
>  enum pgt_level {
>  	pgtable_level_pgd = 1,
>  	pgtable_level_p4d,
> @@ -95,4 +97,9 @@ static inline void unprotect_page(void *vaddr, unsigned long prot)
>  
>  void *get_dat_entry(pgd_t *pgtable, void *vaddr, enum pgt_level level);
>  
> +static inline pgd_t *get_page_root(void)

Only nit I have: naming is hard... do you think it makes sense to reflect that
we're talking about the primary-space root here?

get_primary_page_root()?

It's not like we're using the other spaces much in kut, but maybe we will in the
future.

Re: [kvm-unit-tests 1/5] lib: s390x: Add function to get page root

[Janosch Frank]

On 4/15/26 13:53, Nico Boehr wrote:
> On Wed Apr 15, 2026 at 10:45 AM CEST, Janosch Frank wrote:
> [...]
>> diff --git a/lib/s390x/mmu.h b/lib/s390x/mmu.h
>> index dadc2e60..6ba19d28 100644
>> --- a/lib/s390x/mmu.h
>> +++ b/lib/s390x/mmu.h
>> @@ -10,6 +10,8 @@
>>   #ifndef _S390X_MMU_H_
>>   #define _S390X_MMU_H_
>>   
>> +#include <asm/arch_def.h>
>> +
>>   enum pgt_level {
>>   	pgtable_level_pgd = 1,
>>   	pgtable_level_p4d,
>> @@ -95,4 +97,9 @@ static inline void unprotect_page(void *vaddr, unsigned long prot)
>>   
>>   void *get_dat_entry(pgd_t *pgtable, void *vaddr, enum pgt_level level);
>>   
>> +static inline pgd_t *get_page_root(void)
> 
> Only nit I have: naming is hard... do you think it makes sense to reflect that
> we're talking about the primary-space root here?
> 
> get_primary_page_root()?
> 
> It's not like we're using the other spaces much in kut, but maybe we will in the
> future.

Sure, we can still make it take an enum in the future to switch the crs 
depending on which ASCE we want.

[kvm-unit-tests 2/5] lib: s390x: sie: Allocate physical guest memory via memalign

[Janosch Frank]

alloc_pages_flags() alignes the allocation on the same order as its
requested size. Since we use virtual memory for SIE we can instead
align to 1MB by using memalign() which is less wasteful.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/sie.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
index a397400b..a49c45c7 100644
--- a/lib/s390x/sie.c
+++ b/lib/s390x/sie.c
@@ -171,7 +171,8 @@ uint8_t *sie_guest_alloc(uint64_t guest_size)
 	guest_virt = (uint8_t *)ALIGN(get_ram_size() + guest_counter * 4UL * SZ_1G, SZ_2G);
 	guest_counter++;
 
-	guest_phys = alloc_pages(get_order(guest_size) - 12);
+	guest_phys = memalign_pages_flags(SZ_1M, guest_size, 0);
+	assert(guest_phys);
 	/*
 	 * Establish a new mapping of the guest memory so it can be 2GB aligned
 	 * without actually requiring 2GB physical memory.
-- 
2.51.0

[kvm-unit-tests 3/5] lib: s390x: sie: Free guest memory on destroy

[Janosch Frank]

We never freed the memory that the sie library allocates as the guest
ram on destruction of the VM. Most tests reuse the VM or just leak the
memory since the standard allocation is one megabyte and tests only
use single digit numbers of VMs.

It's time to add automatic freeing to the sie library when a VM is
destroyed.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/sie.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
index a49c45c7..758ead77 100644
--- a/lib/s390x/sie.c
+++ b/lib/s390x/sie.c
@@ -192,4 +192,5 @@ void sie_guest_destroy(struct vm *vm)
 	free_page(vm->sblk);
 	if (vm->sblk->ecb2 & ECB2_ESCA)
 		free_page(vm->sca);
+	free_pages((void *)virt_to_pte_phys(get_page_root(), vm->guest_mem));
 }
-- 
2.51.0

Re: [kvm-unit-tests 3/5] lib: s390x: sie: Free guest memory on destroy

[Nico Boehr]

On Wed Apr 15, 2026 at 10:45 AM CEST, Janosch Frank wrote:
> We never freed the memory that the sie library allocates as the guest
> ram on destruction of the VM. Most tests reuse the VM or just leak the
> memory since the standard allocation is one megabyte and tests only
> use single digit numbers of VMs.
>
> It's time to add automatic freeing to the sie library when a VM is
> destroyed.
>
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  lib/s390x/sie.c | 1 +
>  1 file changed, 1 insertion(+)
>
> diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
> index a49c45c7..758ead77 100644
> --- a/lib/s390x/sie.c
> +++ b/lib/s390x/sie.c
> @@ -192,4 +192,5 @@ void sie_guest_destroy(struct vm *vm)
>  	free_page(vm->sblk);
>  	if (vm->sblk->ecb2 & ECB2_ESCA)
>  		free_page(vm->sca);
> +	free_pages((void *)virt_to_pte_phys(get_page_root(), vm->guest_mem));
>  }

Is there a particular reason why we have a two-step process of calling
sie_guest_alloc() for memory allocation
and then
sie_guest_create() for initializing the sie_block etc

but
sie_guest_destroy()
cleans up both?

[kvm-unit-tests 4/5] lib: s390x: snippet: Add function to create a guest of specific length

[Janosch Frank]

While 1MB is certainly enough to store the guest code, it's often not
enough for memory tests. Let's add a separate function to allow
arbitrary guest sizes.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/snippet.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/lib/s390x/snippet.h b/lib/s390x/snippet.h
index 910849aa..c58ae135 100644
--- a/lib/s390x/snippet.h
+++ b/lib/s390x/snippet.h
@@ -123,13 +123,17 @@ static inline void snippet_pv_init(struct vm *vm, const char *gbin,
 }
 
 /* Allocates and sets up a snippet based guest */
-static inline void snippet_setup_guest(struct vm *vm, bool is_pv)
+static inline void snippet_setup_guest_len(struct vm *vm, bool is_pv,
+					   unsigned long len)
 {
-	const unsigned long guest_size = SZ_1M;
-	uint8_t *guest_start = sie_guest_alloc(guest_size);
+	uint8_t *guest_start;
+
+	/* Guest sizes are specified in megabyte chunks */
+	assert(!(len & ~HPAGE_MASK));
+	guest_start = sie_guest_alloc(len);
 
 	/* Initialize the vm struct and allocate control blocks */
-	sie_guest_create(vm, (uint64_t)guest_start, guest_size);
+	sie_guest_create(vm, (uint64_t)guest_start, len);
 
 	if (is_pv) {
 		/* FMT4 needs a ESCA */
@@ -144,4 +148,10 @@ static inline void snippet_setup_guest(struct vm *vm, bool is_pv)
 	}
 }
 
+/* Allocates and sets up a snippet based guest */
+static inline void snippet_setup_guest(struct vm *vm, bool is_pv)
+{
+	snippet_setup_guest_len(vm, is_pv, SZ_1M);
+}
+
 #endif
-- 
2.51.0

[kvm-unit-tests 5/5] lib: s390x: Remove kvm s390 prefix from sie control block

[Janosch Frank]

We're not kvm.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/asm/sie-arch.h | 2 +-
 lib/s390x/interrupt.c    | 4 ++--
 lib/s390x/sie-icpt.c     | 2 +-
 lib/s390x/sie-icpt.h     | 2 +-
 lib/s390x/sie.h          | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/s390x/asm/sie-arch.h b/lib/s390x/asm/sie-arch.h
index 4911c988..d9d5af81 100644
--- a/lib/s390x/asm/sie-arch.h
+++ b/lib/s390x/asm/sie-arch.h
@@ -29,7 +29,7 @@
 #define CPUSTAT_J          0x00000002
 #define CPUSTAT_P          0x00000001
 
-struct kvm_s390_sie_block {
+struct sie_control_block {
 	uint32_t 	cpuflags;		/* 0x0000 */
 	uint32_t : 1;			/* 0x0004 */
 	uint32_t 	prefix : 18;
diff --git a/lib/s390x/interrupt.c b/lib/s390x/interrupt.c
index a75128b9..7e7371c1 100644
--- a/lib/s390x/interrupt.c
+++ b/lib/s390x/interrupt.c
@@ -227,7 +227,7 @@ static void print_storage_exception_information(void)
 
 static void print_int_regs(struct stack_frame_int *stack, bool sie)
 {
-	struct kvm_s390_sie_block *sblk;
+	struct sie_control_block *sblk;
 
 	printf("\n");
 	printf("%s\n", sie ? "Guest registers:" : "Host registers:");
@@ -240,7 +240,7 @@ static void print_int_regs(struct stack_frame_int *stack, bool sie)
 	       stack->grs0[6], stack->grs0[7], stack->grs0[8], stack->grs0[9]);
 
 	if (sie) {
-		sblk = (struct kvm_s390_sie_block *)stack->grs0[12];
+		sblk = (struct sie_control_block *)stack->grs0[12];
 		printf("%016lx %016lx %016lx %016lx\n",
 		       stack->grs0[10], stack->grs0[11], sblk->gg14, sblk->gg15);
 	} else {
diff --git a/lib/s390x/sie-icpt.c b/lib/s390x/sie-icpt.c
index 17064424..861dce02 100644
--- a/lib/s390x/sie-icpt.c
+++ b/lib/s390x/sie-icpt.c
@@ -7,7 +7,7 @@
 
 #include <sie-icpt.h>
 
-struct diag_itext sblk_ip_as_diag(struct kvm_s390_sie_block *sblk)
+struct diag_itext sblk_ip_as_diag(struct sie_control_block *sblk)
 {
 	union {
 		struct {
diff --git a/lib/s390x/sie-icpt.h b/lib/s390x/sie-icpt.h
index 604a7221..b1edf334 100644
--- a/lib/s390x/sie-icpt.h
+++ b/lib/s390x/sie-icpt.h
@@ -21,7 +21,7 @@ struct diag_itext {
 	uint64_t          : 16;
 };
 
-struct diag_itext sblk_ip_as_diag(struct kvm_s390_sie_block *sblk);
+struct diag_itext sblk_ip_as_diag(struct sie_control_block *sblk);
 
 /**
  * sie_is_diag_icpt() - Check if intercept is due to diagnose instruction
diff --git a/lib/s390x/sie.h b/lib/s390x/sie.h
index 3ec49ed0..b146e386 100644
--- a/lib/s390x/sie.h
+++ b/lib/s390x/sie.h
@@ -32,7 +32,7 @@ struct vm_save_area {
 };
 
 struct vm {
-	struct kvm_s390_sie_block *sblk;
+	struct sie_control_block *sblk;
 	struct vm_save_area save_area;
 	struct esca_block *sca;			/* System Control Area */
 	uint8_t *crycb;				/* Crypto Control Block */
@@ -46,7 +46,7 @@ extern void sie_entry(void);
 extern void sie_exit(void);
 extern void sie_entry_gregs(void);
 extern void sie_exit_gregs(void);
-extern void sie64a(struct kvm_s390_sie_block *sblk, struct vm_save_area *save_area);
+extern void sie64a(struct sie_control_block *sblk, struct vm_save_area *save_area);
 void sie(struct vm *vm);
 void sie_expect_validity(struct vm *vm);
 uint16_t sie_get_validity(struct vm *vm);
-- 
2.51.0