[RFC PATCH 0/2] Add NV Selftest cases

[RFC PATCH 0/2] Add NV Selftest cases

[Ganapatrao Kulkarni]

In this patch seried, modified kvm selftests code to enable
guest code to run in vEL2(As guest Hypervisor).
Also added test cases to test guest code booting in vEL2
and register access of VNCR mapped registers.

This patchset is created as per discussions over ml[1].
Posting RFC patch for the early feedback and to
further explore requirements and test cases.

Ganapatrao Kulkarni (2):
  KVM: arm64: nv: selftests: Add guest hypervisor test
  KVM: arm64: nv: selftests: Access VNCR mapped registers

 tools/testing/selftests/kvm/Makefile.kvm      |   2 +
 .../selftests/kvm/arm64/nv_guest_hypervisor.c |  83 ++++++
 .../selftests/kvm/arm64/nv_vncr_regs_test.c   | 255 ++++++++++++++++++
 .../kvm/include/arm64/kvm_util_arch.h         |   3 +
 .../selftests/kvm/include/arm64/nv_util.h     |  28 ++
 .../testing/selftests/kvm/include/kvm_util.h  |   1 +
 .../selftests/kvm/lib/arm64/processor.c       |  59 +++-
 7 files changed, 417 insertions(+), 14 deletions(-)
 create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
 create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
 create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h

-- 
2.48.1

[RFC PATCH 1/2] KVM: arm64: nv: selftests: Add guest hypervisor test

[Ganapatrao Kulkarni]

This patch adds the required changes to init vcpu in vEL2 context.
Also adds a KVM selftest to execute guest code as a guest hypervisor(L1).

Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
---
 tools/testing/selftests/kvm/Makefile.kvm      |  1 +
 .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 +++++++++++++++++++
 .../kvm/include/arm64/kvm_util_arch.h         |  3 +
 .../selftests/kvm/include/arm64/nv_util.h     | 28 +++++++
 .../testing/selftests/kvm/include/kvm_util.h  |  1 +
 .../selftests/kvm/lib/arm64/processor.c       | 59 +++++++++----
 6 files changed, 161 insertions(+), 14 deletions(-)
 create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
 create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 4277b983cace..a85d3bec9fb1 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -154,6 +154,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq
 TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress
 TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access
 TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3
+TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor
 TEST_GEN_PROGS_arm64 += access_tracking_perf_test
 TEST_GEN_PROGS_arm64 += arch_timer
 TEST_GEN_PROGS_arm64 += coalesced_io_test
diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
new file mode 100644
index 000000000000..5aeefe43aff7
--- /dev/null
+++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025 Ampere Computing LLC
+ */
+#include <kvm_util.h>
+#include <nv_util.h>
+#include <processor.h>
+#include <vgic.h>
+
+static void guest_code(void)
+{
+	if (read_sysreg(CurrentEL) == CurrentEL_EL2)
+		GUEST_PRINTF("Executing guest code in vEL2\n");
+	else
+		GUEST_FAIL("Fail to run in vEL2\n");
+
+	GUEST_DONE();
+}
+
+static void guest_undef_handler(struct ex_regs *regs)
+{
+	GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1));
+}
+
+static void test_run_vcpu(struct kvm_vcpu *vcpu)
+{
+	struct ucall uc;
+
+	do {
+		vcpu_run(vcpu);
+
+		switch (get_ucall(vcpu, &uc)) {
+		case UCALL_ABORT:
+			REPORT_GUEST_ASSERT(uc);
+			break;
+		case UCALL_PRINTF:
+			printf("%s", uc.buffer);
+			break;
+		case UCALL_DONE:
+			printf("Test PASS\n");
+			break;
+		default:
+			TEST_FAIL("Unknown ucall %lu", uc.cmd);
+		}
+	} while (uc.cmd != UCALL_DONE);
+}
+
+static void test_nv_guest_hypervisor(void)
+{
+	struct kvm_vcpu *vcpu;
+	struct kvm_vm *vm;
+	struct kvm_vcpu_init init;
+	int gic_fd;
+
+	vm = vm_create(1);
+	vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
+
+	init.features[0] = 0;
+	init_vcpu_nested(&init);
+	vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
+
+	__TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV");
+
+	vm_init_descriptor_tables(vm);
+	vcpu_init_descriptor_tables(vcpu);
+	gic_fd = vgic_v3_setup(vm, 1, 64);
+	__TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
+
+	vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
+				ESR_ELx_EC_UNKNOWN, guest_undef_handler);
+
+	test_run_vcpu(vcpu);
+	kvm_vm_free(vm);
+}
+
+int main(int argc, char *argv[])
+{
+	TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
+
+	test_nv_guest_hypervisor();
+
+	return 0;
+}
diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
index e43a57d99b56..ab5279c24413 100644
--- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
+++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
@@ -2,6 +2,9 @@
 #ifndef SELFTEST_KVM_UTIL_ARCH_H
 #define SELFTEST_KVM_UTIL_ARCH_H
 
+#define CurrentEL_EL1		(1 << 2)
+#define CurrentEL_EL2		(2 << 2)
+
 struct kvm_vm_arch {};
 
 #endif  // SELFTEST_KVM_UTIL_ARCH_H
diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h
new file mode 100644
index 000000000000..4fecf1f18554
--- /dev/null
+++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2025 Ampere Computing
+ */
+#ifndef SELFTEST_NV_UTIL_H
+#define SELFTEST_NV_UTIL_H
+
+#include <linux/bitmap.h>
+
+/* NV helpers */
+static inline void init_vcpu_nested(struct kvm_vcpu_init *init)
+{
+	init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);
+}
+
+static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init)
+{
+	unsigned long features = init->features[0];
+
+	return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);
+}
+
+static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu)
+{
+	return vcpu->nested;
+}
+
+#endif /* SELFTEST_NV_UTIL_H */
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index 4c4e5a847f67..8c53dbc17f8f 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -58,6 +58,7 @@ struct kvm_vcpu {
 	struct kvm_dirty_gfn *dirty_gfns;
 	uint32_t fetch_index;
 	uint32_t dirty_gfns_count;
+	bool nested;
 };
 
 struct userspace_mem_regions {
diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c
index 7ba3aa3755f3..35ba2ace61a2 100644
--- a/tools/testing/selftests/kvm/lib/arm64/processor.c
+++ b/tools/testing/selftests/kvm/lib/arm64/processor.c
@@ -10,6 +10,7 @@
 
 #include "guest_modes.h"
 #include "kvm_util.h"
+#include "nv_util.h"
 #include "processor.h"
 #include "ucall_common.h"
 
@@ -258,14 +259,47 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 	}
 }
 
+static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,
+			uint64_t tcr_el1, uint64_t ttbr0_el1)
+{
+	uint64_t fpen;
+
+	/*
+	 * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
+	 * registers, which the variable argument list macros do.
+	 */
+	fpen = 3 << 20;
+
+	if (is_vcpu_nested(vcpu)) {
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);
+	} else {
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), fpen);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1);
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id);
+}
+}
+
 void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
 {
 	struct kvm_vcpu_init default_init = { .target = -1, };
 	struct kvm_vm *vm = vcpu->vm;
 	uint64_t sctlr_el1, tcr_el1, ttbr0_el1;
 
-	if (!init)
+	if (!init) {
 		init = &default_init;
+	} else {
+		/* Is this vcpu a Guest-Hypersior */
+		if (kvm_arm_vcpu_has_el2(init))
+			vcpu->nested = true;
+	}
 
 	if (init->target == -1) {
 		struct kvm_vcpu_init preferred;
@@ -275,12 +309,6 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
 
 	vcpu_ioctl(vcpu, KVM_ARM_VCPU_INIT, init);
 
-	/*
-	 * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
-	 * registers, which the variable argument list macros do.
-	 */
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20);
-
 	sctlr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1));
 	tcr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1));
 
@@ -349,11 +377,7 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
 	if (use_lpa2_pte_format(vm))
 		tcr_el1 |= (1ul << 59) /* DS */;
 
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1);
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id);
+	aarch64_vcpu_set_reg(vcpu, sctlr_el1, tcr_el1, ttbr0_el1);
 }
 
 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
@@ -387,7 +411,11 @@ static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
 
 	aarch64_vcpu_setup(vcpu, init);
 
-	vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
+	if (is_vcpu_nested(vcpu))
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SP_EL2), stack_vaddr + stack_size);
+	else
+		vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
+
 	return vcpu;
 }
 
@@ -457,7 +485,10 @@ void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu)
 {
 	extern char vectors;
 
-	vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);
+	if (is_vcpu_nested(vcpu))
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL2), (uint64_t)&vectors);
+	else
+		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);
 }
 
 void route_exception(struct ex_regs *regs, int vector)
-- 
2.48.1

[RFC PATCH 2/2] KVM: arm64: nv: selftests: Access VNCR mapped registers

[Ganapatrao Kulkarni]

With NV2 enabled, some of the EL1/EL2/EL12 register accesses are
transformed to memory accesses. This test code accesses all those
registers in guest code to validate that they are not trapped to L0.

Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
---
 tools/testing/selftests/kvm/Makefile.kvm      |   1 +
 .../selftests/kvm/arm64/nv_vncr_regs_test.c   | 255 ++++++++++++++++++
 2 files changed, 256 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index a85d3bec9fb1..7790e4021013 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -155,6 +155,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress
 TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access
 TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3
 TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor
+TEST_GEN_PROGS_arm64 += arm64/nv_vncr_regs_test
 TEST_GEN_PROGS_arm64 += access_tracking_perf_test
 TEST_GEN_PROGS_arm64 += arch_timer
 TEST_GEN_PROGS_arm64 += coalesced_io_test
diff --git a/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
new file mode 100644
index 000000000000..d05b20b828ff
--- /dev/null
+++ b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025 Ampere Computing LLC
+ *
+ * This is a test to validate Nested Virtualization.
+ */
+#include <kvm_util.h>
+#include <nv_util.h>
+#include <processor.h>
+#include <vgic.h>
+
+#define __check_sr_read(r)					\
+	({							\
+		uint64_t val;					\
+								\
+		handled = false;				\
+		dsb(sy);					\
+		val = read_sysreg_s(SYS_ ## r);			\
+		val;						\
+	})
+
+#define __check_sr_write(r)					\
+	do {							\
+		handled = false;				\
+		dsb(sy);					\
+		write_sysreg_s(0, SYS_ ## r);			\
+		isb();						\
+	} while (0)
+
+
+#define check_sr_read(r)					  \
+	do {							  \
+		__check_sr_read(r);				  \
+		__GUEST_ASSERT(!handled, #r "Read Test Failed");  \
+	} while (0)
+
+#define check_sr_write(r)					  \
+	do {							  \
+		__check_sr_write(r);				  \
+		__GUEST_ASSERT(!handled, #r "Write Test Failed"); \
+	} while (0)
+
+#define check_sr_rw(r)				\
+	do {					\
+		GUEST_PRINTF("%s\n", #r);	\
+		check_sr_write(r);		\
+		check_sr_read(r);		\
+	} while (0)
+
+static void test_vncr_mapped_regs(void);
+static void regs_test_ich_lr(void);
+
+static volatile bool handled;
+
+static void regs_test_ich_lr(void)
+{
+	int nr_lr, lr;
+
+	nr_lr  = (read_sysreg_s(SYS_ICH_VTR_EL2) & 0xf);
+
+	for (lr = 0; lr <= nr_lr;  lr++) {
+		switch (lr) {
+		case 0:
+			check_sr_rw(ICH_LR0_EL2);
+			break;
+		case 1:
+			check_sr_rw(ICH_LR1_EL2);
+			break;
+		case 2:
+			check_sr_rw(ICH_LR2_EL2);
+			break;
+		case 3:
+			check_sr_rw(ICH_LR3_EL2);
+			break;
+		case 4:
+			check_sr_rw(ICH_LR4_EL2);
+			break;
+		case 5:
+			check_sr_rw(ICH_LR5_EL2);
+			break;
+		case 6:
+			check_sr_rw(ICH_LR6_EL2);
+			break;
+		case 7:
+			check_sr_rw(ICH_LR7_EL2);
+			break;
+		case 8:
+			check_sr_rw(ICH_LR8_EL2);
+			break;
+		case 9:
+			check_sr_rw(ICH_LR9_EL2);
+			break;
+		case 10:
+			check_sr_rw(ICH_LR10_EL2);
+			break;
+		case 11:
+			check_sr_rw(ICH_LR11_EL2);
+			break;
+		case 12:
+			check_sr_rw(ICH_LR12_EL2);
+			break;
+		case 13:
+			check_sr_rw(ICH_LR13_EL2);
+			break;
+		case 14:
+			check_sr_rw(ICH_LR14_EL2);
+			break;
+		case 15:
+			check_sr_rw(ICH_LR15_EL2);
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+/*
+ * Validate READ/WRITE to VNCR Mapped registers for NV1=0
+ */
+
+static void test_vncr_mapped_regs(void)
+{
+	/*
+	 * Access all VNCR Mapped registers, and fail if we get an UNDEF.
+	 */
+
+	GUEST_PRINTF("VNCR Mapped registers access test:\n");
+	check_sr_rw(VTTBR_EL2);
+	check_sr_rw(VTCR_EL2);
+	check_sr_rw(VMPIDR_EL2);
+	check_sr_rw(CNTVOFF_EL2);
+	check_sr_rw(HCR_EL2);
+	check_sr_rw(HSTR_EL2);
+	check_sr_rw(VPIDR_EL2);
+	check_sr_rw(TPIDR_EL2);
+	check_sr_rw(VNCR_EL2);
+	check_sr_rw(CPACR_EL12);
+	check_sr_rw(CONTEXTIDR_EL12);
+	check_sr_rw(SCTLR_EL12);
+	check_sr_rw(ACTLR_EL1);
+	check_sr_rw(TCR_EL12);
+	check_sr_rw(AFSR0_EL12);
+	check_sr_rw(AFSR1_EL12);
+	check_sr_rw(ESR_EL12);
+	check_sr_rw(MAIR_EL12);
+	check_sr_rw(AMAIR_EL12);
+	check_sr_rw(MDSCR_EL1);
+	check_sr_rw(SPSR_EL12);
+	check_sr_rw(CNTV_CVAL_EL02);
+	check_sr_rw(CNTV_CTL_EL02);
+	check_sr_rw(CNTP_CVAL_EL02);
+	check_sr_rw(CNTP_CTL_EL02);
+	check_sr_rw(HAFGRTR_EL2);
+	check_sr_rw(TTBR0_EL12);
+	check_sr_rw(TTBR1_EL12);
+	check_sr_rw(FAR_EL12);
+	check_sr_rw(ELR_EL12);
+	check_sr_rw(SP_EL1);
+	check_sr_rw(VBAR_EL12);
+
+	regs_test_ich_lr();
+
+	check_sr_rw(ICH_AP0R0_EL2);
+	check_sr_rw(ICH_AP1R0_EL2);
+	check_sr_rw(ICH_HCR_EL2);
+	check_sr_rw(ICH_VMCR_EL2);
+	check_sr_rw(VDISR_EL2);
+	check_sr_rw(MPAM1_EL12);
+	check_sr_rw(MPAMHCR_EL2);
+	check_sr_rw(MPAMVPMV_EL2);
+	check_sr_rw(MPAMVPM0_EL2);
+	check_sr_rw(MPAMVPM1_EL2);
+	check_sr_rw(MPAMVPM2_EL2);
+	check_sr_rw(MPAMVPM3_EL2);
+	check_sr_rw(MPAMVPM4_EL2);
+	check_sr_rw(MPAMVPM5_EL2);
+	check_sr_rw(MPAMVPM6_EL2);
+	check_sr_rw(MPAMVPM7_EL2);
+}
+
+static void guest_code(void)
+{
+	if (read_sysreg(CurrentEL) != CurrentEL_EL2)
+		GUEST_FAIL("Fail to run in vEL2\n");
+
+	test_vncr_mapped_regs();
+	GUEST_DONE();
+}
+
+static void guest_undef_handler(struct ex_regs *regs)
+{
+	handled = true;
+	regs->pc += 4;
+	GUEST_FAIL("TEST FAIL: register access trap to EL2");
+}
+
+static void test_run_vcpu(struct kvm_vcpu *vcpu)
+{
+	struct ucall uc;
+
+	do {
+		vcpu_run(vcpu);
+
+		switch (get_ucall(vcpu, &uc)) {
+		case UCALL_ABORT:
+			REPORT_GUEST_ASSERT(uc);
+			break;
+		case UCALL_PRINTF:
+			printf("%s", uc.buffer);
+			break;
+		case UCALL_DONE:
+			printf("TEST PASS\n");
+			break;
+		default:
+			TEST_FAIL("Unknown ucall %lu", uc.cmd);
+		}
+	} while (uc.cmd != UCALL_DONE);
+}
+
+static void test_nv_vncr(void)
+{
+	struct kvm_vcpu *vcpu;
+	struct kvm_vm *vm;
+	struct kvm_vcpu_init init;
+	int gic_fd;
+
+	vm = vm_create(1);
+	vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
+
+	init.features[0] = 0;
+	init_vcpu_nested(&init);
+	vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
+
+	__TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV");
+
+	vm_init_descriptor_tables(vm);
+	vcpu_init_descriptor_tables(vcpu);
+	gic_fd = vgic_v3_setup(vm, 1, 64);
+	__TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
+
+	vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
+				ESR_ELx_EC_UNKNOWN, guest_undef_handler);
+
+	test_run_vcpu(vcpu);
+	kvm_vm_free(vm);
+}
+
+int main(int argc, char *argv[])
+{
+	TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
+
+	test_nv_vncr();
+
+	return 0;
+}
-- 
2.48.1

Re: [RFC PATCH 0/2] Add NV Selftest cases

[Ganapatrao Kulkarni]

On 06-02-2025 10:11 pm, Ganapatrao Kulkarni wrote:
> In this patch seried, modified kvm selftests code to enable
> guest code to run in vEL2(As guest Hypervisor).
> Also added test cases to test guest code booting in vEL2
> and register access of VNCR mapped registers.
> 
> This patchset is created as per discussions over ml[1].
> Posting RFC patch for the early feedback and to
> further explore requirements and test cases.
> 

[1] https://lore.kernel.org/linux-arm-kernel/871pz2th4b.wl-maz@kernel.org/

> Ganapatrao Kulkarni (2):
>    KVM: arm64: nv: selftests: Add guest hypervisor test
>    KVM: arm64: nv: selftests: Access VNCR mapped registers
> 
>   tools/testing/selftests/kvm/Makefile.kvm      |   2 +
>   .../selftests/kvm/arm64/nv_guest_hypervisor.c |  83 ++++++
>   .../selftests/kvm/arm64/nv_vncr_regs_test.c   | 255 ++++++++++++++++++
>   .../kvm/include/arm64/kvm_util_arch.h         |   3 +
>   .../selftests/kvm/include/arm64/nv_util.h     |  28 ++
>   .../testing/selftests/kvm/include/kvm_util.h  |   1 +
>   .../selftests/kvm/lib/arm64/processor.c       |  59 +++-
>   7 files changed, 417 insertions(+), 14 deletions(-)
>   create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
>   create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
>   create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
> 

-- 
Thanks,
Ganapat/GK

Re: [RFC PATCH 2/2] KVM: arm64: nv: selftests: Access VNCR mapped registers

[Marc Zyngier]

On Thu, 06 Feb 2025 16:41:20 +0000,
Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com> wrote:
> 
> With NV2 enabled, some of the EL1/EL2/EL12 register accesses are
> transformed to memory accesses. This test code accesses all those
> registers in guest code to validate that they are not trapped to L0.
>

Traps to L0 are invisible to the guest -- by definition. What the
guest can observe is an exception, be it injected by L0 or directly
delivered by the HW.

But the *absence* of an exception doesn't mean things are fine. Quite
the opposite.

> Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
> ---
>  tools/testing/selftests/kvm/Makefile.kvm      |   1 +
>  .../selftests/kvm/arm64/nv_vncr_regs_test.c   | 255 ++++++++++++++++++
>  2 files changed, 256 insertions(+)
>  create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
> 
> diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
> index a85d3bec9fb1..7790e4021013 100644
> --- a/tools/testing/selftests/kvm/Makefile.kvm
> +++ b/tools/testing/selftests/kvm/Makefile.kvm
> @@ -155,6 +155,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress
>  TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access
>  TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3
>  TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor
> +TEST_GEN_PROGS_arm64 += arm64/nv_vncr_regs_test
>  TEST_GEN_PROGS_arm64 += access_tracking_perf_test
>  TEST_GEN_PROGS_arm64 += arch_timer
>  TEST_GEN_PROGS_arm64 += coalesced_io_test
> diff --git a/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
> new file mode 100644
> index 000000000000..d05b20b828ff
> --- /dev/null
> +++ b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
> @@ -0,0 +1,255 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (c) 2025 Ampere Computing LLC
> + *
> + * This is a test to validate Nested Virtualization.
> + */
> +#include <kvm_util.h>
> +#include <nv_util.h>
> +#include <processor.h>
> +#include <vgic.h>
> +
> +#define __check_sr_read(r)					\
> +	({							\
> +		uint64_t val;					\
> +								\
> +		handled = false;				\
> +		dsb(sy);					\
> +		val = read_sysreg_s(SYS_ ## r);			\
> +		val;						\
> +	})
> +
> +#define __check_sr_write(r)					\
> +	do {							\
> +		handled = false;				\
> +		dsb(sy);					\
> +		write_sysreg_s(0, SYS_ ## r);			\
> +		isb();						\
> +	} while (0)
> +
> +
> +#define check_sr_read(r)					  \
> +	do {							  \
> +		__check_sr_read(r);				  \
> +		__GUEST_ASSERT(!handled, #r "Read Test Failed");  \
> +	} while (0)
> +
> +#define check_sr_write(r)					  \
> +	do {							  \
> +		__check_sr_write(r);				  \
> +		__GUEST_ASSERT(!handled, #r "Write Test Failed"); \
> +	} while (0)
> +
> +#define check_sr_rw(r)				\
> +	do {					\
> +		GUEST_PRINTF("%s\n", #r);	\
> +		check_sr_write(r);		\
> +		check_sr_read(r);		\
> +	} while (0)

Instead of lifting things from existing tests, you could move these
things to an include file for everybody's benefit.

> +
> +static void test_vncr_mapped_regs(void);
> +static void regs_test_ich_lr(void);
> +
> +static volatile bool handled;
> +
> +static void regs_test_ich_lr(void)
> +{
> +	int nr_lr, lr;
> +
> +	nr_lr  = (read_sysreg_s(SYS_ICH_VTR_EL2) & 0xf);
> +
> +	for (lr = 0; lr <= nr_lr;  lr++) {
> +		switch (lr) {
> +		case 0:
> +			check_sr_rw(ICH_LR0_EL2);
> +			break;
> +		case 1:
> +			check_sr_rw(ICH_LR1_EL2);
> +			break;
> +		case 2:
> +			check_sr_rw(ICH_LR2_EL2);
> +			break;
> +		case 3:
> +			check_sr_rw(ICH_LR3_EL2);
> +			break;
> +		case 4:
> +			check_sr_rw(ICH_LR4_EL2);
> +			break;
> +		case 5:
> +			check_sr_rw(ICH_LR5_EL2);
> +			break;
> +		case 6:
> +			check_sr_rw(ICH_LR6_EL2);
> +			break;
> +		case 7:
> +			check_sr_rw(ICH_LR7_EL2);
> +			break;
> +		case 8:
> +			check_sr_rw(ICH_LR8_EL2);
> +			break;
> +		case 9:
> +			check_sr_rw(ICH_LR9_EL2);
> +			break;
> +		case 10:
> +			check_sr_rw(ICH_LR10_EL2);
> +			break;
> +		case 11:
> +			check_sr_rw(ICH_LR11_EL2);
> +			break;
> +		case 12:
> +			check_sr_rw(ICH_LR12_EL2);
> +			break;
> +		case 13:
> +			check_sr_rw(ICH_LR13_EL2);
> +			break;
> +		case 14:
> +			check_sr_rw(ICH_LR14_EL2);
> +			break;
> +		case 15:
> +			check_sr_rw(ICH_LR15_EL2);
> +			break;
> +		default:
> +			break;
> +		}
> +	}
> +}
> +
> +/*
> + * Validate READ/WRITE to VNCR Mapped registers for NV1=0
> + */
> +
> +static void test_vncr_mapped_regs(void)
> +{
> +	/*
> +	 * Access all VNCR Mapped registers, and fail if we get an UNDEF.
> +	 */

No. You are accessing a lot of random registers, irrespective of the
configuration exposed to the guest. Being able to access it is not an
indication of correctness. Seeing it UNDEF is not an indication of a
bug.

Also, this is supposed to be an EL2 test that doesn't deal with NV
*itself*. There is no VNCR page in this test. The fact that the host
uses VNCR as a way to virtualise EL2 has nothing to do with the test
at all.

> +
> +	GUEST_PRINTF("VNCR Mapped registers access test:\n");
> +	check_sr_rw(VTTBR_EL2);
> +	check_sr_rw(VTCR_EL2);
> +	check_sr_rw(VMPIDR_EL2);
> +	check_sr_rw(CNTVOFF_EL2);
> +	check_sr_rw(HCR_EL2);
> +	check_sr_rw(HSTR_EL2);
> +	check_sr_rw(VPIDR_EL2);
> +	check_sr_rw(TPIDR_EL2);
> +	check_sr_rw(VNCR_EL2);
> +	check_sr_rw(CPACR_EL12);
> +	check_sr_rw(CONTEXTIDR_EL12);
> +	check_sr_rw(SCTLR_EL12);
> +	check_sr_rw(ACTLR_EL1);
> +	check_sr_rw(TCR_EL12);
> +	check_sr_rw(AFSR0_EL12);
> +	check_sr_rw(AFSR1_EL12);
> +	check_sr_rw(ESR_EL12);
> +	check_sr_rw(MAIR_EL12);
> +	check_sr_rw(AMAIR_EL12);
> +	check_sr_rw(MDSCR_EL1);
> +	check_sr_rw(SPSR_EL12);
> +	check_sr_rw(CNTV_CVAL_EL02);
> +	check_sr_rw(CNTV_CTL_EL02);
> +	check_sr_rw(CNTP_CVAL_EL02);
> +	check_sr_rw(CNTP_CTL_EL02);
> +	check_sr_rw(HAFGRTR_EL2);
> +	check_sr_rw(TTBR0_EL12);
> +	check_sr_rw(TTBR1_EL12);
> +	check_sr_rw(FAR_EL12);
> +	check_sr_rw(ELR_EL12);
> +	check_sr_rw(SP_EL1);
> +	check_sr_rw(VBAR_EL12);
> +
> +	regs_test_ich_lr();
> +
> +	check_sr_rw(ICH_AP0R0_EL2);
> +	check_sr_rw(ICH_AP1R0_EL2);
> +	check_sr_rw(ICH_HCR_EL2);
> +	check_sr_rw(ICH_VMCR_EL2);
> +	check_sr_rw(VDISR_EL2);

This should absolutely UNDEF in the absence of FEAT_RAS exposed to the
guest. And yet it won't. Why? That's because the architecture doesn't
allow this to be trapped. Does it mean being able to access it is
right? Absolutely not. This is an *architecture* bug.

> +	check_sr_rw(MPAM1_EL12);

This should UNDEF in the test if the configuration doesn't expose MPAM
to the guest. And I really hope it explodes or this is a glaring bug,
because MPAM should never be exposed to a guest.

The conclusion is that blindly testing that you can R/W registers buys
us nothing if you are not checking the validity of the access against
the architecture rules.

Any test should take as input:

- the configuration of the guest

- the expected outcome of the access for that particular configuration
  and trap configuration, as described in the ARM ARM pseudocode (or,
  even better, in the BSD-licensed JSON file that has all that
  information)

The result of the access must then be matched against these inputs and
any discrepancy reported, either as fatal (because the outcome of the
access wasn't the expected one) or as an expected, non-fatal failure
(because we know that the architecture is not self-consistent).

Yes, this looks a lot like a full CPU validation suite. What a
surprise.

	M.

-- 
Without deviation from the norm, progress is not possible.

Re: [RFC PATCH 1/2] KVM: arm64: nv: selftests: Add guest hypervisor test

[Marc Zyngier]

On Thu, 06 Feb 2025 16:41:19 +0000,
Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com> wrote:
> 
> This patch adds the required changes to init vcpu in vEL2 context.
> Also adds a KVM selftest to execute guest code as a guest hypervisor(L1).
> 
> Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
> ---
>  tools/testing/selftests/kvm/Makefile.kvm      |  1 +
>  .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 +++++++++++++++++++
>  .../kvm/include/arm64/kvm_util_arch.h         |  3 +
>  .../selftests/kvm/include/arm64/nv_util.h     | 28 +++++++
>  .../testing/selftests/kvm/include/kvm_util.h  |  1 +
>  .../selftests/kvm/lib/arm64/processor.c       | 59 +++++++++----
>  6 files changed, 161 insertions(+), 14 deletions(-)
>  create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
>  create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
> 
> diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
> index 4277b983cace..a85d3bec9fb1 100644
> --- a/tools/testing/selftests/kvm/Makefile.kvm
> +++ b/tools/testing/selftests/kvm/Makefile.kvm
> @@ -154,6 +154,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq
>  TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress
>  TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access
>  TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3
> +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor
>  TEST_GEN_PROGS_arm64 += access_tracking_perf_test
>  TEST_GEN_PROGS_arm64 += arch_timer
>  TEST_GEN_PROGS_arm64 += coalesced_io_test
> diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
> new file mode 100644
> index 000000000000..5aeefe43aff7
> --- /dev/null
> +++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
> @@ -0,0 +1,83 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (c) 2025 Ampere Computing LLC
> + */
> +#include <kvm_util.h>
> +#include <nv_util.h>
> +#include <processor.h>
> +#include <vgic.h>
> +
> +static void guest_code(void)
> +{
> +	if (read_sysreg(CurrentEL) == CurrentEL_EL2)
> +		GUEST_PRINTF("Executing guest code in vEL2\n");
> +	else
> +		GUEST_FAIL("Fail to run in vEL2\n");
> +
> +	GUEST_DONE();
> +}
> +
> +static void guest_undef_handler(struct ex_regs *regs)
> +{
> +	GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1));
> +}
> +
> +static void test_run_vcpu(struct kvm_vcpu *vcpu)
> +{
> +	struct ucall uc;
> +
> +	do {
> +		vcpu_run(vcpu);
> +
> +		switch (get_ucall(vcpu, &uc)) {
> +		case UCALL_ABORT:
> +			REPORT_GUEST_ASSERT(uc);
> +			break;
> +		case UCALL_PRINTF:
> +			printf("%s", uc.buffer);
> +			break;
> +		case UCALL_DONE:
> +			printf("Test PASS\n");
> +			break;
> +		default:
> +			TEST_FAIL("Unknown ucall %lu", uc.cmd);
> +		}
> +	} while (uc.cmd != UCALL_DONE);
> +}
> +
> +static void test_nv_guest_hypervisor(void)
> +{
> +	struct kvm_vcpu *vcpu;
> +	struct kvm_vm *vm;
> +	struct kvm_vcpu_init init;
> +	int gic_fd;
> +
> +	vm = vm_create(1);
> +	vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
> +
> +	init.features[0] = 0;
> +	init_vcpu_nested(&init);
> +	vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
> +
> +	__TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV");
> +
> +	vm_init_descriptor_tables(vm);
> +	vcpu_init_descriptor_tables(vcpu);
> +	gic_fd = vgic_v3_setup(vm, 1, 64);
> +	__TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
> +
> +	vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
> +				ESR_ELx_EC_UNKNOWN, guest_undef_handler);
> +
> +	test_run_vcpu(vcpu);
> +	kvm_vm_free(vm);
> +}
> +
> +int main(int argc, char *argv[])
> +{
> +	TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
> +
> +	test_nv_guest_hypervisor();
> +
> +	return 0;
> +}
> diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
> index e43a57d99b56..ab5279c24413 100644
> --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
> +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
> @@ -2,6 +2,9 @@
>  #ifndef SELFTEST_KVM_UTIL_ARCH_H
>  #define SELFTEST_KVM_UTIL_ARCH_H
>  
> +#define CurrentEL_EL1		(1 << 2)
> +#define CurrentEL_EL2		(2 << 2)
> +
>  struct kvm_vm_arch {};
>  
>  #endif  // SELFTEST_KVM_UTIL_ARCH_H
> diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h
> new file mode 100644
> index 000000000000..4fecf1f18554
> --- /dev/null
> +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h
> @@ -0,0 +1,28 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Copyright (c) 2025 Ampere Computing
> + */
> +#ifndef SELFTEST_NV_UTIL_H
> +#define SELFTEST_NV_UTIL_H
> +
> +#include <linux/bitmap.h>
> +
> +/* NV helpers */
> +static inline void init_vcpu_nested(struct kvm_vcpu_init *init)
> +{
> +	init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);
> +}
> +
> +static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init)
> +{
> +	unsigned long features = init->features[0];
> +
> +	return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);
> +}
> +
> +static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu)
> +{
> +	return vcpu->nested;
> +}
> +
> +#endif /* SELFTEST_NV_UTIL_H */
> diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
> index 4c4e5a847f67..8c53dbc17f8f 100644
> --- a/tools/testing/selftests/kvm/include/kvm_util.h
> +++ b/tools/testing/selftests/kvm/include/kvm_util.h
> @@ -58,6 +58,7 @@ struct kvm_vcpu {
>  	struct kvm_dirty_gfn *dirty_gfns;
>  	uint32_t fetch_index;
>  	uint32_t dirty_gfns_count;
> +	bool nested;
>  };
>  
>  struct userspace_mem_regions {
> diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c
> index 7ba3aa3755f3..35ba2ace61a2 100644
> --- a/tools/testing/selftests/kvm/lib/arm64/processor.c
> +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c
> @@ -10,6 +10,7 @@
>  
>  #include "guest_modes.h"
>  #include "kvm_util.h"
> +#include "nv_util.h"
>  #include "processor.h"
>  #include "ucall_common.h"
>  
> @@ -258,14 +259,47 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
>  	}
>  }
>  
> +static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,
> +			uint64_t tcr_el1, uint64_t ttbr0_el1)
> +{
> +	uint64_t fpen;
> +
> +	/*
> +	 * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
> +	 * registers, which the variable argument list macros do.
> +	 */
> +	fpen = 3 << 20;
> +
> +	if (is_vcpu_nested(vcpu)) {
> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);

How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of
things there do have an impact on how the guest behaves, and relying
on defaults feels like a bad idea.

This also assumes VHE, without trying to enforce it.

Finally, how to you plan to make all the existing tests run as EL2
guests if TPIDR_EL1 isn't populated with the expected value? Surely
you need to change the read side...

	M.

-- 
Without deviation from the norm, progress is not possible.

Re: [RFC PATCH 1/2] KVM: arm64: nv: selftests: Add guest hypervisor test

[Ganapatrao Kulkarni]

Hi Marc,

On 07-02-2025 02:44 am, Marc Zyngier wrote:
> On Thu, 06 Feb 2025 16:41:19 +0000,
> Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com> wrote:
>>
>> This patch adds the required changes to init vcpu in vEL2 context.
>> Also adds a KVM selftest to execute guest code as a guest hypervisor(L1).
>>
>> Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
>> ---
>>   tools/testing/selftests/kvm/Makefile.kvm      |  1 +
>>   .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 +++++++++++++++++++
>>   .../kvm/include/arm64/kvm_util_arch.h         |  3 +
>>   .../selftests/kvm/include/arm64/nv_util.h     | 28 +++++++
>>   .../testing/selftests/kvm/include/kvm_util.h  |  1 +
>>   .../selftests/kvm/lib/arm64/processor.c       | 59 +++++++++----
>>   6 files changed, 161 insertions(+), 14 deletions(-)
>>   create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
>>   create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
>>
>> diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
>> index 4277b983cace..a85d3bec9fb1 100644
>> --- a/tools/testing/selftests/kvm/Makefile.kvm
>> +++ b/tools/testing/selftests/kvm/Makefile.kvm
>> @@ -154,6 +154,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq
>>   TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress
>>   TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access
>>   TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3
>> +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor
>>   TEST_GEN_PROGS_arm64 += access_tracking_perf_test
>>   TEST_GEN_PROGS_arm64 += arch_timer
>>   TEST_GEN_PROGS_arm64 += coalesced_io_test
>> diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
>> new file mode 100644
>> index 000000000000..5aeefe43aff7
>> --- /dev/null
>> +++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c
>> @@ -0,0 +1,83 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/*
>> + * Copyright (c) 2025 Ampere Computing LLC
>> + */
>> +#include <kvm_util.h>
>> +#include <nv_util.h>
>> +#include <processor.h>
>> +#include <vgic.h>
>> +
>> +static void guest_code(void)
>> +{
>> +	if (read_sysreg(CurrentEL) == CurrentEL_EL2)
>> +		GUEST_PRINTF("Executing guest code in vEL2\n");
>> +	else
>> +		GUEST_FAIL("Fail to run in vEL2\n");
>> +
>> +	GUEST_DONE();
>> +}
>> +
>> +static void guest_undef_handler(struct ex_regs *regs)
>> +{
>> +	GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1));
>> +}
>> +
>> +static void test_run_vcpu(struct kvm_vcpu *vcpu)
>> +{
>> +	struct ucall uc;
>> +
>> +	do {
>> +		vcpu_run(vcpu);
>> +
>> +		switch (get_ucall(vcpu, &uc)) {
>> +		case UCALL_ABORT:
>> +			REPORT_GUEST_ASSERT(uc);
>> +			break;
>> +		case UCALL_PRINTF:
>> +			printf("%s", uc.buffer);
>> +			break;
>> +		case UCALL_DONE:
>> +			printf("Test PASS\n");
>> +			break;
>> +		default:
>> +			TEST_FAIL("Unknown ucall %lu", uc.cmd);
>> +		}
>> +	} while (uc.cmd != UCALL_DONE);
>> +}
>> +
>> +static void test_nv_guest_hypervisor(void)
>> +{
>> +	struct kvm_vcpu *vcpu;
>> +	struct kvm_vm *vm;
>> +	struct kvm_vcpu_init init;
>> +	int gic_fd;
>> +
>> +	vm = vm_create(1);
>> +	vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
>> +
>> +	init.features[0] = 0;
>> +	init_vcpu_nested(&init);
>> +	vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
>> +
>> +	__TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV");
>> +
>> +	vm_init_descriptor_tables(vm);
>> +	vcpu_init_descriptor_tables(vcpu);
>> +	gic_fd = vgic_v3_setup(vm, 1, 64);
>> +	__TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
>> +
>> +	vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
>> +				ESR_ELx_EC_UNKNOWN, guest_undef_handler);
>> +
>> +	test_run_vcpu(vcpu);
>> +	kvm_vm_free(vm);
>> +}
>> +
>> +int main(int argc, char *argv[])
>> +{
>> +	TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
>> +
>> +	test_nv_guest_hypervisor();
>> +
>> +	return 0;
>> +}
>> diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
>> index e43a57d99b56..ab5279c24413 100644
>> --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
>> +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h
>> @@ -2,6 +2,9 @@
>>   #ifndef SELFTEST_KVM_UTIL_ARCH_H
>>   #define SELFTEST_KVM_UTIL_ARCH_H
>>   
>> +#define CurrentEL_EL1		(1 << 2)
>> +#define CurrentEL_EL2		(2 << 2)
>> +
>>   struct kvm_vm_arch {};
>>   
>>   #endif  // SELFTEST_KVM_UTIL_ARCH_H
>> diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h
>> new file mode 100644
>> index 000000000000..4fecf1f18554
>> --- /dev/null
>> +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h
>> @@ -0,0 +1,28 @@
>> +/* SPDX-License-Identifier: GPL-2.0-only */
>> +/*
>> + * Copyright (c) 2025 Ampere Computing
>> + */
>> +#ifndef SELFTEST_NV_UTIL_H
>> +#define SELFTEST_NV_UTIL_H
>> +
>> +#include <linux/bitmap.h>
>> +
>> +/* NV helpers */
>> +static inline void init_vcpu_nested(struct kvm_vcpu_init *init)
>> +{
>> +	init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);
>> +}
>> +
>> +static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init)
>> +{
>> +	unsigned long features = init->features[0];
>> +
>> +	return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);
>> +}
>> +
>> +static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu)
>> +{
>> +	return vcpu->nested;
>> +}
>> +
>> +#endif /* SELFTEST_NV_UTIL_H */
>> diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
>> index 4c4e5a847f67..8c53dbc17f8f 100644
>> --- a/tools/testing/selftests/kvm/include/kvm_util.h
>> +++ b/tools/testing/selftests/kvm/include/kvm_util.h
>> @@ -58,6 +58,7 @@ struct kvm_vcpu {
>>   	struct kvm_dirty_gfn *dirty_gfns;
>>   	uint32_t fetch_index;
>>   	uint32_t dirty_gfns_count;
>> +	bool nested;
>>   };
>>   
>>   struct userspace_mem_regions {
>> diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c
>> index 7ba3aa3755f3..35ba2ace61a2 100644
>> --- a/tools/testing/selftests/kvm/lib/arm64/processor.c
>> +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c
>> @@ -10,6 +10,7 @@
>>   
>>   #include "guest_modes.h"
>>   #include "kvm_util.h"
>> +#include "nv_util.h"
>>   #include "processor.h"
>>   #include "ucall_common.h"
>>   
>> @@ -258,14 +259,47 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
>>   	}
>>   }
>>   
>> +static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,
>> +			uint64_t tcr_el1, uint64_t ttbr0_el1)
>> +{
>> +	uint64_t fpen;
>> +
>> +	/*
>> +	 * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
>> +	 * registers, which the variable argument list macros do.
>> +	 */
>> +	fpen = 3 << 20;
>> +
>> +	if (is_vcpu_nested(vcpu)) {
>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);
> 
> How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of
> things there do have an impact on how the guest behaves, and relying
> on defaults feels like a bad idea.

Sure, I will try to have these registers also set to required value 
explicitly.

> 
> This also assumes VHE, without trying to enforce it.

Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.

> 
> Finally, how to you plan to make all the existing tests run as EL2
> guests if TPIDR_EL1 isn't populated with the expected value? Surely
> you need to change the read side...

OK, I have not yet tried running existing tests modifying to run as EL2 
guests. I will try to run them modifying to run in vEL2.

Should we modify them to run as EL2 guests by default, if the host 
supports/detected NV? or command line argument based run? either in 
El1(default) or in EL2?

BTW, I have also ran all existing tests on L1, most of the tests are 
passing(atleast I did not see any failure prints).

arm64/debug-exceptions is failing on L1 and needs to be 
debugged/fixed/skipped.
arm64/arch_timer_edge_cases fails on both L0 and L1.


-- 
Thanks,
Ganapat/GK

Re: [RFC PATCH 1/2] KVM: arm64: nv: selftests: Add guest hypervisor test

[Marc Zyngier]

On Fri, 07 Feb 2025 13:26:41 +0000,
Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com> wrote:
> 
> >> +	if (is_vcpu_nested(vcpu)) {
> >> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
> >> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
> >> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
> >> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
> >> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
> >> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);
> > 
> > How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of
> > things there do have an impact on how the guest behaves, and relying
> > on defaults feels like a bad idea.
> 
> Sure, I will try to have these registers also set to required value
> explicitly.
> 
> > 
> > This also assumes VHE, without trying to enforce it.
> 
> Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
> 
> > 
> > Finally, how to you plan to make all the existing tests run as EL2
> > guests if TPIDR_EL1 isn't populated with the expected value? Surely
> > you need to change the read side...
> 
> OK, I have not yet tried running existing tests modifying to run as
> EL2 guests. I will try to run them modifying to run in vEL2.

You shouldn't try to modify them. Just make them take a parameter so
that the initialisation is done by configuring everything at EL2.

> 
> Should we modify them to run as EL2 guests by default, if the host
> supports/detected NV? or command line argument based run? either in
> El1(default) or in EL2?

EL1 by default.

> 
> BTW, I have also ran all existing tests on L1, most of the tests are
> passing(atleast I did not see any failure prints).
> 
> arm64/debug-exceptions is failing on L1 and needs to be
> debugged/fixed/skipped.
> arm64/arch_timer_edge_cases fails on both L0 and L1.

Then I guess you have some work to do to debug these problems, and it
once more means that NV is not ready for merging.

	M.

-- 
Without deviation from the norm, progress is not possible.

Re: [RFC PATCH 1/2] KVM: arm64: nv: selftests: Add guest hypervisor test

[Ganapatrao Kulkarni]

On 07-02-2025 07:29 pm, Marc Zyngier wrote:
> On Fri, 07 Feb 2025 13:26:41 +0000,
> Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com> wrote:
>>
>>>> +	if (is_vcpu_nested(vcpu)) {
>>>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
>>>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
>>>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
>>>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
>>>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
>>>> +		vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);
>>>
>>> How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of
>>> things there do have an impact on how the guest behaves, and relying
>>> on defaults feels like a bad idea.
>>
>> Sure, I will try to have these registers also set to required value
>> explicitly.
>>
>>>
>>> This also assumes VHE, without trying to enforce it.
>>
>> Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
>>
>>>
>>> Finally, how to you plan to make all the existing tests run as EL2
>>> guests if TPIDR_EL1 isn't populated with the expected value? Surely
>>> you need to change the read side...
>>
>> OK, I have not yet tried running existing tests modifying to run as
>> EL2 guests. I will try to run them modifying to run in vEL2.
> 
> You shouldn't try to modify them. Just make them take a parameter so
> that the initialisation is done by configuring everything at EL2.
> 

OK, make sense, thanks.
I will work on one of the test case and post it in the v2.
>>
>> Should we modify them to run as EL2 guests by default, if the host
>> supports/detected NV? or command line argument based run? either in
>> El1(default) or in EL2?
> 
> EL1 by default.

OK.
> 
>>
>> BTW, I have also ran all existing tests on L1, most of the tests are
>> passing(atleast I did not see any failure prints).
>>
>> arm64/debug-exceptions is failing on L1 and needs to be
>> debugged/fixed/skipped.
>> arm64/arch_timer_edge_cases fails on both L0 and L1.
> 
> Then I guess you have some work to do to debug these problems, and it
> once more means that NV is not ready for merging.

I will debug and share the findings/fix at the earliest.

-- 
Thanks,
Ganapat/GK

Re: [RFC PATCH 1/2] KVM: arm64: nv: selftests: Add guest hypervisor test

[Ganapatrao Kulkarni]

On 07-02-2025 10:16 pm, Ganapatrao Kulkarni wrote:
> 
> 
> On 07-02-2025 07:29 pm, Marc Zyngier wrote:
>> On Fri, 07 Feb 2025 13:26:41 +0000,
>> Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com> wrote:
>>>
>>>>> +    if (is_vcpu_nested(vcpu)) {
>>>>> +        vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
>>>>> +        vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), 
>>>>> sctlr_el1);
>>>>> +        vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
>>>>> +        vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), 
>>>>> DEFAULT_MAIR_EL1);
>>>>> +        vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), 
>>>>> ttbr0_el1);
>>>>> +        vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu- 
>>>>> >id);
>>>>
>>>> How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of
>>>> things there do have an impact on how the guest behaves, and relying
>>>> on defaults feels like a bad idea.
>>>
>>> Sure, I will try to have these registers also set to required value
>>> explicitly.
>>>
>>>>
>>>> This also assumes VHE, without trying to enforce it.
>>>
>>> Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
>>>
>>>>
>>>> Finally, how to you plan to make all the existing tests run as EL2
>>>> guests if TPIDR_EL1 isn't populated with the expected value? Surely
>>>> you need to change the read side...
>>>

IIUC, we need not write to TPIDR_EL2, instead write always to TPIDR_EL1 
and in guest code(like function cpu_copy_el2regs in Linux kernel) copy 
TPIDR_EL1 value to TPIDR_EL2.

OR, Write to both TPIDR_EL1 and TPIDR_EL2 in the test code itself during 
vcpu setup.

-- 
Thanks,
Ganapat/GK