[PATCH 32/33] KVM: selftests: Test sending a vfio-pci device IRQ to a VM

[David Matlack <dmatlack@google.com>]

Add a new selftest called vfio_pci_device_irq_test that routes and
delivers an MSI from a vfio-pci device into a guest.

Note that this test currently uses vfio_pci_irq_trigger(), in which VFIO
emulates a device sending the MSI. In the future we want to replace this
with a real MSI so we can test the hardware path (e.g. VT-d on Intel).

This test only supports x86_64 for now, but can be ported to other
architectures in the future.

Signed-off-by: David Matlack <dmatlack@google.com>
---
 tools/testing/selftests/kvm/Makefile.kvm      |   1 +
 .../testing/selftests/kvm/include/kvm_util.h  |   4 +
 tools/testing/selftests/kvm/lib/kvm_util.c    |  21 +++
 .../selftests/kvm/vfio_pci_device_irq_test.c  | 123 ++++++++++++++++++
 4 files changed, 149 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/vfio_pci_device_irq_test.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 5a9644ce910b..ba6727dcf59d 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -146,6 +146,7 @@ TEST_GEN_PROGS_x86 += rseq_test
 TEST_GEN_PROGS_x86 += steal_time
 TEST_GEN_PROGS_x86 += system_counter_offset_test
 TEST_GEN_PROGS_x86 += pre_fault_memory_test
+TEST_GEN_PROGS_x86 += vfio_pci_device_irq_test
 
 # Compiled outputs used by test targets
 TEST_GEN_PROGS_EXTENDED_x86 += x86/nx_huge_pages_test
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index bee65ca08721..8afa6d6f1435 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -907,6 +907,8 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);
 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 
+void kvm_add_irqfd(struct kvm_vm *vm, u32 gsi, int fd);
+
 #define KVM_MAX_IRQ_ROUTES		4096
 
 struct kvm_irq_routing *kvm_gsi_routing_create(void);
@@ -915,6 +917,8 @@ void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 
+void kvm_route_gsi(struct kvm_vm *vm, struct kvm_irq_routing_entry *entry);
+
 const char *exit_reason_str(unsigned int exit_reason);
 
 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index a055343a7bf7..737252812d15 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -1891,6 +1891,16 @@ void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
 	TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret));
 }
 
+void kvm_add_irqfd(struct kvm_vm *vm, u32 gsi, int fd)
+{
+	struct kvm_irqfd arg = {
+		.gsi = gsi,
+		.fd = fd,
+	};
+
+	vm_ioctl(vm, KVM_IRQFD, &arg);
+}
+
 struct kvm_irq_routing *kvm_gsi_routing_create(void)
 {
 	struct kvm_irq_routing *routing;
@@ -1941,6 +1951,17 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
 	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_GSI_ROUTING, ret));
 }
 
+void kvm_route_gsi(struct kvm_vm *vm, struct kvm_irq_routing_entry *entry)
+{
+	u8 buf[sizeof(struct kvm_irq_routing) + sizeof(*entry)] = {};
+	struct kvm_irq_routing *routes = (void *)&buf;
+
+	routes->nr = 1;
+	routes->entries[0] = *entry;
+
+	vm_ioctl(vm, KVM_SET_GSI_ROUTING, routes);
+}
+
 /*
  * VM Dump
  *
diff --git a/tools/testing/selftests/kvm/vfio_pci_device_irq_test.c b/tools/testing/selftests/kvm/vfio_pci_device_irq_test.c
new file mode 100644
index 000000000000..9b90cf9dd38f
--- /dev/null
+++ b/tools/testing/selftests/kvm/vfio_pci_device_irq_test.c
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "kvm_util.h"
+#include "test_util.h"
+#include "apic.h"
+#include "processor.h"
+
+#include <pthread.h>
+#include <time.h>
+#include <linux/vfio.h>
+#include <vfio_util.h>
+
+static bool guest_ready_for_irq;
+static bool guest_received_irq;
+
+#define TIMEOUT_NS (10ULL * 1000 * 1000 * 1000)
+
+static void guest_enable_interrupts(void)
+{
+	x2apic_enable();
+	asm volatile ("sti");
+}
+
+void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, u8 vector)
+{
+	struct kvm_irq_routing_entry entry = {
+		.gsi = gsi,
+		.type = KVM_IRQ_ROUTING_MSI,
+		.u.msi.address_lo = 0xFEE00000 | (vcpu->id << 12),
+		.u.msi.data = vector,
+	};
+
+	kvm_route_gsi(vm, &entry);
+}
+
+static void guest_irq_handler(struct ex_regs *regs)
+{
+	WRITE_ONCE(guest_received_irq, true);
+	GUEST_DONE();
+}
+
+static void guest_code(void)
+{
+	guest_enable_interrupts();
+	WRITE_ONCE(guest_ready_for_irq, true);
+
+	for (;;)
+		continue;
+}
+
+void *vcpu_thread_main(void *arg)
+{
+	struct kvm_vcpu *vcpu = arg;
+	struct ucall uc;
+
+	vcpu_run(vcpu);
+	TEST_ASSERT_EQ(UCALL_DONE, get_ucall(vcpu, &uc));
+
+	return NULL;
+}
+
+static void help(const char *name)
+{
+	printf("Usage: %s [-i iommu_mode] [segment:bus:device.function]\n", name);
+	exit(KSFT_FAIL);
+}
+
+int main(int argc, char **argv)
+{
+	/* Random non-reserved vector and GSI to use for the device IRQ */
+	const u8 vector = 0xe0;
+	const u32 gsi = 32;
+
+	struct timespec start, elapsed;
+	struct vfio_pci_device *device;
+	const char *iommu_mode = NULL;
+	const char *device_bdf;
+	struct kvm_vcpu *vcpu;
+	pthread_t vcpu_thread;
+	struct kvm_vm *vm;
+	int c;
+
+	device_bdf = vfio_selftests_get_bdf(&argc, argv);
+
+	while ((c = getopt(argc, argv, "i:")) != -1) {
+		switch (c) {
+		case 'i':
+			iommu_mode = optarg;
+			break;
+		default:
+			help(argv[0]);
+		}
+	}
+
+	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+	vm_install_exception_handler(vm, vector, guest_irq_handler);
+
+	device = vfio_pci_device_init(device_bdf, iommu_mode);
+	TEST_REQUIRE(device->msix_info.count > 0);
+
+	vfio_pci_msix_enable(device, 0, 1);
+	kvm_add_irqfd(vm, gsi, device->msi_eventfds[0]);
+	kvm_route_msi(vm, gsi, vcpu, vector);
+
+	pthread_create(&vcpu_thread, NULL, vcpu_thread_main, vcpu);
+
+	while (!READ_ONCE(guest_ready_for_irq))
+		sync_global_from_guest(vm, guest_ready_for_irq);
+
+	vfio_pci_irq_trigger(device, VFIO_PCI_MSIX_IRQ_INDEX, 0);
+
+	clock_gettime(CLOCK_MONOTONIC, &start);
+
+	while (!READ_ONCE(guest_received_irq)) {
+		elapsed = timespec_elapsed(start);
+		TEST_ASSERT(timespec_to_ns(elapsed) < TIMEOUT_NS, "vCPU never received IRQ\n");
+		sync_global_from_guest(vm, guest_received_irq);
+	}
+
+	pthread_join(vcpu_thread, NULL);
+	vfio_pci_device_cleanup(device);
+
+	return 0;
+}
-- 
2.50.0.rc2.701.gf1e915cc24-goog