[Qemu-devel] [RFC 0/3] VFIO Platform device featuring IRQ forwarding

[Qemu-devel] [RFC 0/3] VFIO Platform device featuring IRQ forwarding

[Eric Auger]

This series adds IRQ forwarding support in the VFIO platform device.

The VFIO platform device uses the KVM-VFIO device to enable IRQ forwarding.

Dependency List:
- KVM platform device passthrough PATCH v10 and all its dependencies
  + following kernel dependencies:
- [RFC PATCH 0/9] ARM: Forwarding physical interrupts to a guest VM
  http://lwn.net/Articles/603514/
- [RFC v2 0/4] chip/vgic adaptations for forwarded irq
  http://permalink.gmane.org/gmane.comp.emulators.kvm.devel/132731
- [RFC v4 00/13] KVM-VFIO IRQ forward control
  http://www.spinics.net/lists/arm-kernel/msg398183.html

Repos:
- kernel pieces can be found at:
  http://git.linaro.org/people/eric.auger/linux.git
  (branch irqfd_integ_v9)
- QEMU pieces can be found at:
  http://git.linaro.org/people/eric.auger/qemu.git
  (branch vfio_integ_v10)

The patch series was tested on Calxeda Midway (ARMv7) where one xgmac
is assigned to KVM host while the second one is assigned to the guest.
Reworked PCI device is not tested.

Wiki for Calxeda Midway setup:
https://wiki.linaro.org/LEG/Engineering/Virtualization/Platform_Device_Passthrough_on_Midway

History:
- that code originally was in KVM platform device passthrough series (v8),
  now moved in a separate RFC since dependent on many RFCs.

Best Regards

Eric

Eric Auger (3):
  linux-headers: Update KVM header for KVM-VFIO FORWARD/UNFORWARD
  hw/vfio/common: vfio_kvm_device_fd moved in the common header
  hw/vfio/platform: add forwarded irq support

 hw/vfio/common.c                |  3 +-
 hw/vfio/platform.c              | 64 +++++++++++++++++++++++++++++++++++++++++
 include/hw/vfio/vfio-common.h   |  5 ++++
 include/hw/vfio/vfio-platform.h |  3 ++
 linux-headers/linux/kvm.h       | 23 ++++++++-------
 trace-events                    |  1 +
 6 files changed, 87 insertions(+), 12 deletions(-)

-- 
1.8.3.2

[Qemu-devel] [RFC 1/3] linux-headers: Update KVM header for KVM-VFIO FORWARD/UNFORWARD

[Eric Auger]

Integrate updated KVM-VFIO API related to forwarded IRQ

Update according to header found in
http://git.linaro.org/people/eric.auger/linux.git
branch irqfd_integ_v9

Signed-off-by: Eric Auger <eric.auger@linaro.org>

---

v8 -> v9: use new kvm_vfio_dev_irq struct
---
 linux-headers/linux/kvm.h | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 12045a1..73b8194 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -647,11 +647,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_MP_STATE 14
 #define KVM_CAP_COALESCED_MMIO 15
 #define KVM_CAP_SYNC_MMU 16  /* Changes to host mmap are reflected in guest */
-#define KVM_CAP_DEVICE_ASSIGNMENT 17
 #define KVM_CAP_IOMMU 18
-#ifdef __KVM_HAVE_MSI
-#define KVM_CAP_DEVICE_MSI 20
-#endif
 /* Bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_DESTROY_MEMORY_REGION_WORKS 21
 #define KVM_CAP_USER_NMI 22
@@ -663,10 +659,6 @@ struct kvm_ppc_smmu_info {
 #endif
 #define KVM_CAP_IRQ_ROUTING 25
 #define KVM_CAP_IRQ_INJECT_STATUS 26
-#define KVM_CAP_DEVICE_DEASSIGNMENT 27
-#ifdef __KVM_HAVE_MSIX
-#define KVM_CAP_DEVICE_MSIX 28
-#endif
 #define KVM_CAP_ASSIGN_DEV_IRQ 29
 /* Another bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_JOIN_MEMORY_REGIONS_WORKS 30
@@ -946,6 +938,9 @@ struct kvm_device_attr {
 #define  KVM_DEV_VFIO_GROUP			1
 #define   KVM_DEV_VFIO_GROUP_ADD			1
 #define   KVM_DEV_VFIO_GROUP_DEL			2
+#define  KVM_DEV_VFIO_DEVICE			2
+#define   KVM_DEV_VFIO_DEVICE_FORWARD_IRQ			1
+#define   KVM_DEV_VFIO_DEVICE_UNFORWARD_IRQ			2
 
 enum kvm_device_type {
 	KVM_DEV_TYPE_FSL_MPIC_20	= 1,
@@ -963,6 +958,15 @@ enum kvm_device_type {
 	KVM_DEV_TYPE_MAX,
 };
 
+struct kvm_vfio_dev_irq {
+	__u32	argsz;		/* structure length */
+	__u32	fd;		/* file descriptor of the VFIO device */
+	__u32	index;		/* VFIO device IRQ index */
+	__u32	start;		/* start of subindex range */
+	__u32	count;		/* size of subindex range */
+	__u32	gsi[];		/* gsi, ie. virtual IRQ number */
+};
+
 /*
  * ioctls for VM fds
  */
@@ -1107,9 +1111,6 @@ struct kvm_s390_ucas_mapping {
 #define KVM_X86_SETUP_MCE         _IOW(KVMIO,  0x9c, __u64)
 #define KVM_X86_GET_MCE_CAP_SUPPORTED _IOR(KVMIO,  0x9d, __u64)
 #define KVM_X86_SET_MCE           _IOW(KVMIO,  0x9e, struct kvm_x86_mce)
-/* IA64 stack access */
-#define KVM_IA64_VCPU_GET_STACK   _IOR(KVMIO,  0x9a, void *)
-#define KVM_IA64_VCPU_SET_STACK   _IOW(KVMIO,  0x9b, void *)
 /* Available with KVM_CAP_VCPU_EVENTS */
 #define KVM_GET_VCPU_EVENTS       _IOR(KVMIO,  0x9f, struct kvm_vcpu_events)
 #define KVM_SET_VCPU_EVENTS       _IOW(KVMIO,  0xa0, struct kvm_vcpu_events)
-- 
1.8.3.2

[Qemu-devel] [RFC 2/3] hw/vfio/common: vfio_kvm_device_fd moved in the common header

[Eric Auger]

the device is now used in platform for forwarded IRQ setup

Signed-off-by: Eric Auger <eric.auger@linaro.org>
---
 hw/vfio/common.c              | 3 ++-
 include/hw/vfio/vfio-common.h | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index c5d1551..be72ad6 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -44,9 +44,10 @@ struct vfio_as_head vfio_address_spaces =
  * initialized, this file descriptor is only released on QEMU exit and
  * we'll re-use it should another vfio device be attached before then.
  */
-static int vfio_kvm_device_fd = -1;
+int vfio_kvm_device_fd = -1;
 #endif
 
+
 /*
  * Common VFIO interrupt disable
  */
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 2d1d8b3..8816731 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -41,6 +41,11 @@
 #define VFIO_ALLOW_KVM_MSI 1
 #define VFIO_ALLOW_KVM_MSIX 1
 
+#ifdef CONFIG_KVM
+extern int vfio_kvm_device_fd;
+#endif
+
+
 enum {
     VFIO_DEVICE_TYPE_PCI = 0,
     VFIO_DEVICE_TYPE_PLATFORM = 1,
-- 
1.8.3.2

[Qemu-devel] [RFC 3/3] hw/vfio/platform: add forwarded irq support

[Eric Auger]

Tests whether the forwarded IRQ modality is available.
In the positive device IRQs are forwarded. This control is
achieved with KVM-VFIO device. with such a modality injection
still is handled through irqfds. However end of interrupt is
not trapped anymore. As soon as the guest completes its virtual
IRQ, the corresponding physical IRQ is completed and the same
physical IRQ can hit again.

A new x-forward property enables to force forwarding off although
enabled by the kernel.

Signed-off-by: Eric Auger <eric.auger@linaro.org>

---

v8 -> v9:
- use new kvm_vfio_dev_irq struct
---
 hw/vfio/platform.c              | 64 +++++++++++++++++++++++++++++++++++++++++
 include/hw/vfio/vfio-platform.h |  3 ++
 trace-events                    |  1 +
 3 files changed, 68 insertions(+)

diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
index cadc824..b6ba5b3 100644
--- a/hw/vfio/platform.c
+++ b/hw/vfio/platform.c
@@ -238,6 +238,63 @@ static int vfio_start_eventfd_injection(VFIOINTp *intp)
 }
 
 /*
+ * Functions used with forwarding capability
+ */
+
+#ifdef CONFIG_KVM
+
+static bool has_kvm_vfio_forward_capability(void)
+{
+    struct kvm_device_attr attr = {
+         .group = KVM_DEV_VFIO_DEVICE,
+         .attr = KVM_DEV_VFIO_DEVICE_FORWARD_IRQ};
+
+    if (ioctl(vfio_kvm_device_fd, KVM_HAS_DEVICE_ATTR, &attr) == 0) {
+        return true;
+    } else {
+        return false;
+    }
+}
+
+static int vfio_set_forwarding(VFIOINTp *intp)
+{
+    int argsz;
+    struct kvm_device_attr attr = {
+         .group = KVM_DEV_VFIO_DEVICE,
+         .attr = KVM_DEV_VFIO_DEVICE_FORWARD_IRQ};
+    __u32 *pvirq;
+
+    if (intp->forwarded) {
+        return -EINVAL;
+    }
+
+    argsz = sizeof(*intp->fwd_irq) + sizeof(*pvirq);
+    intp->fwd_irq = g_malloc0(argsz);
+    intp->fwd_irq->argsz = argsz;
+    intp->fwd_irq->fd = intp->vdev->vbasedev.fd;
+    intp->fwd_irq->index = intp->pin;
+    intp->fwd_irq->start = 0;
+    intp->fwd_irq->count = 1;
+    pvirq = (__u32 *)&intp->fwd_irq->gsi;
+    *pvirq = (uint32_t)intp->virtualID;
+
+    attr.addr = (uint64_t)(unsigned long)intp->fwd_irq;
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+            error_report("Failed to forward IRQ %d through KVM VFIO device",
+                         intp->pin);
+            g_free(intp->fwd_irq);
+            return -errno;
+    }
+    trace_vfio_start_fwd_injection(intp->pin);
+    intp->forwarded = true;
+
+    return 0;
+}
+
+#endif
+
+/*
  * Functions used for irqfd
  */
 
@@ -289,6 +346,11 @@ static int vfio_start_irqfd_injection(VFIOINTp *intp)
         .flags = KVM_IRQFD_FLAG_RESAMPLE,
     };
 
+    if (has_kvm_vfio_forward_capability() &&
+                 intp->vdev->forward_allowed) {
+        vfio_set_forwarding(intp);
+    }
+
     if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
         error_report("vfio: Error: Failed to assign the irqfd: %m");
         goto fail_irqfd;
@@ -384,6 +446,7 @@ static VFIOINTp *vfio_init_intp(VFIODevice *vbasedev, unsigned int index)
     intp->vdev = vdev;
     intp->pin = index;
     intp->state = VFIO_IRQ_INACTIVE;
+    intp->forwarded = false;
     sysbus_init_irq(sbdev, &intp->qemuirq);
 
     /* Get an eventfd for trigger */
@@ -716,6 +779,7 @@ static Property vfio_platform_dev_properties[] = {
     DEFINE_PROP_UINT32("mmap-timeout-ms", VFIOPlatformDevice,
                        mmap_timeout, 1100),
     DEFINE_PROP_BOOL("x-irqfd", VFIOPlatformDevice, irqfd_allowed, true),
+    DEFINE_PROP_BOOL("x-forward", VFIOPlatformDevice, forward_allowed, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/include/hw/vfio/vfio-platform.h b/include/hw/vfio/vfio-platform.h
index 097448b..693066d 100644
--- a/include/hw/vfio/vfio-platform.h
+++ b/include/hw/vfio/vfio-platform.h
@@ -42,6 +42,8 @@ typedef struct VFIOINTp {
     bool kvm_accel; /* set when QEMU bypass through KVM enabled */
     uint8_t pin; /* index */
     uint32_t virtualID; /* virtual IRQ */
+    struct kvm_vfio_dev_irq *fwd_irq;
+    bool forwarded;
 } VFIOINTp;
 
 typedef int (*start_irq_fn_t)(VFIOINTp *intp);
@@ -59,6 +61,7 @@ typedef struct VFIOPlatformDevice {
     start_irq_fn_t start_irq_fn;
     QemuMutex  intp_mutex;
     bool irqfd_allowed; /* debug option to force irqfd on/off */
+    bool forward_allowed; /* debug option to force forwarding on/off */
 } VFIOPlatformDevice;
 
 
diff --git a/trace-events b/trace-events
index 7a6a6aa..9a138d6 100644
--- a/trace-events
+++ b/trace-events
@@ -1557,6 +1557,7 @@ vfio_get_device(const char * name, unsigned int flags, unsigned int num_regions,
 vfio_put_base_device(int fd) "close vdev->fd=%d"
 
 # hw/vfio/platform.c
+vfio_start_fwd_injection(int pin) "forwarding set for IRQ pin %d"
 vfio_platform_eoi(int pin, int fd) "EOI IRQ pin %d (fd=%d)"
 vfio_platform_mmap_set_enabled(bool enabled) "fast path = %d"
 vfio_platform_intp_mmap_enable(int pin) "IRQ #%d still active, stay in slow path"
-- 
1.8.3.2