[Qemu-devel] [v3 0/2] Add VT-d Posted-Interrupts support

[Qemu-devel] [v3 0/2] Add VT-d Posted-Interrupts support

[Feng Wu]

VT-d Posted-Interrupts is an enhancement to CPU side Posted-Interrupt.
With VT-d Posted-Interrupts enabled, external interrupts from
direct-assigned devices can be delivered to guests without VMM
intervention when the guest is running in non-root mode.

This patchset uses the new VFIO device attributes KVM_DEV_VFIO_DEVICE
introduced by Eric's vfio patch
"[PATCH v8 00/19] KVM platform device passthrough"

You can find the VT-d Posted-Interrtups Spec. in the following URL:
http://www.intel.com/content/www/us/en/intelligent-systems/intel-technology/vt-directed-io-spec.html

v1->v2:
* Enable this feature based on VFIO mechanism

v2->v3:
* KVM_DEV_VFIO_DEVICE_POSTING_IRQ --> KVM_DEV_VFIO_DEVICE_POST_IRQ
* Add KVM_DEV_VFIO_DEVICE_UPPOST_IRQ attribute in linux header file

Feng Wu (2):
  kvm, vfio: Define new VFIO data structure for VT-d Posted-Interrupts.
  kvm, vfio: Update VT-d Posted-Interrupts related information

 hw/misc/vfio.c            | 60 +++++++++++++++++++++++++++++++++++++++++++++--
 linux-headers/linux/kvm.h | 11 +++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)

-- 
1.9.1

[Qemu-devel] [v3 1/2] kvm, vfio: Define new VFIO data structure for VT-d Posted-Interrupts.

[Feng Wu]

Define new VFIO data structure for VT-d Posted-Interrupts.

Signed-off-by: Feng Wu <feng.wu@intel.com>
---
 linux-headers/linux/kvm.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 1937afa..6463b28 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -949,6 +949,8 @@ struct kvm_device_attr {
 #define  KVM_DEV_VFIO_DEVICE			2
 #define   KVM_DEV_VFIO_DEVICE_FORWARD_IRQ			1
 #define   KVM_DEV_VFIO_DEVICE_UNFORWARD_IRQ			2
+#define   KVM_DEV_VFIO_DEVICE_POST_IRQ				3
+#define   KVM_DEV_VFIO_DEVICE_UNPOST_IRQ			4
 
 enum kvm_device_type {
 	KVM_DEV_TYPE_FSL_MPIC_20	= 1,
@@ -973,6 +975,15 @@ struct kvm_arch_forwarded_irq {
 	__u32 gsi; /* gsi, ie. virtual IRQ number */
 };
 
+struct kvm_vfio_dev_irq {
+	__u32	argsz;
+	__u32	fd;		/* file descriptor of the VFIO device */
+	__u32	index;		/* VFIO device IRQ index */
+	__u32	start;
+	__u32	count;
+	__u32	gsi[];		/* gsi, ie. virtual IRQ number */
+};
+
 /*
  * ioctls for VM fds
  */
-- 
1.9.1

[Qemu-devel] [v3 2/2] kvm, vfio: Update VT-d Posted-Interrupts related information

[Feng Wu]

VT-d Posted-Interrupts(PI) is an enhancement to CPU side Posted-Interrupt.
With VT-d Posted-Interrupts enabled, external interrupts from
direct-assigned devices can be delivered to guests without VMM
involvement when guest is running in non-root mode.

If VT-d PI is supported by KVM, we need to update the IRTE with
the new guest interrupt configuration.

Signed-off-by: Feng Wu <feng.wu@intel.com>
---
 hw/misc/vfio.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 2 deletions(-)

diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index fd318a1..5bdc49c 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -281,6 +281,8 @@ static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
                                   uint32_t val, int len);
 static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
+static void vfio_kvm_device_posting_irq(VFIODevice *vdev, uint32_t index,
+                                        uint32_t start, uint32_t count);
 
 /*
  * Common VFIO interrupt disable
@@ -765,6 +767,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
 {
     VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
     VFIOMSIVector *vector;
+    uint32_t start, count;
     int ret;
 
     DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
@@ -812,6 +815,8 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         if (ret) {
             error_report("vfio: failed to enable vectors, %d", ret);
         }
+        start = 0;
+        count = vdev->nr_vectors;
     } else {
         int argsz;
         struct vfio_irq_set *irq_set;
@@ -824,8 +829,8 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
                          VFIO_IRQ_SET_ACTION_TRIGGER;
         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
-        irq_set->start = nr;
-        irq_set->count = 1;
+        irq_set->start = start = nr;
+        irq_set->count = count = 1;
         pfd = (int32_t *)&irq_set->data;
 
         if (vector->virq >= 0) {
@@ -841,6 +846,11 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         }
     }
 
+    if (msg) {
+        vfio_kvm_device_posting_irq(vdev, VFIO_PCI_MSIX_IRQ_INDEX,
+                                    start, count);
+    }
+
     return 0;
 }
 
@@ -997,6 +1007,9 @@ retry:
         return;
     }
 
+    vfio_kvm_device_posting_irq(vdev, VFIO_PCI_MSI_IRQ_INDEX,
+                                0, vdev->nr_vectors);
+
     DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
             vdev->host.domain, vdev->host.bus, vdev->host.slot,
             vdev->host.function, vdev->nr_vectors);
@@ -1077,6 +1090,9 @@ static void vfio_update_msi(VFIODevice *vdev)
         msg = msi_get_message(&vdev->pdev, i);
         vfio_update_kvm_msi_virq(vector, msg);
     }
+
+    vfio_kvm_device_posting_irq(vdev, VFIO_PCI_MSIX_IRQ_INDEX,
+                                0, vdev->nr_vectors);
 }
 
 /*
@@ -3622,6 +3638,46 @@ static void vfio_kvm_device_del_group(VFIOGroup *group)
 #endif
 }
 
+static void vfio_kvm_device_posting_irq(VFIODevice *vdev, uint32_t index,
+                                        uint32_t start, uint32_t count)
+{
+#ifdef CONFIG_KVM
+    int i, argsz;
+    struct kvm_vfio_dev_irq *pi_info = NULL;
+
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_DEVICE,
+        .attr = KVM_DEV_VFIO_DEVICE_POST_IRQ,
+    };
+
+    if (!kvm_enabled() ||
+        ioctl(vfio_kvm_device_fd, KVM_HAS_DEVICE_ATTR, &attr) != 0) {
+        return;
+    }
+
+    argsz = sizeof(*pi_info) + sizeof(int) * count;
+
+    pi_info = g_malloc0(argsz);
+    pi_info->argsz = argsz;
+    pi_info->fd = vdev->fd;
+    pi_info->index = index;
+    pi_info->start = start;
+    pi_info->count = count;
+
+    for (i = 0; i < count; i++) {
+        pi_info->gsi[i] = vdev->msi_vectors[start+i].virq;;
+    }
+
+    attr.addr = (uint64_t)(unsigned long)pi_info;
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to configure PI for KVM VFIO device: %m\n");
+    }
+
+    g_free(pi_info);
+#endif
+}
+
 static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
 {
     VFIOAddressSpace *space;
-- 
1.9.1

[Qemu-devel] [v3 0/2] Add VT-d Posted-Interrupts support

[Feng Wu]

VT-d Posted-Interrupts is an enhancement to CPU side Posted-Interrupt.
With VT-d Posted-Interrupts enabled, external interrupts from
direct-assigned devices can be delivered to guests without VMM
intervention when the guest is running in non-root mode.

This patchset uses the new VFIO device attributes KVM_DEV_VFIO_DEVICE
introduced by Eric's vfio patch
"[PATCH v8 00/19] KVM platform device passthrough"

You can find the VT-d Posted-Interrtups Spec. in the following URL:
http://www.intel.com/content/www/us/en/intelligent-systems/intel-technology/vt-directed-io-spec.html

v1->v2:
Enable this feature based on VFIO mechanism

v2->v3:
KVM_DEV_VFIO_DEVICE_POSTING_IRQ --> KVM_DEV_VFIO_DEVICE_POST_IRQ

Feng Wu (2):
  kvm, vfio: Define new VFIO data structure for VT-d Posted-Interrupts.
  kvm, vfio: Update VT-d Posted-Interrupts related information

 hw/misc/vfio.c            |   60 +++++++++++++++++++++++++++++++++++++++++++-
 linux-headers/linux/kvm.h |   10 +++++++
 2 files changed, 68 insertions(+), 2 deletions(-)

[Qemu-devel] [v3 1/2] kvm, vfio: Define new VFIO data structure for VT-d Posted-Interrupts.

[Feng Wu]

Define new VFIO data structure for VT-d Posted-Interrupts.

Signed-off-by: Feng Wu <feng.wu@intel.com>
---
 linux-headers/linux/kvm.h |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 1937afa..48edf23 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -949,6 +949,7 @@ struct kvm_device_attr {
 #define  KVM_DEV_VFIO_DEVICE			2
 #define   KVM_DEV_VFIO_DEVICE_FORWARD_IRQ			1
 #define   KVM_DEV_VFIO_DEVICE_UNFORWARD_IRQ			2
+#define   KVM_DEV_VFIO_DEVICE_POST_IRQ				3
 
 enum kvm_device_type {
 	KVM_DEV_TYPE_FSL_MPIC_20	= 1,
@@ -973,6 +974,15 @@ struct kvm_arch_forwarded_irq {
 	__u32 gsi; /* gsi, ie. virtual IRQ number */
 };
 
+struct kvm_vfio_dev_irq {
+	__u32	argsz;
+	__u32	fd;		/* file descriptor of the VFIO device */
+	__u32	index;		/* VFIO device IRQ index */
+	__u32	start;
+	__u32	count;
+	__u32	gsi[];		/* gsi, ie. virtual IRQ number */
+};
+
 /*
  * ioctls for VM fds
  */
-- 
1.7.1

[Qemu-devel] [v3 2/2] kvm, vfio: Update VT-d Posted-Interrupts related information

[Feng Wu]

VT-d Posted-Interrupts(PI) is an enhancement to CPU side Posted-Interrupt.
With VT-d Posted-Interrupts enabled, external interrupts from
direct-assigned devices can be delivered to guests without VMM
involvement when guest is running in non-root mode.

If VT-d PI is supported by KVM, we need to update the IRTE with
the new guest interrupt configuration.

Signed-off-by: Feng Wu <feng.wu@intel.com>
---
 hw/misc/vfio.c |   60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index fd318a1..5bdc49c 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -281,6 +281,8 @@ static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
                                   uint32_t val, int len);
 static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
+static void vfio_kvm_device_posting_irq(VFIODevice *vdev, uint32_t index,
+                                        uint32_t start, uint32_t count);
 
 /*
  * Common VFIO interrupt disable
@@ -765,6 +767,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
 {
     VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
     VFIOMSIVector *vector;
+    uint32_t start, count;
     int ret;
 
     DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
@@ -812,6 +815,8 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         if (ret) {
             error_report("vfio: failed to enable vectors, %d", ret);
         }
+        start = 0;
+        count = vdev->nr_vectors;
     } else {
         int argsz;
         struct vfio_irq_set *irq_set;
@@ -824,8 +829,8 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
                          VFIO_IRQ_SET_ACTION_TRIGGER;
         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
-        irq_set->start = nr;
-        irq_set->count = 1;
+        irq_set->start = start = nr;
+        irq_set->count = count = 1;
         pfd = (int32_t *)&irq_set->data;
 
         if (vector->virq >= 0) {
@@ -841,6 +846,11 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         }
     }
 
+    if (msg) {
+        vfio_kvm_device_posting_irq(vdev, VFIO_PCI_MSIX_IRQ_INDEX,
+                                    start, count);
+    }
+
     return 0;
 }
 
@@ -997,6 +1007,9 @@ retry:
         return;
     }
 
+    vfio_kvm_device_posting_irq(vdev, VFIO_PCI_MSI_IRQ_INDEX,
+                                0, vdev->nr_vectors);
+
     DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
             vdev->host.domain, vdev->host.bus, vdev->host.slot,
             vdev->host.function, vdev->nr_vectors);
@@ -1077,6 +1090,9 @@ static void vfio_update_msi(VFIODevice *vdev)
         msg = msi_get_message(&vdev->pdev, i);
         vfio_update_kvm_msi_virq(vector, msg);
     }
+
+    vfio_kvm_device_posting_irq(vdev, VFIO_PCI_MSIX_IRQ_INDEX,
+                                0, vdev->nr_vectors);
 }
 
 /*
@@ -3622,6 +3638,46 @@ static void vfio_kvm_device_del_group(VFIOGroup *group)
 #endif
 }
 
+static void vfio_kvm_device_posting_irq(VFIODevice *vdev, uint32_t index,
+                                        uint32_t start, uint32_t count)
+{
+#ifdef CONFIG_KVM
+    int i, argsz;
+    struct kvm_vfio_dev_irq *pi_info = NULL;
+
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_DEVICE,
+        .attr = KVM_DEV_VFIO_DEVICE_POST_IRQ,
+    };
+
+    if (!kvm_enabled() ||
+        ioctl(vfio_kvm_device_fd, KVM_HAS_DEVICE_ATTR, &attr) != 0) {
+        return;
+    }
+
+    argsz = sizeof(*pi_info) + sizeof(int) * count;
+
+    pi_info = g_malloc0(argsz);
+    pi_info->argsz = argsz;
+    pi_info->fd = vdev->fd;
+    pi_info->index = index;
+    pi_info->start = start;
+    pi_info->count = count;
+
+    for (i = 0; i < count; i++) {
+        pi_info->gsi[i] = vdev->msi_vectors[start+i].virq;;
+    }
+
+    attr.addr = (uint64_t)(unsigned long)pi_info;
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to configure PI for KVM VFIO device: %m\n");
+    }
+
+    g_free(pi_info);
+#endif
+}
+
 static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
 {
     VFIOAddressSpace *space;
-- 
1.7.1