[PATCH 0/10 v4] MSI/MSI-X capability in userspace

[PATCH 0/10 v4] MSI/MSI-X capability in userspace

[Sheng Yang]

OK, just resend the patchset...

Patch 1-4 is a frame work for capability report in QEmu.

Patch 5-7 is to expose MSI capability to the guest.

Patch 8-10 is to expose MSI-X capability to the guest.

I know it's huge, comments are welcome!

Thanks!

[PATCH 01/10] kvm: Replace force type convert with container_of()

[Sheng Yang]

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/hw/device-assignment.c |   20 ++++++++++++--------
 1 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/qemu/hw/device-assignment.c b/qemu/hw/device-assignment.c
index b7cbcec..c9a3b8e 100644
--- a/qemu/hw/device-assignment.c
+++ b/qemu/hw/device-assignment.c
@@ -144,7 +144,7 @@ static uint32_t assigned_dev_ioport_readl(void *opaque, uint32_t addr)
 static void assigned_dev_iomem_map(PCIDevice *pci_dev, int region_num,
                                    uint32_t e_phys, uint32_t e_size, int type)
 {
-    AssignedDevice *r_dev = (AssignedDevice *) pci_dev;
+    AssignedDevice *r_dev = container_of(pci_dev, AssignedDevice, dev);
     AssignedDevRegion *region = &r_dev->v_addrs[region_num];
     uint32_t old_ephys = region->e_physbase;
     uint32_t old_esize = region->e_size;
@@ -175,7 +175,7 @@ static void assigned_dev_iomem_map(PCIDevice *pci_dev, int region_num,
 static void assigned_dev_ioport_map(PCIDevice *pci_dev, int region_num,
                                     uint32_t addr, uint32_t size, int type)
 {
-    AssignedDevice *r_dev = (AssignedDevice *) pci_dev;
+    AssignedDevice *r_dev = container_of(pci_dev, AssignedDevice, dev);
     AssignedDevRegion *region = &r_dev->v_addrs[region_num];
     int first_map = (region->e_size == 0);
     CPUState *env;
@@ -224,6 +224,7 @@ static void assigned_dev_pci_write_config(PCIDevice *d, uint32_t address,
 {
     int fd;
     ssize_t ret;
+    AssignedDevice *pci_dev = container_of(d, AssignedDevice, dev);
 
     DEBUG("(%x.%x): address=%04x val=0x%08x len=%d\n",
           ((d->devfn >> 3) & 0x1F), (d->devfn & 0x7),
@@ -245,7 +246,7 @@ static void assigned_dev_pci_write_config(PCIDevice *d, uint32_t address,
           ((d->devfn >> 3) & 0x1F), (d->devfn & 0x7),
           (uint16_t) address, val, len);
 
-    fd = ((AssignedDevice *)d)->real_device.config_fd;
+    fd = pci_dev->real_device.config_fd;
 
 again:
     ret = pwrite(fd, &val, len, address);
@@ -266,6 +267,7 @@ static uint32_t assigned_dev_pci_read_config(PCIDevice *d, uint32_t address,
     uint32_t val = 0;
     int fd;
     ssize_t ret;
+    AssignedDevice *pci_dev = container_of(d, AssignedDevice, dev);
 
     if ((address >= 0x10 && address <= 0x24) || address == 0x34 ||
         address == 0x3c || address == 0x3d) {
@@ -279,7 +281,7 @@ static uint32_t assigned_dev_pci_read_config(PCIDevice *d, uint32_t address,
     if (address == 0xFC)
         goto do_log;
 
-    fd = ((AssignedDevice *)d)->real_device.config_fd;
+    fd = pci_dev->real_device.config_fd;
 
 again:
     ret = pread(fd, &val, len, address);
@@ -610,16 +612,18 @@ struct PCIDevice *init_assigned_device(AssignedDevInfo *adev, PCIBus *bus)
 {
     int r;
     AssignedDevice *dev;
+    PCIDevice *pci_dev;
     uint8_t e_device, e_intx;
     struct kvm_assigned_pci_dev assigned_dev_data;
 
     DEBUG("Registering real physical device %s (bus=%x dev=%x func=%x)\n",
           adev->name, adev->bus, adev->dev, adev->func);
 
-    dev = (AssignedDevice *)
-        pci_register_device(bus, adev->name, sizeof(AssignedDevice),
-                            -1, assigned_dev_pci_read_config,
-                            assigned_dev_pci_write_config);
+    pci_dev = pci_register_device(bus, adev->name,
+              sizeof(AssignedDevice), -1, assigned_dev_pci_read_config,
+              assigned_dev_pci_write_config);
+    dev = container_of(pci_dev, AssignedDevice, dev);
+
     if (NULL == dev) {
         fprintf(stderr, "%s: Error: Couldn't register real device %s\n",
                 __func__, adev->name);
-- 
1.5.4.5

[PATCH 02/10] Make device assignment depend on libpci

[Sheng Yang]

Which is used later for capability detection.

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/Makefile.target |    1 +
 qemu/configure       |   20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 0 deletions(-)

diff --git a/qemu/Makefile.target b/qemu/Makefile.target
index efcbe0a..5b6fad4 100644
--- a/qemu/Makefile.target
+++ b/qemu/Makefile.target
@@ -638,6 +638,7 @@ OBJS += msmouse.o
 
 ifeq ($(USE_KVM_DEVICE_ASSIGNMENT), 1)
 OBJS+= device-assignment.o
+LIBS+=-lpci
 endif
 
 ifeq ($(TARGET_BASE_ARCH), i386)
diff --git a/qemu/configure b/qemu/configure
index 902bba3..b6aa7d7 100755
--- a/qemu/configure
+++ b/qemu/configure
@@ -792,6 +792,26 @@ EOF
     fi
 fi
 
+# libpci probe for kvm_cap_device_assignment
+if test $kvm_cap_device_assignment = "yes" ; then
+cat > $TMPC << EOF
+#include <pci/pci.h>
+#ifndef PCI_VENDOR_ID
+#error NO LIBPCI
+#endif
+int main(void) { return 0; }
+EOF
+    if $cc $ARCH_CFLAGS -o $TMPE ${OS_CFLAGS} $TMPC 2>/dev/null ; then
+        :
+    else
+        echo
+        echo "Error: libpci check failed"
+        echo "Disable KVM Device Assignment capability."
+        echo
+        kvm_cap_device_assignment="no"
+    fi
+fi
+
 ##########################################
 # zlib check
 
-- 
1.5.4.5

[PATCH 03/10] Figure out device capability

[Sheng Yang]

Try to figure out device capability in update_dev_cap(). Now we are only care
about MSI capability.

The function pci_find_cap_offset original function wrote by Allen for Xen.
Notice the function need root privilege to work. This depends on libpci to work.

Signed-off-by: Allen Kay <allen.m.kay@intel.com>
Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/hw/device-assignment.c |   29 +++++++++++++++++++++++++++++
 qemu/hw/device-assignment.h |    1 +
 2 files changed, 30 insertions(+), 0 deletions(-)

diff --git a/qemu/hw/device-assignment.c b/qemu/hw/device-assignment.c
index c9a3b8e..fc05ac7 100644
--- a/qemu/hw/device-assignment.c
+++ b/qemu/hw/device-assignment.c
@@ -219,6 +219,35 @@ static void assigned_dev_ioport_map(PCIDevice *pci_dev, int region_num,
                           (r_dev->v_addrs + region_num));
 }
 
+static uint8_t pci_find_cap_offset(struct pci_dev *pci_dev, uint8_t cap)
+{
+    int id;
+    int max_cap = 48;
+    int pos = PCI_CAPABILITY_LIST;
+    int status;
+
+    status = pci_read_byte(pci_dev, PCI_STATUS);
+    if ((status & PCI_STATUS_CAP_LIST) == 0)
+        return 0;
+
+    while (max_cap--) {
+        pos = pci_read_byte(pci_dev, pos);
+        if (pos < 0x40)
+            break;
+
+        pos &= ~3;
+        id = pci_read_byte(pci_dev, pos + PCI_CAP_LIST_ID);
+
+        if (id == 0xff)
+            break;
+        if (id == cap)
+            return pos;
+
+        pos += PCI_CAP_LIST_NEXT;
+    }
+    return 0;
+}
+
 static void assigned_dev_pci_write_config(PCIDevice *d, uint32_t address,
                                           uint32_t val, int len)
 {
diff --git a/qemu/hw/device-assignment.h b/qemu/hw/device-assignment.h
index da775d7..0fd78de 100644
--- a/qemu/hw/device-assignment.h
+++ b/qemu/hw/device-assignment.h
@@ -29,6 +29,7 @@
 #define __DEVICE_ASSIGNMENT_H__
 
 #include <sys/mman.h>
+#include <pci/pci.h>
 #include "qemu-common.h"
 #include "sys-queue.h"
 #include "pci.h"
-- 
1.5.4.5

[PATCH 04/10] Support for device capability

[Sheng Yang]

This framework can be easily extended to support device capability, like
MSI/MSI-x.

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/hw/pci.c |   85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 qemu/hw/pci.h |   30 ++++++++++++++++++++
 2 files changed, 115 insertions(+), 0 deletions(-)

diff --git a/qemu/hw/pci.c b/qemu/hw/pci.c
index ccb4cfa..2225669 100644
--- a/qemu/hw/pci.c
+++ b/qemu/hw/pci.c
@@ -427,11 +427,65 @@ static void pci_update_mappings(PCIDevice *d)
     }
 }
 
+int pci_access_cap_config(PCIDevice *pci_dev, uint32_t address, int len)
+{
+    if (pci_dev->cap.supported && address >= pci_dev->cap.start &&
+            (address + len) < pci_dev->cap.start + pci_dev->cap.length)
+        return 1;
+    return 0;
+}
+
+uint32_t pci_default_cap_read_config(PCIDevice *pci_dev,
+                                     uint32_t address, int len)
+{
+    uint32_t val = 0;
+
+    if (pci_access_cap_config(pci_dev, address, len)) {
+        switch(len) {
+        default:
+        case 4:
+            if (address < pci_dev->cap.start + pci_dev->cap.length - 4) {
+                val = le32_to_cpu(*(uint32_t *)(pci_dev->cap.config
+                            + address - pci_dev->cap.start));
+                break;
+            }
+            /* fall through */
+        case 2:
+            if (address < pci_dev->cap.start + pci_dev->cap.length - 2) {
+                val = le16_to_cpu(*(uint16_t *)(pci_dev->cap.config
+                            + address - pci_dev->cap.start));
+                break;
+            }
+            /* fall through */
+        case 1:
+            val = pci_dev->cap.config[address - pci_dev->cap.start];
+            break;
+        }
+    }
+    return val;
+}
+
+void pci_default_cap_write_config(PCIDevice *pci_dev,
+                                  uint32_t address, uint32_t val, int len)
+{
+    if (pci_access_cap_config(pci_dev, address, len)) {
+        int i;
+        for (i = 0; i < len; i++) {
+            pci_dev->cap.config[address + i - pci_dev->cap.start] = val;
+            val >>= 8;
+        }
+        return;
+    }
+}
+
 uint32_t pci_default_read_config(PCIDevice *d,
                                  uint32_t address, int len)
 {
     uint32_t val;
 
+    if (pci_access_cap_config(d, address, len))
+        return d->cap.config_read(d, address, len);
+
     switch(len) {
     default:
     case 4:
@@ -485,6 +539,11 @@ void pci_default_write_config(PCIDevice *d,
         return;
     }
  default_config:
+    if (pci_access_cap_config(d, address, len)) {
+        d->cap.config_write(d, address, val, len);
+        return;
+    }
+
     /* not efficient, but simple */
     addr = address;
     for(i = 0; i < len; i++) {
@@ -903,3 +962,29 @@ PCIBus *pci_bridge_init(PCIBus *bus, int devfn, uint16_t vid, uint16_t did,
     s->bus = pci_register_secondary_bus(&s->dev, map_irq);
     return s->bus;
 }
+
+void pci_enable_capability_support(PCIDevice *pci_dev,
+                                   uint32_t config_start,
+                                   PCICapConfigReadFunc *config_read,
+                                   PCICapConfigWriteFunc *config_write,
+                                   PCICapConfigInitFunc *config_init)
+{
+    if (!pci_dev)
+        return;
+
+    if (config_start >= 0x40 && config_start < 0xff)
+        pci_dev->cap.start = config_start;
+    else
+        pci_dev->cap.start = PCI_CAPABILITY_CONFIG_DEFAULT_START_ADDR;
+    if (config_read)
+        pci_dev->cap.config_read = config_read;
+    else
+        pci_dev->cap.config_read = pci_default_cap_read_config;
+    if (config_write)
+        pci_dev->cap.config_write = config_write;
+    else
+        pci_dev->cap.config_write = pci_default_cap_write_config;
+    pci_dev->cap.supported = 1;
+    pci_dev->config[0x34] = pci_dev->cap.start;
+    config_init(pci_dev);
+}
diff --git a/qemu/hw/pci.h b/qemu/hw/pci.h
index 227da6e..7f3548b 100644
--- a/qemu/hw/pci.h
+++ b/qemu/hw/pci.h
@@ -133,6 +133,12 @@ typedef void PCIMapIORegionFunc(PCIDevice *pci_dev, int region_num,
                                 uint32_t addr, uint32_t size, int type);
 typedef int PCIUnregisterFunc(PCIDevice *pci_dev);
 
+typedef void PCICapConfigWriteFunc(PCIDevice *pci_dev,
+                                   uint32_t address, uint32_t val, int len);
+typedef uint32_t PCICapConfigReadFunc(PCIDevice *pci_dev,
+                                      uint32_t address, int len);
+typedef void PCICapConfigInitFunc(PCIDevice *pci_dev);
+
 #define PCI_ADDRESS_SPACE_MEM		0x00
 #define PCI_ADDRESS_SPACE_IO		0x01
 #define PCI_ADDRESS_SPACE_MEM_PREFETCH	0x08
@@ -183,6 +189,10 @@ typedef struct PCIIORegion {
 
 #define PCI_COMMAND_RESERVED_MASK_HI (PCI_COMMAND_RESERVED >> 8)
 
+#define PCI_CAPABILITY_CONFIG_MAX_LENGTH 0x60
+#define PCI_CAPABILITY_CONFIG_DEFAULT_START_ADDR 0x40
+#define PCI_CAPABILITY_CONFIG_MSI_LENGTH 0x10
+
 struct PCIDevice {
     /* PCI config space */
     uint8_t config[256];
@@ -205,6 +215,15 @@ struct PCIDevice {
 
     /* Current IRQ levels.  Used internally by the generic PCI code.  */
     int irq_state[4];
+
+    /* Device capability configuration space */
+    struct {
+        int supported;
+        uint8_t config[PCI_CAPABILITY_CONFIG_MAX_LENGTH];
+        unsigned int start, length;
+        PCICapConfigReadFunc *config_read;
+        PCICapConfigWriteFunc *config_write;
+    } cap;
 };
 
 PCIDevice *pci_register_device(PCIBus *bus, const char *name,
@@ -217,6 +236,12 @@ void pci_register_io_region(PCIDevice *pci_dev, int region_num,
                             uint32_t size, int type,
                             PCIMapIORegionFunc *map_func);
 
+void pci_enable_capability_support(PCIDevice *pci_dev,
+                                   uint32_t config_start,
+                                   PCICapConfigReadFunc *config_read,
+                                   PCICapConfigWriteFunc *config_write,
+                                   PCICapConfigInitFunc *config_init);
+
 int pci_map_irq(PCIDevice *pci_dev, int pin);
 uint32_t pci_default_read_config(PCIDevice *d,
                                  uint32_t address, int len);
@@ -224,6 +249,11 @@ void pci_default_write_config(PCIDevice *d,
                               uint32_t address, uint32_t val, int len);
 void pci_device_save(PCIDevice *s, QEMUFile *f);
 int pci_device_load(PCIDevice *s, QEMUFile *f);
+uint32_t pci_default_cap_read_config(PCIDevice *pci_dev,
+                                     uint32_t address, int len);
+void pci_default_cap_write_config(PCIDevice *pci_dev,
+                                  uint32_t address, uint32_t val, int len);
+int pci_access_cap_config(PCIDevice *pci_dev, uint32_t address, int len);
 
 typedef void (*pci_set_irq_fn)(qemu_irq *pic, int irq_num, int level);
 typedef int (*pci_map_irq_fn)(PCIDevice *pci_dev, int irq_num);
-- 
1.5.4.5

[PATCH 05/10] kvm: user interface for MSI type irq routing

[Sheng Yang]

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 libkvm/libkvm.c |   98 ++++++++++++++++++++++++++++++++++++++++++++-----------
 libkvm/libkvm.h |   22 ++++++++++++
 2 files changed, 101 insertions(+), 19 deletions(-)

diff --git a/libkvm/libkvm.c b/libkvm/libkvm.c
index 0ac1c28..050eacd 100644
--- a/libkvm/libkvm.c
+++ b/libkvm/libkvm.c
@@ -1229,11 +1229,12 @@ int kvm_clear_gsi_routes(kvm_context_t kvm)
 #endif
 }
 
-int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
+int kvm_add_routing_entry(kvm_context_t kvm,
+		          struct kvm_irq_routing_entry* entry)
 {
 #ifdef KVM_CAP_IRQ_ROUTING
 	struct kvm_irq_routing *z;
-	struct kvm_irq_routing_entry *e;
+	struct kvm_irq_routing_entry *new;
 	int n, size;
 
 	if (kvm->irq_routes->nr == kvm->nr_allocated_irq_routes) {
@@ -1241,7 +1242,7 @@ int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
 		if (n < 64)
 			n = 64;
 		size = sizeof(struct kvm_irq_routing);
-		size += n * sizeof(*e);
+		size += n * sizeof(*new);
 		z = realloc(kvm->irq_routes, size);
 		if (!z)
 			return -ENOMEM;
@@ -1249,34 +1250,77 @@ int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
 		kvm->irq_routes = z;
 	}
 	n = kvm->irq_routes->nr++;
-	e = &kvm->irq_routes->entries[n];
-	memset(e, 0, sizeof(*e));
-	e->gsi = gsi;
-	e->type = KVM_IRQ_ROUTING_IRQCHIP;
-	e->flags = 0;
-	e->u.irqchip.irqchip = irqchip;
-	e->u.irqchip.pin = pin;
+	new = &kvm->irq_routes->entries[n];
+	memset(new, 0, sizeof(*new));
+	new->gsi = entry->gsi;
+	new->type = entry->type;
+	new->flags = entry->flags;
+	new->u = entry->u;
 	return 0;
 #else
 	return -ENOSYS;
 #endif
 }
 
-int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
+int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
+{
+#ifdef KVM_CAP_IRQ_ROUTING
+	struct kvm_irq_routing_entry e;
+
+	e.gsi = gsi;
+	e.type = KVM_IRQ_ROUTING_IRQCHIP;
+	e.flags = 0;
+	e.u.irqchip.irqchip = irqchip;
+	e.u.irqchip.pin = pin;
+	return kvm_add_routing_entry(kvm, &e);
+#else
+	return -ENOSYS;
+#endif
+}
+
+int kvm_del_routing_entry(kvm_context_t kvm,
+	                  struct kvm_irq_routing_entry* entry)
 {
 #ifdef KVM_CAP_IRQ_ROUTING
 	struct kvm_irq_routing_entry *e, *p;
-	int i;
+	int i, found = 0;
 
 	for (i = 0; i < kvm->irq_routes->nr; ++i) {
 		e = &kvm->irq_routes->entries[i];
-		if (e->type == KVM_IRQ_ROUTING_IRQCHIP
-		    && e->gsi == gsi
-		    && e->u.irqchip.irqchip == irqchip
-		    && e->u.irqchip.pin == pin) {
-			p = &kvm->irq_routes->entries[--kvm->irq_routes->nr];
-			*e = *p;
-			return 0;
+		if (e->type == entry->type
+		    && e->gsi == entry->gsi) {
+			switch (e->type)
+			{
+			case KVM_IRQ_ROUTING_IRQCHIP: {
+				if (e->u.irqchip.irqchip ==
+				    entry->u.irqchip.irqchip
+				    && e->u.irqchip.pin ==
+				    entry->u.irqchip.pin) {
+					p = &kvm->irq_routes->
+					    entries[--kvm->irq_routes->nr];
+					*e = *p;
+					found = 1;
+				}
+				break;
+			}
+			case KVM_IRQ_ROUTING_MSI: {
+				if (e->u.msi.address_lo ==
+				    entry->u.msi.address_lo
+				    && e->u.msi.address_hi ==
+				    entry->u.msi.address_hi
+				    && e->u.msi.data == entry->u.msi.data) {
+					p = &kvm->irq_routes->
+					    entries[--kvm->irq_routes->nr];
+					*e = *p;
+					found = 1;
+				}
+				break;
+			}
+			default:
+				break;
+			}
+			if (found)
+				return 0;
 		}
 	}
 	return -ESRCH;
@@ -1285,6 +1329,22 @@ int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
 #endif
 }
 
+int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
+{
+#ifdef KVM_CAP_IRQ_ROUTING
+	struct kvm_irq_routing_entry e;
+
+	e.gsi = gsi;
+	e.type = KVM_IRQ_ROUTING_IRQCHIP;
+	e.flags = 0;
+	e.u.irqchip.irqchip = irqchip;
+	e.u.irqchip.pin = pin;
+	return kvm_del_routing_entry(kvm, &e);
+#else
+	return -ENOSYS;
+#endif
+}
+
 int kvm_commit_irq_routes(kvm_context_t kvm)
 {
 #ifdef KVM_CAP_IRQ_ROUTING
diff --git a/libkvm/libkvm.h b/libkvm/libkvm.h
index 0239cb6..d9b3ce1 100644
--- a/libkvm/libkvm.h
+++ b/libkvm/libkvm.h
@@ -803,6 +803,28 @@ int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin);
 int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin);
 
 /*!
+ * \brief Adds a routing entry to the temporary irq routing table
+ *
+ * Adds a filled routing entry to the temporary irq routing table. Nothing is
+ * committed to the running VM.
+ *
+ * \param kvm Pointer to the current kvm_context
+ */
+int kvm_add_routing_entry(kvm_context_t kvm,
+                          struct kvm_irq_routing_entry* entry);
+
+/*!
+ * \brief Removes a routing from the temporary irq routing table
+ *
+ * Remove a routing to the temporary irq routing table.  Nothing is
+ * committed to the running VM.
+ *
+ * \param kvm Pointer to the current kvm_context
+ */
+int kvm_del_routing_entry(kvm_context_t kvm,
+		          struct kvm_irq_routing_entry* entry);
+
+/*!
  * \brief Commit the temporary irq routing table
  *
  * Commit the temporary irq routing table to the running VM.
-- 
1.5.4.5

[PATCH 06/10] kvm: libkvm: allocate unused gsi for irq routing

[Sheng Yang]

Notice here is a simple solution, can be replaced later.

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 libkvm/kvm-common.h |    1 +
 libkvm/libkvm.c     |   10 ++++++++++
 libkvm/libkvm.h     |    8 ++++++++
 3 files changed, 19 insertions(+), 0 deletions(-)

diff --git a/libkvm/kvm-common.h b/libkvm/kvm-common.h
index de1ada2..70a95c2 100644
--- a/libkvm/kvm-common.h
+++ b/libkvm/kvm-common.h
@@ -66,6 +66,7 @@ struct kvm_context {
 #ifdef KVM_CAP_IRQ_ROUTING
 	struct kvm_irq_routing *irq_routes;
 	int nr_allocated_irq_routes;
+	int max_used_gsi;
 #endif
 };
 
diff --git a/libkvm/libkvm.c b/libkvm/libkvm.c
index 050eacd..58ac995 100644
--- a/libkvm/libkvm.c
+++ b/libkvm/libkvm.c
@@ -1256,6 +1256,9 @@ int kvm_add_routing_entry(kvm_context_t kvm,
 	new->type = entry->type;
 	new->flags = entry->flags;
 	new->u = entry->u;
+
+	if (entry->gsi > kvm->max_used_gsi)
+		kvm->max_used_gsi = entry->gsi;
 	return 0;
 #else
 	return -ENOSYS;
@@ -1359,3 +1362,10 @@ int kvm_commit_irq_routes(kvm_context_t kvm)
 	return -ENOSYS;
 #endif
 }
+
+int kvm_get_irq_route_gsi(kvm_context_t kvm)
+{
+	return (kvm->max_used_gsi >= KVM_IOAPIC_NUM_PINS) ?
+		(kvm->max_used_gsi + 1) : KVM_IOAPIC_NUM_PINS;
+}
+
diff --git a/libkvm/libkvm.h b/libkvm/libkvm.h
index d9b3ce1..65c4e75 100644
--- a/libkvm/libkvm.h
+++ b/libkvm/libkvm.h
@@ -833,4 +833,12 @@ int kvm_del_routing_entry(kvm_context_t kvm,
  */
 int kvm_commit_irq_routes(kvm_context_t kvm);
 
+/*!
+ * \brief Get unused GSI number for irq routing table
+ *
+ * Get unused GSI number for irq routing table
+ *
+ * \param kvm Pointer to the current kvm_context
+ */
+int kvm_get_irq_route_gsi(kvm_context_t kvm);
 #endif
-- 
1.5.4.5

[PATCH 07/10] kvm: expose MSI capability to guest

[Sheng Yang]

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/hw/device-assignment.c |  112 ++++++++++++++++++++++++++++++++++++++++---
 qemu/hw/device-assignment.h |    7 +++
 2 files changed, 112 insertions(+), 7 deletions(-)

diff --git a/qemu/hw/device-assignment.c b/qemu/hw/device-assignment.c
index fc05ac7..089c5b7 100644
--- a/qemu/hw/device-assignment.c
+++ b/qemu/hw/device-assignment.c
@@ -265,7 +265,8 @@ static void assigned_dev_pci_write_config(PCIDevice *d, uint32_t address,
     }
 
     if ((address >= 0x10 && address <= 0x24) || address == 0x34 ||
-        address == 0x3c || address == 0x3d) {
+        address == 0x3c || address == 0x3d ||
+        pci_access_cap_config(d, address, len)) {
         /* used for update-mappings (BAR emulation) */
         pci_default_write_config(d, address, val, len);
         return;
@@ -299,7 +300,8 @@ static uint32_t assigned_dev_pci_read_config(PCIDevice *d, uint32_t address,
     AssignedDevice *pci_dev = container_of(d, AssignedDevice, dev);
 
     if ((address >= 0x10 && address <= 0x24) || address == 0x34 ||
-        address == 0x3c || address == 0x3d) {
+        address == 0x3c || address == 0x3d ||
+        pci_access_cap_config(d, address, len)) {
         val = pci_default_read_config(d, address, len);
         DEBUG("(%x.%x): address=%04x val=0x%08x len=%d\n",
               (d->devfn >> 3) & 0x1F, (d->devfn & 0x7), address, val, len);
@@ -328,11 +330,13 @@ do_log:
     DEBUG("(%x.%x): address=%04x val=0x%08x len=%d\n",
           (d->devfn >> 3) & 0x1F, (d->devfn & 0x7), address, val, len);
 
-    /* kill the special capabilities */
-    if (address == 4 && len == 4)
-        val &= ~0x100000;
-    else if (address == 6)
-        val &= ~0x10;
+    if (!pci_dev->cap.available) {
+        /* kill the special capabilities */
+        if (address == 4 && len == 4)
+            val &= ~0x100000;
+        else if (address == 6)
+            val &= ~0x10;
+    }
 
     return val;
 }
@@ -637,6 +641,96 @@ void assigned_dev_update_irqs()
     }
 }
 
+#if defined(KVM_CAP_DEVICE_MSI) && defined (KVM_CAP_IRQ_ROUTING)
+static void assigned_dev_update_msi(PCIDevice *pci_dev, unsigned int ctrl_pos)
+{
+    struct kvm_assigned_irq assigned_irq_data;
+    struct kvm_irq_routing_entry gsi_entry;
+    AssignedDevice *assigned_dev = container_of(pci_dev, AssignedDevice, dev);
+    uint8_t ctrl_byte = pci_dev->cap.config[ctrl_pos];
+
+    memset(&assigned_irq_data, 0, sizeof assigned_irq_data);
+    assigned_irq_data.assigned_dev_id  =
+        calc_assigned_dev_id(assigned_dev->h_busnr,
+                (uint8_t)assigned_dev->h_devfn);
+
+    if (ctrl_byte & PCI_MSI_FLAGS_ENABLE) {
+        gsi_entry.u.msi.address_lo = *(uint32_t *)(pci_dev->cap.config +
+                PCI_MSI_ADDRESS_LO);
+        gsi_entry.u.msi.address_hi = 0;
+        gsi_entry.u.msi.data = *(uint16_t *)(pci_dev->cap.config +
+                PCI_MSI_DATA_32);
+        gsi_entry.type = KVM_IRQ_ROUTING_MSI;
+        gsi_entry.gsi = kvm_get_irq_route_gsi(kvm_context);
+        kvm_add_routing_entry(kvm_context, &gsi_entry);
+        if (kvm_commit_irq_routes(kvm_context) < 0) {
+            perror("assigned_dev_enable_msi: kvm_commit_irq_routes");
+            assigned_dev->cap.state &= ~ASSIGNED_DEVICE_MSI_ENABLED;
+            return;
+        }
+        assigned_irq_data.guest_irq = gsi_entry.gsi;
+        assigned_irq_data.flags = KVM_DEV_IRQ_ASSIGN_ENABLE_MSI;
+    } else {
+        assigned_irq_data.guest_irq = assigned_dev->girq;
+    }
+
+    if (kvm_assign_irq(kvm_context, &assigned_irq_data) < 0)
+        perror("assigned_dev_enable_msi");
+    if (assigned_irq_data.flags & KVM_DEV_IRQ_ASSIGN_ENABLE_MSI) {
+        assigned_dev->cap.state |= ASSIGNED_DEVICE_MSI_ENABLED;
+        pci_dev->cap.config[ctrl_pos] |= PCI_MSI_FLAGS_ENABLE;
+    } else {
+        assigned_dev->cap.state &= ~ASSIGNED_DEVICE_MSI_ENABLED;
+        pci_dev->cap.config[ctrl_pos] &= ~PCI_MSI_FLAGS_ENABLE;
+    }
+}
+#endif
+
+void assigned_device_pci_cap_write_config(PCIDevice *pci_dev, uint32_t address,
+                                          uint32_t val, int len)
+{
+    AssignedDevice *assigned_dev = container_of(pci_dev, AssignedDevice, dev);
+    unsigned int pos = pci_dev->cap.start, ctrl_pos;
+
+    pci_default_cap_write_config(pci_dev, address, val, len);
+#if defined(KVM_CAP_DEVICE_MSI) && defined (KVM_CAP_IRQ_ROUTING)
+    if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+        ctrl_pos = pos + PCI_MSI_FLAGS;
+        if (address <= ctrl_pos && address + len > ctrl_pos)
+            assigned_dev_update_msi(pci_dev, ctrl_pos - pci_dev->cap.start);
+        pos += PCI_CAPABILITY_CONFIG_MSI_LENGTH;
+    }
+#endif
+    return;
+}
+
+static void assigned_device_pci_cap_init(PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = container_of(pci_dev, AssignedDevice, dev);
+    int next_cap_pt;
+    struct pci_access *pacc;
+    int h_bus, h_dev, h_func;
+
+    pci_dev->cap.length = 0;
+    h_bus = dev->h_busnr;
+    h_dev = dev->h_devfn >> 3;
+    h_func = dev->h_devfn & 0x07;
+    pacc = pci_alloc();
+    pci_init(pacc);
+    dev->pdev = pci_get_dev(pacc, 0, h_bus, h_dev, h_func);
+    pci_cleanup(pacc);
+#if defined(KVM_CAP_DEVICE_MSI) && defined (KVM_CAP_IRQ_ROUTING)
+    /* Expose MSI capability
+     * MSI capability is the 1st capability in cap.config */
+    if (pci_find_cap_offset(dev->pdev, PCI_CAP_ID_MSI)) {
+        dev->cap.available |= ASSIGNED_DEVICE_CAP_MSI;
+        pci_dev->cap.config[pci_dev->cap.length] = PCI_CAP_ID_MSI;
+        pci_dev->cap.length += PCI_CAPABILITY_CONFIG_MSI_LENGTH;
+        next_cap_pt = 1;
+    }
+#endif
+}
+
 struct PCIDevice *init_assigned_device(AssignedDevInfo *adev, PCIBus *bus)
 {
     int r;
@@ -692,6 +786,10 @@ struct PCIDevice *init_assigned_device(AssignedDevInfo *adev, PCIBus *bus)
     if (r < 0)
         goto assigned_out;
 
+    pci_enable_capability_support(pci_dev, 0, NULL,
+                    assigned_device_pci_cap_write_config,
+                    assigned_device_pci_cap_init);
+
     return &dev->dev;
 
 assigned_out:
diff --git a/qemu/hw/device-assignment.h b/qemu/hw/device-assignment.h
index 0fd78de..39d73d9 100644
--- a/qemu/hw/device-assignment.h
+++ b/qemu/hw/device-assignment.h
@@ -81,6 +81,13 @@ typedef struct {
     unsigned char h_busnr;
     unsigned int h_devfn;
     int bound;
+    struct pci_dev *pdev;
+    struct {
+#define ASSIGNED_DEVICE_CAP_MSI (1 << 0)
+        uint32_t available;
+#define ASSIGNED_DEVICE_MSI_ENABLED (1 << 0)
+        uint32_t state;
+    } cap;
 } AssignedDevice;
 
 typedef struct AssignedDevInfo AssignedDevInfo;
-- 
1.5.4.5

[PATCH 08/10] Add MSI-X related macro to pci.c

[Sheng Yang]

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/hw/pci.h |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/qemu/hw/pci.h b/qemu/hw/pci.h
index 7f3548b..0d988da 100644
--- a/qemu/hw/pci.h
+++ b/qemu/hw/pci.h
@@ -192,6 +192,7 @@ typedef struct PCIIORegion {
 #define PCI_CAPABILITY_CONFIG_MAX_LENGTH 0x60
 #define PCI_CAPABILITY_CONFIG_DEFAULT_START_ADDR 0x40
 #define PCI_CAPABILITY_CONFIG_MSI_LENGTH 0x10
+#define PCI_CAPABILITY_CONFIG_MSIX_LENGTH 0x10
 
 struct PCIDevice {
     /* PCI config space */
-- 
1.5.4.5

[PATCH 09/10] kvm: add ioctl KVM_SET_MSIX_ENTRY_NR and KVM_SET_MSIX_ENTRY

[Sheng Yang]

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 libkvm/libkvm.c |   25 +++++++++++++++++++++++++
 libkvm/libkvm.h |    7 +++++++
 2 files changed, 32 insertions(+), 0 deletions(-)

diff --git a/libkvm/libkvm.c b/libkvm/libkvm.c
index 58ac995..bec1d4a 100644
--- a/libkvm/libkvm.c
+++ b/libkvm/libkvm.c
@@ -1369,3 +1369,28 @@ int kvm_get_irq_route_gsi(kvm_context_t kvm)
 		(kvm->max_used_gsi + 1) : KVM_IOAPIC_NUM_PINS;
 }
 
+#ifdef KVM_CAP_DEVICE_MSIX
+int kvm_assign_set_msix_nr(kvm_context_t kvm,
+                           struct kvm_assigned_msix_nr *msix_nr)
+{
+        int ret;
+
+        ret = ioctl(kvm->vm_fd, KVM_ASSIGN_SET_MSIX_NR, msix_nr);
+        if (ret < 0)
+                return -errno;
+
+        return ret;
+}
+
+int kvm_assign_set_msix_entry(kvm_context_t kvm,
+                              struct kvm_assigned_msix_entry *entry)
+{
+        int ret;
+
+        ret = ioctl(kvm->vm_fd, KVM_ASSIGN_SET_MSIX_ENTRY, entry);
+        if (ret < 0)
+                return -errno;
+
+        return ret;
+}
+#endif
diff --git a/libkvm/libkvm.h b/libkvm/libkvm.h
index 65c4e75..f288dad 100644
--- a/libkvm/libkvm.h
+++ b/libkvm/libkvm.h
@@ -841,4 +841,11 @@ int kvm_commit_irq_routes(kvm_context_t kvm);
  * \param kvm Pointer to the current kvm_context
  */
 int kvm_get_irq_route_gsi(kvm_context_t kvm);
+
+#ifdef KVM_CAP_DEVICE_MSIX
+int kvm_assign_set_msix_nr(kvm_context_t kvm,
+			   struct kvm_assigned_msix_nr *msix_nr);
+int kvm_assign_set_msix_entry(kvm_context_t kvm,
+                              struct kvm_assigned_msix_entry *entry);
+#endif
 #endif
-- 
1.5.4.5

[PATCH 10/10] kvm: enable MSI-X capabilty for assigned device

[Sheng Yang]

The most important part here, is we emulate a page of MMIO region using a
page of memory. That's because MSI-X table was put in the region and we have to
intercept it.

Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
 qemu/hw/device-assignment.c |  286 ++++++++++++++++++++++++++++++++++++++++++-
 qemu/hw/device-assignment.h |    6 +
 2 files changed, 286 insertions(+), 6 deletions(-)

diff --git a/qemu/hw/device-assignment.c b/qemu/hw/device-assignment.c
index 089c5b7..4eb48d0 100644
--- a/qemu/hw/device-assignment.c
+++ b/qemu/hw/device-assignment.c
@@ -146,6 +146,7 @@ static void assigned_dev_iomem_map(PCIDevice *pci_dev, int region_num,
 {
     AssignedDevice *r_dev = container_of(pci_dev, AssignedDevice, dev);
     AssignedDevRegion *region = &r_dev->v_addrs[region_num];
+    PCIRegion *real_region = &r_dev->real_device.regions[region_num];
     uint32_t old_ephys = region->e_physbase;
     uint32_t old_esize = region->e_size;
     int first_map = (region->e_size == 0);
@@ -161,10 +162,27 @@ static void assigned_dev_iomem_map(PCIDevice *pci_dev, int region_num,
 	kvm_destroy_phys_mem(kvm_context, old_ephys,
                              TARGET_PAGE_ALIGN(old_esize));
 
-    if (e_size > 0)
+    if (e_size > 0) {
+        /* deal with MSI-X MMIO page */
+        if (real_region->base_addr <= r_dev->msix_table_addr &&
+                real_region->base_addr + real_region->size >=
+                r_dev->msix_table_addr) {
+            int offset = r_dev->msix_table_addr - real_region->base_addr;
+            ret = munmap(region->u.r_virtbase + offset, TARGET_PAGE_SIZE);
+            if (ret == 0)
+                DEBUG("munmap done, virt_base 0x%p\n",
+                        region->u.r_virtbase + offset);
+            else {
+                fprintf(stderr, "%s: fail munmap msix table!\n", __func__);
+                exit(1);
+            }
+            cpu_register_physical_memory(e_phys + offset,
+                    TARGET_PAGE_SIZE, r_dev->mmio_index);
+        }
 	ret = kvm_register_phys_mem(kvm_context, e_phys,
                                     region->u.r_virtbase,
                                     TARGET_PAGE_ALIGN(e_size), 0);
+    }
 
     if (ret != 0) {
 	fprintf(stderr, "%s: Error: create new mapping failed\n", __func__);
@@ -641,7 +659,9 @@ void assigned_dev_update_irqs()
     }
 }
 
-#if defined(KVM_CAP_DEVICE_MSI) && defined (KVM_CAP_IRQ_ROUTING)
+#ifdef KVM_CAP_IRQ_ROUTING
+
+#ifdef KVM_CAP_DEVICE_MSI
 static void assigned_dev_update_msi(PCIDevice *pci_dev, unsigned int ctrl_pos)
 {
     struct kvm_assigned_irq assigned_irq_data;
@@ -686,14 +706,146 @@ static void assigned_dev_update_msi(PCIDevice *pci_dev, unsigned int ctrl_pos)
 }
 #endif
 
-void assigned_device_pci_cap_write_config(PCIDevice *pci_dev, uint32_t address,
+#ifdef KVM_CAP_DEVICE_MSIX
+static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
+{
+    AssignedDevice *adev = container_of(pci_dev, AssignedDevice, dev);
+    u16 entries_nr = 0, entries_max_nr;
+    int pos = 0, i, r = 0;
+    u32 msg_addr, msg_upper_addr, msg_data, msg_ctrl;
+    struct kvm_assigned_msix_nr msix_nr;
+    struct kvm_assigned_msix_entry msix_entry;
+    struct kvm_irq_routing_entry routing_entry;
+    void *va = adev->msix_table_page;
+
+    if (adev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
+        pos = PCI_CAPABILITY_CONFIG_MSI_LENGTH;
+    entries_max_nr = pci_dev->cap.config[pos + 2];
+    entries_max_nr &= PCI_MSIX_TABSIZE;
+
+    /* Get the usable entry number for allocating */
+    for (i = 0; i < entries_max_nr; i++) {
+        memcpy(&msg_ctrl, va + i * 16 + 12, 4);
+        /* 0x1 is mask bit for per vector */
+        if (msg_ctrl & 0x1)
+            continue;
+        memcpy(&msg_data, va + i * 16 + 8, 4);
+        /* Ignore unused entry even it's unmasked */
+        if (msg_data == 0)
+            continue;
+        entries_nr ++;
+    }
+
+    if (entries_nr == 0) {
+        fprintf(stderr, "MSI-X entry number is zero!\n");
+        return -EINVAL;
+    }
+    msix_nr.assigned_dev_id = calc_assigned_dev_id(adev->h_busnr,
+                                          (uint8_t)adev->h_devfn);
+    msix_nr.entry_nr = entries_nr;
+    r = kvm_assign_set_msix_nr(kvm_context, &msix_nr);
+    if (r != 0) {
+        fprintf(stderr, "fail to set MSI-X entry number for MSIX! %s\n",
+			strerror(-r));
+        return r;
+    }
+
+    msix_entry.assigned_dev_id = msix_nr.assigned_dev_id;
+    entries_nr = 0;
+    for (i = 0; i < entries_max_nr; i++) {
+        if (entries_nr >= msix_nr.entry_nr)
+            break;
+        memcpy(&msg_ctrl, va + i * 16 + 12, 4);
+        if (msg_ctrl & 0x1)
+            continue;
+        memcpy(&msg_data, va + i * 16 + 8, 4);
+        if (msg_data == 0)
+            continue;
+
+        memcpy(&msg_addr, va + i * 16, 4);
+        memcpy(&msg_upper_addr, va + i * 16 + 4, 4);
+
+        routing_entry.gsi = kvm_get_irq_route_gsi(kvm_context);
+        routing_entry.type = KVM_IRQ_ROUTING_MSI;
+        routing_entry.flags = 0;
+        routing_entry.u.msi.address_lo = msg_addr;
+        routing_entry.u.msi.address_hi = msg_upper_addr;
+        routing_entry.u.msi.data = msg_data;
+        DEBUG("MSI-X data 0x%x, MSI-X addr_lo 0x%x\n!", msg_data, msg_addr);
+	kvm_add_routing_entry(kvm_context, &routing_entry);
+
+        msix_entry.gsi = routing_entry.gsi;
+        msix_entry.entry = i;
+        r = kvm_assign_set_msix_entry(kvm_context, &msix_entry);
+        if (r) {
+            fprintf(stderr, "fail to set MSI-X entry! %s\n", strerror(-r));
+            break;
+        }
+        DEBUG("MSI-X entry gsi 0x%x, entry %d\n!",
+                msix_entry.gsi, msix_entry.entry);
+        entries_nr ++;
+    }
+
+    if (r == 0 && kvm_commit_irq_routes(kvm_context) < 0) {
+	    perror("assigned_dev_update_msix_mmio: kvm_commit_irq_routes");
+	    return -EINVAL;
+    }
+
+    return r;
+}
+
+static void assigned_dev_update_msix(PCIDevice *pci_dev, unsigned int ctrl_pos)
+{
+    struct kvm_assigned_irq assigned_irq_data;
+    AssignedDevice *assigned_dev = container_of(pci_dev, AssignedDevice, dev);
+    uint16_t *ctrl_word = (uint16_t *)(pci_dev->cap.config + ctrl_pos);
+
+    memset(&assigned_irq_data, 0, sizeof assigned_irq_data);
+    assigned_irq_data.assigned_dev_id  =
+            calc_assigned_dev_id(assigned_dev->h_busnr,
+                    (uint8_t)assigned_dev->h_devfn);
+
+    if (*ctrl_word & PCI_MSIX_ENABLE) {
+        assigned_irq_data.flags = KVM_DEV_IRQ_ASSIGN_ENABLE_MSIX;
+        if (assigned_dev_update_msix_mmio(pci_dev) < 0) {
+            perror("assigned_dev_update_msix_mmio");
+        }
+    }
+    if (*ctrl_word & PCI_MSIX_MASK)
+        assigned_irq_data.flags |= KVM_DEV_IRQ_ASSIGN_MASK_MSIX;
+
+    if ((*ctrl_word & PCI_MSIX_ENABLE) !=
+		(assigned_dev->cap.state & ASSIGNED_DEVICE_MSIX_ENABLED))
+	    if (kvm_assign_irq(kvm_context, &assigned_irq_data) < 0)
+		    perror("assigned_dev_enable_msix");
+
+    if (assigned_irq_data.flags & KVM_DEV_IRQ_ASSIGN_ENABLE_MSIX) {
+        assigned_dev->cap.state |= ASSIGNED_DEVICE_MSIX_ENABLED;
+        *ctrl_word |= PCI_MSIX_ENABLE;
+    } else {
+        assigned_dev->cap.state &= ~ASSIGNED_DEVICE_MSIX_ENABLED;
+        *ctrl_word &= ~PCI_MSIX_ENABLE;
+    }
+    if (assigned_irq_data.flags & KVM_DEV_IRQ_ASSIGN_MASK_MSIX) {
+        assigned_dev->cap.state |= ASSIGNED_DEVICE_MSIX_MASKED;
+        *ctrl_word |= PCI_MSIX_MASK;
+    } else {
+        assigned_dev->cap.state &= ~ASSIGNED_DEVICE_MSIX_MASKED;
+        *ctrl_word &= ~PCI_MSIX_MASK;
+    }
+}
+#endif
+#endif
+
+static void assigned_device_pci_cap_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = container_of(pci_dev, AssignedDevice, dev);
     unsigned int pos = pci_dev->cap.start, ctrl_pos;
 
     pci_default_cap_write_config(pci_dev, address, val, len);
-#if defined(KVM_CAP_DEVICE_MSI) && defined (KVM_CAP_IRQ_ROUTING)
+#ifdef KVM_CAP_IRQ_ROUTING
+#ifdef KVM_CAP_DEVICE_MSI
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         ctrl_pos = pos + PCI_MSI_FLAGS;
         if (address <= ctrl_pos && address + len > ctrl_pos)
@@ -701,13 +853,25 @@ void assigned_device_pci_cap_write_config(PCIDevice *pci_dev, uint32_t address,
         pos += PCI_CAPABILITY_CONFIG_MSI_LENGTH;
     }
 #endif
+#ifdef KVM_CAP_DEVICE_MSIX
+    if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        ctrl_pos = pos + 3;
+        if (address <= ctrl_pos && address + len > ctrl_pos) {
+            ctrl_pos--; /* control is word long */
+            assigned_dev_update_msix(pci_dev, ctrl_pos - pci_dev->cap.start);
+	}
+        pos += PCI_CAPABILITY_CONFIG_MSIX_LENGTH;
+    }
+#endif
+#endif
     return;
 }
 
 static void assigned_device_pci_cap_init(PCIDevice *pci_dev)
 {
     AssignedDevice *dev = container_of(pci_dev, AssignedDevice, dev);
-    int next_cap_pt;
+    PCIRegion *pci_region = dev->real_device.regions;
+    int next_cap_pt = 0;
     struct pci_access *pacc;
     int h_bus, h_dev, h_func;
 
@@ -719,7 +883,8 @@ static void assigned_device_pci_cap_init(PCIDevice *pci_dev)
     pci_init(pacc);
     dev->pdev = pci_get_dev(pacc, 0, h_bus, h_dev, h_func);
     pci_cleanup(pacc);
-#if defined(KVM_CAP_DEVICE_MSI) && defined (KVM_CAP_IRQ_ROUTING)
+#ifdef KVM_CAP_IRQ_ROUTING
+#ifdef KVM_CAP_DEVICE_MSI
     /* Expose MSI capability
      * MSI capability is the 1st capability in cap.config */
     if (pci_find_cap_offset(dev->pdev, PCI_CAP_ID_MSI)) {
@@ -729,6 +894,110 @@ static void assigned_device_pci_cap_init(PCIDevice *pci_dev)
         next_cap_pt = 1;
     }
 #endif
+#ifdef KVM_CAP_DEVICE_MSIX
+    /* Expose MSI-X capability */
+    if (pci_find_cap_offset(dev->pdev, PCI_CAP_ID_MSIX)) {
+        int pos, entry_nr, bar_nr;
+        u32 msix_table_entry;
+        dev->cap.available |= ASSIGNED_DEVICE_CAP_MSIX;
+        pos = pci_find_cap_offset(dev->pdev, PCI_CAP_ID_MSIX);
+        entry_nr = pci_read_word(dev->pdev, pos + 2) & PCI_MSIX_TABSIZE;
+        pci_dev->cap.config[pci_dev->cap.length] = 0x11;
+        pci_dev->cap.config[pci_dev->cap.length + 2] = entry_nr;
+        msix_table_entry = pci_read_long(dev->pdev, pos + PCI_MSIX_TABLE);
+        *(uint32_t *)(pci_dev->cap.config +
+                      pci_dev->cap.length + PCI_MSIX_TABLE) = msix_table_entry;
+        *(uint32_t *)(pci_dev->cap.config +
+                      pci_dev->cap.length + PCI_MSIX_PBA) =
+                    pci_read_long(dev->pdev, pos + PCI_MSIX_PBA);
+        bar_nr = msix_table_entry & PCI_MSIX_BIR;
+        msix_table_entry &= ~PCI_MSIX_BIR;
+        dev->msix_table_addr = pci_region[bar_nr].base_addr + msix_table_entry;
+        if (next_cap_pt != 0) {
+            pci_dev->cap.config[next_cap_pt] =
+                pci_dev->cap.start + pci_dev->cap.length;
+            next_cap_pt += PCI_CAPABILITY_CONFIG_MSI_LENGTH;
+        } else
+            next_cap_pt = 1;
+        pci_dev->cap.length += PCI_CAPABILITY_CONFIG_MSIX_LENGTH;
+    }
+#endif
+#endif
+}
+
+static uint32_t msix_mmio_readl(void *opaque, target_phys_addr_t addr)
+{
+    AssignedDevice *adev = opaque;
+    unsigned int offset = addr & 0xfff;
+    void *page = adev->msix_table_page;
+    uint32_t val = 0;
+
+    DEBUG("read from MSI-X entry table mmio 0x%lx", addr);
+    memcpy(&val, (void *)((char *)page + offset), 4);
+
+    return val;
+}
+
+static uint32_t msix_mmio_readb(void *opaque, target_phys_addr_t addr)
+{
+    return ((msix_mmio_readl(opaque, addr & ~3)) >>
+            (8 * (addr & 3))) & 0xff;
+}
+
+static uint32_t msix_mmio_readw(void *opaque, target_phys_addr_t addr)
+{
+    return ((msix_mmio_readl(opaque, addr & ~3)) >>
+            (8 * (addr & 3))) & 0xffff;
+}
+
+static void msix_mmio_writel(void *opaque,
+                             target_phys_addr_t addr, uint32_t val)
+{
+    AssignedDevice *adev = opaque;
+    unsigned int offset = addr & 0xfff;
+    void *page = adev->msix_table_page;
+
+    DEBUG("write to MSI-X entry table mmio offset 0x%lx, val 0x%lx\n",
+		    addr, val);
+    memcpy((void *)((char *)page + offset), &val, 4);
+}
+
+static void msix_mmio_writew(void *opaque,
+                             target_phys_addr_t addr, uint32_t val)
+{
+    msix_mmio_writel(opaque, addr & ~3,
+                     (val & 0xffff) << (8*(addr & 3)));
+}
+
+static void msix_mmio_writeb(void *opaque,
+                             target_phys_addr_t addr, uint32_t val)
+{
+    msix_mmio_writel(opaque, addr & ~3,
+                     (val & 0xff) << (8*(addr & 3)));
+}
+
+static CPUWriteMemoryFunc *msix_mmio_write[] = {
+    msix_mmio_writeb,	msix_mmio_writew,	msix_mmio_writel
+};
+
+static CPUReadMemoryFunc *msix_mmio_read[] = {
+    msix_mmio_readb,	msix_mmio_readw,	msix_mmio_readl
+};
+
+static int assigned_dev_register_msix_mmio(AssignedDevice *dev)
+{
+    dev->msix_table_page = mmap(NULL, 0x1000,
+                                PROT_READ|PROT_WRITE,
+                                MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+    memset(dev->msix_table_page, 0, 0x1000);
+    if (dev->msix_table_page == MAP_FAILED) {
+        fprintf(stderr, "fail allocate msix_table_page! %s\n",
+                strerror(errno));
+        return -EFAULT;
+    }
+    dev->mmio_index = cpu_register_io_memory(0,
+                        msix_mmio_read, msix_mmio_write, dev);
+    return 0;
 }
 
 struct PCIDevice *init_assigned_device(AssignedDevInfo *adev, PCIBus *bus)
@@ -790,6 +1059,11 @@ struct PCIDevice *init_assigned_device(AssignedDevInfo *adev, PCIBus *bus)
                     assigned_device_pci_cap_write_config,
                     assigned_device_pci_cap_init);
 
+    /* intercept MSI-X entry page in the MMIO */
+    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX)
+        if (assigned_dev_register_msix_mmio(dev))
+            return NULL;
+
     return &dev->dev;
 
 assigned_out:
diff --git a/qemu/hw/device-assignment.h b/qemu/hw/device-assignment.h
index 39d73d9..fc1af8f 100644
--- a/qemu/hw/device-assignment.h
+++ b/qemu/hw/device-assignment.h
@@ -84,10 +84,16 @@ typedef struct {
     struct pci_dev *pdev;
     struct {
 #define ASSIGNED_DEVICE_CAP_MSI (1 << 0)
+#define ASSIGNED_DEVICE_CAP_MSIX (1 << 1)
         uint32_t available;
 #define ASSIGNED_DEVICE_MSI_ENABLED (1 << 0)
+#define ASSIGNED_DEVICE_MSIX_ENABLED (1 << 1)
+#define ASSIGNED_DEVICE_MSIX_MASKED (1 << 2)
         uint32_t state;
     } cap;
+    void *msix_table_page;
+    target_phys_addr_t msix_table_addr;
+    int mmio_index;
 } AssignedDevice;
 
 typedef struct AssignedDevInfo AssignedDevInfo;
-- 
1.5.4.5

Re: [PATCH 04/10] Support for device capability

[Marcelo Tosatti]

Hi Sheng,

On Mon, Mar 02, 2009 at 04:29:27PM +0800, Sheng Yang wrote:
> This framework can be easily extended to support device capability, like
> MSI/MSI-x.
> 
> Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> ---
>  qemu/hw/pci.c |   85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  qemu/hw/pci.h |   30 ++++++++++++++++++++
>  2 files changed, 115 insertions(+), 0 deletions(-)
> 

> @@ -205,6 +215,15 @@ struct PCIDevice {
>  
>      /* Current IRQ levels.  Used internally by the generic PCI code.  */
>      int irq_state[4];
> +
> +    /* Device capability configuration space */
> +    struct {
> +        int supported;
> +        uint8_t config[PCI_CAPABILITY_CONFIG_MAX_LENGTH];
> +        unsigned int start, length;
> +        PCICapConfigReadFunc *config_read;
> +        PCICapConfigWriteFunc *config_write;
> +    } cap;
>  };

Why do you have a copy of the capabilities config space? Why not just
access PCIDevice->config directly?

Re: [PATCH 03/10] Figure out device capability

[Marcelo Tosatti]

On Mon, Mar 02, 2009 at 04:29:26PM +0800, Sheng Yang wrote:
> Try to figure out device capability in update_dev_cap(). Now we are only care
> about MSI capability.
> 
> The function pci_find_cap_offset original function wrote by Allen for Xen.
> Notice the function need root privilege to work. This depends on libpci to work.
> 
> Signed-off-by: Allen Kay <allen.m.kay@intel.com>
> Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> ---
>  qemu/hw/device-assignment.c |   29 +++++++++++++++++++++++++++++
>  qemu/hw/device-assignment.h |    1 +
>  2 files changed, 30 insertions(+), 0 deletions(-)
> 
> diff --git a/qemu/hw/device-assignment.c b/qemu/hw/device-assignment.c
> index c9a3b8e..fc05ac7 100644
> --- a/qemu/hw/device-assignment.c
> +++ b/qemu/hw/device-assignment.c
> @@ -219,6 +219,35 @@ static void assigned_dev_ioport_map(PCIDevice *pci_dev, int region_num,
>                            (r_dev->v_addrs + region_num));
>  }
>  
> +static uint8_t pci_find_cap_offset(struct pci_dev *pci_dev, uint8_t cap)
> +{
> +    int id;
> +    int max_cap = 48;
> +    int pos = PCI_CAPABILITY_LIST;

Don't you need to handle cardbus capability list offset?

Re: [PATCH 04/10] Support for device capability

[Sheng Yang]

On Tuesday 03 March 2009 20:42:07 Marcelo Tosatti wrote:
> Hi Sheng,
>
> On Mon, Mar 02, 2009 at 04:29:27PM +0800, Sheng Yang wrote:
> > This framework can be easily extended to support device capability, like
> > MSI/MSI-x.
> >
> > Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> > ---
> >  qemu/hw/pci.c |   85
> > +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ qemu/hw/pci.h |
> >   30 ++++++++++++++++++++
> >  2 files changed, 115 insertions(+), 0 deletions(-)
> >
> >
> > @@ -205,6 +215,15 @@ struct PCIDevice {
> >
> >      /* Current IRQ levels.  Used internally by the generic PCI code.  */
> >      int irq_state[4];
> > +
> > +    /* Device capability configuration space */
> > +    struct {
> > +        int supported;
> > +        uint8_t config[PCI_CAPABILITY_CONFIG_MAX_LENGTH];
> > +        unsigned int start, length;
> > +        PCICapConfigReadFunc *config_read;
> > +        PCICapConfigWriteFunc *config_write;
> > +    } cap;
> >  };
>
> Why do you have a copy of the capabilities config space? Why not just
> access PCIDevice->config directly?

I am not sure upstream would accept which. Separate the logic seems more clear 
and easy to accept to me...

-- 
regards
Yang, Sheng

Re: [PATCH 03/10] Figure out device capability

[Sheng Yang]

On Tuesday 03 March 2009 20:50:16 Marcelo Tosatti wrote:
> On Mon, Mar 02, 2009 at 04:29:26PM +0800, Sheng Yang wrote:
> > Try to figure out device capability in update_dev_cap(). Now we are only
> > care about MSI capability.
> >
> > The function pci_find_cap_offset original function wrote by Allen for
> > Xen. Notice the function need root privilege to work. This depends on
> > libpci to work.
> >
> > Signed-off-by: Allen Kay <allen.m.kay@intel.com>
> > Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> > ---
> >  qemu/hw/device-assignment.c |   29 +++++++++++++++++++++++++++++
> >  qemu/hw/device-assignment.h |    1 +
> >  2 files changed, 30 insertions(+), 0 deletions(-)
> >
> > diff --git a/qemu/hw/device-assignment.c b/qemu/hw/device-assignment.c
> > index c9a3b8e..fc05ac7 100644
> > --- a/qemu/hw/device-assignment.c
> > +++ b/qemu/hw/device-assignment.c
> > @@ -219,6 +219,35 @@ static void assigned_dev_ioport_map(PCIDevice
> > *pci_dev, int region_num, (r_dev->v_addrs + region_num));
> >  }
> >
> > +static uint8_t pci_find_cap_offset(struct pci_dev *pci_dev, uint8_t cap)
> > +{
> > +    int id;
> > +    int max_cap = 48;
> > +    int pos = PCI_CAPABILITY_LIST;
>
> Don't you need to handle cardbus capability list offset?

I think cardbus and PCI bridge are out of our scope for now... Maybe we should 
prohibit from assigning them.

-- 
regards
Yang, Sheng

Re: [PATCH 04/10] Support for device capability

[Marcelo Tosatti]

On Wed, Mar 04, 2009 at 10:56:36AM +0800, Sheng Yang wrote:
> On Tuesday 03 March 2009 20:42:07 Marcelo Tosatti wrote:
> > Hi Sheng,
> >
> > On Mon, Mar 02, 2009 at 04:29:27PM +0800, Sheng Yang wrote:
> > > This framework can be easily extended to support device capability, like
> > > MSI/MSI-x.
> > >
> > > Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> > > ---
> > >  qemu/hw/pci.c |   85
> > > +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ qemu/hw/pci.h |
> > >   30 ++++++++++++++++++++
> > >  2 files changed, 115 insertions(+), 0 deletions(-)
> > >
> > >
> > > @@ -205,6 +215,15 @@ struct PCIDevice {
> > >
> > >      /* Current IRQ levels.  Used internally by the generic PCI code.  */
> > >      int irq_state[4];
> > > +
> > > +    /* Device capability configuration space */
> > > +    struct {
> > > +        int supported;
> > > +        uint8_t config[PCI_CAPABILITY_CONFIG_MAX_LENGTH];
> > > +        unsigned int start, length;
> > > +        PCICapConfigReadFunc *config_read;
> > > +        PCICapConfigWriteFunc *config_write;
> > > +    } cap;
> > >  };
> >
> > Why do you have a copy of the capabilities config space? Why not just
> > access PCIDevice->config directly?
> 
> I am not sure upstream would accept which. Separate the logic seems more clear 
> and easy to accept to me...

You have to maintain two copies of the same data. Whats the point? The
logic can be separate with a unified config space, no?