[Qemu-devel] [PATCH v3 0/8] uq/master: MSI support for in-kernel irqchip mode

[Qemu-devel] [PATCH v3 0/8] uq/master: MSI support for in-kernel irqchip mode

[Jan Kiszka]

Version 3 is mostly a rebase + the final consolidation over the direct
MSI inject mechanism that made it into kvm.git recently. I also
introduced an MSIMessage structure as it will be helpful for the
follow-up work on irqfd-driven MSI injection under KVM. And, finally,
this flips the default for kernel_irqchip to on.

CC: Stefano Stabellini <stefano.stabellini@eu.citrix.com>

Jan Kiszka (8):
  kvm: Refactor KVMState::max_gsi to gsi_count
  Introduce MSIMessage structure
  kvm: Introduce basic MSI support for in-kernel irqchips
  pc: Enable MSI support at APIC level
  kvm: x86: Wire up MSI support for in-kernel irqchip
  kvm: Update kernel headers
  kvm: Add support for direct MSI injections
  kvm: Enable in-kernel irqchip support by default

 hw/apic.c                 |    3 +
 hw/kvm/apic.c             |   34 +++++++++-
 hw/msi.h                  |    5 ++
 hw/pc.c                   |    9 ---
 hw/xen.h                  |   10 ---
 hw/xen_apic.c             |    5 ++
 kvm-all.c                 |  165 +++++++++++++++++++++++++++++++++++++++++++--
 kvm.h                     |    1 +
 linux-headers/linux/kvm.h |   38 ++++++++++
 qemu-common.h             |    1 +
 10 files changed, 244 insertions(+), 27 deletions(-)

-- 
1.7.3.4

[Qemu-devel] [PATCH v3 1/8] kvm: Refactor KVMState::max_gsi to gsi_count

[Jan Kiszka]

Instead of the bitmap size, store the maximum of GSIs the kernel
support. Move the GSI limit assertion to the API function
kvm_irqchip_add_route and make it stricter.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 kvm-all.c |    8 ++++----
 1 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/kvm-all.c b/kvm-all.c
index 9b73ccf..2d82d54 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -86,7 +86,7 @@ struct KVMState
     struct kvm_irq_routing *irq_routes;
     int nr_allocated_irq_routes;
     uint32_t *used_gsi_bitmap;
-    unsigned int max_gsi;
+    unsigned int gsi_count;
 #endif
 };
 
@@ -859,8 +859,6 @@ int kvm_irqchip_set_irq(KVMState *s, int irq, int level)
 #ifdef KVM_CAP_IRQ_ROUTING
 static void set_gsi(KVMState *s, unsigned int gsi)
 {
-    assert(gsi < s->max_gsi);
-
     s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
 }
 
@@ -875,7 +873,7 @@ static void kvm_init_irq_routing(KVMState *s)
         /* Round up so we can search ints using ffs */
         gsi_bits = ALIGN(gsi_count, 32);
         s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
-        s->max_gsi = gsi_bits;
+        s->gsi_count = gsi_count;
 
         /* Mark any over-allocated bits as already in use */
         for (i = gsi_count; i < gsi_bits; i++) {
@@ -920,6 +918,8 @@ void kvm_irqchip_add_route(KVMState *s, int irq, int irqchip, int pin)
 {
     struct kvm_irq_routing_entry e;
 
+    assert(pin < s->gsi_count);
+
     e.gsi = irq;
     e.type = KVM_IRQ_ROUTING_IRQCHIP;
     e.flags = 0;
-- 
1.7.3.4

[Qemu-devel] [PATCH v3 2/8] Introduce MSIMessage structure

[Jan Kiszka]

Will be used for generating and distributing MSI messages, both in
emulation mode and under KVM.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 hw/msi.h      |    5 +++++
 qemu-common.h |    1 +
 2 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/hw/msi.h b/hw/msi.h
index 3040bb0..75747ab 100644
--- a/hw/msi.h
+++ b/hw/msi.h
@@ -24,6 +24,11 @@
 #include "qemu-common.h"
 #include "pci.h"
 
+struct MSIMessage {
+    uint64_t address;
+    uint32_t data;
+};
+
 extern bool msi_supported;
 
 bool msi_enabled(const PCIDevice *dev);
diff --git a/qemu-common.h b/qemu-common.h
index 50f659a..57fe28f 100644
--- a/qemu-common.h
+++ b/qemu-common.h
@@ -248,6 +248,7 @@ typedef struct PCIEAERLog PCIEAERLog;
 typedef struct PCIEAERErr PCIEAERErr;
 typedef struct PCIEPort PCIEPort;
 typedef struct PCIESlot PCIESlot;
+typedef struct MSIMessage MSIMessage;
 typedef struct SerialState SerialState;
 typedef struct IRQState *qemu_irq;
 typedef struct PCMCIACardState PCMCIACardState;
-- 
1.7.3.4

[Qemu-devel] [PATCH v3 3/8] kvm: Introduce basic MSI support for in-kernel irqchips

[Jan Kiszka]

This patch basically adds kvm_irqchip_send_msi, a service for sending
arbitrary MSI messages to KVM's in-kernel irqchip models.

As the current KVI API requires us to establish a static route from a
pseudo GSI to the target MSI message and inject the MSI via toggling
that GSI, we need to play some tricks to make this unfortunately
interface transparent. We create those routes on demand and keep them
in a hash table. Succeeding messages can then search for an existing
route in the table first and reuse it whenever possible. If we should
run out of limited GSIs, we simply flush the table and rebuild it as
messages are sent.

This approach is rather simple and could be optimized further. However,
future kernels will contain a more efficient MSI injection interface
that will obsolete the GSI-based dynamic injection.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 kvm-all.c |  139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 kvm.h     |    1 +
 2 files changed, 139 insertions(+), 1 deletions(-)

diff --git a/kvm-all.c b/kvm-all.c
index 2d82d54..e7ed510 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -24,6 +24,7 @@
 #include "qemu-barrier.h"
 #include "sysemu.h"
 #include "hw/hw.h"
+#include "hw/msi.h"
 #include "gdbstub.h"
 #include "kvm.h"
 #include "bswap.h"
@@ -48,6 +49,8 @@
     do { } while (0)
 #endif
 
+#define KVM_MSI_HASHTAB_SIZE    256
+
 typedef struct KVMSlot
 {
     target_phys_addr_t start_addr;
@@ -59,6 +62,11 @@ typedef struct KVMSlot
 
 typedef struct kvm_dirty_log KVMDirtyLog;
 
+typedef struct KVMMSIRoute {
+    struct kvm_irq_routing_entry kroute;
+    QTAILQ_ENTRY(KVMMSIRoute) entry;
+} KVMMSIRoute;
+
 struct KVMState
 {
     KVMSlot slots[32];
@@ -87,6 +95,7 @@ struct KVMState
     int nr_allocated_irq_routes;
     uint32_t *used_gsi_bitmap;
     unsigned int gsi_count;
+    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
 #endif
 };
 
@@ -862,9 +871,14 @@ static void set_gsi(KVMState *s, unsigned int gsi)
     s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
 }
 
+static void clear_gsi(KVMState *s, unsigned int gsi)
+{
+    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
+}
+
 static void kvm_init_irq_routing(KVMState *s)
 {
-    int gsi_count;
+    int gsi_count, i;
 
     gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
     if (gsi_count > 0) {
@@ -884,6 +898,10 @@ static void kvm_init_irq_routing(KVMState *s)
     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
     s->nr_allocated_irq_routes = 0;
 
+    for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
+        QTAILQ_INIT(&s->msi_hashtab[i]);
+    }
+
     kvm_arch_init_irq_routing(s);
 }
 
@@ -934,11 +952,130 @@ int kvm_irqchip_commit_routes(KVMState *s)
     return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
 }
 
+static void kvm_release_gsi(KVMState *s, int gsi)
+{
+    struct kvm_irq_routing_entry *e;
+    int i;
+
+    for (i = 0; i < s->irq_routes->nr; i++) {
+        e = &s->irq_routes->entries[i];
+        if (e->gsi == gsi) {
+            s->irq_routes->nr--;
+            *e = s->irq_routes->entries[s->irq_routes->nr];
+        }
+    }
+    clear_gsi(s, gsi);
+}
+
+static unsigned int kvm_hash_msi(uint32_t data)
+{
+    /* This is optimized for IA32 MSI layout. However, no other arch shall
+     * repeat the mistake of not providing a direct MSI injection API. */
+    return data & 0xff;
+}
+
+static void kvm_flush_dynamic_msi_routes(KVMState *s)
+{
+    KVMMSIRoute *route, *next;
+    unsigned int hash;
+
+    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
+        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
+            kvm_release_gsi(s, route->kroute.gsi);
+            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
+            g_free(route);
+        }
+    }
+}
+
+static int kvm_get_pseudo_gsi(KVMState *s)
+{
+    uint32_t *word = s->used_gsi_bitmap;
+    int max_words = ALIGN(s->gsi_count, 32) / 32;
+    int i, bit;
+    bool retry = true;
+
+again:
+    /* Return the lowest unused GSI in the bitmap */
+    for (i = 0; i < max_words; i++) {
+        bit = ffs(~word[i]);
+        if (!bit) {
+            continue;
+        }
+
+        return bit - 1 + i * 32;
+    }
+    if (retry) {
+        retry = false;
+        kvm_flush_dynamic_msi_routes(s);
+        goto again;
+    }
+    return -ENOSPC;
+
+}
+
+static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
+{
+    unsigned int hash = kvm_hash_msi(msg.data);
+    KVMMSIRoute *route;
+
+    QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
+        if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
+            route->kroute.u.msi.address_hi == (msg.address >> 32) &&
+            route->kroute.u.msi.data == msg.data) {
+            return route;
+        }
+    }
+    return NULL;
+}
+
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
+{
+    KVMMSIRoute *route;
+
+    route = kvm_lookup_msi_route(s, msg);
+    if (!route) {
+        int gsi, ret;
+
+        gsi = kvm_get_pseudo_gsi(s);
+        if (gsi < 0) {
+            return gsi;
+        }
+
+        route = g_malloc(sizeof(KVMMSIRoute));
+        route->kroute.gsi = gsi;
+        route->kroute.type = KVM_IRQ_ROUTING_MSI;
+        route->kroute.flags = 0;
+        route->kroute.u.msi.address_lo = (uint32_t)msg.address;
+        route->kroute.u.msi.address_hi = msg.address >> 32;
+        route->kroute.u.msi.data = msg.data;
+
+        kvm_add_routing_entry(s, &route->kroute);
+
+        QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
+                           entry);
+
+        ret = kvm_irqchip_commit_routes(s);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
+
+    return kvm_irqchip_set_irq(s, route->kroute.gsi, 1);
+}
+
 #else /* !KVM_CAP_IRQ_ROUTING */
 
 static void kvm_init_irq_routing(KVMState *s)
 {
 }
+
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
+{
+    abort();
+}
 #endif /* !KVM_CAP_IRQ_ROUTING */
 
 static int kvm_irqchip_create(KVMState *s)
diff --git a/kvm.h b/kvm.h
index 4ccae8c..7857dbf 100644
--- a/kvm.h
+++ b/kvm.h
@@ -132,6 +132,7 @@ int kvm_arch_on_sigbus(int code, void *addr);
 void kvm_arch_init_irq_routing(KVMState *s);
 
 int kvm_irqchip_set_irq(KVMState *s, int irq, int level);
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg);
 
 void kvm_irqchip_add_route(KVMState *s, int gsi, int irqchip, int pin);
 int kvm_irqchip_commit_routes(KVMState *s);
-- 
1.7.3.4

Re: [Qemu-devel] [PATCH v3 3/8] kvm: Introduce basic MSI support for in-kernel irqchips

[Marcelo Tosatti]

On Thu, May 10, 2012 at 06:02:52PM -0300, Jan Kiszka wrote:
> This patch basically adds kvm_irqchip_send_msi, a service for sending
> arbitrary MSI messages to KVM's in-kernel irqchip models.
> 
> As the current KVI API requires us to establish a static route from a
> pseudo GSI to the target MSI message and inject the MSI via toggling
> that GSI, we need to play some tricks to make this unfortunately
> interface transparent. We create those routes on demand and keep them
> in a hash table. Succeeding messages can then search for an existing
> route in the table first and reuse it whenever possible. If we should
> run out of limited GSIs, we simply flush the table and rebuild it as
> messages are sent.
> 
> This approach is rather simple and could be optimized further. However,
> future kernels will contain a more efficient MSI injection interface
> that will obsolete the GSI-based dynamic injection.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
>  kvm-all.c |  139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>  kvm.h     |    1 +
>  2 files changed, 139 insertions(+), 1 deletions(-)
> 
> diff --git a/kvm-all.c b/kvm-all.c
> index 2d82d54..e7ed510 100644
> --- a/kvm-all.c
> +++ b/kvm-all.c
> @@ -24,6 +24,7 @@
>  #include "qemu-barrier.h"
>  #include "sysemu.h"
>  #include "hw/hw.h"
> +#include "hw/msi.h"
>  #include "gdbstub.h"
>  #include "kvm.h"
>  #include "bswap.h"
> @@ -48,6 +49,8 @@
>      do { } while (0)
>  #endif
>  
> +#define KVM_MSI_HASHTAB_SIZE    256
> +
>  typedef struct KVMSlot
>  {
>      target_phys_addr_t start_addr;
> @@ -59,6 +62,11 @@ typedef struct KVMSlot
>  
>  typedef struct kvm_dirty_log KVMDirtyLog;
>  
> +typedef struct KVMMSIRoute {
> +    struct kvm_irq_routing_entry kroute;
> +    QTAILQ_ENTRY(KVMMSIRoute) entry;
> +} KVMMSIRoute;
> +
>  struct KVMState
>  {
>      KVMSlot slots[32];
> @@ -87,6 +95,7 @@ struct KVMState
>      int nr_allocated_irq_routes;
>      uint32_t *used_gsi_bitmap;
>      unsigned int gsi_count;
> +    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>  #endif
>  };
>  
> @@ -862,9 +871,14 @@ static void set_gsi(KVMState *s, unsigned int gsi)
>      s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
>  }
>  
> +static void clear_gsi(KVMState *s, unsigned int gsi)
> +{
> +    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
> +}
> +
>  static void kvm_init_irq_routing(KVMState *s)
>  {
> -    int gsi_count;
> +    int gsi_count, i;
>  
>      gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
>      if (gsi_count > 0) {
> @@ -884,6 +898,10 @@ static void kvm_init_irq_routing(KVMState *s)
>      s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
>      s->nr_allocated_irq_routes = 0;
>  
> +    for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
> +        QTAILQ_INIT(&s->msi_hashtab[i]);
> +    }
> +
>      kvm_arch_init_irq_routing(s);
>  }
>  
> @@ -934,11 +952,130 @@ int kvm_irqchip_commit_routes(KVMState *s)
>      return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
>  }
>  
> +static void kvm_release_gsi(KVMState *s, int gsi)
> +{
> +    struct kvm_irq_routing_entry *e;
> +    int i;
> +
> +    for (i = 0; i < s->irq_routes->nr; i++) {
> +        e = &s->irq_routes->entries[i];
> +        if (e->gsi == gsi) {
> +            s->irq_routes->nr--;
> +            *e = s->irq_routes->entries[s->irq_routes->nr];
> +        }
> +    }
> +    clear_gsi(s, gsi);
> +}
> +
> +static unsigned int kvm_hash_msi(uint32_t data)
> +{
> +    /* This is optimized for IA32 MSI layout. However, no other arch shall
> +     * repeat the mistake of not providing a direct MSI injection API. */
> +    return data & 0xff;
> +}
> +
> +static void kvm_flush_dynamic_msi_routes(KVMState *s)
> +{
> +    KVMMSIRoute *route, *next;
> +    unsigned int hash;
> +
> +    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
> +        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
> +            kvm_release_gsi(s, route->kroute.gsi);
> +            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
> +            g_free(route);
> +        }
> +    }
> +}
> +
> +static int kvm_get_pseudo_gsi(KVMState *s)
> +{
> +    uint32_t *word = s->used_gsi_bitmap;
> +    int max_words = ALIGN(s->gsi_count, 32) / 32;
> +    int i, bit;
> +    bool retry = true;
> +
> +again:
> +    /* Return the lowest unused GSI in the bitmap */
> +    for (i = 0; i < max_words; i++) {
> +        bit = ffs(~word[i]);
> +        if (!bit) {
> +            continue;
> +        }
> +
> +        return bit - 1 + i * 32;
> +    }
> +    if (retry) {
> +        retry = false;
> +        kvm_flush_dynamic_msi_routes(s);
> +        goto again;
> +    }

Guests with more MSI vectors than GSIs available, multiplexing are going
to be so slow that its best to disallow this completly (set_irq_route ->
synchronize_rcu can take 10ms or more, each).

MSI vectors must be allocated before use, can't it report #nr of
available ones in MSI Multiple Message Capable field? Or allow one MSI
per device? Or any other way to avoid this multiplexing.

Re: [Qemu-devel] [PATCH v3 3/8] kvm: Introduce basic MSI support for in-kernel irqchips

[Jan Kiszka]

On 2012-05-16 00:27, Marcelo Tosatti wrote:
> On Thu, May 10, 2012 at 06:02:52PM -0300, Jan Kiszka wrote:
>> This patch basically adds kvm_irqchip_send_msi, a service for sending
>> arbitrary MSI messages to KVM's in-kernel irqchip models.
>>
>> As the current KVI API requires us to establish a static route from a
>> pseudo GSI to the target MSI message and inject the MSI via toggling
>> that GSI, we need to play some tricks to make this unfortunately
>> interface transparent. We create those routes on demand and keep them
>> in a hash table. Succeeding messages can then search for an existing
>> route in the table first and reuse it whenever possible. If we should
>> run out of limited GSIs, we simply flush the table and rebuild it as
>> messages are sent.
>>
>> This approach is rather simple and could be optimized further. However,
>> future kernels will contain a more efficient MSI injection interface
>> that will obsolete the GSI-based dynamic injection.
>>
>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>> ---
>>  kvm-all.c |  139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>>  kvm.h     |    1 +
>>  2 files changed, 139 insertions(+), 1 deletions(-)
>>
>> diff --git a/kvm-all.c b/kvm-all.c
>> index 2d82d54..e7ed510 100644
>> --- a/kvm-all.c
>> +++ b/kvm-all.c
>> @@ -24,6 +24,7 @@
>>  #include "qemu-barrier.h"
>>  #include "sysemu.h"
>>  #include "hw/hw.h"
>> +#include "hw/msi.h"
>>  #include "gdbstub.h"
>>  #include "kvm.h"
>>  #include "bswap.h"
>> @@ -48,6 +49,8 @@
>>      do { } while (0)
>>  #endif
>>  
>> +#define KVM_MSI_HASHTAB_SIZE    256
>> +
>>  typedef struct KVMSlot
>>  {
>>      target_phys_addr_t start_addr;
>> @@ -59,6 +62,11 @@ typedef struct KVMSlot
>>  
>>  typedef struct kvm_dirty_log KVMDirtyLog;
>>  
>> +typedef struct KVMMSIRoute {
>> +    struct kvm_irq_routing_entry kroute;
>> +    QTAILQ_ENTRY(KVMMSIRoute) entry;
>> +} KVMMSIRoute;
>> +
>>  struct KVMState
>>  {
>>      KVMSlot slots[32];
>> @@ -87,6 +95,7 @@ struct KVMState
>>      int nr_allocated_irq_routes;
>>      uint32_t *used_gsi_bitmap;
>>      unsigned int gsi_count;
>> +    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>>  #endif
>>  };
>>  
>> @@ -862,9 +871,14 @@ static void set_gsi(KVMState *s, unsigned int gsi)
>>      s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
>>  }
>>  
>> +static void clear_gsi(KVMState *s, unsigned int gsi)
>> +{
>> +    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
>> +}
>> +
>>  static void kvm_init_irq_routing(KVMState *s)
>>  {
>> -    int gsi_count;
>> +    int gsi_count, i;
>>  
>>      gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
>>      if (gsi_count > 0) {
>> @@ -884,6 +898,10 @@ static void kvm_init_irq_routing(KVMState *s)
>>      s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
>>      s->nr_allocated_irq_routes = 0;
>>  
>> +    for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
>> +        QTAILQ_INIT(&s->msi_hashtab[i]);
>> +    }
>> +
>>      kvm_arch_init_irq_routing(s);
>>  }
>>  
>> @@ -934,11 +952,130 @@ int kvm_irqchip_commit_routes(KVMState *s)
>>      return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
>>  }
>>  
>> +static void kvm_release_gsi(KVMState *s, int gsi)
>> +{
>> +    struct kvm_irq_routing_entry *e;
>> +    int i;
>> +
>> +    for (i = 0; i < s->irq_routes->nr; i++) {
>> +        e = &s->irq_routes->entries[i];
>> +        if (e->gsi == gsi) {
>> +            s->irq_routes->nr--;
>> +            *e = s->irq_routes->entries[s->irq_routes->nr];
>> +        }
>> +    }
>> +    clear_gsi(s, gsi);
>> +}
>> +
>> +static unsigned int kvm_hash_msi(uint32_t data)
>> +{
>> +    /* This is optimized for IA32 MSI layout. However, no other arch shall
>> +     * repeat the mistake of not providing a direct MSI injection API. */
>> +    return data & 0xff;
>> +}
>> +
>> +static void kvm_flush_dynamic_msi_routes(KVMState *s)
>> +{
>> +    KVMMSIRoute *route, *next;
>> +    unsigned int hash;
>> +
>> +    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
>> +        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
>> +            kvm_release_gsi(s, route->kroute.gsi);
>> +            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
>> +            g_free(route);
>> +        }
>> +    }
>> +}
>> +
>> +static int kvm_get_pseudo_gsi(KVMState *s)
>> +{
>> +    uint32_t *word = s->used_gsi_bitmap;
>> +    int max_words = ALIGN(s->gsi_count, 32) / 32;
>> +    int i, bit;
>> +    bool retry = true;
>> +
>> +again:
>> +    /* Return the lowest unused GSI in the bitmap */
>> +    for (i = 0; i < max_words; i++) {
>> +        bit = ffs(~word[i]);
>> +        if (!bit) {
>> +            continue;
>> +        }
>> +
>> +        return bit - 1 + i * 32;
>> +    }
>> +    if (retry) {
>> +        retry = false;
>> +        kvm_flush_dynamic_msi_routes(s);
>> +        goto again;
>> +    }
> 
> Guests with more MSI vectors than GSIs available, multiplexing are going
> to be so slow that its best to disallow this completly (set_irq_route ->
> synchronize_rcu can take 10ms or more, each).

Well, it's still way better than exit(1) - what happens so far (but
obviously not in practice, or we would have heard about it). Note that
this code path is also required for MSI garbage collection, even if we
limited the vectors somehow.

> 
> MSI vectors must be allocated before use, can't it report #nr of
> available ones in MSI Multiple Message Capable field? Or allow one MSI
> per device? Or any other way to avoid this multiplexing.

I can't follow yet. All we have is the maximum number of vectors per
device on MSI[-X] init. There is no spec-conforming interface to
retrieve the actual vector usage from the guest, nor is there anything
to signal resource shortage once we exposed the MSI[-X] capability.
Also, in this design, we do not track vector origins back to the device,
so we can't apply per-device quotas.

But given that we haven't heard about such an overflow scenario in
practice so far (or did you?), I think there is no urgent need to worry.
And we could still increase the number of GSIs.

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

Re: [Qemu-devel] [PATCH v3 3/8] kvm: Introduce basic MSI support for in-kernel irqchips

[Marcelo Tosatti]

On Wed, May 16, 2012 at 08:15:37AM -0300, Jan Kiszka wrote:
> On 2012-05-16 00:27, Marcelo Tosatti wrote:
> > On Thu, May 10, 2012 at 06:02:52PM -0300, Jan Kiszka wrote:
> >> This patch basically adds kvm_irqchip_send_msi, a service for sending
> >> arbitrary MSI messages to KVM's in-kernel irqchip models.
> >>
> >> As the current KVI API requires us to establish a static route from a
> >> pseudo GSI to the target MSI message and inject the MSI via toggling
> >> that GSI, we need to play some tricks to make this unfortunately
> >> interface transparent. We create those routes on demand and keep them
> >> in a hash table. Succeeding messages can then search for an existing
> >> route in the table first and reuse it whenever possible. If we should
> >> run out of limited GSIs, we simply flush the table and rebuild it as
> >> messages are sent.
> >>
> >> This approach is rather simple and could be optimized further. However,
> >> future kernels will contain a more efficient MSI injection interface
> >> that will obsolete the GSI-based dynamic injection.
> >>
> >> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> >> ---
> >>  kvm-all.c |  139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
> >>  kvm.h     |    1 +
> >>  2 files changed, 139 insertions(+), 1 deletions(-)
> >>
> >> diff --git a/kvm-all.c b/kvm-all.c
> >> index 2d82d54..e7ed510 100644
> >> --- a/kvm-all.c
> >> +++ b/kvm-all.c
> >> @@ -24,6 +24,7 @@
> >>  #include "qemu-barrier.h"
> >>  #include "sysemu.h"
> >>  #include "hw/hw.h"
> >> +#include "hw/msi.h"
> >>  #include "gdbstub.h"
> >>  #include "kvm.h"
> >>  #include "bswap.h"
> >> @@ -48,6 +49,8 @@
> >>      do { } while (0)
> >>  #endif
> >>  
> >> +#define KVM_MSI_HASHTAB_SIZE    256
> >> +
> >>  typedef struct KVMSlot
> >>  {
> >>      target_phys_addr_t start_addr;
> >> @@ -59,6 +62,11 @@ typedef struct KVMSlot
> >>  
> >>  typedef struct kvm_dirty_log KVMDirtyLog;
> >>  
> >> +typedef struct KVMMSIRoute {
> >> +    struct kvm_irq_routing_entry kroute;
> >> +    QTAILQ_ENTRY(KVMMSIRoute) entry;
> >> +} KVMMSIRoute;
> >> +
> >>  struct KVMState
> >>  {
> >>      KVMSlot slots[32];
> >> @@ -87,6 +95,7 @@ struct KVMState
> >>      int nr_allocated_irq_routes;
> >>      uint32_t *used_gsi_bitmap;
> >>      unsigned int gsi_count;
> >> +    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
> >>  #endif
> >>  };
> >>  
> >> @@ -862,9 +871,14 @@ static void set_gsi(KVMState *s, unsigned int gsi)
> >>      s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
> >>  }
> >>  
> >> +static void clear_gsi(KVMState *s, unsigned int gsi)
> >> +{
> >> +    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
> >> +}
> >> +
> >>  static void kvm_init_irq_routing(KVMState *s)
> >>  {
> >> -    int gsi_count;
> >> +    int gsi_count, i;
> >>  
> >>      gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
> >>      if (gsi_count > 0) {
> >> @@ -884,6 +898,10 @@ static void kvm_init_irq_routing(KVMState *s)
> >>      s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
> >>      s->nr_allocated_irq_routes = 0;
> >>  
> >> +    for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
> >> +        QTAILQ_INIT(&s->msi_hashtab[i]);
> >> +    }
> >> +
> >>      kvm_arch_init_irq_routing(s);
> >>  }
> >>  
> >> @@ -934,11 +952,130 @@ int kvm_irqchip_commit_routes(KVMState *s)
> >>      return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
> >>  }
> >>  
> >> +static void kvm_release_gsi(KVMState *s, int gsi)
> >> +{
> >> +    struct kvm_irq_routing_entry *e;
> >> +    int i;
> >> +
> >> +    for (i = 0; i < s->irq_routes->nr; i++) {
> >> +        e = &s->irq_routes->entries[i];
> >> +        if (e->gsi == gsi) {
> >> +            s->irq_routes->nr--;
> >> +            *e = s->irq_routes->entries[s->irq_routes->nr];
> >> +        }
> >> +    }
> >> +    clear_gsi(s, gsi);
> >> +}
> >> +
> >> +static unsigned int kvm_hash_msi(uint32_t data)
> >> +{
> >> +    /* This is optimized for IA32 MSI layout. However, no other arch shall
> >> +     * repeat the mistake of not providing a direct MSI injection API. */
> >> +    return data & 0xff;
> >> +}
> >> +
> >> +static void kvm_flush_dynamic_msi_routes(KVMState *s)
> >> +{
> >> +    KVMMSIRoute *route, *next;
> >> +    unsigned int hash;
> >> +
> >> +    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
> >> +        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
> >> +            kvm_release_gsi(s, route->kroute.gsi);
> >> +            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
> >> +            g_free(route);
> >> +        }
> >> +    }
> >> +}
> >> +
> >> +static int kvm_get_pseudo_gsi(KVMState *s)
> >> +{
> >> +    uint32_t *word = s->used_gsi_bitmap;
> >> +    int max_words = ALIGN(s->gsi_count, 32) / 32;
> >> +    int i, bit;
> >> +    bool retry = true;
> >> +
> >> +again:
> >> +    /* Return the lowest unused GSI in the bitmap */
> >> +    for (i = 0; i < max_words; i++) {
> >> +        bit = ffs(~word[i]);
> >> +        if (!bit) {
> >> +            continue;
> >> +        }
> >> +
> >> +        return bit - 1 + i * 32;
> >> +    }
> >> +    if (retry) {
> >> +        retry = false;
> >> +        kvm_flush_dynamic_msi_routes(s);
> >> +        goto again;
> >> +    }
> > 
> > Guests with more MSI vectors than GSIs available, multiplexing are going
> > to be so slow that its best to disallow this completly (set_irq_route ->
> > synchronize_rcu can take 10ms or more, each).
> 
> Well, it's still way better than exit(1) - what happens so far (but
> obviously not in practice, or we would have heard about it). Note that
> this code path is also required for MSI garbage collection, even if we
> limited the vectors somehow.
> 
> > 
> > MSI vectors must be allocated before use, can't it report #nr of
> > available ones in MSI Multiple Message Capable field? Or allow one MSI
> > per device? Or any other way to avoid this multiplexing.
> 
> I can't follow yet. All we have is the maximum number of vectors per
> device on MSI[-X] init. There is no spec-conforming interface to
> retrieve the actual vector usage from the guest, nor is there anything
> to signal resource shortage once we exposed the MSI[-X] capability.
> Also, in this design, we do not track vector origins back to the device,
> so we can't apply per-device quotas.

Wishful thinking.

> But given that we haven't heard about such an overflow scenario in
> practice so far (or did you?), I think there is no urgent need to worry.
> And we could still increase the number of GSIs.

I did not. With no route cache hits, guest will be limited to 100
interrupts per second (and hang waiting on synchronize_rcu during 
that second). Perhaps it is not a bad idea to exit().

Will apply the patch anyway, thanks.

Re: [Qemu-devel] [PATCH v3 3/8] kvm: Introduce basic MSI support for in-kernel irqchips

[Jan Kiszka]

On 2012-05-16 08:15, Jan Kiszka wrote:
> On 2012-05-16 00:27, Marcelo Tosatti wrote:
>> On Thu, May 10, 2012 at 06:02:52PM -0300, Jan Kiszka wrote:
>>> This patch basically adds kvm_irqchip_send_msi, a service for sending
>>> arbitrary MSI messages to KVM's in-kernel irqchip models.
>>>
>>> As the current KVI API requires us to establish a static route from a
>>> pseudo GSI to the target MSI message and inject the MSI via toggling
>>> that GSI, we need to play some tricks to make this unfortunately
>>> interface transparent. We create those routes on demand and keep them
>>> in a hash table. Succeeding messages can then search for an existing
>>> route in the table first and reuse it whenever possible. If we should
>>> run out of limited GSIs, we simply flush the table and rebuild it as
>>> messages are sent.
>>>
>>> This approach is rather simple and could be optimized further. However,
>>> future kernels will contain a more efficient MSI injection interface
>>> that will obsolete the GSI-based dynamic injection.
>>>
>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>>> ---
>>>  kvm-all.c |  139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>>>  kvm.h     |    1 +
>>>  2 files changed, 139 insertions(+), 1 deletions(-)
>>>
>>> diff --git a/kvm-all.c b/kvm-all.c
>>> index 2d82d54..e7ed510 100644
>>> --- a/kvm-all.c
>>> +++ b/kvm-all.c
>>> @@ -24,6 +24,7 @@
>>>  #include "qemu-barrier.h"
>>>  #include "sysemu.h"
>>>  #include "hw/hw.h"
>>> +#include "hw/msi.h"
>>>  #include "gdbstub.h"
>>>  #include "kvm.h"
>>>  #include "bswap.h"
>>> @@ -48,6 +49,8 @@
>>>      do { } while (0)
>>>  #endif
>>>  
>>> +#define KVM_MSI_HASHTAB_SIZE    256
>>> +
>>>  typedef struct KVMSlot
>>>  {
>>>      target_phys_addr_t start_addr;
>>> @@ -59,6 +62,11 @@ typedef struct KVMSlot
>>>  
>>>  typedef struct kvm_dirty_log KVMDirtyLog;
>>>  
>>> +typedef struct KVMMSIRoute {
>>> +    struct kvm_irq_routing_entry kroute;
>>> +    QTAILQ_ENTRY(KVMMSIRoute) entry;
>>> +} KVMMSIRoute;
>>> +
>>>  struct KVMState
>>>  {
>>>      KVMSlot slots[32];
>>> @@ -87,6 +95,7 @@ struct KVMState
>>>      int nr_allocated_irq_routes;
>>>      uint32_t *used_gsi_bitmap;
>>>      unsigned int gsi_count;
>>> +    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>>>  #endif
>>>  };
>>>  
>>> @@ -862,9 +871,14 @@ static void set_gsi(KVMState *s, unsigned int gsi)
>>>      s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
>>>  }
>>>  
>>> +static void clear_gsi(KVMState *s, unsigned int gsi)
>>> +{
>>> +    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
>>> +}
>>> +
>>>  static void kvm_init_irq_routing(KVMState *s)
>>>  {
>>> -    int gsi_count;
>>> +    int gsi_count, i;
>>>  
>>>      gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
>>>      if (gsi_count > 0) {
>>> @@ -884,6 +898,10 @@ static void kvm_init_irq_routing(KVMState *s)
>>>      s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
>>>      s->nr_allocated_irq_routes = 0;
>>>  
>>> +    for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
>>> +        QTAILQ_INIT(&s->msi_hashtab[i]);
>>> +    }
>>> +
>>>      kvm_arch_init_irq_routing(s);
>>>  }
>>>  
>>> @@ -934,11 +952,130 @@ int kvm_irqchip_commit_routes(KVMState *s)
>>>      return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
>>>  }
>>>  
>>> +static void kvm_release_gsi(KVMState *s, int gsi)
>>> +{
>>> +    struct kvm_irq_routing_entry *e;
>>> +    int i;
>>> +
>>> +    for (i = 0; i < s->irq_routes->nr; i++) {
>>> +        e = &s->irq_routes->entries[i];
>>> +        if (e->gsi == gsi) {
>>> +            s->irq_routes->nr--;
>>> +            *e = s->irq_routes->entries[s->irq_routes->nr];
>>> +        }
>>> +    }
>>> +    clear_gsi(s, gsi);
>>> +}
>>> +
>>> +static unsigned int kvm_hash_msi(uint32_t data)
>>> +{
>>> +    /* This is optimized for IA32 MSI layout. However, no other arch shall
>>> +     * repeat the mistake of not providing a direct MSI injection API. */
>>> +    return data & 0xff;
>>> +}
>>> +
>>> +static void kvm_flush_dynamic_msi_routes(KVMState *s)
>>> +{
>>> +    KVMMSIRoute *route, *next;
>>> +    unsigned int hash;
>>> +
>>> +    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
>>> +        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
>>> +            kvm_release_gsi(s, route->kroute.gsi);
>>> +            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
>>> +            g_free(route);
>>> +        }
>>> +    }
>>> +}
>>> +
>>> +static int kvm_get_pseudo_gsi(KVMState *s)
>>> +{
>>> +    uint32_t *word = s->used_gsi_bitmap;
>>> +    int max_words = ALIGN(s->gsi_count, 32) / 32;
>>> +    int i, bit;
>>> +    bool retry = true;
>>> +
>>> +again:
>>> +    /* Return the lowest unused GSI in the bitmap */
>>> +    for (i = 0; i < max_words; i++) {
>>> +        bit = ffs(~word[i]);
>>> +        if (!bit) {
>>> +            continue;
>>> +        }
>>> +
>>> +        return bit - 1 + i * 32;
>>> +    }
>>> +    if (retry) {
>>> +        retry = false;
>>> +        kvm_flush_dynamic_msi_routes(s);
>>> +        goto again;
>>> +    }
>>
>> Guests with more MSI vectors than GSIs available, multiplexing are going
>> to be so slow that its best to disallow this completly (set_irq_route ->
>> synchronize_rcu can take 10ms or more, each).
> 
> Well, it's still way better than exit(1) - what happens so far (but
> obviously not in practice, or we would have heard about it). Note that
> this code path is also required for MSI garbage collection, even if we
> limited the vectors somehow.
> 
>>
>> MSI vectors must be allocated before use, can't it report #nr of
>> available ones in MSI Multiple Message Capable field? Or allow one MSI
>> per device? Or any other way to avoid this multiplexing.
> 
> I can't follow yet. All we have is the maximum number of vectors per
> device on MSI[-X] init. There is no spec-conforming interface to
> retrieve the actual vector usage from the guest, nor is there anything
> to signal resource shortage once we exposed the MSI[-X] capability.
> Also, in this design, we do not track vector origins back to the device,
> so we can't apply per-device quotas.
> 
> But given that we haven't heard about such an overflow scenario in
> practice so far (or did you?), I think there is no urgent need to worry.
> And we could still increase the number of GSIs.

Oh, and forgot to remind that this route flushing code path will be
unused on current kernels with KVM_CAP_SIGNAL_MSI.

I will push a new version of this series soon as this one as a small
bisactability issue.

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

[Qemu-devel] [PATCH v3 4/8] pc: Enable MSI support at APIC level

[Jan Kiszka]

Push msi_supported enabling to the APIC implementations where we can
encapsulate the decision more cleanly, hiding the details from the
generic code.

CC: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 hw/apic.c     |    2 ++
 hw/pc.c       |    9 ---------
 hw/xen.h      |   10 ----------
 hw/xen_apic.c |    5 +++++
 4 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/hw/apic.c b/hw/apic.c
index 4eeaf88..a337790 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -862,6 +862,8 @@ static void apic_init(APICCommonState *s)
 
     s->timer = qemu_new_timer_ns(vm_clock, apic_timer, s);
     local_apics[s->idx] = s;
+
+    msi_supported = true;
 }
 
 static void apic_class_init(ObjectClass *klass, void *data)
diff --git a/hw/pc.c b/hw/pc.c
index 4d34a33..6691b18 100644
--- a/hw/pc.c
+++ b/hw/pc.c
@@ -911,15 +911,6 @@ static DeviceState *apic_init(void *env, uint8_t apic_id)
         apic_mapped = 1;
     }
 
-    /* KVM does not support MSI yet. */
-    if (!kvm_irqchip_in_kernel()) {
-        msi_supported = true;
-    }
-
-    if (xen_msi_support()) {
-        msi_supported = true;
-    }
-
     return dev;
 }
 
diff --git a/hw/xen.h b/hw/xen.h
index 3ae4cd0..e5926b7 100644
--- a/hw/xen.h
+++ b/hw/xen.h
@@ -57,14 +57,4 @@ void xen_register_framebuffer(struct MemoryRegion *mr);
 #  define HVM_MAX_VCPUS 32
 #endif
 
-static inline int xen_msi_support(void)
-{
-#if defined(CONFIG_XEN_CTRL_INTERFACE_VERSION) \
-    && CONFIG_XEN_CTRL_INTERFACE_VERSION >= 420
-    return xen_enabled();
-#else
-    return 0;
-#endif
-}
-
 #endif /* QEMU_HW_XEN_H */
diff --git a/hw/xen_apic.c b/hw/xen_apic.c
index 1725ff6..a9e101f 100644
--- a/hw/xen_apic.c
+++ b/hw/xen_apic.c
@@ -40,6 +40,11 @@ static void xen_apic_init(APICCommonState *s)
 {
     memory_region_init_io(&s->io_memory, &xen_apic_io_ops, s, "xen-apic-msi",
                           MSI_SPACE_SIZE);
+
+#if defined(CONFIG_XEN_CTRL_INTERFACE_VERSION) \
+    && CONFIG_XEN_CTRL_INTERFACE_VERSION >= 420
+    msi_supported = true;
+#endif
 }
 
 static void xen_apic_set_base(APICCommonState *s, uint64_t val)
-- 
1.7.3.4

Re: [Qemu-devel] [PATCH v3 4/8] pc: Enable MSI support at APIC level

[Stefano Stabellini]

On Thu, 10 May 2012, Jan Kiszka wrote:
> Push msi_supported enabling to the APIC implementations where we can
> encapsulate the decision more cleanly, hiding the details from the
> generic code.
> 
> CC: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

This patch is missing the KVM part, but assuming that it is done in a
later patch:

Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>



>  hw/apic.c     |    2 ++
>  hw/pc.c       |    9 ---------
>  hw/xen.h      |   10 ----------
>  hw/xen_apic.c |    5 +++++
>  4 files changed, 7 insertions(+), 19 deletions(-)
> 
> diff --git a/hw/apic.c b/hw/apic.c
> index 4eeaf88..a337790 100644
> --- a/hw/apic.c
> +++ b/hw/apic.c
> @@ -862,6 +862,8 @@ static void apic_init(APICCommonState *s)
>  
>      s->timer = qemu_new_timer_ns(vm_clock, apic_timer, s);
>      local_apics[s->idx] = s;
> +
> +    msi_supported = true;
>  }
>  
>  static void apic_class_init(ObjectClass *klass, void *data)
> diff --git a/hw/pc.c b/hw/pc.c
> index 4d34a33..6691b18 100644
> --- a/hw/pc.c
> +++ b/hw/pc.c
> @@ -911,15 +911,6 @@ static DeviceState *apic_init(void *env, uint8_t apic_id)
>          apic_mapped = 1;
>      }
>  
> -    /* KVM does not support MSI yet. */
> -    if (!kvm_irqchip_in_kernel()) {
> -        msi_supported = true;
> -    }
> -
> -    if (xen_msi_support()) {
> -        msi_supported = true;
> -    }
> -
>      return dev;
>  }
>  
> diff --git a/hw/xen.h b/hw/xen.h
> index 3ae4cd0..e5926b7 100644
> --- a/hw/xen.h
> +++ b/hw/xen.h
> @@ -57,14 +57,4 @@ void xen_register_framebuffer(struct MemoryRegion *mr);
>  #  define HVM_MAX_VCPUS 32
>  #endif
>  
> -static inline int xen_msi_support(void)
> -{
> -#if defined(CONFIG_XEN_CTRL_INTERFACE_VERSION) \
> -    && CONFIG_XEN_CTRL_INTERFACE_VERSION >= 420
> -    return xen_enabled();
> -#else
> -    return 0;
> -#endif
> -}
> -
>  #endif /* QEMU_HW_XEN_H */
> diff --git a/hw/xen_apic.c b/hw/xen_apic.c
> index 1725ff6..a9e101f 100644
> --- a/hw/xen_apic.c
> +++ b/hw/xen_apic.c
> @@ -40,6 +40,11 @@ static void xen_apic_init(APICCommonState *s)
>  {
>      memory_region_init_io(&s->io_memory, &xen_apic_io_ops, s, "xen-apic-msi",
>                            MSI_SPACE_SIZE);
> +
> +#if defined(CONFIG_XEN_CTRL_INTERFACE_VERSION) \
> +    && CONFIG_XEN_CTRL_INTERFACE_VERSION >= 420
> +    msi_supported = true;
> +#endif
>  }
>  
>  static void xen_apic_set_base(APICCommonState *s, uint64_t val)
> -- 
> 1.7.3.4
> 

Re: [Qemu-devel] [PATCH v3 4/8] pc: Enable MSI support at APIC level

[Jan Kiszka]

On 2012-05-11 13:24, Stefano Stabellini wrote:
> On Thu, 10 May 2012, Jan Kiszka wrote:
>> Push msi_supported enabling to the APIC implementations where we can
>> encapsulate the decision more cleanly, hiding the details from the
>> generic code.
>>
>> CC: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> 
> This patch is missing the KVM part, but assuming that it is done in a
> later patch:

It is as the KVM APIC is not yet ready for MSI activation at this point.

> 
> Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
> 

Thanks,
Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

[Qemu-devel] [PATCH v3 5/8] kvm: x86: Wire up MSI support for in-kernel irqchip

[Jan Kiszka]

Catch writes to the MSI MMIO region in the KVM APIC and forward them to
the kernel. Provide the kernel support GSI routing, this allows to
enable MSI support also for in-kernel irqchip mode.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 hw/apic.c     |    1 +
 hw/kvm/apic.c |   34 ++++++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/hw/apic.c b/hw/apic.c
index a337790..5fbf01c 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -19,6 +19,7 @@
 #include "apic_internal.h"
 #include "apic.h"
 #include "ioapic.h"
+#include "msi.h"
 #include "host-utils.h"
 #include "trace.h"
 #include "pc.h"
diff --git a/hw/kvm/apic.c b/hw/kvm/apic.c
index ffe7a52..8ba4079 100644
--- a/hw/kvm/apic.c
+++ b/hw/kvm/apic.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 #include "hw/apic_internal.h"
+#include "hw/msi.h"
 #include "kvm.h"
 
 static inline void kvm_apic_set_reg(struct kvm_lapic_state *kapic,
@@ -145,10 +146,39 @@ static void kvm_apic_external_nmi(APICCommonState *s)
     run_on_cpu(s->cpu_env, do_inject_external_nmi, s);
 }
 
+static uint64_t kvm_apic_mem_read(void *opaque, target_phys_addr_t addr,
+                                  unsigned size)
+{
+    return ~(uint64_t)0;
+}
+
+static void kvm_apic_mem_write(void *opaque, target_phys_addr_t addr,
+                               uint64_t data, unsigned size)
+{
+    MSIMessage msg = { .address = addr, .data = data };
+    int ret;
+
+    ret = kvm_irqchip_send_msi(kvm_state, msg);
+    if (ret < 0) {
+        fprintf(stderr, "KVM: injection failed, MSI lost (%s)\n",
+                strerror(-ret));
+    }
+}
+
+static const MemoryRegionOps kvm_apic_io_ops = {
+    .read = kvm_apic_mem_read,
+    .write = kvm_apic_mem_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
 static void kvm_apic_init(APICCommonState *s)
 {
-    memory_region_init_reservation(&s->io_memory, "kvm-apic-msi",
-                                   MSI_SPACE_SIZE);
+    memory_region_init_io(&s->io_memory, &kvm_apic_io_ops, s, "kvm-apic-msi",
+                          MSI_SPACE_SIZE);
+
+    if (kvm_has_gsi_routing()) {
+        msi_supported = true;
+    }
 }
 
 static void kvm_apic_class_init(ObjectClass *klass, void *data)
-- 
1.7.3.4

[Qemu-devel] [PATCH v3 6/8] kvm: Update kernel headers

[Jan Kiszka]

Corresponding kvm.git hash: f2569053e0

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 linux-headers/linux/kvm.h |   38 ++++++++++++++++++++++++++++++++++++++
 1 files changed, 38 insertions(+), 0 deletions(-)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index ee7bd9c..c4426ec 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -449,6 +449,30 @@ struct kvm_ppc_pvinfo {
 	__u8  pad[108];
 };
 
+/* for KVM_PPC_GET_SMMU_INFO */
+#define KVM_PPC_PAGE_SIZES_MAX_SZ	8
+
+struct kvm_ppc_one_page_size {
+	__u32 page_shift;	/* Page shift (or 0) */
+	__u32 pte_enc;		/* Encoding in the HPTE (>>12) */
+};
+
+struct kvm_ppc_one_seg_page_size {
+	__u32 page_shift;	/* Base page shift of segment (or 0) */
+	__u32 slb_enc;		/* SLB encoding for BookS */
+	struct kvm_ppc_one_page_size enc[KVM_PPC_PAGE_SIZES_MAX_SZ];
+};
+
+#define KVM_PPC_PAGE_SIZES_REAL		0x00000001
+#define KVM_PPC_1T_SEGMENTS		0x00000002
+
+struct kvm_ppc_smmu_info {
+	__u64 flags;
+	__u32 slb_size;
+	__u32 pad;
+	struct kvm_ppc_one_seg_page_size sps[KVM_PPC_PAGE_SIZES_MAX_SZ];
+};
+
 #define KVMIO 0xAE
 
 /* machine type bits, to be used as argument to KVM_CREATE_VM */
@@ -590,6 +614,8 @@ struct kvm_ppc_pvinfo {
 #define KVM_CAP_SYNC_REGS 74
 #define KVM_CAP_PCI_2_3 75
 #define KVM_CAP_KVMCLOCK_CTRL 76
+#define KVM_CAP_SIGNAL_MSI 77
+#define KVM_CAP_PPC_GET_SMMU_INFO 78
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -715,6 +741,14 @@ struct kvm_one_reg {
 	__u64 addr;
 };
 
+struct kvm_msi {
+	__u32 address_lo;
+	__u32 address_hi;
+	__u32 data;
+	__u32 flags;
+	__u8  pad[16];
+};
+
 /*
  * ioctls for VM fds
  */
@@ -789,6 +823,10 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_PCI_2_3 */
 #define KVM_ASSIGN_SET_INTX_MASK  _IOW(KVMIO,  0xa4, \
 				       struct kvm_assigned_pci_dev)
+/* Available with KVM_CAP_SIGNAL_MSI */
+#define KVM_SIGNAL_MSI            _IOW(KVMIO,  0xa5, struct kvm_msi)
+/* Available with KVM_CAP_PPC_GET_SMMU_INFO */
+#define KVM_PPC_GET_SMMU_INFO	  _IOR(KVMIO,  0xa6, struct kvm_ppc_smmu_info)
 
 /*
  * ioctls for vcpu fds
-- 
1.7.3.4

[Qemu-devel] [PATCH v3 7/8] kvm: Add support for direct MSI injections

[Jan Kiszka]

If the kernel supports KVM_SIGNAL_MSI, we can avoid the route-based
MSI injection mechanism.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 kvm-all.c |   22 +++++++++++++++++++---
 1 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/kvm-all.c b/kvm-all.c
index e7ed510..4e8c8f7 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -96,6 +96,7 @@ struct KVMState
     uint32_t *used_gsi_bitmap;
     unsigned int gsi_count;
     QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
+    bool direct_msi;
 #endif
 };
 
@@ -898,8 +899,10 @@ static void kvm_init_irq_routing(KVMState *s)
     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
     s->nr_allocated_irq_routes = 0;
 
-    for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
-        QTAILQ_INIT(&s->msi_hashtab[i]);
+    if (!s->direct_msi) {
+        for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
+            QTAILQ_INIT(&s->msi_hashtab[i]);
+        }
     }
 
     kvm_arch_init_irq_routing(s);
@@ -1005,7 +1008,7 @@ again:
 
         return bit - 1 + i * 32;
     }
-    if (retry) {
+    if (!s->direct_msi && retry) {
         retry = false;
         kvm_flush_dynamic_msi_routes(s);
         goto again;
@@ -1031,8 +1034,19 @@ static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
 
 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
 {
+    struct kvm_msi msi;
     KVMMSIRoute *route;
 
+    if (s->direct_msi) {
+        msi.address_lo = (uint32_t)msg.address;
+        msi.address_hi = msg.address >> 32;
+        msi.data = msg.data;
+        msi.flags = 0;
+        memset(msi.pad, 0, sizeof(msi.pad));
+
+        return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
+    }
+
     route = kvm_lookup_msi_route(s, msg);
     if (!route) {
         int gsi, ret;
@@ -1209,6 +1223,8 @@ int kvm_init(void)
     s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
 #endif
 
+    s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
+
     ret = kvm_arch_init(s);
     if (ret < 0) {
         goto err;
-- 
1.7.3.4

[Qemu-devel] [PATCH v3 8/8] kvm: Enable in-kernel irqchip support by default

[Jan Kiszka]

As MSI is now fully supported by KVM (/wrt available features in
upstream), we can finally enable the in-kernel irqchip by default.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 kvm-all.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/kvm-all.c b/kvm-all.c
index 4e8c8f7..8f8cec5 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -1099,7 +1099,7 @@ static int kvm_irqchip_create(KVMState *s)
 
     if (QTAILQ_EMPTY(&list->head) ||
         !qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
-                           "kernel_irqchip", false) ||
+                           "kernel_irqchip", true) ||
         !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
         return 0;
     }
-- 
1.7.3.4