[Qemu-devel] [PATCH] APIC/IOAPIC EOI callback

[Qemu-devel] [PATCH] APIC/IOAPIC EOI callback

[Alex Williamson]

For device assignment, we need to know when the VM writes an end
of interrupt to the APIC, which allows us to re-enable interrupts
on the physical device.  Add a new wrapper for ioapic generated
interrupts with a callback on eoi and create an interface for
drivers to be notified on eoi.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---

Note that the notifier and notifier_enabled eoi_client fields aren't
used here yet.  I'll send an RFC patch showing how we make use of
these with the proposed KVM_EOI_EVENTFD patches.

 hw/apic.c   |   18 ++++++++++++++++--
 hw/apic.h   |    4 ++++
 hw/ioapic.c |   38 ++++++++++++++++++++++++++++++++++++--
 hw/pc.h     |   16 +++++++++++++++-
 4 files changed, 71 insertions(+), 5 deletions(-)

diff --git a/hw/apic.c b/hw/apic.c
index 63d62c7..a24117b 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -22,6 +22,7 @@
 #include "host-utils.h"
 #include "sysbus.h"
 #include "trace.h"
+#include "pc.h"
 
 /* APIC Local Vector Table */
 #define APIC_LVT_TIMER   0
@@ -103,6 +104,7 @@ struct APICState {
     int wait_for_sipi;
 };
 
+static uint8_t vector_to_gsi_map[256] = { 0xff };
 static APICState *local_apics[MAX_APICS + 1];
 static int apic_irq_delivered;
 
@@ -292,6 +294,15 @@ void apic_deliver_irq(uint8_t dest, uint8_t dest_mode,
                      trigger_mode);
 }
 
+void apic_deliver_ioapic_irq(uint8_t dest, uint8_t dest_mode,
+                             uint8_t delivery_mode, uint8_t vector_num,
+                             uint8_t polarity, uint8_t trigger_mode, int gsi)
+{
+    vector_to_gsi_map[vector_num] = gsi;
+    apic_deliver_irq(dest, dest_mode, delivery_mode,
+                     vector_num, polarity, trigger_mode);
+}
+
 void cpu_set_apic_base(DeviceState *d, uint64_t val)
 {
     APICState *s = DO_UPCAST(APICState, busdev.qdev, d);
@@ -420,8 +431,11 @@ static void apic_eoi(APICState *s)
     if (isrv < 0)
         return;
     reset_bit(s->isr, isrv);
-    /* XXX: send the EOI packet to the APIC bus to allow the I/O APIC to
-            set the remote IRR bit for level triggered interrupts. */
+  
+    if (vector_to_gsi_map[isrv] != 0xff) {
+        ioapic_eoi(vector_to_gsi_map[isrv]);
+        vector_to_gsi_map[isrv] = 0xff;
+    }
     apic_update_irq(s);
 }
 
diff --git a/hw/apic.h b/hw/apic.h
index 8a0c9d0..59d0e37 100644
--- a/hw/apic.h
+++ b/hw/apic.h
@@ -8,6 +8,10 @@ void apic_deliver_irq(uint8_t dest, uint8_t dest_mode,
                              uint8_t delivery_mode,
                              uint8_t vector_num, uint8_t polarity,
                              uint8_t trigger_mode);
+void apic_deliver_ioapic_irq(uint8_t dest, uint8_t dest_mode,
+                             uint8_t delivery_mode,
+                             uint8_t vector_num, uint8_t polarity,
+                             uint8_t trigger_mode, int gsi);
 int apic_accept_pic_intr(DeviceState *s);
 void apic_deliver_pic_intr(DeviceState *s, int level);
 int apic_get_interrupt(DeviceState *s);
diff --git a/hw/ioapic.c b/hw/ioapic.c
index 5ae21e9..ffd1c92 100644
--- a/hw/ioapic.c
+++ b/hw/ioapic.c
@@ -26,6 +26,7 @@
 #include "qemu-timer.h"
 #include "host-utils.h"
 #include "sysbus.h"
+#include "qlist.h"
 
 //#define DEBUG_IOAPIC
 
@@ -61,6 +62,39 @@ struct IOAPICState {
     uint64_t ioredtbl[IOAPIC_NUM_PINS];
 };
 
+static QLIST_HEAD(ioapic_eoi_client_list,
+                  ioapic_eoi_client) ioapic_eoi_client_list =
+                  QLIST_HEAD_INITIALIZER(ioapic_eoi_client_list);
+
+int ioapic_register_eoi_client(ioapic_eoi_client *client)
+{
+    QLIST_INSERT_HEAD(&ioapic_eoi_client_list, client, list);
+    return 0;
+}
+
+void ioapic_unregister_eoi_client(ioapic_eoi_client *client)
+{
+    QLIST_REMOVE(client, list);
+}
+
+int ioapic_eoi_client_get_fd(ioapic_eoi_client *client)
+{
+    if (!client->notifier_enabled) {
+        return -ENODEV;
+    }
+    return event_notifier_get_fd(&client->notifier);
+}
+
+void ioapic_eoi(int gsi)
+{
+    ioapic_eoi_client *client;
+    QLIST_FOREACH(client, &ioapic_eoi_client_list, list) {
+        if (client->irq == gsi) {
+            client->eoi(client);
+        }
+    }
+}
+
 static void ioapic_service(IOAPICState *s)
 {
     uint8_t i;
@@ -90,8 +124,8 @@ static void ioapic_service(IOAPICState *s)
                 else
                     vector = entry & 0xff;
 
-                apic_deliver_irq(dest, dest_mode, delivery_mode,
-                                 vector, polarity, trig_mode);
+                apic_deliver_ioapic_irq(dest, dest_mode, delivery_mode,
+                                        vector, polarity, trig_mode, i);
             }
         }
     }
diff --git a/hw/pc.h b/hw/pc.h
index 63b0249..5945bff 100644
--- a/hw/pc.h
+++ b/hw/pc.h
@@ -5,6 +5,7 @@
 #include "ioport.h"
 #include "isa.h"
 #include "fdc.h"
+#include "event_notifier.h"
 
 /* PC-style peripherals (also used by other machines).  */
 
@@ -48,8 +49,21 @@ typedef struct isa_irq_state {
 
 void isa_irq_handler(void *opaque, int n, int level);
 
-/* i8254.c */
+struct ioapic_eoi_client;
+typedef struct ioapic_eoi_client ioapic_eoi_client;
+struct ioapic_eoi_client {
+    void (*eoi)(struct ioapic_eoi_client *client);
+    int irq;
+    EventNotifier notifier;
+    QLIST_ENTRY(ioapic_eoi_client) list;
+    bool notifier_enabled;
+};
+int ioapic_register_eoi_client(ioapic_eoi_client *client);
+void ioapic_unregister_eoi_client(ioapic_eoi_client *client);
+int ioapic_eoi_client_get_fd(ioapic_eoi_client *client);
+void ioapic_eoi(int gsi);
 
+/* i8254.c */
 #define PIT_FREQ 1193182
 
 typedef struct PITState PITState;

[Qemu-devel] [RFC PATCH] kvm: KVM_EOI_EVENTFD support for eoi_client

[Alex Williamson]

With the KVM irqchip, we need to get the EOI via an eventfd.  This
adds support for that, abstracting the details to the caller.

The get_fd function allows drivers to make further optimizations
in handling the EOI.  For instance with VFIO, we can make use of
an irqfd-like mechanism to have the VFIO kernel module consume
the EOI directly, bypassing qemu userspace.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---

 hw/ioapic.c |   52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 52 insertions(+), 0 deletions(-)

diff --git a/hw/ioapic.c b/hw/ioapic.c
index c43be3a..707f2a2 100644
--- a/hw/ioapic.c
+++ b/hw/ioapic.c
@@ -72,14 +72,66 @@ static QLIST_HEAD(ioapic_eoi_client_list,
                   ioapic_eoi_client) ioapic_eoi_client_list =
                   QLIST_HEAD_INITIALIZER(ioapic_eoi_client_list);
 
+#ifdef KVM_EOI_EVENTFD
+static void ioapic_eoi_callback(void *opaque)
+{
+    ioapic_eoi_client *client = opaque;
+
+    if (!event_notifier_test_and_clear(&client->notifier)) {
+        return;
+    }
+
+    client->eoi(client);
+}
+#endif
+
 int ioapic_register_eoi_client(ioapic_eoi_client *client)
 {
     QLIST_INSERT_HEAD(&ioapic_eoi_client_list, client, list);
+
+#ifdef KVM_EOI_EVENTFD
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        int ret, fd;
+
+        ret = event_notifier_init(&client->notifier, 0);
+        if (ret) {
+            fprintf(stderr, "%s notifier init failed %d\n", __FUNCTION__, ret);
+            return ret;
+        }
+
+        fd = event_notifier_get_fd(&client->notifier);
+        qemu_set_fd_handler(fd, ioapic_eoi_callback, NULL, client);
+
+        ret = kvm_eoi_eventfd(client->irq, fd, KVM_EOI_EVENTFD_FLAG_DEASSERT);
+        if (ret) {
+            fprintf(stderr, "%s eoi eventfd failed %d\n", __FUNCTION__, ret);
+            return ret;
+        }
+        client->notifier_enabled = true;
+    }
+#endif
     return 0;
 }
 
 void ioapic_unregister_eoi_client(ioapic_eoi_client *client)
 {
+#ifdef KVM_EOI_EVENTFD
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        int ret, fd;
+
+        fd = event_notifier_get_fd(&client->notifier);
+
+        ret = kvm_eoi_eventfd(client->irq, fd, KVM_EOI_EVENTFD_FLAG_DEASSIGN);
+        if (ret) {
+            fprintf(stderr, "%s eoi eventfd failed %d\n", __FUNCTION__, ret);
+        }
+
+        qemu_set_fd_handler(fd, NULL, NULL, NULL);
+
+        event_notifier_cleanup(&client->notifier);
+        client->notifier_enabled = false;
+    }
+#endif
     QLIST_REMOVE(client, list);
 }
 

[Qemu-devel] Re: [PATCH] APIC/IOAPIC EOI callback

[Anthony Liguori]

On 10/29/2010 12:56 PM, Alex Williamson wrote:
> For device assignment, we need to know when the VM writes an end
> of interrupt to the APIC, which allows us to re-enable interrupts
> on the physical device.  Add a new wrapper for ioapic generated
> interrupts with a callback on eoi and create an interface for
> drivers to be notified on eoi.
>
> Signed-off-by: Alex Williamson<alex.williamson@redhat.com>
> ---
>
> Note that the notifier and notifier_enabled eoi_client fields aren't
> used here yet.  I'll send an RFC patch showing how we make use of
> these with the proposed KVM_EOI_EVENTFD patches.
>
>   hw/apic.c   |   18 ++++++++++++++++--
>   hw/apic.h   |    4 ++++
>   hw/ioapic.c |   38 ++++++++++++++++++++++++++++++++++++--
>   hw/pc.h     |   16 +++++++++++++++-
>   4 files changed, 71 insertions(+), 5 deletions(-)
>
> diff --git a/hw/apic.c b/hw/apic.c
> index 63d62c7..a24117b 100644
> --- a/hw/apic.c
> +++ b/hw/apic.c
> @@ -22,6 +22,7 @@
>   #include "host-utils.h"
>   #include "sysbus.h"
>   #include "trace.h"
> +#include "pc.h"
>
>   /* APIC Local Vector Table */
>   #define APIC_LVT_TIMER   0
> @@ -103,6 +104,7 @@ struct APICState {
>       int wait_for_sipi;
>   };
>
> +static uint8_t vector_to_gsi_map[256] = { 0xff };
>   static APICState *local_apics[MAX_APICS + 1];
>   static int apic_irq_delivered;
>
> @@ -292,6 +294,15 @@ void apic_deliver_irq(uint8_t dest, uint8_t dest_mode,
>                        trigger_mode);
>   }
>
> +void apic_deliver_ioapic_irq(uint8_t dest, uint8_t dest_mode,
> +                             uint8_t delivery_mode, uint8_t vector_num,
> +                             uint8_t polarity, uint8_t trigger_mode, int gsi)
> +{
> +    vector_to_gsi_map[vector_num] = gsi;
> +    apic_deliver_irq(dest, dest_mode, delivery_mode,
> +                     vector_num, polarity, trigger_mode);
> +}
> +
>   void cpu_set_apic_base(DeviceState *d, uint64_t val)
>   {
>       APICState *s = DO_UPCAST(APICState, busdev.qdev, d);
> @@ -420,8 +431,11 @@ static void apic_eoi(APICState *s)
>       if (isrv<  0)
>           return;
>       reset_bit(s->isr, isrv);
> -    /* XXX: send the EOI packet to the APIC bus to allow the I/O APIC to
> -            set the remote IRR bit for level triggered interrupts. */
> +
> +    if (vector_to_gsi_map[isrv] != 0xff) {
> +        ioapic_eoi(vector_to_gsi_map[isrv]);
> +        vector_to_gsi_map[isrv] = 0xff;
> +    }
>       apic_update_irq(s);
>   }
>
> diff --git a/hw/apic.h b/hw/apic.h
> index 8a0c9d0..59d0e37 100644
> --- a/hw/apic.h
> +++ b/hw/apic.h
> @@ -8,6 +8,10 @@ void apic_deliver_irq(uint8_t dest, uint8_t dest_mode,
>                                uint8_t delivery_mode,
>                                uint8_t vector_num, uint8_t polarity,
>                                uint8_t trigger_mode);
> +void apic_deliver_ioapic_irq(uint8_t dest, uint8_t dest_mode,
> +                             uint8_t delivery_mode,
> +                             uint8_t vector_num, uint8_t polarity,
> +                             uint8_t trigger_mode, int gsi);
>   int apic_accept_pic_intr(DeviceState *s);
>   void apic_deliver_pic_intr(DeviceState *s, int level);
>   int apic_get_interrupt(DeviceState *s);
> diff --git a/hw/ioapic.c b/hw/ioapic.c
> index 5ae21e9..ffd1c92 100644
> --- a/hw/ioapic.c
> +++ b/hw/ioapic.c
> @@ -26,6 +26,7 @@
>   #include "qemu-timer.h"
>   #include "host-utils.h"
>   #include "sysbus.h"
> +#include "qlist.h"
>
>   //#define DEBUG_IOAPIC
>
> @@ -61,6 +62,39 @@ struct IOAPICState {
>       uint64_t ioredtbl[IOAPIC_NUM_PINS];
>   };
>
> +static QLIST_HEAD(ioapic_eoi_client_list,
> +                  ioapic_eoi_client) ioapic_eoi_client_list =
> +                  QLIST_HEAD_INITIALIZER(ioapic_eoi_client_list);
> +
> +int ioapic_register_eoi_client(ioapic_eoi_client *client)
> +{
> +    QLIST_INSERT_HEAD(&ioapic_eoi_client_list, client, list);
> +    return 0;
> +}
> +
> +void ioapic_unregister_eoi_client(ioapic_eoi_client *client)
> +{
> +    QLIST_REMOVE(client, list);
> +}
> +
> +int ioapic_eoi_client_get_fd(ioapic_eoi_client *client)
> +{
> +    if (!client->notifier_enabled) {
> +        return -ENODEV;
> +    }
> +    return event_notifier_get_fd(&client->notifier);
> +}
> +
> +void ioapic_eoi(int gsi)
> +{
> +    ioapic_eoi_client *client;
> +    QLIST_FOREACH(client,&ioapic_eoi_client_list, list) {
> +        if (client->irq == gsi) {
> +            client->eoi(client);
> +        }
> +    }
> +}
>    

I think this all goes away with a NotifierList.

Regards,

Anthony Liguori

> +
>   static void ioapic_service(IOAPICState *s)
>   {
>       uint8_t i;
> @@ -90,8 +124,8 @@ static void ioapic_service(IOAPICState *s)
>                   else
>                       vector = entry&  0xff;
>
> -                apic_deliver_irq(dest, dest_mode, delivery_mode,
> -                                 vector, polarity, trig_mode);
> +                apic_deliver_ioapic_irq(dest, dest_mode, delivery_mode,
> +                                        vector, polarity, trig_mode, i);
>               }
>           }
>       }
> diff --git a/hw/pc.h b/hw/pc.h
> index 63b0249..5945bff 100644
> --- a/hw/pc.h
> +++ b/hw/pc.h
> @@ -5,6 +5,7 @@
>   #include "ioport.h"
>   #include "isa.h"
>   #include "fdc.h"
> +#include "event_notifier.h"
>
>   /* PC-style peripherals (also used by other machines).  */
>
> @@ -48,8 +49,21 @@ typedef struct isa_irq_state {
>
>   void isa_irq_handler(void *opaque, int n, int level);
>
> -/* i8254.c */
> +struct ioapic_eoi_client;
> +typedef struct ioapic_eoi_client ioapic_eoi_client;
> +struct ioapic_eoi_client {
> +    void (*eoi)(struct ioapic_eoi_client *client);
> +    int irq;
> +    EventNotifier notifier;
> +    QLIST_ENTRY(ioapic_eoi_client) list;
> +    bool notifier_enabled;
> +};
> +int ioapic_register_eoi_client(ioapic_eoi_client *client);
> +void ioapic_unregister_eoi_client(ioapic_eoi_client *client);
> +int ioapic_eoi_client_get_fd(ioapic_eoi_client *client);
> +void ioapic_eoi(int gsi);
>
> +/* i8254.c */
>   #define PIT_FREQ 1193182
>
>   typedef struct PITState PITState;
>
>    

[Qemu-devel] Re: [PATCH] APIC/IOAPIC EOI callback

[Alex Williamson]

On Fri, 2010-10-29 at 14:19 -0500, Anthony Liguori wrote:
> On 10/29/2010 12:56 PM, Alex Williamson wrote:
> > For device assignment, we need to know when the VM writes an end
> > of interrupt to the APIC, which allows us to re-enable interrupts
> > on the physical device.  Add a new wrapper for ioapic generated
> > interrupts with a callback on eoi and create an interface for
> > drivers to be notified on eoi.
> >
> > Signed-off-by: Alex Williamson<alex.williamson@redhat.com>
> > ---
> >
> > Note that the notifier and notifier_enabled eoi_client fields aren't
> > used here yet.  I'll send an RFC patch showing how we make use of
> > these with the proposed KVM_EOI_EVENTFD patches.
> >
> >   hw/apic.c   |   18 ++++++++++++++++--
> >   hw/apic.h   |    4 ++++
> >   hw/ioapic.c |   38 ++++++++++++++++++++++++++++++++++++--
> >   hw/pc.h     |   16 +++++++++++++++-
> >   4 files changed, 71 insertions(+), 5 deletions(-)
> >
> > diff --git a/hw/apic.c b/hw/apic.c
> > index 63d62c7..a24117b 100644
> > --- a/hw/apic.c
> > +++ b/hw/apic.c
> > @@ -22,6 +22,7 @@
> >   #include "host-utils.h"
> >   #include "sysbus.h"
> >   #include "trace.h"
> > +#include "pc.h"
> >
> >   /* APIC Local Vector Table */
> >   #define APIC_LVT_TIMER   0
> > @@ -103,6 +104,7 @@ struct APICState {
> >       int wait_for_sipi;
> >   };
> >
> > +static uint8_t vector_to_gsi_map[256] = { 0xff };
> >   static APICState *local_apics[MAX_APICS + 1];
> >   static int apic_irq_delivered;
> >
> > @@ -292,6 +294,15 @@ void apic_deliver_irq(uint8_t dest, uint8_t dest_mode,
> >                        trigger_mode);
> >   }
> >
> > +void apic_deliver_ioapic_irq(uint8_t dest, uint8_t dest_mode,
> > +                             uint8_t delivery_mode, uint8_t vector_num,
> > +                             uint8_t polarity, uint8_t trigger_mode, int gsi)
> > +{
> > +    vector_to_gsi_map[vector_num] = gsi;
> > +    apic_deliver_irq(dest, dest_mode, delivery_mode,
> > +                     vector_num, polarity, trigger_mode);
> > +}
> > +
> >   void cpu_set_apic_base(DeviceState *d, uint64_t val)
> >   {
> >       APICState *s = DO_UPCAST(APICState, busdev.qdev, d);
> > @@ -420,8 +431,11 @@ static void apic_eoi(APICState *s)
> >       if (isrv<  0)
> >           return;
> >       reset_bit(s->isr, isrv);
> > -    /* XXX: send the EOI packet to the APIC bus to allow the I/O APIC to
> > -            set the remote IRR bit for level triggered interrupts. */
> > +
> > +    if (vector_to_gsi_map[isrv] != 0xff) {
> > +        ioapic_eoi(vector_to_gsi_map[isrv]);
> > +        vector_to_gsi_map[isrv] = 0xff;
> > +    }
> >       apic_update_irq(s);
> >   }
> >
> > diff --git a/hw/apic.h b/hw/apic.h
> > index 8a0c9d0..59d0e37 100644
> > --- a/hw/apic.h
> > +++ b/hw/apic.h
> > @@ -8,6 +8,10 @@ void apic_deliver_irq(uint8_t dest, uint8_t dest_mode,
> >                                uint8_t delivery_mode,
> >                                uint8_t vector_num, uint8_t polarity,
> >                                uint8_t trigger_mode);
> > +void apic_deliver_ioapic_irq(uint8_t dest, uint8_t dest_mode,
> > +                             uint8_t delivery_mode,
> > +                             uint8_t vector_num, uint8_t polarity,
> > +                             uint8_t trigger_mode, int gsi);
> >   int apic_accept_pic_intr(DeviceState *s);
> >   void apic_deliver_pic_intr(DeviceState *s, int level);
> >   int apic_get_interrupt(DeviceState *s);
> > diff --git a/hw/ioapic.c b/hw/ioapic.c
> > index 5ae21e9..ffd1c92 100644
> > --- a/hw/ioapic.c
> > +++ b/hw/ioapic.c
> > @@ -26,6 +26,7 @@
> >   #include "qemu-timer.h"
> >   #include "host-utils.h"
> >   #include "sysbus.h"
> > +#include "qlist.h"
> >
> >   //#define DEBUG_IOAPIC
> >
> > @@ -61,6 +62,39 @@ struct IOAPICState {
> >       uint64_t ioredtbl[IOAPIC_NUM_PINS];
> >   };
> >
> > +static QLIST_HEAD(ioapic_eoi_client_list,
> > +                  ioapic_eoi_client) ioapic_eoi_client_list =
> > +                  QLIST_HEAD_INITIALIZER(ioapic_eoi_client_list);
> > +
> > +int ioapic_register_eoi_client(ioapic_eoi_client *client)
> > +{
> > +    QLIST_INSERT_HEAD(&ioapic_eoi_client_list, client, list);
> > +    return 0;
> > +}
> > +
> > +void ioapic_unregister_eoi_client(ioapic_eoi_client *client)
> > +{
> > +    QLIST_REMOVE(client, list);
> > +}
> > +
> > +int ioapic_eoi_client_get_fd(ioapic_eoi_client *client)
> > +{
> > +    if (!client->notifier_enabled) {
> > +        return -ENODEV;
> > +    }
> > +    return event_notifier_get_fd(&client->notifier);
> > +}
> > +
> > +void ioapic_eoi(int gsi)
> > +{
> > +    ioapic_eoi_client *client;
> > +    QLIST_FOREACH(client,&ioapic_eoi_client_list, list) {
> > +        if (client->irq == gsi) {
> > +            client->eoi(client);
> > +        }
> > +    }
> > +}
> >    
> 
> I think this all goes away with a NotifierList.

Are you thinking of an array of 256 NotifierLists, one per irq?  I'm not
sure how else I could keep track of which Notifier is associated with
which irq.  That seems a little excessive unless we start getting a lot
more users of this callback.  Thanks,

Alex