[Qemu-devel] [PATCH] qxl: async I/O

[Alon Levy <alevy@redhat.com>]

From: Gerd Hoffmann <kraxel@redhat.com>

Some of the QXL port i/o commands are waiting for the spice server to
complete certain actions.  Add async versions for these commands, so we
don't block the vcpu while the spice server processses the command.
Instead the qxl device will raise an IRQ when done.

The async command processing relies on an added QXLInterface::async_complete.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
Signed-off-by: Alon Levy     <alevy@redhat.com>
---
 hw/qxl.c           |  130 ++++++++++++++++++++++++++++++++++++++++++++--------
 hw/qxl.h           |   11 ++++
 ui/spice-display.c |   41 ++++++++++++++---
 ui/spice-display.h |    4 ++
 4 files changed, 159 insertions(+), 27 deletions(-)

diff --git a/hw/qxl.c b/hw/qxl.c
index a5c2d69..54e28b9 100644
--- a/hw/qxl.c
+++ b/hw/qxl.c
@@ -151,29 +151,51 @@ void qxl_guest_bug(PCIQXLDevice *qxl, const char *msg, ...)
     }
 }
 
+void qxl_spice_update_area_async(PCIQXLDevice *qxl, uint32_t surface_id,
+                           struct QXLRect *area, struct QXLRect *dirty_rects,
+                           uint32_t num_dirty_rects, uint32_t clear_dirty_region,
+                           int async)
+{
+    qemu_mutex_lock(&qxl->ssd.wlock);
+    if (async) {
+        qxl->ssd.worker->update_area_async(qxl->ssd.worker, surface_id, area, dirty_rects,
+                                 num_dirty_rects, clear_dirty_region);
+    } else {
+        qxl->ssd.worker->update_area(qxl->ssd.worker, surface_id, area, dirty_rects,
+                                 num_dirty_rects, clear_dirty_region);
+    }
+    qemu_mutex_unlock(&qxl->ssd.wlock);
+}
 
 void qxl_spice_update_area(PCIQXLDevice *qxl, uint32_t surface_id,
                            struct QXLRect *area, struct QXLRect *dirty_rects,
                            uint32_t num_dirty_rects, uint32_t clear_dirty_region)
 {
-    qemu_mutex_lock(&qxl->ssd.wlock);
-    qxl->ssd.worker->update_area(qxl->ssd.worker, surface_id, area, dirty_rects,
-                             num_dirty_rects, clear_dirty_region);
-    qemu_mutex_unlock(&qxl->ssd.wlock);
+    qxl_spice_update_area_async(qxl, surface_id, area, dirty_rects, num_dirty_rects,
+                          clear_dirty_region, 0);
 }
 
-void qxl_spice_destroy_surface_wait(PCIQXLDevice *qxl, uint32_t id)
+void qxl_spice_destroy_surface_wait_async(PCIQXLDevice *qxl, uint32_t id, int async)
 {
     qemu_mutex_lock(&qxl->ssd.wlock);
     qemu_mutex_lock(&qxl->track_lock);
     PANIC_ON(id >= NUM_SURFACES);
-    qxl->ssd.worker->destroy_surface_wait(qxl->ssd.worker, id);
+    if (async) {
+        qxl->ssd.worker->destroy_surface_wait_async(qxl->ssd.worker, id);
+    } else {
+        qxl->ssd.worker->destroy_surface_wait(qxl->ssd.worker, id);
+    }
     qxl->guest_surfaces.cmds[id] = 0;
     qxl->guest_surfaces.count--;
     qemu_mutex_unlock(&qxl->track_lock);
     qemu_mutex_unlock(&qxl->ssd.wlock);
 }
 
+void qxl_spice_destroy_surface_wait(PCIQXLDevice *qxl, uint32_t id)
+{
+    qxl_spice_destroy_surface_wait_async(qxl, id, 0);
+}
+
 void qxl_spice_loadvm_commands(PCIQXLDevice *qxl, struct QXLCommandExt *ext,
                                uint32_t count)
 {
@@ -182,13 +204,22 @@ void qxl_spice_loadvm_commands(PCIQXLDevice *qxl, struct QXLCommandExt *ext,
     qemu_mutex_unlock(&qxl->ssd.wlock);
 }
 
-void qxl_spice_oom(PCIQXLDevice *qxl)
+void qxl_spice_oom_async(PCIQXLDevice *qxl, int async)
 {
     qemu_mutex_lock(&qxl->ssd.wlock);
-    qxl->ssd.worker->oom(qxl->ssd.worker);
+    if (async) {
+        qxl->ssd.worker->oom_async(qxl->ssd.worker);
+    } else {
+        qxl->ssd.worker->oom(qxl->ssd.worker);
+    }
     qemu_mutex_unlock(&qxl->ssd.wlock);
 }
 
+void qxl_spice_oom(PCIQXLDevice *qxl)
+{
+    qxl_spice_oom_async(qxl, 0);
+}
+
 void qxl_spice_reset_memslots(PCIQXLDevice *qxl)
 {
     qemu_mutex_lock(&qxl->ssd.wlock);
@@ -196,17 +227,26 @@ void qxl_spice_reset_memslots(PCIQXLDevice *qxl)
     qemu_mutex_unlock(&qxl->ssd.wlock);
 }
 
-void qxl_spice_destroy_surfaces(PCIQXLDevice *qxl)
+void qxl_spice_destroy_surfaces_async(PCIQXLDevice *qxl, int async)
 {
     qemu_mutex_lock(&qxl->ssd.wlock);
     qemu_mutex_lock(&qxl->track_lock);
-    qxl->ssd.worker->destroy_surfaces(qxl->ssd.worker);
+    if (async) {
+        qxl->ssd.worker->destroy_surfaces_async(qxl->ssd.worker);
+    } else {
+        qxl->ssd.worker->destroy_surfaces(qxl->ssd.worker);
+    }
     memset(&qxl->guest_surfaces.cmds, 0, sizeof(qxl->guest_surfaces.cmds));
     qxl->guest_surfaces.count = 0;
     qemu_mutex_unlock(&qxl->track_lock);
     qemu_mutex_unlock(&qxl->ssd.wlock);
 }
 
+void qxl_spice_destroy_surfaces(PCIQXLDevice *qxl)
+{
+    qxl_spice_destroy_surfaces_async(qxl, 0);
+}
+
 void qxl_spice_reset_image_cache(PCIQXLDevice *qxl)
 {
     qemu_mutex_lock(&qxl->ssd.wlock);
@@ -739,6 +779,16 @@ static int interface_flush_resources(QXLInstance *sin)
     return ret;
 }
 
+/* called from spice server thread context only */
+static void interface_async_complete(QXLInstance *sin)
+{
+    PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
+
+    dprint(qxl, 1, "async_complete: %d done\n", qxl->current_async);
+    qxl->current_async = QXL_UNDEFINED_IO;
+    qxl_send_events(qxl, QXL_INTERRUPT_IO_CMD);
+}
+
 static const QXLInterface qxl_interface = {
     .base.type               = SPICE_INTERFACE_QXL,
     .base.description        = "qxl gpu",
@@ -758,6 +808,7 @@ static const QXLInterface qxl_interface = {
     .req_cursor_notification = interface_req_cursor_notification,
     .notify_update           = interface_notify_update,
     .flush_resources         = interface_flush_resources,
+    .async_complete          = interface_async_complete,
 };
 
 static void qxl_enter_vga_mode(PCIQXLDevice *d)
@@ -1090,13 +1141,16 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
 {
     PCIQXLDevice *d = opaque;
     uint32_t io_port = addr - d->io_base;
+    int async = 0;
 
     switch (io_port) {
     case QXL_IO_RESET:
     case QXL_IO_SET_MODE:
     case QXL_IO_MEMSLOT_ADD:
+    case QXL_IO_MEMSLOT_ADD_ASYNC:
     case QXL_IO_MEMSLOT_DEL:
     case QXL_IO_CREATE_PRIMARY:
+    case QXL_IO_CREATE_PRIMARY_ASYNC:
     case QXL_IO_UPDATE_IRQ:
     case QXL_IO_LOG:
         break;
@@ -1105,15 +1159,40 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
             break;
         dprint(d, 1, "%s: unexpected port 0x%x (%s) in vga mode\n",
             __FUNCTION__, io_port, io_port_to_string(io_port));
+        /* be nice to buggy guest drivers */
+        if (io_port >= QXL_IO_UPDATE_AREA_ASYNC &&
+            io_port <= QXL_IO_DESTROY_ALL_SURFACES_ASYNC) {
+            qxl_send_events(d, QXL_INTERRUPT_IO_CMD);
+        }
         return;
     }
 
     switch (io_port) {
+    case QXL_IO_UPDATE_AREA_ASYNC:
+    case QXL_IO_NOTIFY_OOM_ASYNC:
+    case QXL_IO_MEMSLOT_ADD_ASYNC:
+    case QXL_IO_CREATE_PRIMARY_ASYNC:
+    case QXL_IO_DESTROY_PRIMARY_ASYNC:
+    case QXL_IO_DESTROY_SURFACE_ASYNC:
+    case QXL_IO_DESTROY_ALL_SURFACES_ASYNC:
+        async = 1;
+        if (d->current_async != QXL_UNDEFINED_IO) {
+            qxl_guest_bug(d, "%d async started before last (%d) complete\n",
+                io_port, d->current_async);
+        }
+        d->current_async = io_port;
+        dprint(d, 1, "start async %d\n", d->current_async);
+        break;
+    default:
+        break;
+    }
+
+    switch (io_port) {
+    case QXL_IO_UPDATE_AREA_ASYNC:
     case QXL_IO_UPDATE_AREA:
     {
-        QXLRect update = d->ram->update_area;
-        qxl_spice_update_area(d, d->ram->update_surface,
-                              &update, NULL, 0, 0);
+        qxl_spice_update_area_async(d, d->ram->update_surface,
+                              &d->ram->update_area, NULL, 0, 0, async);
         break;
     }
     case QXL_IO_NOTIFY_CMD:
@@ -1125,6 +1204,7 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
     case QXL_IO_UPDATE_IRQ:
         qxl_set_irq(d);
         break;
+    case QXL_IO_NOTIFY_OOM_ASYNC:
     case QXL_IO_NOTIFY_OOM:
         if (!SPICE_RING_IS_EMPTY(&d->ram->release_ring)) {
             break;
@@ -1134,7 +1214,7 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
             break;
         }
         d->oom_running = 1;
-        qxl_spice_oom(d);
+        qxl_spice_oom_async(d, async);
         d->oom_running = 0;
         break;
     case QXL_IO_SET_MODE:
@@ -1151,6 +1231,7 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
         dprint(d, 1, "QXL_IO_RESET\n");
         qxl_hard_reset(d, 0);
         break;
+    case QXL_IO_MEMSLOT_ADD_ASYNC:
     case QXL_IO_MEMSLOT_ADD:
     {
         QXLDevMemSlot memslot;
@@ -1165,7 +1246,7 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
         }
         d->guest_slots[val].slot = d->ram->mem_slot;
         qxl_add_memslot(d, val, 0, &memslot);
-        qemu_spice_add_memslot(&d->ssd, &memslot);
+        qemu_spice_add_memslot_async(&d->ssd, &memslot, async);
         break;
     }
     case QXL_IO_MEMSLOT_DEL:
@@ -1175,6 +1256,7 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
         }
         qxl_del_memslot(d, val);
         break;
+    case QXL_IO_CREATE_PRIMARY_ASYNC:
     case QXL_IO_CREATE_PRIMARY:
     {
         QXLDevSurfaceCreate surface;
@@ -1183,12 +1265,13 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
             qxl_guest_bug(d, "QXL_IO_CREATE_PRIMARY: val != 0\n");
             break;
         }
-        dprint(d, 1, "QXL_IO_CREATE_PRIMARY\n");
+        dprint(d, 1, "QXL_IO_CREATE_PRIMARY async=%d\n", async);
         d->guest_primary.surface = d->ram->create_surface;
         qxl_create_guest_primary(d, 0, &surface);
-        qemu_spice_create_primary_surface(&d->ssd, 0, &surface);
+        qemu_spice_create_primary_surface_async(&d->ssd, 0, &surface, async);
         break;
     }
+    case QXL_IO_DESTROY_PRIMARY_ASYNC:
     case QXL_IO_DESTROY_PRIMARY:
         if (val != 0) {
             qxl_guest_bug(d, "QXL_IO_DESTROY_PRIMARY: val != 0\n");
@@ -1197,14 +1280,21 @@ static void ioport_write(void *opaque, uint32_t addr, uint32_t val)
         dprint(d, 1, "QXL_IO_DESTROY_PRIMARY (%s)\n", qxl_mode_to_string(d->mode));
         if (d->mode != QXL_MODE_UNDEFINED) {
             d->mode = QXL_MODE_UNDEFINED;
-            qemu_spice_destroy_primary_surface(&d->ssd, 0);
+            qemu_spice_destroy_primary_surface_async(&d->ssd, 0, async);
+        } else {
+            if (async) {
+                qxl_send_events(d, QXL_INTERRUPT_IO_CMD);
+            }
         }
         break;
+    case QXL_IO_DESTROY_SURFACE_ASYNC:
     case QXL_IO_DESTROY_SURFACE_WAIT:
-        qxl_spice_destroy_surface_wait(d, val);
+        qxl_spice_destroy_surface_wait_async(d, val, async);
         break;
+    case QXL_IO_DESTROY_ALL_SURFACES_ASYNC:
     case QXL_IO_DESTROY_ALL_SURFACES:
-        qxl_spice_destroy_surfaces(d);
+        d->mode = QXL_MODE_UNDEFINED;
+        qxl_spice_destroy_surfaces_async(d, async);
         break;
     default:
         fprintf(stderr, "%s: ioport=0x%x, abort()\n", __FUNCTION__, io_port);
diff --git a/hw/qxl.h b/hw/qxl.h
index e361bc6..e4134b9 100644
--- a/hw/qxl.h
+++ b/hw/qxl.h
@@ -15,6 +15,8 @@ enum qxl_mode {
     QXL_MODE_NATIVE,
 };
 
+#define QXL_UNDEFINED_IO UINT32_MAX
+
 typedef struct PCIQXLDevice {
     PCIDevice          pci;
     SimpleSpiceDisplay ssd;
@@ -30,6 +32,8 @@ typedef struct PCIQXLDevice {
     int32_t            num_memslots;
     int32_t            num_surfaces;
 
+    uint32_t           current_async;
+
     struct guest_slots {
         QXLMemSlot     slot;
         void           *ptr;
@@ -104,12 +108,19 @@ void qxl_guest_bug(PCIQXLDevice *qxl, const char *msg, ...);
 void qxl_spice_update_area(PCIQXLDevice *qxl, uint32_t surface_id,
                            struct QXLRect *area, struct QXLRect *dirty_rects,
                            uint32_t num_dirty_rects, uint32_t clear_dirty_region);
+void qxl_spice_update_area_async(PCIQXLDevice *qxl, uint32_t surface_id,
+                           struct QXLRect *area, struct QXLRect *dirty_rects,
+                           uint32_t num_dirty_rects, uint32_t clear_dirty_region,
+                           int async);
 void qxl_spice_destroy_surface_wait(PCIQXLDevice *qxl, uint32_t id);
+void qxl_spice_destroy_surface_wait_async(PCIQXLDevice *qxl, uint32_t id, int async);
 void qxl_spice_loadvm_commands(PCIQXLDevice *qxl, struct QXLCommandExt *ext,
                                uint32_t count);
 void qxl_spice_oom(PCIQXLDevice *qxl);
+void qxl_spice_oom_async(PCIQXLDevice *qxl, int async);
 void qxl_spice_reset_memslots(PCIQXLDevice *qxl);
 void qxl_spice_destroy_surfaces(PCIQXLDevice *qxl);
+void qxl_spice_destroy_surfaces_async(PCIQXLDevice *qxl, int async);
 void qxl_spice_reset_image_cache(PCIQXLDevice *qxl);
 void qxl_spice_reset_cursor(PCIQXLDevice *qxl);
 
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 5bd7460..8cbdd0a 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -62,14 +62,22 @@ void qemu_spice_rect_union(QXLRect *dest, const QXLRect *r)
     dest->right = MAX(dest->right, r->right);
 }
 
-
-void qemu_spice_add_memslot(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot)
+void qemu_spice_add_memslot_async(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot, int async)
 {
     qemu_mutex_lock(&ssd->wlock);
-    ssd->worker->add_memslot(ssd->worker, memslot);
+    if (async) {
+        ssd->worker->add_memslot_async(ssd->worker, memslot);
+    } else {
+        ssd->worker->add_memslot(ssd->worker, memslot);
+    }
     qemu_mutex_unlock(&ssd->wlock);
 }
 
+void qemu_spice_add_memslot(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot)
+{
+    qemu_spice_add_memslot_async(ssd, memslot, 0);
+}
+
 void qemu_spice_del_memslot(SimpleSpiceDisplay *ssd, uint32_t gid, uint32_t sid)
 {
     qemu_mutex_lock(&ssd->wlock);
@@ -77,19 +85,38 @@ void qemu_spice_del_memslot(SimpleSpiceDisplay *ssd, uint32_t gid, uint32_t sid)
     qemu_mutex_unlock(&ssd->wlock);
 }
 
+void qemu_spice_create_primary_surface_async(SimpleSpiceDisplay *ssd, uint32_t id,
+                                       QXLDevSurfaceCreate *surface, int async)
+{
+    qemu_mutex_lock(&ssd->wlock);
+    if (async) {
+        ssd->worker->create_primary_surface_async(ssd->worker, id, surface);
+    } else {
+        ssd->worker->create_primary_surface(ssd->worker, id, surface);
+    }
+    qemu_mutex_unlock(&ssd->wlock);
+}
+
 void qemu_spice_create_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id,
                                        QXLDevSurfaceCreate *surface)
 {
+    qemu_spice_create_primary_surface_async(ssd, id, surface, 0);
+}
+
+void qemu_spice_destroy_primary_surface_async(SimpleSpiceDisplay *ssd, uint32_t id, int async)
+{
     qemu_mutex_lock(&ssd->wlock);
-    ssd->worker->create_primary_surface(ssd->worker, id, surface);
+    if (async) {
+        ssd->worker->destroy_primary_surface_async(ssd->worker, id);
+    } else {
+        ssd->worker->destroy_primary_surface(ssd->worker, id);
+    }
     qemu_mutex_unlock(&ssd->wlock);
 }
 
 void qemu_spice_destroy_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id)
 {
-    qemu_mutex_lock(&ssd->wlock);
-    ssd->worker->destroy_primary_surface(ssd->worker, id);
-    qemu_mutex_unlock(&ssd->wlock);
+    qemu_spice_destroy_primary_surface_async(ssd, id, 0);
 }
 
 void qemu_spice_wakeup(SimpleSpiceDisplay *ssd)
diff --git a/ui/spice-display.h b/ui/spice-display.h
index e84847f..5696e0c 100644
--- a/ui/spice-display.h
+++ b/ui/spice-display.h
@@ -84,10 +84,14 @@ void qemu_spice_display_resize(SimpleSpiceDisplay *ssd);
 void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd);
 
 void qemu_spice_add_memslot(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot);
+void qemu_spice_add_memslot_async(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot, int async);
 void qemu_spice_del_memslot(SimpleSpiceDisplay *ssd, uint32_t gid, uint32_t sid);
 void qemu_spice_create_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id,
                                        QXLDevSurfaceCreate *surface);
+void qemu_spice_create_primary_surface_async(SimpleSpiceDisplay *ssd, uint32_t id,
+                                       QXLDevSurfaceCreate *surface, int async);
 void qemu_spice_destroy_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id);
+void qemu_spice_destroy_primary_surface_async(SimpleSpiceDisplay *ssd, uint32_t id, int async);
 void qemu_spice_wakeup(SimpleSpiceDisplay *ssd);
 void qemu_spice_start(SimpleSpiceDisplay *ssd);
 void qemu_spice_stop(SimpleSpiceDisplay *ssd);
-- 
1.7.5.4