[PATCH v2 08/15] libxl: convert libxl_device_disk_add to an asyn op

[Roger Pau Monne <roger.pau@citrix.com>]

This patch converts libxl_device_disk_add to an ao operation that
waits for device backend to reach state XenbusStateInitWait and then
marks the operation as completed. This is not really useful now, but
will be used by latter patches that will launch hotplug scripts after
we reached the desired xenbus state.

As usual, libxl_device_disk_add callers have been modified, and the
internal function libxl__device_disk_add has been used if the call was
inside an already running ao.

Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
---
 tools/libxl/Makefile         |    2 +-
 tools/libxl/libxl.c          |   50 ++++++++++++++++++++++++++--------
 tools/libxl/libxl.h          |    4 ++-
 tools/libxl/libxl_create.c   |   41 ++++++++++++++++++++++------
 tools/libxl/libxl_device.c   |   57 +++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_dm.c       |   61 +++++++++++++++++++++++++++++++----------
 tools/libxl/libxl_internal.h |   26 ++++++++++++++++++
 tools/libxl/xl_cmdimpl.c     |    2 +-
 8 files changed, 204 insertions(+), 39 deletions(-)

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 5d9227e..81b467e 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -13,7 +13,7 @@ XLUMINOR = 0
 
 CFLAGS += -Werror -Wno-format-zero-length -Wmissing-declarations \
 	-Wno-declaration-after-statement -Wformat-nonliteral
-CFLAGS += -I. -fPIC
+CFLAGS += -I. -fPIC -O0
 
 ifeq ($(CONFIG_Linux),y)
 LIBUUID_LIBS += -luuid
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index b44ae75..46d13a7 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -1480,13 +1480,31 @@ static int libxl__device_from_disk(libxl__gc *gc, uint32_t domid,
     return 0;
 }
 
-int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk)
+int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid,
+                          libxl_device_disk *disk,
+                          const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
+    AO_CREATE(ctx, domid, ao_how);
+    libxl__ao_device *device;
+
+    GCNEW(device);
+    libxl__init_ao_device(device, ao, NULL);
+    device->callback = libxl__device_cb;
+    libxl__device_disk_add(egc, domid, disk, device);
+
+    return AO_INPROGRESS;
+}
+
+void libxl__device_disk_add(libxl__egc *egc, uint32_t domid,
+                            libxl_device_disk *disk,
+                            libxl__ao_device *aoadd)
+{
+    STATE_AO_GC(aoadd->ao);
+    libxl_ctx *ctx = CTX;
     flexarray_t *front;
     flexarray_t *back;
     char *dev;
-    libxl__device device;
+    libxl__device *device;
     int major, minor, rc;
 
     rc = libxl__device_disk_setdefault(gc, disk);
@@ -1510,7 +1528,8 @@ int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *dis
         goto out_free;
     }
 
-    rc = libxl__device_from_disk(gc, domid, disk, &device);
+    GCNEW(device);
+    rc = libxl__device_from_disk(gc, domid, disk, device);
     if (rc != 0) {
         LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "Invalid or unsupported"
                " virtual disk identifier %s", disk->vdev);
@@ -1528,7 +1547,7 @@ int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *dis
             flexarray_append(back, "params");
             flexarray_append(back, dev);
 
-            assert(device.backend_kind == LIBXL__DEVICE_KIND_VBD);
+            assert(device->backend_kind == LIBXL__DEVICE_KIND_VBD);
             break;
         case LIBXL_DISK_BACKEND_TAP:
             dev = libxl__blktap_devpath(gc, disk->pdev_path, disk->format);
@@ -1549,7 +1568,7 @@ int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *dis
             flexarray_append(back, "params");
             flexarray_append(back, libxl__sprintf(gc, "%s:%s",
                           libxl__device_disk_string_of_format(disk->format), disk->pdev_path));
-            assert(device.backend_kind == LIBXL__DEVICE_KIND_QDISK);
+            assert(device->backend_kind == LIBXL__DEVICE_KIND_QDISK);
             break;
         default:
             LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "unrecognized disk backend type: %d\n", disk->backend);
@@ -1581,22 +1600,27 @@ int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *dis
     flexarray_append(front, "state");
     flexarray_append(front, libxl__sprintf(gc, "%d", 1));
     flexarray_append(front, "virtual-device");
-    flexarray_append(front, libxl__sprintf(gc, "%d", device.devid));
+    flexarray_append(front, libxl__sprintf(gc, "%d", device->devid));
     flexarray_append(front, "device-type");
     flexarray_append(front, disk->is_cdrom ? "cdrom" : "disk");
 
-    libxl__device_generic_add(gc, &device,
+    libxl__device_generic_add(gc, device,
                              libxl__xs_kvs_of_flexarray(gc, back, back->count),
                              libxl__xs_kvs_of_flexarray(gc, front, front->count));
 
+    aoadd->dev = device;
+    aoadd->action = DEVICE_CONNECT;
+    libxl__initiate_device_add(egc, aoadd);
+
     rc = 0;
 
 out_free:
     flexarray_free(back);
     flexarray_free(front);
 out:
-    GC_FREE;
-    return rc;
+    aoadd->rc = rc;
+    if (rc) aoadd->callback(egc, aoadd);
+    return;
 }
 
 int libxl_device_disk_remove(libxl_ctx *ctx, uint32_t domid,
@@ -1850,11 +1874,13 @@ int libxl_cdrom_insert(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk)
     ret = 0;
 
     libxl_device_disk_remove(ctx, domid, disks + i, 0);
-    libxl_device_disk_add(ctx, domid, disk);
+    /* fixme-ao */
+    libxl_device_disk_add(ctx, domid, disk, 0);
     stubdomid = libxl_get_stubdom_id(ctx, domid);
     if (stubdomid) {
         libxl_device_disk_remove(ctx, stubdomid, disks + i, 0);
-        libxl_device_disk_add(ctx, stubdomid, disk);
+        /* fixme-ao */
+        libxl_device_disk_add(ctx, stubdomid, disk, 0);
     }
 out:
     for (i = 0; i < num; i++)
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 39a8c58..c14e832 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -663,7 +663,9 @@ void libxl_vminfo_list_free(libxl_vminfo *list, int nr);
  */
 
 /* Disks */
-int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk);
+int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid,
+                          libxl_device_disk *disk,
+                          const libxl_asyncop_how *ao_how);
 int libxl_device_disk_remove(libxl_ctx *ctx, uint32_t domid,
                              libxl_device_disk *disk,
                              const libxl_asyncop_how *ao_how);
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 63f34fa..4378f48 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -575,6 +575,8 @@ static void domcreate_bootloader_done(libxl__egc *egc,
                                       libxl__bootloader_state *bl,
                                       int rc);
 
+static void domcreate_disk_connected(libxl__egc *egc, libxl__ao_device *aorm);
+
 static void domcreate_console_available(libxl__egc *egc,
                                         libxl__domain_create_state *dcs);
 
@@ -670,7 +672,6 @@ static void domcreate_bootloader_done(libxl__egc *egc,
 {
     libxl__domain_create_state *dcs = CONTAINER_OF(bl, *dcs, bl);
     STATE_AO_GC(bl->ao);
-    int i;
 
     /* convenience aliases */
     const uint32_t domid = dcs->guest_domid;
@@ -704,15 +705,37 @@ static void domcreate_bootloader_done(libxl__egc *egc,
 
     store_libxl_entry(gc, domid, &d_config->b_info);
 
-    for (i = 0; i < d_config->num_disks; i++) {
-        ret = libxl_device_disk_add(ctx, domid, &d_config->disks[i]);
-        if (ret) {
-            LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
-                       "cannot add disk %d to domain: %d", i, ret);
-            ret = ERROR_FAIL;
-            goto error_out;
-        }
+    dcs->num_devices = d_config->num_disks;
+    libxl__add_disks(egc, ao, domid, d_config, &dcs->devices,
+                     domcreate_disk_connected);
+
+    return;
+
+ error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
+static void domcreate_disk_connected(libxl__egc *egc, libxl__ao_device *aorm)
+{
+    STATE_AO_GC(aorm->ao);
+    libxl__domain_create_state *dcs = CONTAINER_OF(aorm->base, *dcs, devices);
+    int i, last, ret = 0;
+
+    /* convenience aliases */
+    const uint32_t domid = dcs->guest_domid;
+    libxl_domain_config *const d_config = dcs->guest_config;
+    libxl__domain_build_state *const state = &dcs->build_state;
+    libxl_ctx *const ctx = CTX;
+
+    ret = libxl__ao_device_check_last(gc, aorm, dcs->devices,
+                                      dcs->num_devices, &last);
+    if (!last) return;
+    if (ret) {
+        LOG(ERROR, "error connecting disk devices");
+        goto error_out;
     }
+
     for (i = 0; i < d_config->num_vifs; i++) {
         ret = libxl_device_nic_add(ctx, domid, &d_config->vifs[i]);
         if (ret) {
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 020e934..8058afa 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -426,6 +426,24 @@ int libxl__ao_device_check_last(libxl__gc *gc, libxl__ao_device *device,
     return ret;
 }
 
+void libxl__add_disks(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
+                      libxl_domain_config *d_config,
+                      libxl__ao_device **devices,
+                      libxl__device_callback callback)
+{
+    AO_GC;
+    GCNEW_ARRAY(*devices, d_config->num_disks);
+    libxl__ao_device *aodev = *devices;
+    int i;
+
+    for (i = 0; i < d_config->num_disks; i++) {
+        libxl__init_ao_device(&aodev[i], ao, devices);
+        aodev[i].callback = callback;
+        libxl__device_disk_add(egc, domid, &d_config->disks[i],
+                               &aodev[i]);
+    }
+}
+
 int libxl__device_destroy(libxl__gc *gc, libxl__device *dev)
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
@@ -533,6 +551,45 @@ static void device_backend_callback(libxl__egc *egc, libxl__ev_devstate *ds,
 
 static void device_backend_cleanup(libxl__gc *gc, libxl__ao_device *aodev);
 
+void libxl__initiate_device_add(libxl__egc *egc, libxl__ao_device *aoadd)
+{
+    STATE_AO_GC(aoadd->ao);
+    char *be_path = libxl__device_backend_path(gc, aoadd->dev);
+    char *state_path = libxl__sprintf(gc, "%s/state", be_path);
+    char *state = libxl__xs_read(gc, XBT_NULL, state_path);
+    int rc = 0;
+
+    if (aoadd->dev->backend_kind == LIBXL__DEVICE_KIND_QDISK) {
+        aoadd->callback(egc, aoadd);
+        return;
+    }
+
+    if (atoi(state) == XenbusStateInitWait)
+        goto out_ok;
+
+    libxl__ev_devstate_init(&aoadd->ds);
+    rc = libxl__ev_devstate_wait(gc, &aoadd->ds, device_backend_callback,
+                                 state_path, XenbusStateInitWait,
+                                 LIBXL_INIT_TIMEOUT * 1000);
+    if (rc) {
+        LOGE(ERROR, "unable to initialize device %s", be_path);
+        goto out_fail;
+    }
+
+    return;
+
+out_fail:
+    assert(rc);
+    aoadd->rc = rc;
+    aoadd->callback(egc, aoadd);
+    return;
+
+out_ok:
+    assert(!rc);
+    aoadd->callback(egc, aoadd);
+    return;
+}
+
 void libxl__initiate_device_remove(libxl__egc *egc,
                                    libxl__ao_device *aorm)
 {
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 6604549..670d1dd 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -673,6 +673,8 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,
                                 libxl__dm_spawn_state *stubdom_dmss,
                                 int rc);
 
+static void spawn_stub_disk_connected(libxl__egc *egc, libxl__ao_device *aorm);
+
 static void spaw_stubdom_pvqemu_destroy_cb(libxl__egc *egc,
                                            libxl__destroy_domid_state *dis,
                                            int rc);
@@ -681,8 +683,7 @@ void libxl__spawn_stub_dm(libxl__egc *egc, libxl__stub_dm_spawn_state *sdss)
 {
     STATE_AO_GC(sdss->dm.spawn.ao);
     libxl_ctx *ctx = libxl__gc_owner(gc);
-    int i, num_console = STUBDOM_SPECIAL_CONSOLES, ret;
-    libxl__device_console *console;
+    int ret;
     libxl_device_vfb vfb;
     libxl_device_vkb vkb;
     char **args;
@@ -800,22 +801,55 @@ retry_transaction:
         if (errno == EAGAIN)
             goto retry_transaction;
 
-    for (i = 0; i < dm_config->num_disks; i++) {
-        ret = libxl_device_disk_add(ctx, dm_domid, &dm_config->disks[i]);
-        if (ret)
-            goto out_free;
-    }
+    sdss->num_devices = dm_config->num_disks;
+    libxl__add_disks(egc, ao, dm_domid, dm_config, &sdss->devices,
+                     spawn_stub_disk_connected);
+
+    free(args);
+    return;
+
+out_free:
+    free(args);
+out:
+    assert(ret);
+    spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
+}
+
+static void spawn_stub_disk_connected(libxl__egc *egc, libxl__ao_device *aorm)
+{
+    STATE_AO_GC(aorm->ao);
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+    libxl__stub_dm_spawn_state *sdss = CONTAINER_OF(aorm->base, *sdss, devices);
+    int i, num_console = STUBDOM_SPECIAL_CONSOLES, ret = 0, last;
+    libxl__device_console *console;
+
+    /* convenience aliases */
+    libxl_domain_config *const dm_config = &sdss->dm_config;
+    libxl_domain_config *const guest_config = sdss->dm.guest_config;
+    const int guest_domid = sdss->dm.guest_domid;
+    libxl__domain_build_state *const d_state = sdss->dm.build_state;
+    libxl__domain_build_state *const stubdom_state = &sdss->dm_state;
+    uint32_t dm_domid = sdss->pvqemu.guest_domid;
+
+    ret = libxl__ao_device_check_last(gc, aorm, sdss->devices,
+                                      sdss->num_devices, &last);
+    if (!last) return;
+    if (ret) {
+        LOG(ERROR, "error connecting disk devices");
+        goto out;
+     }
+
     for (i = 0; i < dm_config->num_vifs; i++) {
         ret = libxl_device_nic_add(ctx, dm_domid, &dm_config->vifs[i]);
         if (ret)
-            goto out_free;
+            goto out;
     }
     ret = libxl_device_vfb_add(ctx, dm_domid, &dm_config->vfbs[0]);
     if (ret)
-        goto out_free;
+        goto out;
     ret = libxl_device_vkb_add(ctx, dm_domid, &dm_config->vkbs[0]);
     if (ret)
-        goto out_free;
+        goto out;
 
     if (guest_config->b_info.u.hvm.serial)
         num_console++;
@@ -823,7 +857,7 @@ retry_transaction:
     console = libxl__calloc(gc, num_console, sizeof(libxl__device_console));
     if (!console) {
         ret = ERROR_NOMEM;
-        goto out_free;
+        goto out;
     }
 
     for (i = 0; i < num_console; i++) {
@@ -859,7 +893,7 @@ retry_transaction:
         ret = libxl__device_console_add(gc, dm_domid, &console[i],
                         i == STUBDOM_CONSOLE_LOGGING ? stubdom_state : NULL);
         if (ret)
-            goto out_free;
+            goto out;
     }
 
     sdss->pvqemu.guest_domid = dm_domid;
@@ -869,11 +903,8 @@ retry_transaction:
 
     libxl__spawn_local_dm(egc, &sdss->pvqemu);
 
-    free(args);
     return;
 
-out_free:
-    free(args);
 out:
     assert(ret);
     spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 68d076c..a5fc092 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -70,6 +70,7 @@
 #include "_libxl_types_internal.h"
 #include "_libxl_types_internal_json.h"
 
+#define LIBXL_INIT_TIMEOUT 10
 #define LIBXL_DESTROY_TIMEOUT 10
 #define LIBXL_DEVICE_MODEL_START_TIMEOUT 10
 #define LIBXL_XENCONSOLE_LIMIT 1048576
@@ -1790,6 +1791,19 @@ struct libxl__ao_device {
     void *base;
 };
 
+/* Internal AO operation to connect a disk device */
+_hidden void libxl__device_disk_add(libxl__egc *egc, uint32_t domid,
+                                    libxl_device_disk *disk,
+                                    libxl__ao_device *aorm);
+
+/* Arranges that dev will be added to the guest, and the
+ * hotplug scripts will be executed (if necessary). When
+ * this is done (or an error happens), the callback in
+ * aorm->callback will be called.
+ */
+_hidden void libxl__initiate_device_add(libxl__egc*, libxl__ao_device *aorm);
+
+
 /* Arranges that dev will be removed to the guest, and the
  * hotplug scripts will be executed (if necessary). When
  * this is done (or an error happens), the callback in
@@ -1880,6 +1894,12 @@ _hidden void libxl__destroy_domid(libxl__egc *egc,
 _hidden void libxl__devices_destroy(libxl__egc *egc,
                                     libxl__devices_remove_state *drs);
 
+/* Helper function to add a bunch of disks */
+void libxl__add_disks(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
+                      libxl_domain_config *d_config,
+                      libxl__ao_device **devices,
+                      libxl__device_callback callback);
+
 /*----- device model creation -----*/
 
 /* First layer; wraps libxl__spawn_spawn. */
@@ -1914,6 +1934,9 @@ typedef struct {
     libxl__domain_build_state dm_state;
     libxl__dm_spawn_state pvqemu;
     libxl__destroy_domid_state dis;
+    /* used to store the state of devices being connected */
+    libxl__ao_device *devices;
+    int num_devices;
 } libxl__stub_dm_spawn_state;
 
 _hidden void libxl__spawn_stub_dm(libxl__egc *egc, libxl__stub_dm_spawn_state*);
@@ -1942,6 +1965,9 @@ struct libxl__domain_create_state {
          * for the non-stubdom device model. */
     /* necessary if the domain creation failed and we have to destroy it */
     libxl__domain_destroy_state dds;
+    /* used to store the state of devices being connected */
+    libxl__ao_device *devices;
+    int num_devices;
 };
 
 
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 3c02b69..69ce45a 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -5081,7 +5081,7 @@ int main_blockattach(int argc, char **argv)
         return 0;
     }
 
-    if (libxl_device_disk_add(ctx, fe_domid, &disk)) {
+    if (libxl_device_disk_add(ctx, fe_domid, &disk, 0)) {
         fprintf(stderr, "libxl_device_disk_add failed.\n");
     }
     return 0;
-- 
1.7.7.5 (Apple Git-26)