[PATCH 09/13] libxl: convert libxl_device_nic_add to an async operation

[Roger Pau Monne <roger.pau@citrix.com>]

This patch converts libxl_device_nic_add to an ao operation that
waits for device backend to reach state XenbusStateInitWait and then
marks the operation as completed. This is not really useful now, but
will be used by latter patches that will launch hotplug scripts after
we reached the desired xenbus state.

Calls to libxl_device_nic_add have also been moved to occur after the
device model has been launched, so when hotplug scripts are called
from this functions the interfaces already exists.

As usual, libxl_device_nic_add callers have been modified, and the
internal function libxl__device_disk_add has been used if the call was
inside an already running ao.

Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
---
 tools/libxl/libxl.c          |   15 +++++---
 tools/libxl/libxl.h          |    3 +-
 tools/libxl/libxl_create.c   |   73 ++++++++++++++++++++++++++++++++++++++----
 tools/libxl/libxl_device.c   |   23 ++++++++++---
 tools/libxl/libxl_dm.c       |   58 +++++++++++++++++++++++++++++++--
 tools/libxl/libxl_internal.h |    7 +++-
 tools/libxl/xl_cmdimpl.c     |    2 +-
 7 files changed, 155 insertions(+), 26 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 5edfbdc..2490138 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -1641,15 +1641,18 @@ int libxl__device_nic_setdefault(libxl__gc *gc, libxl_device_nic *nic)
     return 0;
 }
 
-int libxl_device_nic_add(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic)
+int libxl_device_nic_add(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic,
+                         const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
-    int rc;
+    AO_CREATE(ctx, domid, ao_how);
+    libxl__ao_device *device;
 
-    rc = libxl__device_nic_add(gc, domid, nic);
+    GCNEW(device);
+    libxl__init_ao_device(device, ao, NULL);
+    device->callback = libxl__device_cb;
+    libxl__device_nic_add(egc, domid, nic, device);
 
-    GC_FREE;
-    return rc;
+    return AO_INPROGRESS;
 }
 
 int libxl_device_nic_remove(libxl_ctx *ctx, uint32_t domid,
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index f76b2e6..fc8a1a1 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -690,7 +690,8 @@ char * libxl_device_disk_local_attach(libxl_ctx *ctx, libxl_device_disk *disk);
 int libxl_device_disk_local_detach(libxl_ctx *ctx, libxl_device_disk *disk);
 
 /* Network Interfaces */
-int libxl_device_nic_add(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic);
+int libxl_device_nic_add(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic,
+                         const libxl_asyncop_how *ao_how);
 int libxl_device_nic_remove(libxl_ctx *ctx, uint32_t domid,
                             libxl_device_nic *nic,
                             const libxl_asyncop_how *ao_how);
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 9a65c45..2134645 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -577,6 +577,11 @@ static void domcreate_bootloader_done(libxl__egc *egc,
 
 static void domcreate_disk_connected(libxl__egc *egc, libxl__ao_device *aorm);
 
+static void domcreate_nics_connected(libxl__egc *egc, libxl__ao_device *aorm);
+
+static void domcreate_attach_pci(libxl__egc *egc,
+                                 libxl__domain_create_state *dcs);
+
 static void domcreate_console_available(libxl__egc *egc,
                                         libxl__domain_create_state *dcs);
 
@@ -752,13 +757,11 @@ static void domcreate_disk_connected(libxl__egc *egc, libxl__ao_device *aorm)
     dcs->dmss.callback = domcreate_devmodel_started;
 
     for (i = 0; i < d_config->num_vifs; i++) {
-        ret = libxl_device_nic_add(ctx, domid, &d_config->vifs[i]);
-        if (ret) {
-            LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
-                       "cannot add nic %d to domain: %d", i, ret);
-            ret = ERROR_FAIL;
-            goto error_out;
-        }
+        /* We have to init the nic here, because we still haven't
+         * called libxl_device_nic_add at this point, but qemu needs
+         * the nic information to be complete.
+         */
+        libxl__device_nic_setdefault(gc, &d_config->vifs[i]);
     }
     switch (d_config->c_info.type) {
     case LIBXL_DOMAIN_TYPE_HVM:
@@ -851,6 +854,62 @@ static void domcreate_devmodel_started(libxl__egc *egc,
         }
     }
 
+    /* Plug nic interfaces */
+    if (!ret && d_config->num_vifs > 0) {
+        /* Attach nics */
+        GCNEW_ARRAY(dcs->devices, d_config->num_vifs);
+        dcs->num_devices = d_config->num_vifs;
+        for (i = 0; i < d_config->num_vifs; i++) {
+            libxl__init_ao_device(&dcs->devices[i], ao, &dcs->devices);
+            dcs->devices[i].callback = domcreate_nics_connected;
+            libxl__device_nic_add(egc, domid, &d_config->vifs[i],
+                                  &dcs->devices[i]);
+        }
+        return;
+    }
+
+    domcreate_attach_pci(egc, dcs);
+    return;
+
+error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
+static void domcreate_nics_connected(libxl__egc *egc, libxl__ao_device *aorm)
+{
+    STATE_AO_GC(aorm->ao);
+    libxl__domain_create_state *dcs = CONTAINER_OF(aorm->base, *dcs, devices);
+    int last, ret = 0;
+
+    ret = libxl__ao_device_check_last(gc, aorm, dcs->devices,
+                                      dcs->num_devices, &last);
+
+    if (!last) return;
+    if (last && ret) {
+        LOGE(ERROR, "error connecting nics devices");
+        goto error_out;
+    }
+
+    domcreate_attach_pci(egc, dcs);
+    return;
+
+error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
+static void domcreate_attach_pci(libxl__egc *egc,
+                                 libxl__domain_create_state *dcs)
+{
+    STATE_AO_GC(dcs->ao);
+    int i, ret = 0;
+    libxl_ctx *ctx = CTX;
+    int domid = dcs->guest_domid;
+
+    /* convenience aliases */
+    libxl_domain_config *const d_config = dcs->guest_config;
+
     for (i = 0; i < d_config->num_pcidevs; i++)
         libxl__device_pci_add(gc, domid, &d_config->pcidevs[i], 1);
 
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 96e6ec4..6b0ce95 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -467,11 +467,13 @@ int libxl__device_from_nic(libxl__gc *gc, uint32_t domid,
     return 0;
 }
 
-int libxl__device_nic_add(libxl__gc *gc, uint32_t domid, libxl_device_nic *nic)
+void libxl__device_nic_add(libxl__egc *egc, uint32_t domid,
+                           libxl_device_nic *nic, libxl__ao_device *aorm)
 {
+    STATE_AO_GC(aorm->ao);
     flexarray_t *front;
     flexarray_t *back;
-    libxl__device device;
+    libxl__device *device;
     char *dompath, **l;
     unsigned int nb, rc;
 
@@ -504,7 +506,8 @@ int libxl__device_nic_add(libxl__gc *gc, uint32_t domid, libxl_device_nic *nic)
         }
     }
 
-    rc = libxl__device_from_nic(gc, domid, nic, &device);
+    GCNEW(device);
+    rc = libxl__device_from_nic(gc, domid, nic, device);
     if ( rc != 0 ) goto out_free;
 
     flexarray_append(back, "frontend-id");
@@ -545,6 +548,9 @@ int libxl__device_nic_add(libxl__gc *gc, uint32_t domid, libxl_device_nic *nic)
     flexarray_append(back, libxl__strdup(gc, nic->bridge));
     flexarray_append(back, "handle");
     flexarray_append(back, GCSPRINTF("%d", nic->devid));
+    flexarray_append(back, "type");
+    flexarray_append(back, GCSPRINTF("%s",
+                                     libxl_nic_type_to_string(nic->nictype)));
 
     flexarray_append(front, "backend-id");
     flexarray_append(front, GCSPRINTF("%d", nic->backend_domid));
@@ -555,17 +561,22 @@ int libxl__device_nic_add(libxl__gc *gc, uint32_t domid, libxl_device_nic *nic)
     flexarray_append(front, "mac");
     flexarray_append(front, GCSPRINTF(LIBXL_MAC_FMT,
                                       LIBXL_MAC_BYTES(nic->mac)));
-    libxl__device_generic_add(gc, &device,
+    libxl__device_generic_add(gc, device,
                         libxl__xs_kvs_of_flexarray(gc, back, back->count),
                         libxl__xs_kvs_of_flexarray(gc, front, front->count));
 
-    /* FIXME: wait for plug */
+    aorm->dev = device;
+    aorm->action = DEVICE_CONNECT;
+    libxl__initiate_device_add(egc, aorm);
+
     rc = 0;
 out_free:
     flexarray_free(back);
     flexarray_free(front);
 out:
-    return rc;
+    aorm->rc = rc;
+    if (aorm->rc) aorm->callback(egc, aorm);
+    return;
 }
 
 int libxl__device_physdisk_major_minor(const char *physpath, int *major, int *minor)
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 2b63a15..f63cf31 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -675,6 +675,12 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,
 
 static void spawn_stub_disk_connected(libxl__egc *egc, libxl__ao_device *aorm);
 
+static void stubdom_nics_connected(libxl__egc *egc, libxl__ao_device *aorm);
+
+static void stubdom_pvqemu_cb(libxl__egc *egc,
+                              libxl__dm_spawn_state *stubdom_dmss,
+                              int rc);
+
 static void spaw_stubdom_pvqemu_destroy_cb(libxl__egc *egc,
                                            libxl__destroy_domid_state *dis,
                                            int rc);
@@ -845,9 +851,11 @@ static void spawn_stub_disk_connected(libxl__egc *egc, libxl__ao_device *aorm)
     }
 
     for (i = 0; i < dm_config->num_vifs; i++) {
-        ret = libxl_device_nic_add(ctx, dm_domid, &dm_config->vifs[i]);
-        if (ret)
-            goto out;
+        /* We have to init the nic here, because we still haven't
+         * called libxl_device_nic_add at this point, but qemu needs
+         * the nic information to be complete.
+         */
+        libxl__device_nic_setdefault(gc, &dm_config->vifs[i]);
     }
     ret = libxl_device_vfb_add(ctx, dm_domid, &dm_config->vfbs[0]);
     if (ret)
@@ -923,9 +931,53 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,
         CONTAINER_OF(stubdom_dmss, *sdss, pvqemu);
     STATE_AO_GC(sdss->dm.spawn.ao);
     uint32_t dm_domid = sdss->pvqemu.guest_domid;
+    libxl_domain_config *d_config = stubdom_dmss->guest_config;
+    int i;
 
     if (rc) goto out;
 
+    if (!rc && d_config->num_vifs > 0) {
+        GCNEW_ARRAY(sdss->devices, d_config->num_vifs);
+        sdss->num_devices = d_config->num_vifs;
+        for (i = 0; i < d_config->num_vifs; i++) {
+            libxl__init_ao_device(&sdss->devices[i], ao, &sdss->devices);
+            sdss->devices[i].callback = stubdom_nics_connected;
+            libxl__device_nic_add(egc, dm_domid, &d_config->vifs[i],
+                                        &sdss->devices[i]);
+        }
+        return;
+    }
+
+out:
+    stubdom_pvqemu_cb(egc, stubdom_dmss, rc);
+}
+
+static void stubdom_nics_connected(libxl__egc *egc, libxl__ao_device *aorm)
+{
+    STATE_AO_GC(aorm->ao);
+    libxl__stub_dm_spawn_state *sdss = CONTAINER_OF(aorm->base, *sdss, devices);
+    int last, ret = 0;
+
+    ret = libxl__ao_device_check_last(gc, aorm, sdss->devices,
+                                      sdss->num_devices, &last);
+
+    if (!last) return;
+    if (last && ret)
+        LOGE(ERROR, "error connecting nics devices");
+
+    stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
+    return;
+}
+
+static void stubdom_pvqemu_cb(libxl__egc *egc,
+                              libxl__dm_spawn_state *stubdom_dmss,
+                              int rc)
+{
+    libxl__stub_dm_spawn_state *sdss =
+        CONTAINER_OF(stubdom_dmss, *sdss, pvqemu);
+    STATE_AO_GC(sdss->dm.spawn.ao);
+    uint32_t dm_domid = sdss->pvqemu.guest_domid;
+
     rc = libxl_domain_unpause(CTX, dm_domid);
     if (rc) goto out;
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 2500b86..725975d 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -788,8 +788,6 @@ _hidden int libxl__device_from_disk(libxl__gc *gc, uint32_t domid,
 _hidden int libxl__device_from_nic(libxl__gc *gc, uint32_t domid,
                                    libxl_device_nic *nic,
                                    libxl__device *device);
-_hidden int libxl__device_nic_add(libxl__gc *gc, uint32_t domid,
-                                  libxl_device_nic *nic);
 
 _hidden int libxl__device_physdisk_major_minor(const char *physpath, int *major, int *minor);
 _hidden int libxl__device_disk_dev_number(const char *virtpath,
@@ -1821,6 +1819,11 @@ _hidden void libxl__device_disk_add(libxl__egc *egc, uint32_t domid,
                                     libxl_device_disk *disk,
                                     libxl__ao_device *aorm);
 
+/* Internal AO operation to connect a nic device */
+_hidden void libxl__device_nic_add(libxl__egc *egc, uint32_t domid,
+                                   libxl_device_nic *nic,
+                                   libxl__ao_device *aorm);
+
 /* Arranges that dev will be added to the guest, and the
  * hotplug scripts will be executed (if necessary). When
  * this is done (or an error happens), the callback in
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 69ce45a..46af9fb 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -4970,7 +4970,7 @@ int main_networkattach(int argc, char **argv)
         return 0;
     }
 
-    if (libxl_device_nic_add(ctx, domid, &nic)) {
+    if (libxl_device_nic_add(ctx, domid, &nic, 0)) {
         fprintf(stderr, "libxl_device_nic_add failed.\n");
         return 1;
     }
-- 
1.7.7.5 (Apple Git-26)