[PATCH 11/11] libxl: Convert to asynchronous: device removal

[Ian Jackson <ian.jackson@eu.citrix.com>]

Convert libxl_FOO_device_remove, and the function which does the bulk
of the work, libxl__device_remove, to the new async ops scheme.

Adjust all callers.

Also remove libxl__wait_for_device_state which is now obsolete.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 tools/libxl/libxl.c                  |   60 +++++++++++-------
 tools/libxl/libxl.h                  |   16 ++++-
 tools/libxl/libxl_device.c           |  113 +++++++++------------------------
 tools/libxl/libxl_internal.h         |   30 ++--------
 tools/libxl/xl_cmdimpl.c             |    4 +-
 tools/ocaml/libs/xl/xenlight_stubs.c |    8 +-
 6 files changed, 92 insertions(+), 139 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 59e8cb8..ec7b3ee 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -1311,19 +1311,23 @@ out:
 }
 
 int libxl_device_disk_remove(libxl_ctx *ctx, uint32_t domid,
-                             libxl_device_disk *disk)
+                             libxl_device_disk *disk,
+                             const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
+    AO_CREATE(ctx, domid, ao_how);
     libxl__device device;
     int rc;
 
     rc = libxl__device_from_disk(gc, domid, disk, &device);
     if (rc != 0) goto out;
 
-    rc = libxl__device_remove(gc, &device, 1);
+    rc = libxl__initiate_device_remove(ao, &device);
+    if (rc) goto out;
+
+    return AO_INPROGRESS;
+
 out:
-    GC_FREE;
-    return rc;
+    return AO_ABORT(rc);
 }
 
 int libxl_device_disk_destroy(libxl_ctx *ctx, uint32_t domid,
@@ -1537,11 +1541,11 @@ int libxl_cdrom_insert(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk)
 
     ret = 0;
 
-    libxl_device_disk_remove(ctx, domid, disks + i);
+    libxl_device_disk_remove(ctx, domid, disks + i, 0);
     libxl_device_disk_add(ctx, domid, disk);
     stubdomid = libxl_get_stubdom_id(ctx, domid);
     if (stubdomid) {
-        libxl_device_disk_remove(ctx, stubdomid, disks + i);
+        libxl_device_disk_remove(ctx, stubdomid, disks + i, 0);
         libxl_device_disk_add(ctx, stubdomid, disk);
     }
 out:
@@ -1760,19 +1764,23 @@ out:
 }
 
 int libxl_device_nic_remove(libxl_ctx *ctx, uint32_t domid,
-                            libxl_device_nic *nic)
+                            libxl_device_nic *nic,
+                            const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
+    AO_CREATE(ctx, domid, ao_how);
     libxl__device device;
     int rc;
 
     rc = libxl__device_from_nic(gc, domid, nic, &device);
     if (rc != 0) goto out;
 
-    rc = libxl__device_remove(gc, &device, 1);
+    rc = libxl__initiate_device_remove(ao, &device);
+    if (rc) goto out;
+
+    return AO_INPROGRESS;
+
 out:
-    GC_FREE;
-    return rc;
+    return AO_ABORT(rc);
 }
 
 int libxl_device_nic_destroy(libxl_ctx *ctx, uint32_t domid,
@@ -2100,19 +2108,23 @@ out:
 }
 
 int libxl_device_vkb_remove(libxl_ctx *ctx, uint32_t domid,
-                            libxl_device_vkb *vkb)
+                            libxl_device_vkb *vkb,
+                            const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
+    AO_CREATE(ctx, domid, ao_how);
     libxl__device device;
     int rc;
 
     rc = libxl__device_from_vkb(gc, domid, vkb, &device);
     if (rc != 0) goto out;
 
-    rc = libxl__device_remove(gc, &device, 1);
+    rc = libxl__initiate_device_remove(ao, &device);
+    if (rc) goto out;
+
+    return AO_INPROGRESS;
+
 out:
-    GC_FREE;
-    return rc;
+    return AO_ABORT(rc);
 }
 
 int libxl_device_vkb_destroy(libxl_ctx *ctx, uint32_t domid,
@@ -2217,19 +2229,23 @@ out:
 }
 
 int libxl_device_vfb_remove(libxl_ctx *ctx, uint32_t domid,
-                            libxl_device_vfb *vfb)
+                            libxl_device_vfb *vfb,
+                            const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
+    AO_CREATE(ctx, domid, ao_how);
     libxl__device device;
     int rc;
 
     rc = libxl__device_from_vfb(gc, domid, vfb, &device);
     if (rc != 0) goto out;
 
-    rc = libxl__device_remove(gc, &device, 1);
+    rc = libxl__initiate_device_remove(ao, &device);
+    if (rc) goto out;
+
+    return AO_INPROGRESS;
+
 out:
-    GC_FREE;
-    return rc;
+    return AO_ABORT(rc);
 }
 
 int libxl_device_vfb_destroy(libxl_ctx *ctx, uint32_t domid,
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index fa79c24..508f9a5 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -467,7 +467,9 @@ libxl_vminfo * libxl_list_vm(libxl_ctx *ctx, int *nb_vm);
 /* Disks */
 int libxl_device_disk_init(libxl_ctx *ctx, libxl_device_disk *disk);
 int libxl_device_disk_add(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk);
-int libxl_device_disk_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk);
+int libxl_device_disk_remove(libxl_ctx *ctx, uint32_t domid,
+                             libxl_device_disk *disk,
+                             const libxl_asyncop_how *ao_how);
 int libxl_device_disk_destroy(libxl_ctx *ctx, uint32_t domid,
                               libxl_device_disk *disk);
 
@@ -491,7 +493,9 @@ int libxl_device_disk_local_detach(libxl_ctx *ctx, libxl_device_disk *disk);
 /* Network Interfaces */
 int libxl_device_nic_init(libxl_ctx *ctx, libxl_device_nic *nic);
 int libxl_device_nic_add(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic);
-int libxl_device_nic_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic);
+int libxl_device_nic_remove(libxl_ctx *ctx, uint32_t domid,
+                            libxl_device_nic *nic,
+                            const libxl_asyncop_how *ao_how);
 int libxl_device_nic_destroy(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic);
 
 libxl_device_nic *libxl_device_nic_list(libxl_ctx *ctx, uint32_t domid, int *num);
@@ -501,13 +505,17 @@ int libxl_device_nic_getinfo(libxl_ctx *ctx, uint32_t domid,
 /* Keyboard */
 int libxl_device_vkb_init(libxl_ctx *ctx, libxl_device_vkb *vkb);
 int libxl_device_vkb_add(libxl_ctx *ctx, uint32_t domid, libxl_device_vkb *vkb);
-int libxl_device_vkb_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_vkb *vkb);
+int libxl_device_vkb_remove(libxl_ctx *ctx, uint32_t domid,
+                            libxl_device_vkb *vkb,
+                            const libxl_asyncop_how *ao_how);
 int libxl_device_vkb_destroy(libxl_ctx *ctx, uint32_t domid, libxl_device_vkb *vkb);
 
 /* Framebuffer */
 int libxl_device_vfb_init(libxl_ctx *ctx, libxl_device_vfb *vfb);
 int libxl_device_vfb_add(libxl_ctx *ctx, uint32_t domid, libxl_device_vfb *vfb);
-int libxl_device_vfb_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_vfb *vfb);
+int libxl_device_vfb_remove(libxl_ctx *ctx, uint32_t domid,
+                            libxl_device_vfb *vfb,
+                            const libxl_asyncop_how *ao_how);
 int libxl_device_vfb_destroy(libxl_ctx *ctx, uint32_t domid, libxl_device_vfb *vfb);
 
 /* PCI Passthrough */
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 5d05e90..9af1a17 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -357,85 +357,36 @@ int libxl__device_disk_dev_number(const char *virtpath, int *pdisk,
     return -1;
 }
 
-/*
- * Returns 0 if a device is removed, ERROR_* if an error
- * or timeout occurred.
- */
-int libxl__wait_for_device_state(libxl__gc *gc, struct timeval *tv,
-                                 XenbusState state,
-                                 libxl__device_state_handler handler)
-{
-    libxl_ctx *ctx = libxl__gc_owner(gc);
-    int nfds, rc;
-    unsigned int n;
-    fd_set rfds;
-    char **l1 = NULL;
-
-start:
-    rc = 1;
-    nfds = xs_fileno(ctx->xsh) + 1;
-    FD_ZERO(&rfds);
-    FD_SET(xs_fileno(ctx->xsh), &rfds);
-    switch (select(nfds, &rfds, NULL, NULL, tv)) {
-        case -1:
-            if (errno == EINTR)
-                goto start;
-            rc = ERROR_FAIL;
-            break;
-        case 0:
-            rc = ERROR_TIMEDOUT;
-            break;
-        default:
-            l1 = xs_read_watch(ctx->xsh, &n);
-            if (l1 != NULL) {
-                char *sstate = libxl__xs_read(gc, XBT_NULL,
-                                             l1[XS_WATCH_PATH]);
-                if (!sstate || atoi(sstate) == state) {
-                    /* Call handler function if present */
-                    if (handler)
-                        rc = handler(gc, l1, sstate);
-                } else {
-                    /* State is different than expected, continue waiting... */
-                    goto start;
-                }
-                free(l1);
-            } else {
-                rc = ERROR_FAIL;
-            }
-            break;
-    }
-    return rc;
-}
 
-/*
- * Handler function for device destruction to be passed to
- * libxl__wait_for_device_state
- */
-static int destroy_device(libxl__gc *gc, char **l1, char *state)
-{
-    libxl_ctx *ctx = libxl__gc_owner(gc);
-
-    xs_unwatch(ctx->xsh, l1[0], l1[1]);
-    xs_rm(ctx->xsh, XBT_NULL, l1[XS_WATCH_TOKEN]);
-    LIBXL__LOG(ctx, LIBXL__LOG_DEBUG,
-               "Destroyed device backend at %s",
-               l1[XS_WATCH_TOKEN]);
+typedef struct {
+    libxl__ao *ao;
+    libxl__ev_devstate ds;
+} libxl__ao_device_remove;
+
+static void device_remove_cleanup(libxl__gc *gc,
+                                  libxl__ao_device_remove *aorm) {
+    if (!aorm) return;
+    libxl__ev_devstate_cancel(gc, &aorm->ds);
+}
 
-    return 0;
+static void device_remove_callback(libxl__egc *egc, libxl__ev_devstate *ds,
+                                   int rc) {
+    libxl__ao_device_remove *aorm = CONTAINER_OF(ds, *aorm, ds);
+    libxl__gc *gc = &aorm->ao->gc;
+    libxl__ao_complete(egc, aorm->ao, rc);
+    device_remove_cleanup(gc, aorm);
 }
 
-/*
- * Returns 0 (device already destroyed) or 1 (caller must
- * wait_for_dev_destroy) on success, ERROR_* on fail.
- */
-int libxl__device_remove(libxl__gc *gc, libxl__device *dev, int wait)
+int libxl__initiate_device_remove(libxl__ao *ao, libxl__device *dev)
 {
+    AO_GC;
     libxl_ctx *ctx = libxl__gc_owner(gc);
     xs_transaction_t t;
     char *be_path = libxl__device_backend_path(gc, dev);
     char *state_path = libxl__sprintf(gc, "%s/state", be_path);
     char *state = libxl__xs_read(gc, XBT_NULL, state_path);
     int rc = 0;
+    libxl__ao_device_remove *aorm = 0;
 
     if (!state)
         goto out;
@@ -458,23 +409,21 @@ retry_transaction:
         }
     }
 
-    xs_watch(ctx->xsh, state_path, be_path);
     libxl__device_destroy_tapdisk(gc, be_path);
 
-    if (wait) {
-        struct timeval tv;
-        tv.tv_sec = LIBXL_DESTROY_TIMEOUT;
-        tv.tv_usec = 0;
-        rc = libxl__wait_for_device_state(gc, &tv, XenbusStateClosed,
-                                          destroy_device);
-        if (rc < 0) /* an error or timeout occurred, clear watches */
-            xs_unwatch(ctx->xsh, state_path, be_path);
-        xs_rm(ctx->xsh, XBT_NULL, libxl__device_frontend_path(gc, dev));
-    } else {
-        rc = 1; /* Caller must wait_for_dev_destroy */
-    }
+    aorm = libxl__zalloc(gc, sizeof(*aorm));
+    aorm->ao = ao;
+    libxl__ev_devstate_init(&aorm->ds);
 
-out:
+    rc = libxl__ev_devstate_wait(gc, &aorm->ds, device_remove_callback,
+                                 state_path, XenbusStateClosed,
+                                 LIBXL_DESTROY_TIMEOUT * 1000);
+    if (rc) goto out;
+
+    return 0;
+
+ out:
+    device_remove_cleanup(gc, aorm);
     return rc;
 }
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 87490c1..3834c5c 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -655,35 +655,15 @@ _hidden char *libxl__device_backend_path(libxl__gc *gc, libxl__device *device);
 _hidden char *libxl__device_frontend_path(libxl__gc *gc, libxl__device *device);
 _hidden int libxl__parse_backend_path(libxl__gc *gc, const char *path,
                                       libxl__device *dev);
-_hidden int libxl__device_remove(libxl__gc *gc, libxl__device *dev, int wait);
 _hidden int libxl__device_destroy(libxl__gc *gc, libxl__device *dev);
 _hidden int libxl__devices_destroy(libxl__gc *gc, uint32_t domid);
 _hidden int libxl__wait_for_backend(libxl__gc *gc, char *be_path, char *state);
 
-/* Handler for the libxl__wait_for_device_state callback */
-/*
- * libxl__device_state_handler - Handler for the libxl__wait_for_device_state
- * gc: allocation pool
- * l1: array containing the path and token
- * state: string that contains the state of the device
- *
- * Returns 0 on success, and < 0 on error.
- */
-typedef int libxl__device_state_handler(libxl__gc *gc, char **l1, char *state);
-
-/*
- * libxl__wait_for_device_state - waits a given time for a device to
- * reach a given state
- * gc: allocation pool
- * tv: timeval struct containing the maximum time to wait
- * state: state to wait for (check xen/io/xenbus.h)
- * handler: callback function to execute when state is reached
- *
- * Returns 0 on success, and < 0 on error.
- */
-_hidden int libxl__wait_for_device_state(libxl__gc *gc, struct timeval *tv,
-                                         XenbusState state,
-                                         libxl__device_state_handler handler);
+/* Arranges that dev will be removed from its guest.  When
+ * this is done, the ao will be completed.  An error
+ * return from libxl__initiate_device_remove means that the ao
+ * will _not_ be completed and the caller must do so. */
+_hidden int libxl__initiate_device_remove(libxl__ao*, libxl__device *dev);
 
 /*
  * libxl__ev_devstate - waits a given time for a device to
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 92bb275..b42f296 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -4646,7 +4646,7 @@ int main_networkdetach(int argc, char **argv)
             return 1;
         }
     }
-    if (libxl_device_nic_remove(ctx, domid, &nic)) {
+    if (libxl_device_nic_remove(ctx, domid, &nic, 0)) {
         fprintf(stderr, "libxl_device_nic_del failed.\n");
         return 1;
     }
@@ -4741,7 +4741,7 @@ int main_blockdetach(int argc, char **argv)
         fprintf(stderr, "Error: Device %s not connected.\n", argv[optind+1]);
         return 1;
     }
-    if (libxl_device_disk_remove(ctx, domid, &disk)) {
+    if (libxl_device_disk_remove(ctx, domid, &disk, 0)) {
         fprintf(stderr, "libxl_device_disk_remove failed.\n");
     }
     return 0;
diff --git a/tools/ocaml/libs/xl/xenlight_stubs.c b/tools/ocaml/libs/xl/xenlight_stubs.c
index 0944c56..59329f1 100644
--- a/tools/ocaml/libs/xl/xenlight_stubs.c
+++ b/tools/ocaml/libs/xl/xenlight_stubs.c
@@ -247,7 +247,7 @@ value stub_xl_device_disk_del(value info, value domid)
 	device_disk_val(&gc, &lg, &c_info, info);
 
 	INIT_CTX();
-	ret = libxl_device_disk_remove(ctx, Int_val(domid), &c_info);
+	ret = libxl_device_disk_remove(ctx, Int_val(domid), &c_info, 0);
 	if (ret != 0)
 		failwith_xl("disk_del", &lg);
 	FREE_CTX();
@@ -281,7 +281,7 @@ value stub_xl_device_nic_del(value info, value domid)
 	device_nic_val(&gc, &lg, &c_info, info);
 
 	INIT_CTX();
-	ret = libxl_device_nic_remove(ctx, Int_val(domid), &c_info);
+	ret = libxl_device_nic_remove(ctx, Int_val(domid), &c_info, 0);
 	if (ret != 0)
 		failwith_xl("nic_del", &lg);
 	FREE_CTX();
@@ -316,7 +316,7 @@ value stub_xl_device_vkb_remove(value info, value domid)
 	device_vkb_val(&gc, &lg, &c_info, info);
 
 	INIT_CTX();
-	ret = libxl_device_vkb_remove(ctx, Int_val(domid), &c_info);
+	ret = libxl_device_vkb_remove(ctx, Int_val(domid), &c_info, 0);
 	if (ret != 0)
 		failwith_xl("vkb_clean_shutdown", &lg);
 	FREE_CTX();
@@ -370,7 +370,7 @@ value stub_xl_device_vfb_remove(value info, value domid)
 	device_vfb_val(&gc, &lg, &c_info, info);
 
 	INIT_CTX();
-	ret = libxl_device_vfb_remove(ctx, Int_val(domid), &c_info);
+	ret = libxl_device_vfb_remove(ctx, Int_val(domid), &c_info, 0);
 	if (ret != 0)
 		failwith_xl("vfb_clean_shutdown", &lg);
 	FREE_CTX();
-- 
1.7.2.5