[PATCH v11 11/11] (lib)xl: soft reset support

[Vitaly Kuznetsov <vkuznets@redhat.com>]

Use existing create/restore path to perform 'soft reset' for HVM
domains. Tear everything down, e.g. destroy domain's device model,
remove the domain from xenstore, save toolstack record and start
over.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
Changes since v10:
- Adapt to 'migration v2' changes.
- Use LIBXL_DEVICE_MODEL_SAVE_FILE as Qemu save file (and rename it to
  LIBXL_DEVICE_MODEL_RESTORE_FILE later) to support stubdom case (as
  we connect consoles to both files on create.
- Fix coding style, minor description change in xl.cfg.pod.5 [Wei Liu]

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 docs/man/xl.cfg.pod.5        |   8 +-
 tools/libxl/libxl.c          |  22 ++++-
 tools/libxl/libxl.h          |  15 ++++
 tools/libxl/libxl_create.c   | 192 ++++++++++++++++++++++++++++++++++++++-----
 tools/libxl/libxl_internal.h |   4 +
 tools/libxl/libxl_types.idl  |   2 +
 tools/libxl/xl.h             |   1 +
 tools/libxl/xl_cmdimpl.c     |  25 +++++-
 8 files changed, 242 insertions(+), 27 deletions(-)

diff --git a/docs/man/xl.cfg.pod.5 b/docs/man/xl.cfg.pod.5
index c6345b8..d8c4186 100644
--- a/docs/man/xl.cfg.pod.5
+++ b/docs/man/xl.cfg.pod.5
@@ -349,6 +349,12 @@ destroy the domain.
 write a "coredump" of the domain to F</var/lib/xen/dump/NAME> and then
 restart the domain.
 
+=item B<soft-reset>
+
+Reset all Xen specific interfaces for the Xen-aware HVM domain allowing
+it to reestablish these interfaces and continue executing the domain. PV
+guest is not supported.
+
 =back
 
 The default for C<on_poweroff> is C<destroy>.
@@ -370,7 +376,7 @@ Action to take if the domain crashes.  Default is C<destroy>.
 =item B<on_soft_reset="ACTION">
 
 Action to take if the domain performs 'soft reset' (e.g. does kexec).
-Default is C<restart>.
+Default is C<soft-reset>.
 
 =back
 
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 4f2eb24..a9b8b8b 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -1489,6 +1489,7 @@ void libxl__domain_destroy(libxl__egc *egc, libxl__domain_destroy_state *dds)
         dds->stubdom.ao = ao;
         dds->stubdom.domid = stubdomid;
         dds->stubdom.callback = stubdom_destroy_callback;
+        dds->stubdom.soft_reset = false;
         libxl__destroy_domid(egc, &dds->stubdom);
     } else {
         dds->stubdom_finished = 1;
@@ -1497,6 +1498,7 @@ void libxl__domain_destroy(libxl__egc *egc, libxl__domain_destroy_state *dds)
     dds->domain.ao = ao;
     dds->domain.domid = dds->domid;
     dds->domain.callback = domain_destroy_callback;
+    dds->domain.soft_reset = dds->soft_reset;
     libxl__destroy_domid(egc, &dds->domain);
 }
 
@@ -1677,10 +1679,14 @@ static void devices_destroy_cb(libxl__egc *egc,
 
     /* Clean up qemu-save and qemu-resume files. They are
      * intermediate files created by libxc. Unfortunately they
-     * don't fit in existing userdata scheme very well.
+     * don't fit in existing userdata scheme very well. In soft reset
+     * case we need to keep the file.
      */
-    rc = libxl__remove_file(gc, libxl__device_model_savefile(gc, domid));
-    if (rc < 0) goto out;
+    if (!dis->soft_reset) {
+        rc = libxl__remove_file(gc,
+                                libxl__device_model_savefile(gc, domid));
+        if (rc < 0) goto out;
+    }
     rc = libxl__remove_file(gc,
              GCSPRINTF(LIBXL_DEVICE_MODEL_RESTORE_FILE".%u", domid));
     if (rc < 0) goto out;
@@ -1691,7 +1697,15 @@ static void devices_destroy_cb(libxl__egc *egc,
         ctx->xch = xc_interface_open(ctx->lg,0,0);
         if (!ctx->xch) goto badchild;
 
-        rc = xc_domain_destroy(ctx->xch, domid);
+        if (!dis->soft_reset) {
+            rc = xc_domain_destroy(ctx->xch, domid);
+        } else {
+            rc = xc_domain_pause(ctx->xch, domid);
+            if (rc < 0) goto badchild;
+            rc = xc_domain_soft_reset(ctx->xch, domid);
+            if (rc < 0) goto badchild;
+            rc = xc_domain_unpause(ctx->xch, domid);
+        }
         if (rc < 0) goto badchild;
         _exit(0);
 
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5f9047c..4ee9fbf 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -205,6 +205,13 @@
 #define LIBXL_HAVE_BUILDINFO_ARM_GIC_VERSION 1
 
 /*
+ * LIBXL_HAVE_SOFT_RESET indicates that libxl supports performing
+ * 'soft reset' for domains and there is 'soft_reset' shutdown reason
+ * in enum libxl_shutdown_reason.
+ */
+#define LIBXL_HAVE_SOFT_RESET 1
+
+/*
  * libxl ABI compatibility
  *
  * The only guarantee which libxl makes regarding ABI compatibility
@@ -1132,6 +1139,14 @@ int static inline libxl_domain_create_restore_0x040200(
 
 #endif
 
+int libxl_domain_soft_reset(libxl_ctx *ctx,
+                            libxl_domain_config *d_config,
+                            uint32_t domid,
+                            const libxl_asyncop_how *ao_how,
+                            const libxl_asyncprogress_how
+                            *aop_console_how)
+                            LIBXL_EXTERNAL_CALLERS_ONLY;
+
   /* A progress report will be made via ao_console_how, of type
    * domain_create_console_available, when the domain's primary
    * console is available and can be connected to.
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 5128160..86bf9dc 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -496,8 +496,6 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_config *d_config,
     /* convenience aliases */
     libxl_domain_create_info *info = &d_config->c_info;
 
-    assert(!libxl_domid_valid_guest(*domid));
-
     uuid_string = libxl__uuid2string(gc, info->uuid);
     if (!uuid_string) {
         rc = ERROR_NOMEM;
@@ -518,7 +516,6 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_config *d_config,
         }
         flags |= XEN_DOMCTL_CDF_hap;
     }
-    *domid = -1;
 
     /* Ultimately, handle is an array of 16 uint8_t, same as uuid */
     libxl_uuid_copy(ctx, (libxl_uuid *)handle, &info->uuid);
@@ -530,13 +527,17 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_config *d_config,
         goto out;
     }
 
-    ret = xc_domain_create_config(ctx->xch, info->ssidref,
-                                  handle, flags, domid,
-                                  xc_config);
-    if (ret < 0) {
-        LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "domain creation fail");
-        rc = ERROR_FAIL;
-        goto out;
+    /* Valid domid here means we're soft resetting. */
+    if (!libxl_domid_valid_guest(*domid)) {
+        ret = xc_domain_create_config(ctx->xch, info->ssidref,
+                                      handle, flags, domid,
+                                      xc_config);
+        if (ret < 0) {
+            LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR,
+                             "domain creation fail");
+            rc = ERROR_FAIL;
+            goto out;
+        }
     }
 
     rc = libxl__arch_domain_save_config(gc, d_config, xc_config);
@@ -771,9 +772,8 @@ static void initiate_domain_create(libxl__egc *egc,
     libxl_domain_config *const d_config = dcs->guest_config;
     libxl__domain_build_state *const state = &dcs->build_state;
     const int restore_fd = dcs->restore_fd;
-    memset(&dcs->build_state, 0, sizeof(dcs->build_state));
 
-    domid = 0;
+    domid = dcs->domid_soft_reset;
 
     if (d_config->c_info.ssid_label) {
         char *s = d_config->c_info.ssid_label;
@@ -937,7 +937,7 @@ static void initiate_domain_create(libxl__egc *egc,
             d_config->nics[i].devid = ++last_devid;
     }
 
-    if (restore_fd >= 0) {
+    if (restore_fd >= 0 || dcs->domid_soft_reset != INVALID_DOMID) {
         LOG(DEBUG, "restoring, not running bootloader");
         domcreate_bootloader_done(egc, &dcs->bl, 0);
     } else  {
@@ -1010,7 +1010,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->dmss.dm.callback = domcreate_devmodel_started;
     dcs->dmss.callback = domcreate_devmodel_started;
 
-    if ( restore_fd < 0 ) {
+    if (restore_fd < 0 && dcs->domid_soft_reset == INVALID_DOMID) {
         rc = libxl__domain_build(gc, d_config, domid, state);
         domcreate_rebuild_done(egc, dcs, rc);
         return;
@@ -1030,8 +1030,10 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->srs.completion_callback = domcreate_stream_done;
     dcs->srs.checkpoint_callback = remus_checkpoint_stream_done;
 
-    libxl__stream_read_start(egc, &dcs->srs);
-    return;
+    if (restore_fd >= 0) {
+        libxl__stream_read_start(egc, &dcs->srs);
+        return;
+    }
 
  out:
     domcreate_stream_done(egc, &dcs->srs, rc);
@@ -1120,9 +1122,12 @@ out:
         libxl__file_reference_unmap(&state->pv_ramdisk);
     }
 
-    esave = errno;
-    libxl_fd_set_nonblock(ctx, fd, 0);
-    errno = esave;
+    /* fd == -1 here means we're doing soft reset. */
+    if (fd != -1) {
+        esave = errno;
+        libxl_fd_set_nonblock(ctx, fd, 0);
+        errno = esave;
+    }
     domcreate_rebuild_done(egc, dcs, ret);
 }
 
@@ -1543,6 +1548,14 @@ typedef struct {
     uint32_t *domid_out;
 } libxl__app_domain_create_state;
 
+typedef struct {
+    libxl__app_domain_create_state cdcs;
+    libxl__domain_destroy_state dds;
+    libxl__domain_suspend_state dss;
+    char *toolstack_buf;
+    uint32_t toolstack_len;
+} libxl__domain_soft_reset_state;
+
 static void domain_create_cb(libxl__egc *egc,
                              libxl__domain_create_state *dcs,
                              int rc, uint32_t domid);
@@ -1565,6 +1578,7 @@ static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
     if (restore_fd > -1)
         cdcs->dcs.restore_params = *params;
     cdcs->dcs.callback = domain_create_cb;
+    cdcs->dcs.domid_soft_reset = INVALID_DOMID;
     libxl__ao_progress_gethow(&cdcs->dcs.aop_console_how, aop_console_how);
     cdcs->domid_out = domid;
 
@@ -1573,6 +1587,131 @@ static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
     return AO_INPROGRESS;
 }
 
+static void domain_soft_reset_cb(libxl__egc *egc,
+                                 libxl__domain_destroy_state *dds,
+                                 int rc)
+{
+    STATE_AO_GC(dds->ao);
+    libxl__domain_soft_reset_state *srs = CONTAINER_OF(dds, *srs, dds);
+    libxl__app_domain_create_state *cdcs = &srs->cdcs;
+    char *savefile, *restorefile;
+
+    if (rc) {
+        LOG(ERROR, "destruction of domain %u failed.", dds->domid);
+        goto error;
+    }
+
+    cdcs->dcs.guest_domid = dds->domid;
+    rc = libxl__restore_emulator_xenstore_data(&cdcs->dcs, srs->toolstack_buf,
+                                               srs->toolstack_len);
+    if (rc) {
+        LOG(ERROR, "failed to restore toolstack record.");
+        goto error;
+    }
+
+    savefile = GCSPRINTF(LIBXL_DEVICE_MODEL_SAVE_FILE".%d", dds->domid);
+    restorefile = GCSPRINTF(LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", dds->domid);
+    rc = rename(savefile, restorefile);
+    if (rc) {
+        LOG(ERROR, "failed to rename dm save file.");
+        goto error;
+    }
+
+    initiate_domain_create(egc, &cdcs->dcs);
+    return;
+
+error:
+    domcreate_complete(egc, &cdcs->dcs, rc);
+}
+
+static int do_domain_soft_reset(libxl_ctx *ctx,
+                                libxl_domain_config *d_config,
+                                uint32_t domid_soft_reset,
+                                const libxl_asyncop_how *ao_how,
+                                const libxl_asyncprogress_how
+                                *aop_console_how)
+{
+    AO_CREATE(ctx, 0, ao_how);
+    libxl__domain_soft_reset_state *srs;
+    libxl__app_domain_create_state *cdcs;
+    libxl__domain_create_state *dcs;
+    libxl__domain_build_state *state;
+    libxl__domain_suspend_state *dss;
+    char *dom_path, *xs_store_mfn, *xs_console_mfn;
+    uint32_t domid_out;
+    int rc;
+
+    GCNEW(srs);
+    cdcs = &srs->cdcs;
+    dcs = &cdcs->dcs;
+    state = &dcs->build_state;
+    dss = &srs->dss;
+
+    srs->cdcs.dcs.ao = ao;
+    srs->cdcs.dcs.guest_config = d_config;
+    libxl_domain_config_init(&srs->cdcs.dcs.guest_config_saved);
+    libxl_domain_config_copy(ctx, &srs->cdcs.dcs.guest_config_saved,
+                             d_config);
+    cdcs->dcs.restore_fd = -1;
+    cdcs->dcs.domid_soft_reset = domid_soft_reset;
+    cdcs->dcs.callback = domain_create_cb;
+    libxl__ao_progress_gethow(&srs->cdcs.dcs.aop_console_how,
+                              aop_console_how);
+    cdcs->domid_out = &domid_out;
+
+    dom_path = libxl__xs_get_dompath(gc, domid_soft_reset);
+    if (!dom_path) {
+        LOG(ERROR, "failed to read domain path");
+        return AO_CREATE_FAIL(ERROR_FAIL);
+    }
+
+    xs_store_mfn = xs_read(ctx->xsh, XBT_NULL,
+                           GCSPRINTF("%s/store/ring-ref", dom_path),
+                           NULL);
+    state->store_mfn = xs_store_mfn ? atol(xs_store_mfn): 0;
+    free(xs_store_mfn);
+
+    xs_console_mfn = xs_read(ctx->xsh, XBT_NULL,
+                             GCSPRINTF("%s/console/ring-ref", dom_path),
+                             NULL);
+    state->console_mfn = xs_console_mfn ? atol(xs_console_mfn): 0;
+    free(xs_console_mfn);
+
+    dss->ao = ao;
+    dss->domid = domid_soft_reset;
+    dss->dm_savefile = GCSPRINTF(LIBXL_DEVICE_MODEL_SAVE_FILE".%d",
+                                 domid_soft_reset);
+
+    rc = libxl__save_emulator_xenstore_data(dss, &srs->toolstack_buf,
+                                            &srs->toolstack_len);
+    if (rc) {
+        LOG(ERROR, "failed to save toolstack record.");
+        return AO_CREATE_FAIL(ERROR_FAIL);
+    }
+
+    rc = libxl__domain_suspend_device_model(gc, dss);
+    if (rc) {
+        LOG(ERROR, "failed to suspend device model.");
+        return AO_CREATE_FAIL(ERROR_FAIL);
+    }
+
+    /*
+     * Ask all backends to disconnect by removing the domain from
+     * xenstore. On the creation path the domain will be introduced to
+     * xenstore again with probably different store/console/...
+     * channels.
+     */
+    xs_release_domain(ctx->xsh, cdcs->dcs.domid_soft_reset);
+
+    srs->dds.ao = ao;
+    srs->dds.domid = domid_soft_reset;
+    srs->dds.callback = domain_soft_reset_cb;
+    srs->dds.soft_reset = true;
+    libxl__domain_destroy(egc, &srs->dds);
+
+    return AO_INPROGRESS;
+}
+
 static void domain_create_cb(libxl__egc *egc,
                              libxl__domain_create_state *dcs,
                              int rc, uint32_t domid)
@@ -1604,6 +1743,21 @@ int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
                             ao_how, aop_console_how);
 }
 
+int libxl_domain_soft_reset(libxl_ctx *ctx,
+                            libxl_domain_config *d_config,
+                            uint32_t domid,
+                            const libxl_asyncop_how *ao_how,
+                            const libxl_asyncprogress_how
+                            *aop_console_how)
+{
+    libxl_domain_build_info *const info = &d_config->b_info;
+
+    if (info->type != LIBXL_DOMAIN_TYPE_HVM) return ERROR_INVAL;
+
+    return do_domain_soft_reset(ctx, d_config, domid, ao_how,
+                                aop_console_how);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index c3c3b7b..0a81ca8 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -111,6 +111,7 @@
 #define STUBDOM_SPECIAL_CONSOLES 3
 #define TAP_DEVICE_SUFFIX "-emu"
 #define DOMID_XS_PATH "domid"
+#define INVALID_DOMID ~0
 
 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
 
@@ -3214,6 +3215,7 @@ struct libxl__destroy_domid_state {
     /* private to implementation */
     libxl__devices_remove_state drs;
     libxl__ev_child destroyer;
+    bool soft_reset;
 };
 
 struct libxl__domain_destroy_state {
@@ -3228,6 +3230,7 @@ struct libxl__domain_destroy_state {
     int stubdom_finished;
     libxl__destroy_domid_state domain;
     int domain_finished;
+    bool soft_reset;
 };
 
 /*
@@ -3392,6 +3395,7 @@ struct libxl__domain_create_state {
     libxl_domain_config guest_config_saved; /* vanilla config */
     int restore_fd, libxc_fd;
     libxl_domain_restore_params restore_params;
+    uint32_t domid_soft_reset;
     libxl__domain_create_cb *callback;
     libxl_asyncprogress_how aop_console_how;
     /* private to domain_create */
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 9f6ec00..87d8255 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -138,6 +138,8 @@ libxl_action_on_shutdown = Enumeration("action_on_shutdown", [
 
     (5, "COREDUMP_DESTROY"),
     (6, "COREDUMP_RESTART"),
+
+    (7, "SOFT_RESET"),
     ], init_val = "LIBXL_ACTION_ON_SHUTDOWN_DESTROY")
 
 libxl_trigger = Enumeration("trigger", [
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index 6c19c0d..0021112 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -194,6 +194,7 @@ typedef enum {
     DOMAIN_RESTART_NONE = 0,     /* No domain restart */
     DOMAIN_RESTART_NORMAL,       /* Domain should be restarted */
     DOMAIN_RESTART_RENAME,       /* Domain should be renamed and restarted */
+    DOMAIN_RESTART_SOFT_RESET,   /* Soft reset should be performed */
 } domain_restart_type;
 
 extern void printf_info_sexp(int domid, libxl_domain_config *d_config, FILE *fh);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index c9bd839..2414d11 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -133,6 +133,8 @@ static const char *action_on_shutdown_names[] = {
 
     [LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_DESTROY] = "coredump-destroy",
     [LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_RESTART] = "coredump-restart",
+
+    [LIBXL_ACTION_ON_SHUTDOWN_SOFT_RESET] = "soft-reset",
 };
 
 /* Optional data, in order:
@@ -1420,7 +1422,7 @@ static void parse_config_data(const char *config_source,
     }
 
     if (xlu_cfg_get_string (config, "on_soft_reset", &buf, 0))
-        buf = "restart";
+        buf = "soft-reset";
     if (!parse_action_on_shutdown(buf, &d_config->on_soft_reset)) {
         fprintf(stderr, "Unknown on_soft_reset action \"%s\" specified\n", buf);
         exit(1);
@@ -2485,6 +2487,11 @@ static domain_restart_type handle_domain_death(uint32_t *r_domid,
         *r_domid = INVALID_DOMID;
         break;
 
+    case LIBXL_ACTION_ON_SHUTDOWN_SOFT_RESET:
+        reload_domain_config(*r_domid, d_config);
+        restart = DOMAIN_RESTART_SOFT_RESET;
+        break;
+
     case LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_DESTROY:
     case LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_RESTART:
         /* Already handled these above. */
@@ -2661,6 +2668,7 @@ static uint32_t create_domain(struct domain_create *dom_info)
     int restore_fd_to_close = -1;
     const libxl_asyncprogress_how *autoconnect_console_how;
     struct save_file_header hdr;
+    uint32_t domid_soft_reset = INVALID_DOMID;
 
     int restoring = (restore_file || (migrate_fd >= 0));
 
@@ -2875,7 +2883,13 @@ start:
          * restore/migrate-receive it again.
          */
         restoring = 0;
-    }else{
+    } else if (domid_soft_reset != INVALID_DOMID) {
+        /* Do soft reset. */
+        ret = libxl_domain_soft_reset(ctx, &d_config, domid_soft_reset,
+                                      0, autoconnect_console_how);
+        domid = domid_soft_reset;
+        domid_soft_reset = INVALID_DOMID;
+    } else {
         ret = libxl_domain_create_new(ctx, &d_config, &domid,
                                       0, autoconnect_console_how);
     }
@@ -2943,8 +2957,13 @@ start:
                 event->u.domain_shutdown.shutdown_reason,
                 event->u.domain_shutdown.shutdown_reason);
             switch (handle_domain_death(&domid, event, &d_config)) {
+            case DOMAIN_RESTART_SOFT_RESET:
+                domid_soft_reset = domid;
+                domid = INVALID_DOMID;
+                /* fall through */
             case DOMAIN_RESTART_RENAME:
-                if (!preserve_domain(&domid, event, &d_config)) {
+                if (domid_soft_reset == INVALID_DOMID &&
+                    !preserve_domain(&domid, event, &d_config)) {
                     /* If we fail then exit leaving the old domain in place. */
                     ret = -1;
                     goto out;
-- 
2.4.3