[PATCH RFC/WIP 4/4] libxl: rebuild guest domains on kexec

[Vitaly Kuznetsov <vkuznets@redhat.com>]

When a PVHVM linux guest performs kexec there are lots of things which
require taking care of:
- shared info, vcpu_info
- grants
- event channels
- ...
Instead of taking care of all these things we can rebuild the domain
performing kexec from scratch doing so-called soft-reboot. Here is a naive
and simple implementation which leverages save/restore (migrate) path. It
actually copies all the memory through a pipe so it is not very fast and
the host is required to have free memory to host two copies of the guest
doing kexec.

The idea was suggested by David Vrabel, Jan Beulich, and Konrad Rzeszutek Wilk.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 tools/libxl/xl_cmdimpl.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 2 deletions(-)

diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 4fc46eb..a9bce17 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -1774,7 +1774,8 @@ static void reload_domain_config(uint32_t domid,
 }
 
 /* Returns 1 if domain should be restarted,
- * 2 if domain should be renamed then restarted, or 0
+ * 2 if domain should be renamed then restarted,
+ * 3 if domain performed kexec/kdump, or 0
  * Can update r_domid if domain is destroyed etc */
 static int handle_domain_death(uint32_t *r_domid,
                                libxl_event *event,
@@ -1801,6 +1802,9 @@ static int handle_domain_death(uint32_t *r_domid,
     case LIBXL_SHUTDOWN_REASON_WATCHDOG:
         action = d_config->on_watchdog;
         break;
+    case LIBXL_SHUTDOWN_REASON_KEXEC:
+        LOG("Domain kexec-ed.");
+        return 3;
     default:
         LOG("Unknown shutdown reason code %d. Destroying domain.",
             event->u.domain_shutdown.shutdown_reason);
@@ -2038,6 +2042,19 @@ static void evdisable_disk_ejects(libxl_evgen_disk_eject **diskws,
     }
 }
 
+static void domain_kexec_finish(libxl_ctx *ctx_cb, int rc, void *domid_arg)
+{
+    uint32_t domid = *(uint32_t *)domid_arg;
+    if (!rc) {
+        /* successful kexec, destroy the domain */
+        LOG("Domain kexec finished, destroying");
+        libxl_domain_destroy(ctx_cb, domid, 0);
+    }
+    else
+        LOG("Domain kexec failed, manual intervention required");
+    free(domid_arg);
+}
+
 static uint32_t create_domain(struct domain_create *dom_info)
 {
     uint32_t domid = INVALID_DOMID;
@@ -2066,6 +2083,9 @@ static uint32_t create_domain(struct domain_create *dom_info)
     int restore_fd = -1;
     const libxl_asyncprogress_how *autoconnect_console_how;
     struct save_file_header hdr;
+    int pipefd[2];
+
+    pipefd[0] = pipefd[1] = -1;
 
     int restoring = (restore_file || (migrate_fd >= 0));
 
@@ -2250,6 +2270,12 @@ start:
          * restore/migrate-receive it again.
          */
         restoring = 0;
+        if (pipefd[0] != -1) {
+            /* we were doing kexec, close pipes */
+            close(pipefd[0]);
+            close(pipefd[1]);
+            pipefd[0] = pipefd[1] = -1;
+        }
     }else{
         ret = libxl_domain_create_new(ctx, &d_config, &domid,
                                       0, autoconnect_console_how);
@@ -2339,6 +2365,10 @@ start:
     }
     while (1) {
         libxl_event *event;
+        libxl_asyncop_how ao_how;
+        uint32_t domid_old;
+        void *domid_arg;
+
         ret = domain_wait_event(domid, &event);
         if (ret) goto out;
 
@@ -2351,6 +2381,33 @@ start:
             switch (handle_domain_death(&domid, event,
                                         (uint8_t **)&config_data, &config_len,
                                         &d_config)) {
+            case 3:
+                domid_old = domid;
+                if (!preserve_domain(&domid, event, &d_config)) {
+                    ret = -1;
+                    goto out;
+                }
+                if (libxl_pipe(ctx, pipefd)) {
+                    ret = -1;
+                    goto out;
+                }
+
+                domid_arg = (void *)malloc(sizeof(domid_old));
+                if (!domid_arg) {
+                    ret = -1;
+                    goto out;
+                }
+                *(uint32_t *)domid_arg = domid_old;
+
+                ao_how.callback = domain_kexec_finish;
+                ao_how.u.for_callback = domid_arg;
+
+                ret = libxl_domain_suspend(ctx, domid_old, pipefd[1], LIBXL_SUSPEND_KEXEC, &ao_how);
+
+                restoring = 1;
+                restore_fd = pipefd[0];
+
+                goto restart;
             case 2:
                 if (!preserve_domain(&domid, event, &d_config)) {
                     /* If we fail then exit leaving the old domain in place. */
@@ -2359,6 +2416,7 @@ start:
                 }
 
                 /* Otherwise fall through and restart. */
+restart:
             case 1:
                 libxl_event_free(ctx, event);
                 libxl_evdisable_domain_death(ctx, deathw);
@@ -3287,7 +3345,7 @@ static void list_domains(int verbose, int context, int claim, int numa,
                          const libxl_dominfo *info, int nb_domain)
 {
     int i;
-    static const char shutdown_reason_letters[]= "-rscw";
+    static const char shutdown_reason_letters[]= "-rscwk";
     libxl_bitmap nodemap;
     libxl_physinfo physinfo;
 
-- 
1.9.3