[PATCH V3 13/13] xl: introduce "xl-json" format

[Wei Liu <wei.liu2@citrix.com>]

Originally xl verbatimly copies the domain config file in its user data
store. This patch adds the functionality to transform text domain config
file to JSON object and save that object in user data store.

What this patch does:
1. add a mandatory flag to save protocol to indicate whether the saved
   config is a JSON object
2. register a new private data type "xl-json" in libxl.h
3. modify xl to save / load "xl-json" file where necessary

After this change xl supports both "xl" format and "xl-json" format.
The user-supplied config file is still restricted to normal text config
file format ("xl"), as xl has more sanity checks when parsing text
config file. "xl-json" format is only used internally. The saved config
file is always in "xl-json" format.

Tests done so far (xl.{new,old} denotes xl with{,out} "xl_json"
support):
1. xl.new create then hexdump saved file: domain config saved in JSON format
2. xl.new create then xl.old restore: failed on mandatory flag check
3. xl.new create then xl.new restore: succeeded
4. xl.old create then xl.new restore: succeeded
5. xl.new create then local migrate, receiving end xl.new: succeeded
6. xl.old create then local migrate, receiving end xl.new: succeeded

The only drawback is that when restoring a domain, xl cannot
automatically spawn a vncviewer anymore. That's because that option is
part of domain_create info not a domain configuration thus it's not
saved in the JSON config. A warning is printed out and documentation
is updated.

Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
 docs/man/xl.cfg.pod.5    |    3 +-
 tools/libxl/libxl.h      |    1 +
 tools/libxl/xl_cmdimpl.c |  112 ++++++++++++++++++++++++++++++++++++----------
 3 files changed, 92 insertions(+), 24 deletions(-)

diff --git a/docs/man/xl.cfg.pod.5 b/docs/man/xl.cfg.pod.5
index a6663b9..9e11cdb 100644
--- a/docs/man/xl.cfg.pod.5
+++ b/docs/man/xl.cfg.pod.5
@@ -1106,7 +1106,8 @@ other VNC-related settings.  The default is to enable this.
 
 =item B<vncviewer=BOOLEAN>
 
-Automatically spawn a vncviewer when creating/restoring a guest.
+Automatically spawn a vncviewer when creating a guest. This option has
+no effect on restoring a guest and it might be removed in the future.
 
 =item B<vnclisten="ADDRESS[:DISPLAYNUM]">
 
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 7bface3..44f8897 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1057,6 +1057,7 @@ void libxl_cpuid_set(libxl_ctx *ctx, uint32_t domid,
  *
  *  userid        Data contents
  *  "xl"          domain config file in xl format, Unix line endings
+ *  "xl-json"     domain config in JSON format generated by xl
  *  "libvirt-xml" domain config file in libvirt XML format.  See
  *                http://libvirt.org/formatdomain.html
  *
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 0b38b32..d72388b 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -110,6 +110,8 @@ static const char migrate_report[]=
    *            from target to source
    */
 
+#define XL_MANDATORY_FLAG_JSON (1U << 0) /* config data is in JSON format  */
+#define XL_MANDATORY_FLAG_ALL  (XL_MANDATORY_FLAG_JSON)
 struct save_file_header {
     char magic[32]; /* savefileheader_magic */
     /* All uint32_ts are in domain's byte order. */
@@ -725,6 +727,24 @@ static void parse_top_level_sdl_options(XLU_Config *config,
     xlu_cfg_replace_string (config, "xauthority", &sdl->xauthority, 0);
 }
 
+static void parse_config_data_json(const char *config_data,
+                                   int config_len,
+                                   libxl_domain_config *d_config)
+{
+    int ret;
+
+    if (!config_len) {
+        fprintf(stderr, "Config data stream empty\n");
+        exit(1);
+    }
+
+    ret = libxl_domain_config_from_json(ctx, d_config, config_data);
+    if (ret) {
+        fprintf(stderr, "Failed to parse config\n");
+        exit(1);
+    }
+}
+
 static void parse_config_data(const char *config_source,
                               const char *config_data,
                               int config_len,
@@ -989,6 +1009,8 @@ static void parse_config_data(const char *config_source,
         b_info->rtc_timeoffset = l;
 
     if (dom_info && !xlu_cfg_get_long(config, "vncviewer", &l, 0)) {
+        fprintf(stderr, "WARNING: \"vncviewer\" option found. It might be removed in future release. "
+            "Use \"-V\" option of \"xl create\" to automatically spawn vncviewer.\n");
         /* Command line arguments must take precedence over what's
          * specified in the configuration file. */
         if (!dom_info->vnc)
@@ -1769,12 +1791,13 @@ skip_vfb:
 }
 
 static void reload_domain_config(uint32_t domid,
-                                 uint8_t **config_data, int *config_len)
+                                 uint8_t **config_data, int *config_len,
+                                 bool *config_in_json)
 {
     uint8_t *t_data;
     int ret, t_len;
 
-    ret = libxl_userdata_retrieve(ctx, domid, "xl", &t_data, &t_len);
+    ret = libxl_userdata_retrieve(ctx, domid, "xl-json", &t_data, &t_len);
     if (ret) {
         LOG("failed to retrieve guest configuration (rc=%d). "
             "reusing old configuration", ret);
@@ -1784,6 +1807,7 @@ static void reload_domain_config(uint32_t domid,
     free(*config_data);
     *config_data = t_data;
     *config_len = t_len;
+    *config_in_json = true;
 }
 
 /* Returns 1 if domain should be restarted,
@@ -1792,6 +1816,7 @@ static void reload_domain_config(uint32_t domid,
 static int handle_domain_death(uint32_t *r_domid,
                                libxl_event *event,
                                uint8_t **config_data, int *config_len,
+                               bool *config_in_json,
                                libxl_domain_config *d_config)
 
 {
@@ -1849,12 +1874,14 @@ static int handle_domain_death(uint32_t *r_domid,
         break;
 
     case LIBXL_ACTION_ON_SHUTDOWN_RESTART_RENAME:
-        reload_domain_config(*r_domid, config_data, config_len);
+        reload_domain_config(*r_domid, config_data, config_len,
+                             config_in_json);
         restart = 2;
         break;
 
     case LIBXL_ACTION_ON_SHUTDOWN_RESTART:
-        reload_domain_config(*r_domid, config_data, config_len);
+        reload_domain_config(*r_domid, config_data, config_len,
+                             config_in_json);
 
         restart = 1;
         /* fall-through */
@@ -2040,6 +2067,7 @@ static uint32_t create_domain(struct domain_create *dom_info)
     uint32_t domid = INVALID_DOMID;
 
     libxl_domain_config d_config;
+    char *d_config_json = NULL;
 
     int debug = dom_info->debug;
     int daemonize = dom_info->daemonize;
@@ -2060,6 +2088,7 @@ static uint32_t create_domain(struct domain_create *dom_info)
     libxl_evgen_disk_eject **diskws = NULL; /* one per disk */
     void *config_data = 0;
     int config_len = 0;
+    bool config_in_json = false;
     int restore_fd = -1;
     const libxl_asyncprogress_how *autoconnect_console_how;
     struct save_file_header hdr;
@@ -2106,7 +2135,7 @@ static uint32_t create_domain(struct domain_create *dom_info)
                 restore_source, hdr.mandatory_flags, hdr.optional_flags,
                 hdr.optional_data_len);
 
-        badflags = hdr.mandatory_flags & ~( 0 /* none understood yet */ );
+        badflags = hdr.mandatory_flags & ~XL_MANDATORY_FLAG_ALL;
         if (badflags) {
             fprintf(stderr, "Savefile has mandatory flag(s) 0x%"PRIx32" "
                     "which are not supported; need newer xl\n",
@@ -2134,7 +2163,10 @@ static uint32_t create_domain(struct domain_create *dom_info)
         optdata_here = optdata_begin;
 
         if (OPTDATA_LEFT) {
-            fprintf(stderr, " Savefile contains xl domain config\n");
+            config_in_json =
+                !!(hdr.mandatory_flags & XL_MANDATORY_FLAG_JSON);
+            fprintf(stderr, " Savefile contains xl domain config%s\n",
+                    config_in_json ? " in JSON format" : "");
             WITH_OPTDATA(4, {
                 memcpy(u32buf.b, optdata_here, 4);
                 config_len = u32buf.u32;
@@ -2174,6 +2206,8 @@ static uint32_t create_domain(struct domain_create *dom_info)
                 extra_config);
         }
         config_source=config_file;
+        /* User supplied file is just a text config file */
+        config_in_json = false;
     } else {
         if (!config_data) {
             fprintf(stderr, "Config file not specified and"
@@ -2186,7 +2220,11 @@ static uint32_t create_domain(struct domain_create *dom_info)
     if (!dom_info->quiet)
         printf("Parsing config from %s\n", config_source);
 
-    parse_config_data(config_source, config_data, config_len, &d_config, dom_info);
+    if (config_in_json)
+        parse_config_data_json(config_data, config_len, &d_config);
+    else
+        parse_config_data(config_source, config_data, config_len,
+                          &d_config, dom_info);
 
     if (migrate_fd >= 0) {
         if (d_config.c_info.name) {
@@ -2213,6 +2251,10 @@ static uint32_t create_domain(struct domain_create *dom_info)
     if (dom_info->dryrun)
         goto out;
 
+    d_config_json = libxl_domain_config_to_json(ctx, &d_config);
+    if (!d_config_json)
+        goto out;
+
 start:
     assert(domid == INVALID_DOMID);
 
@@ -2281,8 +2323,9 @@ start:
         free(vcpu_to_pcpu); vcpu_to_pcpu = NULL;
     }
 
-    ret = libxl_userdata_store(ctx, domid, "xl",
-                                    config_data, config_len);
+    ret = libxl_userdata_store(ctx, domid, "xl-json",
+                               (const uint8_t *)d_config_json,
+                               strlen(d_config_json));
     if (ret) {
         perror("cannot save config file");
         ret = ERROR_FAIL;
@@ -2347,7 +2390,7 @@ start:
                 event->u.domain_shutdown.shutdown_reason);
             switch (handle_domain_death(&domid, event,
                                         (uint8_t **)&config_data, &config_len,
-                                        &d_config)) {
+                                        &config_in_json, &d_config)) {
             case 2:
                 if (!preserve_domain(&domid, event, &d_config)) {
                     /* If we fail then exit leaving the old domain in place. */
@@ -2387,8 +2430,11 @@ start:
                 /* Reparse the configuration in case it has changed */
                 libxl_domain_config_dispose(&d_config);
                 libxl_domain_config_init(&d_config);
-                parse_config_data(config_source, config_data, config_len,
-                                  &d_config, dom_info);
+                if (config_in_json)
+                    parse_config_data_json(config_data, config_len, &d_config);
+                else
+                    parse_config_data(config_source, config_data, config_len,
+                                      &d_config, dom_info);
 
                 /*
                  * XXX FIXME: If this sleep is not there then domain
@@ -2444,6 +2490,8 @@ out:
 
     free(config_data);
 
+    free(d_config_json);
+
     console_child_report(child_console);
 
     if (deathw)
@@ -3194,12 +3242,14 @@ static void list_domains_details(const libxl_dominfo *info, int nb_domain)
         /* no detailed info available on dom0 */
         if (info[i].domid == 0)
             continue;
-        rc = libxl_userdata_retrieve(ctx, info[i].domid, "xl", &data, &len);
+        rc = libxl_userdata_retrieve(ctx, info[i].domid, "xl-json",
+                                     &data, &len);
         if (rc)
             continue;
         CHK_SYSCALL(asprintf(&config_source, "<domid %d data>", info[i].domid));
         libxl_domain_config_init(&d_config);
-        parse_config_data(config_source, (char *)data, len, &d_config, NULL);
+        /* Saved config file is in JSON format */
+        parse_config_data_json((const char *)data, len, &d_config);
         if (default_output_format == OUTPUT_FORMAT_JSON)
             s = printf_info_one_json(hand, info[i].domid, &d_config);
         else
@@ -3407,7 +3457,7 @@ static void save_domain_core_begin(uint32_t domid,
                                       &config_v, config_len_r);
         *config_data_r = config_v;
     } else {
-        rc = libxl_userdata_retrieve(ctx, domid, "xl",
+        rc = libxl_userdata_retrieve(ctx, domid, "xl-json",
                                      config_data_r, config_len_r);
     }
     if (rc) {
@@ -3417,7 +3467,8 @@ static void save_domain_core_begin(uint32_t domid,
 }
 
 static void save_domain_core_writeconfig(int fd, const char *source,
-                                  const uint8_t *config_data, int config_len)
+                                         const uint8_t *config_data, int config_len,
+                                         bool config_in_json)
 {
     struct save_file_header hdr;
     uint8_t *optdata_begin;
@@ -3441,6 +3492,8 @@ static void save_domain_core_writeconfig(int fd, const char *source,
     u32buf.u32 = config_len;
     ADD_OPTDATA(u32buf.b,    4);
     ADD_OPTDATA(config_data, config_len);
+    if (config_in_json)
+        hdr.mandatory_flags |= XL_MANDATORY_FLAG_JSON;
 
     /* that's the optional data */
 
@@ -3464,12 +3517,17 @@ static int save_domain(uint32_t domid, const char *filename, int checkpoint,
     int fd;
     uint8_t *config_data;
     int config_len;
+    /* If user doesn't provide override_config_file, we use saved
+     * config file which is in JSON format.
+     */
+    bool config_in_json = (override_config_file == NULL);
 
     save_domain_core_begin(domid, override_config_file,
                            &config_data, &config_len);
 
     if (!config_len) {
         fputs(" Savefile will not contain xl domain config\n", stderr);
+        config_in_json = false;
     }
 
     fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0644);
@@ -3478,7 +3536,8 @@ static int save_domain(uint32_t domid, const char *filename, int checkpoint,
         exit(2);
     }
 
-    save_domain_core_writeconfig(fd, filename, config_data, config_len);
+    save_domain_core_writeconfig(fd, filename, config_data, config_len,
+                                 config_in_json);
 
     int rc = libxl_domain_suspend(ctx, domid, fd, 0, NULL);
     close(fd);
@@ -3619,7 +3678,7 @@ static void migration_child_report(int recv_fd) {
 
 static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
                                 uint8_t *config_data, int config_len,
-                                const char *rune)
+                                bool config_in_json, const char *rune)
 {
     int rc = 0;
 
@@ -3638,7 +3697,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
     }
 
     save_domain_core_writeconfig(send_fd, "migration stream",
-                                 config_data, config_len);
+                                 config_data, config_len, config_in_json);
 
 }
 
@@ -3652,6 +3711,7 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
     char rc_buf;
     uint8_t *config_data;
     int config_len, flags = LIBXL_SUSPEND_LIVE;
+    bool config_in_json = (override_config_file == NULL);
 
     save_domain_core_begin(domid, override_config_file,
                            &config_data, &config_len);
@@ -3665,7 +3725,7 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
     child = create_migration_child(rune, &send_fd, &recv_fd);
 
     migrate_do_preamble(send_fd, recv_fd, child, config_data, config_len,
-                        rune);
+                        config_in_json, rune);
 
     xtl_stdiostream_adjust_flags(logger, XTL_STDIOSTREAM_HIDE_PROGRESS, 0);
 
@@ -4507,19 +4567,24 @@ int main_config_update(int argc, char **argv)
 
     libxl_domain_config_init(&d_config);
 
+    /* User supplied config is just text config file */
     parse_config_data(filename, config_data, config_len, &d_config, NULL);
 
     if (debug || dryrun_only)
         printf_info(default_output_format, -1, &d_config);
 
     if (!dryrun_only) {
+        char *d_config_json = NULL;
         fprintf(stderr, "setting dom%d configuration\n", domid);
-        rc = libxl_userdata_store(ctx, domid, "xl",
-                                   config_data, config_len);
+        d_config_json = libxl_domain_config_to_json(ctx, &d_config);
+        rc = libxl_userdata_store(ctx, domid, "xl-json",
+                                  (const uint8_t*)d_config_json,
+                                  strlen(d_config_json));
         if (rc) {
             fprintf(stderr, "failed to update configuration\n");
             exit(1);
         }
+        free(d_config_json);
     }
 
     libxl_domain_config_dispose(&d_config);
@@ -7333,7 +7398,8 @@ int main_remus(int argc, char **argv)
 
         child = create_migration_child(rune, &send_fd, &recv_fd);
 
-        migrate_do_preamble(send_fd, recv_fd, child, config_data, config_len,
+        migrate_do_preamble(send_fd, recv_fd, child,
+                            config_data, config_len, true /* config_in_json */,
                             rune);
 
         if (ssh_command[0])
-- 
1.7.10.4