[PATCH v13.1 20/26] Support colo mode for qemu disk

[Changlong Xie <xiecl.fnst@cn.fujitsu.com>]

 From 468ff9fb2f6699314c28f30a7d7d09eac9aa6756 Mon Sep 17 00:00:00 2001
From: Wen Congyang <wency@cn.fujitsu.com>
Date: Mon, 21 Mar 2016 15:38:30 +0800
Subject: [PATCH v13.1 20/26] Support colo mode for qemu disk

Usage: disk = 
['...,colo,colo-host=xxx,colo-port=xxx,colo-export=xxx,active-disk=xxx,hidden-disk=xxx...']
For QEMU block replication details:
http://wiki.qemu.org/Features/BlockReplication

Note: we just introduce COLO framework, but don't implement COLO
operations in this patch.

Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <hongyang.yang@easystack.cn>
Signed-off-by: Changlong Xie <xiecl.fnst@cn.fujitsu.com>
---
  docs/man/xl.pod.1                   |  41 +++++++++++--
  docs/misc/xl-disk-configuration.txt |  55 ++++++++++++++++++
  tools/libxl/libxl.c                 |  51 +++++++++++++++-
  tools/libxl/libxl_create.c          |  26 ++++++++-
  tools/libxl/libxl_device.c          |  11 ++++
  tools/libxl/libxl_dm.c              | 113 
+++++++++++++++++++++++++++++++++++-
  tools/libxl/libxl_types.idl         |   9 +++
  tools/libxl/libxlu_disk_l.l         |  19 ++++++
  8 files changed, 317 insertions(+), 8 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index a992a45..2664402 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -450,12 +450,43 @@ Print huge (!) amount of debug during the 
migration process.
  Enable Remus HA or COLO HA for domain. By default B<xl> relies on ssh as a
  transport mechanism between the two hosts.

-N.B: Remus support in xl is still in experimental (proof-of-concept) phase.
-     Disk replication support is limited to DRBD disks.
+B<NOTES>
+
+=over 4
+
+Remus support in xl is still in experimental (proof-of-concept) phase.
+Disk replication support is limited to DRBD disks.
+
+COLO support in xl is still in experimental (proof-of-concept) phase.
+There is no support for network, so the guest will confuse its network
+peers at the moment.

-     COLO support in xl is still in experimental (proof-of-concept) phase.
-     There is no support for network or disk, so the guest will corrupt its
-     disk and confuse its network peers at the moment.
+=back
+
+B<EXAMPLE>
+
+=over 4
+
+(a) An example for COLO replication's configuration: disk 
=['...,colo,colo-host
+=xxx,colo-port=xxx,colo-export=xxx,active-disk=xxx,hidden-disk=xxx...']
+
+=item B<colo-host>      :Secondary host's ip address.
+
+=item B<colo-port>      :Secondary host's port, we will run a nbd server on
+secondary host, and the nbd server will listen this port.
+
+=item B<colo-export>    :Nbd server's disk export name of secondary host.
+
+=item B<active-disk>    :Secondary's guest write will be buffered in 
this disk,
+and it's used by secondary.
+
+=item B<hidden-disk>    :Primary's modified contents will be buffered 
in this
+disk, and it's used by secondary.
+
+Note that the COLO configuration settings should be considered 
unstable. They
+may change incompatibly in future versions of Xen.
+
+=back

  B<OPTIONS>

diff --git a/docs/misc/xl-disk-configuration.txt 
b/docs/misc/xl-disk-configuration.txt
index 29f6ddb..b3402bc 100644
--- a/docs/misc/xl-disk-configuration.txt
+++ b/docs/misc/xl-disk-configuration.txt
@@ -234,6 +234,61 @@ were intentionally created non-sparse to avoid 
fragmentation of the
  file.


+===============
+COLO PARAMETERS
+===============
+
+
+colo
+----
+
+Enable COLO HA for disk. For better understanding block replication on
+QEMU, please refer to:
+http://wiki.qemu.org/Features/BlockReplication
+Note that the COLO configuration settings should be considered unstable.
+They may change incompatibly in future versions of Xen.
+
+
+colo-host
+---------
+
+Description:           Secondary host's address
+Mandatory:             Yes when COLO enabled
+
+
+colo-port
+---------
+
+Description:           Secondary port
+                       We will run a nbd server on secondary host,
+                       and the nbd server will listen this port.
+Mandatory:             Yes when COLO enabled
+
+
+colo-export
+-----------
+
+Description:           We will run a nbd server on secondary host,
+                       exportname is the nbd server's disk export name.
+Mandatory:             Yes when COLO enabled
+
+
+active-disk
+-----------
+
+Description:           This is used by secondary. Secondary guest's write
+                       will be buffered in this disk.
+Mandatory:             Yes when COLO enabled
+
+
+hidden-disk
+-----------
+
+Description:           This is used by secondary. It buffers the original
+                       content that is modified by the primary VM.
+Mandatory:             Yes when COLO enabled
+
+
  ============================================
  DEPRECATED PARAMETERS, PREFIXES AND SYNTAXES
  ============================================
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 349a3c6..63fbe16 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -2306,6 +2306,8 @@ int libxl__device_disk_setdefault(libxl__gc *gc, 
libxl_device_disk *disk)
      int rc;

      libxl_defbool_setdefault(&disk->discard_enable, !!disk->readwrite);
+    libxl_defbool_setdefault(&disk->colo_enable, false);
+    libxl_defbool_setdefault(&disk->colo_restore_enable, false);

      rc = libxl__resolve_domid(gc, disk->backend_domname, 
&disk->backend_domid);
      if (rc < 0) return rc;
@@ -2504,6 +2506,18 @@ static void device_disk_add(libxl__egc *egc, 
uint32_t domid,
                  flexarray_append(back, "params");
                  flexarray_append(back, GCSPRINTF("%s:%s",
 
libxl__device_disk_string_of_format(disk->format), disk->pdev_path));
+                if (libxl_defbool_val(disk->colo_enable)) {
+                    flexarray_append(back, "colo-host");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->colo_host));
+                    flexarray_append(back, "colo-port");
+                    flexarray_append(back, libxl__sprintf(gc, "%d", 
disk->colo_port));
+                    flexarray_append(back, "colo-export");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->colo_export));
+                    flexarray_append(back, "active-disk");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->active_disk));
+                    flexarray_append(back, "hidden-disk");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->hidden_disk));
+                }
                  assert(device->backend_kind == LIBXL__DEVICE_KIND_QDISK);
                  break;
              default:
@@ -2619,7 +2633,12 @@ static int 
libxl__device_disk_from_xs_be(libxl__gc *gc,
          goto cleanup;
      }

-    /* "params" may not be present; but everything else must be. */
+    /*
+     * "params" may not be present; but everything else must be.
+     * colo releated entries(colo-host, colo-port, colo-export,
+     * active-disk and hidden-disk) are present only if colo is
+     * enabled.
+     */
      tmp = xs_read(ctx->xsh, XBT_NULL,
                    GCSPRINTF("%s/params", be_path), &len);
      if (tmp && strchr(tmp, ':')) {
@@ -2629,6 +2648,36 @@ static int 
libxl__device_disk_from_xs_be(libxl__gc *gc,
          disk->pdev_path = tmp;
      }

+    tmp = xs_read(ctx->xsh, XBT_NULL,
+                  GCSPRINTF("%s/colo-host", be_path), &len);
+    if (tmp) {
+        libxl_defbool_set(&disk->colo_enable, true);
+        disk->colo_host = tmp;
+
+        tmp = xs_read(ctx->xsh, XBT_NULL,
+                      GCSPRINTF("%s/colo-port", be_path), &len);
+        if (!tmp) {
+            LOG(ERROR, "Missing xenstore node %s/colo-port", be_path);
+            goto cleanup;
+        }
+        disk->colo_port = atoi(tmp);
+
+#define XS_READ_COLO(param, item) do {                                  \
+        tmp = xs_read(ctx->xsh, XBT_NULL,                               \
+                      GCSPRINTF("%s/"#param"", be_path), &len);         \
+        if (!tmp) {                                                     \
+            LOG(ERROR, "Missing xenstore node %s/"#param"", be_path);   \
+            goto cleanup;                                               \
+        }                                                               \
+        disk->item = tmp;                                               \
+} while (0)
+        XS_READ_COLO(colo-export, colo_export);
+        XS_READ_COLO(active-disk, active_disk);
+        XS_READ_COLO(hidden-disk, hidden_disk);
+#undef XS_READ_COLO
+    } else {
+        libxl_defbool_set(&disk->colo_enable, false);
+    }

      tmp = libxl__xs_read(gc, XBT_NULL,
                           GCSPRINTF("%s/type", be_path));
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index be604e5..e2ec25c 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1876,12 +1876,30 @@ static void domain_create_cb(libxl__egc *egc,

      libxl__ao_complete(egc, ao, rc);
  }
-
+
+
+static void set_disk_colo_restore(libxl_domain_config *d_config)
+{
+    int i;
+
+    for (i = 0; i < d_config->num_disks; i++)
+        libxl_defbool_set(&d_config->disks[i].colo_restore_enable, true);
+}
+
+static void unset_disk_colo_restore(libxl_domain_config *d_config)
+{
+    int i;
+
+    for (i = 0; i < d_config->num_disks; i++)
+        libxl_defbool_set(&d_config->disks[i].colo_restore_enable, false);
+}
+
  int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
                              uint32_t *domid,
                              const libxl_asyncop_how *ao_how,
                              const libxl_asyncprogress_how 
*aop_console_how)
  {
+    unset_disk_colo_restore(d_config);
      return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
                              ao_how, aop_console_how);
  }
@@ -1893,6 +1911,12 @@ int libxl_domain_create_restore(libxl_ctx *ctx, 
libxl_domain_config *d_config,
                                  const libxl_asyncop_how *ao_how,
                                  const libxl_asyncprogress_how 
*aop_console_how)
  {
+    if (params->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO) {
+        set_disk_colo_restore(d_config);
+    } else {
+        unset_disk_colo_restore(d_config);
+    }
+
      return do_domain_create(ctx, d_config, domid, restore_fd, 
send_back_fd,
                              params, ao_how, aop_console_how);
  }
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 4ced9b6..6a411c6 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -196,6 +196,9 @@ static int disk_try_backend(disk_try_backend_args *a,
              goto bad_format;
          }

+        if (libxl_defbool_val(a->disk->colo_enable))
+            goto bad_colo;
+
          if (a->disk->backend_domid != LIBXL_TOOLSTACK_DOMID) {
              LOG(DEBUG, "Disk vdev=%s, is using a storage driver domain, "
                         "skipping physical device check", a->disk->vdev);
@@ -218,6 +221,9 @@ static int disk_try_backend(disk_try_backend_args *a,
      case LIBXL_DISK_BACKEND_TAP:
          if (a->disk->script) goto bad_script;

+        if (libxl_defbool_val(a->disk->colo_enable))
+            goto bad_colo;
+
          if (a->disk->is_cdrom) {
              LOG(DEBUG, "Disk vdev=%s, backend tap unsuitable for cdroms",
                         a->disk->vdev);
@@ -256,6 +262,11 @@ static int disk_try_backend(disk_try_backend_args *a,
      LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with script=...",
          a->disk->vdev, libxl_disk_backend_to_string(backend));
      return 0;
+
+ bad_colo:
+    LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with colo",
+        a->disk->vdev, libxl_disk_backend_to_string(backend));
+    return 0;
  }

  int libxl__device_disk_set_backend(libxl__gc *gc, libxl_device_disk 
*disk) {
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 1d1b25b..4c3dff8 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -754,6 +754,8 @@ static int libxl__dm_runas_helper(libxl__gc *gc, 
const char *username)
  /* colo mode */
  enum {
      LIBXL__COLO_NONE = 0,
+    LIBXL__COLO_PRIMARY,
+    LIBXL__COLO_SECONDARY,
  };

  static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char 
*pdev_path,
@@ -762,6 +764,9 @@ static char *qemu_disk_scsi_drive_string(libxl__gc 
*gc, const char *pdev_path,
                                           int colo_mode)
  {
      char *drive = NULL;
+    const char *exportname = disk->colo_export;
+    const char *active_disk = disk->active_disk;
+    const char *hidden_disk = disk->hidden_disk;

      switch (colo_mode) {
      case LIBXL__COLO_NONE:
@@ -769,6 +774,45 @@ static char *qemu_disk_scsi_drive_string(libxl__gc 
*gc, const char *pdev_path,
              (gc, 
"file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback",
               pdev_path, unit, format);
          break;
+    case LIBXL__COLO_PRIMARY:
+        /*
+         * primary:
+         *  -dirve if=scsi,bus=0,unit=x,cache=writeback,driver=quorum,\
+         *  id=exportname,\
+         *  children.0.file.filename=pdev_path,\
+         *  children.0.driver=format,\
+         *  read-pattern=fifo,\
+         *  vote-threshold=1
+         */
+        drive = GCSPRINTF(
+            "if=scsi,bus=0,unit=%d,cache=writeback,driver=quorum,"
+            "id=%s,"
+            "children.0.file.filename=%s,"
+            "children.0.driver=%s,"
+            "read-pattern=fifo,"
+            "vote-threshold=1",
+            unit, exportname, pdev_path, format);
+        break;
+    case LIBXL__COLO_SECONDARY:
+        /*
+         * secondary:
+         *  -drive 
if=scsi,bus=0,unit=x,cache=writeback,driver=replication,\
+         *  mode=secondary,\
+         *  file.driver=qcow2,\
+         *  file.file.filename=active_disk,\
+         *  file.backing.driver=qcow2,\
+         *  file.backing.file.filename=hidden_disk,\
+         *  file.backing.backing=exportname,
+         */
+        drive = GCSPRINTF(
+            "if=scsi,bus=0,unit=%d,cache=writeback,driver=replication,"
+            "mode=secondary,"
+            "file.driver=qcow2,"
+            "file.file.filename=%s,"
+            "file.backing.driver=qcow2,"
+            "file.backing.file.filename=%s,"
+            "file.backing.backing=%s",
+            unit, active_disk, hidden_disk, exportname);
      default:
           abort();
      }
@@ -782,6 +826,9 @@ static char *qemu_disk_ide_drive_string(libxl__gc 
*gc, const char *pdev_path,
                                          int colo_mode)
  {
      char *drive = NULL;
+    const char *exportname = disk->colo_export;
+    const char *active_disk = disk->active_disk;
+    const char *hidden_disk = disk->hidden_disk;

      switch (colo_mode) {
      case LIBXL__COLO_NONE:
@@ -789,6 +836,46 @@ static char *qemu_disk_ide_drive_string(libxl__gc 
*gc, const char *pdev_path,
 
("file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback",
               pdev_path, unit, format);
          break;
+    case LIBXL__COLO_PRIMARY:
+        /*
+         * primary:
+         *  -dirve 
if=ide,index=x,media=disk,cache=writeback,driver=quorum,\
+         *  id=exportname,\
+         *  children.0.file.filename=pdev_path,\
+         *  children.0.driver=format,\
+         *  read-pattern=fifo,\
+         *  vote-threshold=1
+         */
+        drive = GCSPRINTF(
+            "if=ide,index=%d,media=disk,cache=writeback,driver=quorum,"
+            "id=%s,"
+            "children.0.file.filename=%s,"
+            "children.0.driver=%s,"
+            "read-pattern=fifo,"
+            "vote-threshold=1",
+             unit, exportname, pdev_path, format);
+        break;
+    case LIBXL__COLO_SECONDARY:
+        /*
+         * secondary:
+         *  -drive 
if=ide,index=x,media=disk,cache=writeback,driver=replication,\
+         *  mode=secondary,\
+         *  file.driver=qcow2,\
+         *  file.file.filename=active_disk,\
+         *  file.backing.driver=qcow2,\
+         *  file.backing.file.filename=hidden_disk,\
+         *  file.backing.backing=exportname,
+         */
+        drive = GCSPRINTF(
+ 
"if=ide,index=%d,media=disk,cache=writeback,driver=replication,"
+            "mode=secondary,"
+            "file.driver=qcow2,"
+            "file.file.filename=%s,"
+            "file.backing.driver=qcow2,"
+            "file.backing.file.filename=%s,"
+            "file.backing.backing=%s",
+            unit, active_disk, hidden_disk, exportname);
+        break;
      default:
           abort();
      }
@@ -1261,8 +1348,24 @@ static int 
libxl__build_device_model_args_new(libxl__gc *gc,
                   * hd[a-d] and ignore the rest.
                   */

-                colo_mode = LIBXL__COLO_NONE;
+                if (libxl_defbool_val(disks[i].colo_enable)) {
+                    if (libxl_defbool_val(disks[i].colo_restore_enable))
+                        colo_mode = LIBXL__COLO_SECONDARY;
+                    else
+                        colo_mode = LIBXL__COLO_PRIMARY;
+                } else {
+                    colo_mode = LIBXL__COLO_NONE;
+                }
+
                  if (strncmp(disks[i].vdev, "sd", 2) == 0) {
+                    if (colo_mode == LIBXL__COLO_SECONDARY) {
+                        drive = libxl__sprintf
+                            (gc, "if=none,driver=%s,file=%s,id=%s",
+                             format, pdev_path, disks[i].colo_export);
+
+                        flexarray_append(dm_args, "-drive");
+                        flexarray_append(dm_args, drive);
+                    }
                      drive = qemu_disk_scsi_drive_string(gc, pdev_path, 
disk,
                                                          format,
                                                          &disks[i],
@@ -1289,6 +1392,14 @@ static int 
libxl__build_device_model_args_new(libxl__gc *gc,
                          LOG(ERROR, "qemu-xen doesn't support read-only 
IDE disk drivers");
                          return ERROR_INVAL;
                      }
+                    if (colo_mode == LIBXL__COLO_SECONDARY) {
+                        drive = libxl__sprintf
+                            (gc, "if=none,driver=%s,file=%s,id=%s",
+                             format, pdev_path, disks[i].colo_export);
+
+                        flexarray_append(dm_args, "-drive");
+                        flexarray_append(dm_args, drive);
+                    }
                      drive = qemu_disk_ide_drive_string(gc, pdev_path, 
disk,
                                                         format,
                                                         &disks[i],
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 95efd82..8335291 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -576,6 +576,15 @@ libxl_device_disk = Struct("device_disk", [
      ("is_cdrom", integer),
      ("direct_io_safe", bool),
      ("discard_enable", libxl_defbool),
+    # Note that the COLO configuration settings should be considered 
unstable.
+    # They may change incompatibly in future versions of Xen.
+    ("colo_enable", libxl_defbool),
+    ("colo_restore_enable", libxl_defbool),
+    ("colo_host", string),
+    ("colo_port", integer),
+    ("colo_export", string),
+    ("active_disk", string),
+    ("hidden_disk", string)
      ])

  libxl_device_nic = Struct("device_nic", [
diff --git a/tools/libxl/libxlu_disk_l.l b/tools/libxl/libxlu_disk_l.l
index 1a5deb5..5b6db22 100644
--- a/tools/libxl/libxlu_disk_l.l
+++ b/tools/libxl/libxlu_disk_l.l
@@ -113,6 +113,16 @@ static void setbackendtype(DiskParseContext *dpc, 
const char *str) {
      else xlu__disk_err(dpc,str,"unknown value for backendtype");
  }

+/* Sets ->colo-port from the string.  COLO need this. */
+static void setcoloport(DiskParseContext *dpc, const char *str) {
+    int port = atoi(str);
+    if (port) {
+       dpc->disk->colo_port = port;
+    } else {
+	xlu__disk_err(dpc,str,"unknown value for colo_port");
+    }
+}
+
  #define DEPRECATE(usewhatinstead) /* not currently reported */

  /* Handles a vdev positional parameter which includes a devtype. */
@@ -176,6 +186,15 @@ script=[^,]*,?	{ STRIP(','); SAVESTRING("script", 
script, FROMEQUALS); }
  direct-io-safe,? { DPC->disk->direct_io_safe = 1; }
  discard,?	{ libxl_defbool_set(&DPC->disk->discard_enable, true); }
  no-discard,?	{ libxl_defbool_set(&DPC->disk->discard_enable, false); }
+ /* Note that the COLO configuration settings should be considered 
unstable.
+  * They may change incompatibly in future versions of Xen. */
+colo,?		{ libxl_defbool_set(&DPC->disk->colo_enable, true); }
+no-colo,?	{ libxl_defbool_set(&DPC->disk->colo_enable, false); }
+colo-host=[^,]*,?	{ STRIP(','); SAVESTRING("colo-host", colo_host, 
FROMEQUALS); }
+colo-port=[^,]*,?	{ STRIP(','); setcoloport(DPC, FROMEQUALS); }
+colo-export=[^,]*,?	{ STRIP(','); SAVESTRING("colo-export", 
colo_export, FROMEQUALS); }
+active-disk=[^,]*,?	{ STRIP(','); SAVESTRING("active-disk", 
active_disk, FROMEQUALS); }
+hidden-disk=[^,]*,?	{ STRIP(','); SAVESTRING("hidden-disk", 
hidden_disk, FROMEQUALS); }

   /* the target magic parameter, eats the rest of the string */

-- 
1.9.3




_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel