[PATCH RFC] remus: implement remus replicated checkpointing disk

[Lai Jiangshan <laijs@cn.fujitsu.com>]

This patch implements remus replicated checkpointing disk.
It includes two parts:
  generic remus replicated checkpointing disks framework
  drbd replicated checkpointing disks
They will be split into different files in next round.

The patch is still simple due to disk-setup-teardown-script is
still under implementing. I need to use libxl_ao to implement it,
but libxl_ao is hard to use. The work sequence is needed to ugly split
to serveral callbacks like device_hotplug().

And becuase the remus disk script is unimplemented, the drbd_setup() code
can't check the disk now. So it just assumes the user config the disk correctly.

This patch is *UNTESTED*.
(there is a problem with xl&drbd(without remus) in my BOXes).

I request *comments* as many as possible.

Thanks,
Lai

Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
 tools/libxl/Makefile                      |    1 +
 tools/libxl/libxl_dom.c                   |   19 +++-
 tools/libxl/libxl_internal.h              |   10 ++
 tools/libxl/libxl_remus.c                 |    2 +
 tools/libxl/libxl_remus_replicated_disk.c |  219 +++++++++++++++++++++++++++++
 5 files changed, 249 insertions(+), 2 deletions(-)
 create mode 100644 tools/libxl/libxl_remus_replicated_disk.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 218f55e..dbf5dd9 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -53,6 +53,7 @@ LIBXL_OBJS-y += libxl_nonetbuffer.o
 endif
 
 LIBXL_OBJS-y += libxl_remus.o
+LIBXL_OBJS-y += libxl_remus_replicated_disk.o
 
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index a4ffdfd..858f5be 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1251,9 +1251,14 @@ static int libxl__remus_domain_suspend_callback(void *data)
 
     STATE_AO_GC(dss->ao);
 
-    /* REMUS TODO: Issue disk checkpoint reqs. */
     int ok = libxl__domain_suspend_common_callback(data);
 
+    /* Issue disk checkpoint reqs. */
+    if (libxl__remus_disks_postsuspend(remus_state)) {
+        ok = 0;
+        goto out;
+    }
+
     if (!remus_state->netbuf_state || !ok) goto out;
 
     /* The domain was suspended successfully. Start a new network
@@ -1279,7 +1284,10 @@ static int libxl__remus_domain_resume_callback(void *data)
     if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1))
         return 0;
 
-    /* REMUS TODO: Deal with disk. */
+    /* Deal with disk. */
+    if (libxl__remus_disks_preresume(dss->remus_state))
+        return 0;
+
     return 1;
 }
 
@@ -1326,6 +1334,13 @@ static void remus_checkpoint_dm_saved(libxl__egc *egc,
         goto out;
     }
 
+    rc = libxl__remus_disks_commit(remus_state);
+    if (rc) {
+        LOG(ERROR, "Failed to commit disks state"
+            " Terminating Remus..");
+        goto out;
+    }
+
     if (remus_state->netbuf_state) {
         rc = libxl__remus_netbuf_release_prev_epoch(gc, dss->domid,
                                                     remus_state);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 1bd2bba..8933e5f 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2309,6 +2309,10 @@ typedef struct libxl__remus_state {
     void *netbuf_state;
     libxl__ev_time timeout;
     libxl__ev_child child;
+
+    /* remus disks state */
+    uint32_t nr_disks;
+    struct libxl__remus_disk **disks;
 } libxl__remus_state;
 
 _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
@@ -2336,6 +2340,12 @@ _hidden int libxl__remus_netbuf_start_new_epoch(libxl__gc *gc, uint32_t domid,
 _hidden int libxl__remus_netbuf_release_prev_epoch(libxl__gc *gc, uint32_t domid,
                                                   libxl__remus_state *remus_state);
 
+_hidden int libxl__remus_disks_postsuspend(libxl__remus_state *state);
+_hidden int libxl__remus_disks_preresume(libxl__remus_state *state);
+_hidden int libxl__remus_disks_commit(libxl__remus_state *state);
+_hidden int libxl__remus_disks_setup(libxl__egc *egc, libxl__domain_suspend_state *dss);
+_hidden void libxl__remus_disks_teardown(libxl__remus_state *state);
+
 _hidden void libxl__remus_setup_initiate(libxl__egc *egc,
                                          libxl__domain_suspend_state *dss);
 
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index cdc1c16..92eb36a 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -23,6 +23,7 @@ void libxl__remus_setup_initiate(libxl__egc *egc,
                                  libxl__domain_suspend_state *dss)
 {
     libxl__ev_time_init(&dss->remus_state->timeout);
+    libxl__remus_disks_setup(egc, dss);
     if (!dss->remus_state->netbufscript)
         libxl__remus_setup_done(egc, dss, 0);
     else
@@ -51,6 +52,7 @@ void libxl__remus_teardown_initiate(libxl__egc *egc,
     /* stash rc somewhere before invoking teardown ops. */
     dss->remus_state->saved_rc = rc;
 
+    libxl__remus_disks_teardown(dss->remus_state);
     if (!dss->remus_state->netbuf_state)
         libxl__remus_teardown_done(egc, dss);
     else
diff --git a/tools/libxl/libxl_remus_replicated_disk.c b/tools/libxl/libxl_remus_replicated_disk.c
new file mode 100644
index 0000000..4b16403
--- /dev/null
+++ b/tools/libxl/libxl_remus_replicated_disk.c
@@ -0,0 +1,219 @@
+/*
+ * Copyright (C) 2013
+ * Author Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+typedef struct libxl__remus_disk
+{
+    const struct libxl_device_disk *disk;
+    const struct libxl__remus_disk_type *type;
+
+    /* ao callbacks for setup & teardown script */
+    int (*setup_cb)(struct libxl__remus_disk *d);
+    int (*teardown_cb)(struct libxl__remus_disk *d);
+} libxl__remus_disk;
+
+typedef struct libxl__remus_disk_type
+{
+    /* checkpointing */
+    int (*postsuspend)(libxl__remus_disk *d);
+    int (*preresume)(libxl__remus_disk *d);
+    int (*commit)(libxl__remus_disk *d);
+
+    /* setup & teardown */
+    libxl__remus_disk *(*setup)(libxl__gc *gc, libxl_device_disk *disk);
+    void (*teardown)(libxl__remus_disk *d);
+} libxl__remus_disk_type;
+
+
+/*** drbd implementation ***/
+const int DRBD_SEND_CHECKPOINT = 20;
+const int DRBD_WAIT_CHECKPOINT_ACK = 30;
+typedef struct libxl__remus_drbd_disk
+{
+    libxl__remus_disk remus_disk;
+    int ctl_fd;
+    int ackwait;
+} libxl__remus_drbd_disk;
+
+static int drbd_postsuspend(libxl__remus_disk *d)
+{
+    struct libxl__remus_drbd_disk *drbd = CONTAINER_OF(d, *drbd, remus_disk);
+
+    if (!drbd->ackwait) {
+        if (ioctl(drbd->ctl_fd, DRBD_SEND_CHECKPOINT, 0) <= 0)
+            drbd->ackwait = 1;
+    }
+
+    return 0;
+}
+
+static int drbd_preresume(libxl__remus_disk *d)
+{
+    struct libxl__remus_drbd_disk *drbd = CONTAINER_OF(d, *drbd, remus_disk);
+
+    if (drbd->ackwait) {
+        ioctl(drbd->ctl_fd, DRBD_WAIT_CHECKPOINT_ACK, 0);
+        drbd->ackwait = 0;
+    }
+
+    return 0;
+}
+
+static int drbd_commit(libxl__remus_disk *d)
+{
+    /* nothing to do, all work are done by DRBD's protocal-D. */
+    return 0;
+}
+
+static libxl__remus_disk *drbd_setup(libxl__gc *gc, libxl_device_disk *disk)
+{
+    libxl__remus_drbd_disk *drbd;
+    //if (!(drbd && protocal-D)) // TODO: need to run script async to check
+    //  return NULL
+
+    GCNEW(drbd);
+
+    drbd->ctl_fd = open(GCSPRINTF("/dev/drbd/by-res/%s", disk->pdev_path), O_RDONLY);
+    drbd->ackwait = 0;
+
+    if (drbd->ctl_fd < 0)
+        return NULL;
+
+    return &drbd->remus_disk;
+}
+
+static void drbd_teardown(libxl__remus_disk *d)
+{
+    struct libxl__remus_drbd_disk *drbd = CONTAINER_OF(d, *drbd, remus_disk);
+
+    close(drbd->ctl_fd);
+}
+
+static const libxl__remus_disk_type drbd_disk_type = {
+  .postsuspend = drbd_postsuspend,
+  .preresume = drbd_preresume,
+  .commit = drbd_commit,
+  .setup = drbd_setup,
+  .teardown = drbd_teardown,
+};
+
+/*** checkpoint disks states and callbacks ***/
+static const libxl__remus_disk_type *remus_disk_types[] =
+{
+    &drbd_disk_type,
+};
+
+int libxl__remus_disks_postsuspend(libxl__remus_state *state)
+{
+    int i;
+    int rc = 0;
+
+    for (i = 0; rc == 0 && i < state->nr_disks; i++)
+        rc = state->disks[i]->type->postsuspend(state->disks[i]);
+
+    return rc;
+}
+
+int libxl__remus_disks_preresume(libxl__remus_state *state)
+{
+    int i;
+    int rc = 0;
+
+    for (i = 0; rc == 0 && i < state->nr_disks; i++)
+        rc = state->disks[i]->type->preresume(state->disks[i]);
+
+    return rc;
+}
+
+int libxl__remus_disks_commit(libxl__remus_state *state)
+{
+    int i;
+    int rc = 0;
+
+    for (i = 0; rc == 0 && i < state->nr_disks; i++)
+        rc = state->disks[i]->type->commit(state->disks[i]);
+
+    return rc;
+}
+
+#if 0
+/* TODO: implement disk setup/teardown script */
+static void disk_exec_timeout_cb(libxl__egc *egc, libxl__ev_time *ev,
+                                      const struct timeval *requested_abs)
+{
+    libxl__remus_disks_state *state = CONTAINER_OF(ev, *aodev, timeout);
+    STATE_AO_GC(state->ao);
+
+    libxl__ev_time_deregister(gc, &state->timeout);
+
+    assert(libxl__ev_child_inuse(&state->child));
+    if (kill(state->child.pid, SIGKILL)) {
+    }
+
+    return;
+}
+
+int libxl__remus_disks_exec_script(libxl__gc *gc,
+    libxl__remus_disks_state *state)
+{
+}
+#endif
+
+int libxl__remus_disks_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
+{
+    libxl__remus_state *remus_state = dss->remus_state;
+    int i, j, nr_disks;
+    libxl_device_disk *disks;
+    libxl__remus_disk *remus_disk;
+    const libxl__remus_disk_type *type;
+
+    STATE_AO_GC(dss->ao);
+    disks = libxl_device_disk_list(CTX, dss->domid, &nr_disks);
+    remus_state->nr_disks = nr_disks;
+    GCNEW_ARRAY(remus_state->disks, nr_disks);
+
+    for (i = 0; i < nr_disks; i++) {
+        remus_disk = NULL;
+        for (j = 0; j < ARRAY_SIZE(remus_disk_types); j++) {
+            type = remus_disk_types[j];
+            remus_disk = type->setup(gc, &disks[i]);
+            if (!remus_disk)
+                break;
+
+            remus_state->disks[i] = remus_disk;
+            remus_disk->disk = &disks[i];
+            remus_disk->type = type;
+        }
+        if (!remus_disk) {
+            remus_state->nr_disks = i;
+            libxl__remus_disks_teardown(remus_state);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+void libxl__remus_disks_teardown(libxl__remus_state *state)
+{
+    int i;
+
+    for (i = 0; i < state->nr_disks; i++)
+        state->disks[i]->type->teardown(state->disks[i]);
+    state->nr_disks = 0;
+}
+
-- 
1.7.1