[RFC][PATCH 07/13] Kemari: add Kemari support to python

[Yoshiaki Tamura <tamura.yoshiaki@lab.ntt.co.jp>]

This is an updated version of the following patch.  Followed the
changes in live migration code.

http://lists.xensource.com/archives/html/xen-devel/2009-03/msg00376.html

Signed-off-by: Yoshisato Yanagisawa <yanagisawa.yoshisato@lab.ntt.co.jp>
Signed-off-by: Yoshi Tamura <tamura.yoshiaki@lab.ntt.co.jp>
---
 tools/python/xen/xend/XendAPI.py        |    3 -
 tools/python/xen/xend/XendCheckpoint.py |   86 +++++++++++++++++++++++++++-----
 tools/python/xen/xend/XendDomain.py     |    6 +-
 tools/python/xen/xm/migrate.py          |   10 ++-
 4 files changed, 88 insertions(+), 17 deletions(-)

diff -r b249f3e979a5 -r cf6a910e3663 tools/python/xen/xm/migrate.py
--- a/tools/python/xen/xm/migrate.py	Mon Mar 09 10:32:24 2009 +0000
+++ b/tools/python/xen/xm/migrate.py	Wed Mar 11 18:03:47 2009 +0900
@@ -51,6 +51,10 @@
           fn=set_true, default=None,
           use="Use ssl connection for migration.")

+gopts.opt('kemari', short='k',
+          fn=set_true, default=None,
+          use="Use the Kemari fault tolerant migration.")
+
 def help():
     return str(gopts)

@@ -70,7 +74,8 @@
         other_config = {
             "port":     opts.vals.port,
             "node":     opts.vals.node,
-            "ssl":      opts.vals.ssl
+            "ssl":      opts.vals.ssl,
+            "kemari":   opts.vals.kemari
             }
         server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live),
                                  other_config)
@@ -78,4 +83,5 @@
         server.xend.domain.migrate(dom, dst, opts.vals.live,
                                    opts.vals.port,
                                    opts.vals.node,
-                                   opts.vals.ssl)
+                                   opts.vals.ssl,
+                                   opts.vals.kemari)
diff -r b249f3e979a5 -r cf6a910e3663 tools/python/xen/xend/XendAPI.py
--- a/tools/python/xen/xend/XendAPI.py	Mon Mar 09 10:32:24 2009 +0000
+++ b/tools/python/xen/xend/XendAPI.py	Wed Mar 11 18:03:47 2009 +0900
@@ -1797,9 +1797,10 @@
         port = other_config.get("port", 0)
         node = other_config.get("node", -1)
         ssl = other_config.get("ssl", None)
+        kemari = other_config.get("kemari", None)

         xendom.domain_migrate(xeninfo.getDomid(), destination_url,
-                              bool(live), port, node, ssl)
+                              bool(live), port, node, ssl, kemari)
         return xen_api_success_void()

     def VM_save(self, _, vm_ref, dest, checkpoint):
diff -r b249f3e979a5 -r cf6a910e3663 tools/python/xen/xend/XendCheckpoint.py
--- a/tools/python/xen/xend/XendCheckpoint.py	Mon Mar 09 10:32:24 2009 +0000
+++ b/tools/python/xen/xend/XendCheckpoint.py	Wed Mar 11 18:03:47 2009 +0900
@@ -29,6 +29,8 @@
 dm_batch = 512
 XC_SAVE = "xc_save"
 XC_RESTORE = "xc_restore"
+XC_KEMARI_SAVE = "xc_kemari_save"
+XC_KEMARI_RESTORE = "xc_kemari_restore"


 sizeof_int = calcsize("i")
@@ -64,8 +66,15 @@
               list.insert (i+1, value)
     return

+def get_dev_info(info, n):
+    i = 0
+    while i < len(info):
+        if (info[i][0] == n):
+            return [n, info[i][1]]
+        i = i + 1
+    return [n, '']

-def save(fd, dominfo, network, live, dst, checkpoint=False, node=-1):
+def save(fd, dominfo, network, live, dst, checkpoint=False, node=-1, kemari=False):
     try:
         if not os.path.isdir("/var/lib/xen"):
             os.makedirs("/var/lib/xen")
@@ -76,6 +85,30 @@
     write_exact(fd, SIGNATURE, "could not write guest state file: signature")

     sxprep = dominfo.sxpr()
+
+    # Add kemari option if enabled.
+    if kemari:
+        sxprep.append(['kemari', kemari])
+        pv_devlist = []
+        pv_devs = dominfo.getDeviceSxprs('vbd')
+        for x in pv_devs:
+            devinfo = []
+            for n in ['event-channel', 'ring-ref']:
+                devinfo.append(get_dev_info(x[1], n))
+            pv_devlist.append([x[0], devinfo])
+        pv_devs = dominfo.getDeviceSxprs('vif')
+        for x in pv_devs:
+            devinfo = []
+            for n in ['event-channel', 'tx-ring-ref', 'rx-ring-ref',
+                'request-rx-copy', 'feature-rx-notify', 'feature-sg',
+                'feature-gso-tcpv4']:
+                devinfo.append(get_dev_info(x[1], n))
+            pv_devlist.append([x[0], devinfo])
+        sxprep.append(['kemari-device-info', pv_devlist])
+
+    # Add kemari option if enabled.
+    if kemari:
+        sxprep.append(['kemari', kemari])

     if node > -1:
         insert_after(sxprep,'vcpus',['node', str(node)])
@@ -104,9 +137,17 @@
         # enabled. Passing "0" simply uses the defaults compiled into
         # libxenguest; see the comments and/or code in xc_linux_save() for
         # more information.
-        cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd),
-               str(dominfo.getDomid()), "0", "0",
-               str(int(live) | (int(hvm) << 2)) ]
+        if kemari:
+            if not hvm:
+                raise XendError("You can only use kemari on HVM domain.")
+
+            cmd = [xen.util.auxbin.pathTo(XC_KEMARI_SAVE), str(fd),
+                   str(dominfo.getDomid()), "0", "0",
+                   str(int(live) | (int(hvm) << 2)) ]
+        else:
+            cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd),
+                   str(dominfo.getDomid()), "0", "0",
+                   str(int(live) | (int(hvm) << 2)) ]
         log.debug("[xc_save]: %s", string.join(cmd))

         def saveInputHandler(line, tochild):
@@ -132,7 +173,7 @@
         forkHelper(cmd, fd, saveInputHandler, False)

         # put qemu device model state
-        if os.path.exists("/var/lib/xen/qemu-save.%d" % dominfo.getDomid()):
+        if not kemari and os.path.exists("/var/lib/xen/qemu-save.%d" % dominfo.getDomid()):
             write_exact(fd, QEMU_SIGNATURE, "could not write qemu signature")
             qemu_fd = os.open("/var/lib/xen/qemu-save.%d" % dominfo.getDomid(),
                               os.O_RDONLY)
@@ -198,6 +239,16 @@
         raise XendError("not a valid guest state file: config parse")

     vmconfig = p.get_val()
+
+    # Checks if kemari is enabled or not.
+    # Since Xen do not know kemari option, this option will not be migrated.
+    is_kemari = False
+    kemari_device_info = []
+    for v in vmconfig:
+        if v[0] == 'kemari' and v[1]:
+            is_kemari = True
+        if v[0] == 'kemari-device-info' and v[1]:
+            kemari_device_info = v[1]

     if not relocating:
         domconfig = XendConfig(sxp_obj = vmconfig)
@@ -272,14 +323,21 @@
         shadow_cur = xc.shadow_mem_control(dominfo.getDomid(), shadow / 1024)
         dominfo.info['shadow_memory'] = shadow_cur

-        cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE),
-                        fd, dominfo.getDomid(),
-                        store_port, console_port, int(is_hvm), pae, apic])
+        if is_kemari:
+            cmd = map(str, [xen.util.auxbin.pathTo(XC_KEMARI_RESTORE),
+                            fd, dominfo.getDomid(),
+                            store_port, console_port, int(is_hvm), pae, apic])
+        else:
+            cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE),
+                            fd, dominfo.getDomid(),
+                            store_port, console_port, int(is_hvm), pae, apic])
         log.debug("[xc_restore]: %s", string.join(cmd))

         handler = RestoreInputHandler()

         forkHelper(cmd, fd, handler.handler, True)
+        if is_kemari:
+            os.close(fd)

         # We don't want to pass this fd to any other children -- we
         # might need to recover the disk space that backs it.
@@ -299,7 +357,7 @@
         # get qemu state and create a tmp file for dm restore
         # Even PV guests may have QEMU stat, but its not currently
         # used so only bother with HVM currently.
-        if is_hvm:
+        if is_hvm and not is_kemari:
             qemu_signature = read_exact(fd, len(QEMU_SIGNATURE),
                                         "invalid device model signature read")
             if qemu_signature != QEMU_SIGNATURE:
@@ -318,7 +376,10 @@
             restore_image.setCpuid()


-        os.read(fd, 1)           # Wait for source to close connection
+        if is_kemari:
+            restore_image.setCpuid()
+        else:
+            os.read(fd, 1)           # Wait for source to close connection

         dominfo.completeRestore(handler.store_mfn, handler.console_mfn)

@@ -336,7 +397,10 @@
             lock = False;

         try:
-            dominfo.waitForDevices() # Wait for backends to set up
+            if is_kemari:
+                dominfo.waitForAttachedDevices(kemari_device_info)
+            else:
+                dominfo.waitForDevices() # Wait for backends to set up
         except Exception, exn:
             log.exception(exn)

diff -r b249f3e979a5 -r cf6a910e3663 tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py	Mon Mar 09 10:32:24 2009 +0000
+++ b/tools/python/xen/xend/XendDomain.py	Wed Mar 11 18:03:47 2009 +0900
@@ -1273,7 +1273,7 @@

         return val

-    def domain_migrate(self, domid, dst, live=False, port=0, node=-1, ssl=None):
+    def domain_migrate(self, domid, dst, live=False, port=0, node=-1, ssl=None, kemari=None):
         """Start domain migration.

         @param domid: Domain ID or Name
@@ -1338,7 +1338,7 @@

             try:
                 XendCheckpoint.save(p2cwrite, dominfo, True, live, dst,
-                                    node=node)
+                                    node=node, kemari=kemari)
             finally:
                 sock.shutdown()
                 sock.close()
@@ -1364,7 +1364,7 @@

             try:
                 XendCheckpoint.save(sock.fileno(), dominfo, True, live,
-                                    dst, node=node)
+                                    dst, node=node, kemari=kemari)
             finally:
                 sock.close()