[PATCH V3 7/7] COLO-Proxy: Use socket to get checkpoint event.

[Zhang Chen <zhangchen.fnst@cn.fujitsu.com>]

We use kernel colo proxy's way to get the checkpoint event
from qemu colo-compare.
Qemu colo-compare need add a API to support this(I will add this in qemu).

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
---
 tools/libxl/libxl_colo.h         |  2 +
 tools/libxl/libxl_colo_proxy.c   | 85 +++++++++++++++++++++++++++++++++++++---
 tools/libxl/libxl_colo_restore.c | 11 ++++--
 tools/libxl/libxl_colo_save.c    | 22 +++++++----
 tools/libxl/libxl_nic.c          |  4 ++
 tools/libxl/libxl_types.idl      |  4 +-
 tools/libxl/xl_cmdimpl.c         |  4 ++
 7 files changed, 114 insertions(+), 18 deletions(-)

diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h
index 4746d8c..6c01b55 100644
--- a/tools/libxl/libxl_colo.h
+++ b/tools/libxl/libxl_colo.h
@@ -69,6 +69,8 @@ struct libxl__colo_proxy_state {
      *          False means use kernel colo proxy.
      */
     bool is_userspace_proxy;
+    const char *checkpoint_host;
+    const char *checkpoint_port;
 };
 
 struct libxl__colo_save_state {
diff --git a/tools/libxl/libxl_colo_proxy.c b/tools/libxl/libxl_colo_proxy.c
index dd902fc..a6436b0 100644
--- a/tools/libxl/libxl_colo_proxy.c
+++ b/tools/libxl/libxl_colo_proxy.c
@@ -18,9 +18,13 @@
 #include "libxl_internal.h"
 
 #include <netlink/netlink.h>
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
 
 /* Consistent with the new COLO netlink channel in kernel side */
 #define NETLINK_COLO 28
+#define COLO_DEFAULT_WAIT_TIME 500000
 
 enum colo_netlink_op {
     COLO_QUERY_CHECKPOINT = (NLMSG_MIN_TYPE + 1),
@@ -76,6 +80,26 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, uint8_t *buff,
     return ret;
 }
 
+static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps,
+                                     char *buff,
+                                     unsigned int timeout_us)
+{
+    struct timeval tv;
+    int ret;
+
+    STATE_AO_GC(cps->ao);
+
+    if (timeout_us) {
+        tv.tv_sec = timeout_us / 1000000;
+        tv.tv_usec = timeout_us % 1000000;
+        setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
+    }
+
+    ret = recv(cps->sock_fd, buff, sizeof(buff),0);
+
+    return ret;
+}
+
 /* error: return -1, otherwise return 0 */
 static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff,
                                unsigned int timeout_us)
@@ -153,8 +177,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps)
     STATE_AO_GC(cps->ao);
 
     /* If enable userspace proxy mode, we don't need setup kernel proxy */
-    if (cps->is_userspace_proxy)
+    if (cps->is_userspace_proxy) {
+        struct sockaddr_in addr;
+        int port;
+        char recvbuff[1024];
+        char sendbuf[] = "COLO_USERSPACE_PROXY_INIT";
+
+        memset(&addr, 0, sizeof(addr));
+        port = atoi(cps->checkpoint_port);
+        addr.sin_family = AF_INET;
+        addr.sin_port = htons(port);
+        addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host);
+
+        skfd = socket(AF_INET, SOCK_STREAM, 0);
+        if (skfd < 0) {
+            LOGD(ERROR, ao->domid, "can not create a TCP socket: %s",
+                 strerror(errno));
+            goto out;
+        }
+
+        cps->sock_fd = skfd;
+
+        if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+            LOGD(ERROR, ao->domid, "connect error");
+            goto out;
+        }
+
+        ret = send(skfd, sendbuf, strlen(sendbuf),0);
+        if (ret < 0)
+            goto out;
+
+        ret = colo_userspace_proxy_recv(cps, recvbuff, COLO_DEFAULT_WAIT_TIME);
+        if (ret < 0) {
+            LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s",
+                 strerror(errno));
+            goto out;
+        }
+
         return 0;
+    }
 
     skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO);
     if (skfd < 0) {
@@ -247,8 +308,11 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps)
      * If enable userspace proxy mode,
      * we don't need preresume kernel proxy
      */
-    if (cps->is_userspace_proxy)
+    if (cps->is_userspace_proxy) {
+        char sendbuf[] = "COLO_CHECKPOINT";
+        send(cps->sock_fd, sendbuf, strlen(sendbuf),0);
         return;
+    }
 
     colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT);
     /* TODO: need to handle if the call fails... */
@@ -277,16 +341,25 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
     struct nlmsghdr *h;
     struct colo_msg *m;
     int ret = -1;
+    char recvbuff[1024];
 
     STATE_AO_GC(cps->ao);
 
     /*
-     * enable userspace proxy mode, tmp sleep.
-     * then we will add qemu API support this func.
+     * enable userspace proxy mode.
+     * Then we will add qemu API support for this func.
      */
     if (cps->is_userspace_proxy) {
-        sleep(timeout_us / 1000000);
-        return 0;
+        ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us);
+        if (ret <= 0)
+            return 0;
+
+        if (!strcmp(recvbuff, "DO_CHECKPOINT")) {
+            return 1;
+        } else {
+            LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint error");
+            return -1;
+        }
     }
 
     size = colo_proxy_recv(cps, &buff, timeout_us);
diff --git a/tools/libxl/libxl_colo_restore.c b/tools/libxl/libxl_colo_restore.c
index c6d239a..065ea00 100644
--- a/tools/libxl/libxl_colo_restore.c
+++ b/tools/libxl/libxl_colo_restore.c
@@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc,
         }
     }
 
-    colo_proxy_preresume(&crs->cps);
+    if (!crs->cps.is_userspace_proxy)
+        colo_proxy_preresume(&crs->cps);
 
     colo_restore_resume_vm(egc, crcs);
 
@@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc,
     cds->ops = colo_restore_ops;
 
     crs->cps.ao = ao;
-    if (colo_proxy_setup(&crs->cps)) {
-        LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
-        goto out;
+    if (!crs->cps.is_userspace_proxy) {
+        if (colo_proxy_setup(&crs->cps)) {
+            LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
+            goto out;
+        }
     }
 
     if (init_device_subkind(cds))
diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c
index 91e3fce..a0cfc5a 100644
--- a/tools/libxl/libxl_colo_save.c
+++ b/tools/libxl/libxl_colo_save.c
@@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
     libxl__checkpoint_devices_state *const cds = &dss->cds;
     libxl__srm_save_autogen_callbacks *const callbacks =
         &dss->sws.shs.callbacks.save.a;
+    libxl_device_nic *nics;
 
     STATE_AO_GC(dss->ao);
 
@@ -110,24 +111,31 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
         css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup",
                                            libxl__xen_script_dir_path());
 
-    /* If enable userspace proxy mode, we don't need VIF */
-    if (css->cps.is_userspace_proxy)
-        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
-    else
-        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
-                                 (1 << LIBXL__DEVICE_KIND_VBD);
-
     cds->ops = colo_ops;
     cds->callback = colo_save_setup_done;
     cds->ao = ao;
     cds->domid = dss->domid;
     cds->concrete_data = css;
 
+    /* If enable userspace proxy mode, we don't need VIF */
+    if (css->cps.is_userspace_proxy) {
+        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
+
+        /* Use this args we can connect to qemu colo-compare */
+        nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
+        css->cps.checkpoint_host = nics->colo_checkpoint_host;
+        css->cps.checkpoint_port = nics->colo_checkpoint_port;
+    } else {
+        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
+                                 (1 << LIBXL__DEVICE_KIND_VBD);
+    }
+
     css->srs.ao = ao;
     css->srs.fd = css->recv_fd;
     css->srs.back_channel = true;
     libxl__stream_read_start(egc, &css->srs);
     css->cps.ao = ao;
+
     if (colo_proxy_setup(&css->cps)) {
         LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
         goto out;
diff --git a/tools/libxl/libxl_nic.c b/tools/libxl/libxl_nic.c
index 5e1fecd..6bc6146 100644
--- a/tools/libxl/libxl_nic.c
+++ b/tools/libxl/libxl_nic.c
@@ -246,6 +246,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t domid,
     MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev);
     MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev);
     MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue);
+    MAYBE_ADD_COLO_ARGS(checkpoint_host);
+    MAYBE_ADD_COLO_ARGS(checkpoint_port);
 
 #undef MAYBE_ADD_COLO_ARGS
 
@@ -451,6 +453,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc,
     CHECK_COLO_ARGS(filter_sec_redirector1_indev);
     CHECK_COLO_ARGS(filter_sec_redirector1_outdev);
     CHECK_COLO_ARGS(filter_sec_rewriter0_queue);
+    CHECK_COLO_ARGS(checkpoint_host);
+    CHECK_COLO_ARGS(checkpoint_port);
 
 #undef CHECK_COLO_ARGS
 
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 47e96b1..0b412ee 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -671,7 +671,9 @@ libxl_device_nic = Struct("device_nic", [
     ("colo_filter_sec_redirector1_queue", string),
     ("colo_filter_sec_redirector1_indev", string),
     ("colo_filter_sec_redirector1_outdev", string),
-    ("colo_filter_sec_rewriter0_queue", string)
+    ("colo_filter_sec_rewriter0_queue", string),
+    ("colo_checkpoint_host", string),
+    ("colo_checkpoint_port", string)
     ])
 
 libxl_device_pci = Struct("device_pci", [
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 32a47f6..ad5e193 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -1145,6 +1145,10 @@ static int parse_nic_config(libxl_device_nic *nic, XLU_Config **config, char *to
         replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg);
     } else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) {
         replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg);
+    } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) {
+        replace_string(&nic->colo_checkpoint_host, oparg);
+    } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) {
+        replace_string(&nic->colo_checkpoint_port, oparg);
     } else if (MATCH_OPTION("accel", token, oparg)) {
         fprintf(stderr, "the accel parameter for vifs is currently not supported\n");
     } else {
-- 
2.7.4




_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel