Re: [PATCH RFC V2 7/7] COLO-Proxy: Use socket to get checkpoint event.

[Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>]

On Wed, Feb 15, 2017 at 05:54:33PM +0800, Zhang Chen wrote:
> We use kernel colo proxy's way to get the checkpoint event
> from qemu colo-compare.
> Qemu colo-compare need add a API to support this(I will add this in qemu).
> 
> Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
> ---
>  tools/libxl/libxl_colo.h         |  2 +
>  tools/libxl/libxl_colo_proxy.c   | 84 +++++++++++++++++++++++++++++++++++++---
>  tools/libxl/libxl_colo_restore.c | 11 ++++--
>  tools/libxl/libxl_colo_save.c    | 22 +++++++----
>  tools/libxl/libxl_nic.c          |  4 ++
>  tools/libxl/libxl_types.idl      |  4 +-
>  tools/libxl/xl_cmdimpl.c         |  4 ++
>  7 files changed, 113 insertions(+), 18 deletions(-)
> 
> diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h
> index 4746d8c..6c01b55 100644
> --- a/tools/libxl/libxl_colo.h
> +++ b/tools/libxl/libxl_colo.h
> @@ -69,6 +69,8 @@ struct libxl__colo_proxy_state {
>       *          False means use kernel colo proxy.
>       */
>      bool is_userspace_proxy;
> +    const char *checkpoint_host;
> +    const char *checkpoint_port;
>  };
>  
>  struct libxl__colo_save_state {
> diff --git a/tools/libxl/libxl_colo_proxy.c b/tools/libxl/libxl_colo_proxy.c
> index dd902fc..9d21cf1 100644
> --- a/tools/libxl/libxl_colo_proxy.c
> +++ b/tools/libxl/libxl_colo_proxy.c
> @@ -18,6 +18,9 @@
>  #include "libxl_internal.h"
>  
>  #include <netlink/netlink.h>
> +#include <arpa/inet.h>
> +#include <sys/socket.h>
> +#include <netinet/in.h>
>  
>  /* Consistent with the new COLO netlink channel in kernel side */
>  #define NETLINK_COLO 28
> @@ -76,6 +79,26 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, uint8_t *buff,
>      return ret;
>  }
>  
> +static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps,
> +                                     char *buff,
> +                                     unsigned int timeout_us)
> +{
> +    struct timeval tv;
> +    int ret;
> +
> +    STATE_AO_GC(cps->ao);
> +
> +    if (timeout_us) {
> +        tv.tv_sec = timeout_us / 1000000;
> +        tv.tv_usec = timeout_us % 1000000;
> +        setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
> +    }
> +
> +    ret = recv(cps->sock_fd, buff, sizeof(buff),0);
> +
> +    return ret;
> +}
> +
>  /* error: return -1, otherwise return 0 */
>  static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff,
>                                 unsigned int timeout_us)
> @@ -153,8 +176,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps)
>      STATE_AO_GC(cps->ao);
>  
>      /* If enable userspace proxy mode, we don't need setup kernel proxy */
> -    if (cps->is_userspace_proxy)
> +    if (cps->is_userspace_proxy) {
> +        struct sockaddr_in addr;
> +        int port;
> +        char recvbuff[1024];
> +
> +        memset(&addr, 0, sizeof(addr));
> +        port = atoi(cps->checkpoint_port);
> +        addr.sin_family = AF_INET;
> +        addr.sin_port = htons(port);
> +        addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host);
> +
> +        skfd = socket(AF_INET, SOCK_STREAM, 0);
> +        if (skfd < 0) {
> +            LOGD(ERROR, ao->domid, "can not create a TCP socket: %s",
> +                 strerror(errno));
> +            goto out;
> +        }
> +
> +        cps->sock_fd = skfd;
> +
> +        if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
> +            LOGD(ERROR, ao->domid, "connect error");
> +            goto out;
> +        }
> +
> +        char sendbuf[] = "COLO_USERSPACE_PROXY_INIT";

Um, could you move it right around 'char recvbuff' ?

> +        ret = send(skfd, sendbuf, strlen(sendbuf),0);
> +        if (ret < 0)
> +            goto out;
> +
> +        ret = colo_userspace_proxy_recv(cps, recvbuff, 500000);

That 500000 looks like a good candidate for a #define?

> +        if (size < 0) {
> +            LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s",
> +                 strerror(errno));
> +            goto out;
> +        }
> +
>          return 0;
> +    }
>  
>      skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO);
>      if (skfd < 0) {
> @@ -247,8 +307,11 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps)
>       * If enable userspace proxy mode,
>       * we don't need preresume kernel proxy
>       */
> -    if (cps->is_userspace_proxy)
> +    if (cps->is_userspace_proxy) {
> +        char sendbuf[] = "COLO_CHECKPOINT";
> +        send(cps->sock_fd, sendbuf, strlen(sendbuf),0);
>          return;
> +    }
>  
>      colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT);
>      /* TODO: need to handle if the call fails... */
> @@ -277,16 +340,25 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
>      struct nlmsghdr *h;
>      struct colo_msg *m;
>      int ret = -1;
> +    char recvbuff[1024];
>  
>      STATE_AO_GC(cps->ao);
>  
>      /*
> -     * enable userspace proxy mode, tmp sleep.
> -     * then we will add qemu API support this func.
> +     * enable userspace proxy mode.
> +     * Then we will add qemu API support for this func.
>       */
>      if (cps->is_userspace_proxy) {
> -        sleep(timeout_us / 1000000);
> -        return 0;
> +        ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us);
> +        if (ret <= 0)
> +            return 0;
> +
> +        if (!strcmp(recvbuff, "DO_CHECKPOINT")) {
> +            return 1;
> +        } else {
> +            LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint error");
> +            return -1;
> +        }
>      }
>  
>      size = colo_proxy_recv(cps, &buff, timeout_us);
> diff --git a/tools/libxl/libxl_colo_restore.c b/tools/libxl/libxl_colo_restore.c
> index c6d239a..065ea00 100644
> --- a/tools/libxl/libxl_colo_restore.c
> +++ b/tools/libxl/libxl_colo_restore.c
> @@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc,
>          }
>      }
>  
> -    colo_proxy_preresume(&crs->cps);
> +    if (!crs->cps.is_userspace_proxy)
> +        colo_proxy_preresume(&crs->cps);
>  
>      colo_restore_resume_vm(egc, crcs);
>  
> @@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc,
>      cds->ops = colo_restore_ops;
>  
>      crs->cps.ao = ao;
> -    if (colo_proxy_setup(&crs->cps)) {
> -        LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
> -        goto out;
> +    if (!crs->cps.is_userspace_proxy) {
> +        if (colo_proxy_setup(&crs->cps)) {
> +            LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
> +            goto out;
> +        }
>      }
>  
>      if (init_device_subkind(cds))
> diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c
> index 91e3fce..a0cfc5a 100644
> --- a/tools/libxl/libxl_colo_save.c
> +++ b/tools/libxl/libxl_colo_save.c
> @@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
>      libxl__checkpoint_devices_state *const cds = &dss->cds;
>      libxl__srm_save_autogen_callbacks *const callbacks =
>          &dss->sws.shs.callbacks.save.a;
> +    libxl_device_nic *nics;
>  
>      STATE_AO_GC(dss->ao);
>  
> @@ -110,24 +111,31 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
>          css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup",
>                                             libxl__xen_script_dir_path());
>  
> -    /* If enable userspace proxy mode, we don't need VIF */
> -    if (css->cps.is_userspace_proxy)
> -        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
> -    else
> -        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
> -                                 (1 << LIBXL__DEVICE_KIND_VBD);
> -
>      cds->ops = colo_ops;
>      cds->callback = colo_save_setup_done;
>      cds->ao = ao;
>      cds->domid = dss->domid;
>      cds->concrete_data = css;
>  
> +    /* If enable userspace proxy mode, we don't need VIF */
> +    if (css->cps.is_userspace_proxy) {
> +        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
> +
> +        /* Use this args we can connect to qemu colo-compare */
> +        nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
> +        css->cps.checkpoint_host = nics->colo_checkpoint_host;
> +        css->cps.checkpoint_port = nics->colo_checkpoint_port;
> +    } else {
> +        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
> +                                 (1 << LIBXL__DEVICE_KIND_VBD);
> +    }
> +
>      css->srs.ao = ao;
>      css->srs.fd = css->recv_fd;
>      css->srs.back_channel = true;
>      libxl__stream_read_start(egc, &css->srs);
>      css->cps.ao = ao;
> +
>      if (colo_proxy_setup(&css->cps)) {
>          LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
>          goto out;
> diff --git a/tools/libxl/libxl_nic.c b/tools/libxl/libxl_nic.c
> index 5e1fecd..6bc6146 100644
> --- a/tools/libxl/libxl_nic.c
> +++ b/tools/libxl/libxl_nic.c
> @@ -246,6 +246,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t domid,
>      MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev);
>      MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev);
>      MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue);
> +    MAYBE_ADD_COLO_ARGS(checkpoint_host);
> +    MAYBE_ADD_COLO_ARGS(checkpoint_port);
>  
>  #undef MAYBE_ADD_COLO_ARGS
>  
> @@ -451,6 +453,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc,
>      CHECK_COLO_ARGS(filter_sec_redirector1_indev);
>      CHECK_COLO_ARGS(filter_sec_redirector1_outdev);
>      CHECK_COLO_ARGS(filter_sec_rewriter0_queue);
> +    CHECK_COLO_ARGS(checkpoint_host);
> +    CHECK_COLO_ARGS(checkpoint_port);
>  
>  #undef CHECK_COLO_ARGS
>  
> diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
> index 47e96b1..0b412ee 100644
> --- a/tools/libxl/libxl_types.idl
> +++ b/tools/libxl/libxl_types.idl
> @@ -671,7 +671,9 @@ libxl_device_nic = Struct("device_nic", [
>      ("colo_filter_sec_redirector1_queue", string),
>      ("colo_filter_sec_redirector1_indev", string),
>      ("colo_filter_sec_redirector1_outdev", string),
> -    ("colo_filter_sec_rewriter0_queue", string)
> +    ("colo_filter_sec_rewriter0_queue", string),
> +    ("colo_checkpoint_host", string),
> +    ("colo_checkpoint_port", string)
>      ])
>  
>  libxl_device_pci = Struct("device_pci", [
> diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
> index 32a47f6..ad5e193 100644
> --- a/tools/libxl/xl_cmdimpl.c
> +++ b/tools/libxl/xl_cmdimpl.c
> @@ -1145,6 +1145,10 @@ static int parse_nic_config(libxl_device_nic *nic, XLU_Config **config, char *to
>          replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg);
>      } else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) {
>          replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg);
> +    } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) {
> +        replace_string(&nic->colo_checkpoint_host, oparg);
> +    } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) {
> +        replace_string(&nic->colo_checkpoint_port, oparg);
>      } else if (MATCH_OPTION("accel", token, oparg)) {
>          fprintf(stderr, "the accel parameter for vifs is currently not supported\n");
>      } else {
> -- 
> 2.7.4
> 
> 
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> https://lists.xen.org/xen-devel

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel