[Qemu-devel] [PATCH v2 0/3] rdma: IPv6 bugfixes and cleanups

[Qemu-devel] [PATCH v2 0/3] rdma: IPv6 bugfixes and cleanups

[mrhines]

From: "Michael R. Hines" <mrhines@us.ibm.com>

Changes:
1. IPv6 support was broken under libvirt.
2. incorrect use of error_setg()
3. DPRINTF flag was not disabled


Michael R. Hines (3):
  rdma: bugfix: make IPv6 support work
  rdma: forgot to turn off the debugging flag
  rdma: correct newlines in error statements

 migration-rdma.c |   99 +++++++++++++++++++++++++++++-------------------------
 1 file changed, 54 insertions(+), 45 deletions(-)

-- 
1.7.10.4

[Qemu-devel] [PATCH v2 1/3] rdma: bugfix: make IPv6 support work

[mrhines]

From: "Michael R. Hines" <mrhines@us.ibm.com>

RDMA does not use sockets, so we cannot use many of the socket
helper functions, but we *do* use inet_parse() which gives
RDMA all the necessary details of the connection parameters.

However, when testing with libvirt, a simple IPv6 migration test failed
because we were not using getaddrinfo() properly.

This makes IPv6 migration over RDMA work.

Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>
---
 migration-rdma.c |   33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/migration-rdma.c b/migration-rdma.c
index d044830..9cf73e3 100644
--- a/migration-rdma.c
+++ b/migration-rdma.c
@@ -392,6 +392,7 @@ typedef struct RDMAContext {
     uint64_t unregistrations[RDMA_SIGNALED_SEND_MAX];
 
     GHashTable *blockmap;
+    bool ipv6;
 } RDMAContext;
 
 /*
@@ -744,6 +745,7 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
     char port_str[16];
     struct rdma_cm_event *cm_event;
     char ip[40] = "unknown";
+    int af = rdma->ipv6 ? PF_INET6 : PF_INET;
 
     if (rdma->host == NULL || !strcmp(rdma->host, "")) {
         ERROR(errp, "RDMA hostname has not been set\n");
@@ -773,7 +775,7 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
         goto err_resolve_get_addr;
     }
 
-    inet_ntop(AF_INET, &((struct sockaddr_in *) res->ai_addr)->sin_addr,
+    inet_ntop(af, &((struct sockaddr_in *) res->ai_addr)->sin_addr,
                                 ip, sizeof ip);
     DPRINTF("%s => %s\n", rdma->host, ip);
 
@@ -2236,9 +2238,12 @@ err_rdma_source_connect:
 static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
 {
     int ret = -EINVAL, idx;
+    int af = rdma->ipv6 ? PF_INET6 : PF_INET;
     struct sockaddr_in sin;
     struct rdma_cm_id *listen_id;
     char ip[40] = "unknown";
+    struct addrinfo *res;
+    char port_str[16];
 
     for (idx = 0; idx <= RDMA_WRID_MAX; idx++) {
         rdma->wr_data[idx].control_len = 0;
@@ -2266,27 +2271,30 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
     }
 
     memset(&sin, 0, sizeof(sin));
-    sin.sin_family = AF_INET;
+    sin.sin_family = af;
     sin.sin_port = htons(rdma->port);
+    snprintf(port_str, 16, "%d", rdma->port);
+    port_str[15] = '\0';
 
     if (rdma->host && strcmp("", rdma->host)) {
-        struct hostent *dest_addr;
-        dest_addr = gethostbyname(rdma->host);
-        if (!dest_addr) {
-            ERROR(errp, "migration could not gethostbyname!\n");
-            ret = -EINVAL;
+        ret = getaddrinfo(rdma->host, port_str, NULL, &res);
+        if (ret < 0) {
+            ERROR(errp, "could not getaddrinfo address %s\n", rdma->host);
             goto err_dest_init_bind_addr;
         }
-        memcpy(&sin.sin_addr.s_addr, dest_addr->h_addr,
-                dest_addr->h_length);
-        inet_ntop(AF_INET, dest_addr->h_addr, ip, sizeof ip);
+
+
+        inet_ntop(af, &((struct sockaddr_in *) res->ai_addr)->sin_addr,
+                                    ip, sizeof ip);
     } else {
-        sin.sin_addr.s_addr = INADDR_ANY;
+        ERROR(errp, "migration host and port not specified!\n");
+        ret = -EINVAL;
+        goto err_dest_init_bind_addr;
     }
 
     DPRINTF("%s => %s\n", rdma->host, ip);
 
-    ret = rdma_bind_addr(listen_id, (struct sockaddr *)&sin);
+    ret = rdma_bind_addr(listen_id, res->ai_addr);
     if (ret) {
         ERROR(errp, "Error: could not rdma_bind_addr!\n");
         goto err_dest_init_bind_addr;
@@ -2321,6 +2329,7 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp)
         if (addr != NULL) {
             rdma->port = atoi(addr->port);
             rdma->host = g_strdup(addr->host);
+            rdma->ipv6 = addr->ipv6;
         } else {
             ERROR(errp, "bad RDMA migration address '%s'", host_port);
             g_free(rdma);
-- 
1.7.10.4

[Qemu-devel] [PATCH v2 2/3] rdma: forgot to turn off the debugging flag

[mrhines]

From: "Michael R. Hines" <mrhines@us.ibm.com>

Ooops. We forgot to turn off the flag.

Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>
---
 migration-rdma.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migration-rdma.c b/migration-rdma.c
index 9cf73e3..fe6118d 100644
--- a/migration-rdma.c
+++ b/migration-rdma.c
@@ -27,7 +27,7 @@
 #include <string.h>
 #include <rdma/rdma_cma.h>
 
-#define DEBUG_RDMA
+//#define DEBUG_RDMA
 //#define DEBUG_RDMA_VERBOSE
 //#define DEBUG_RDMA_REALLY_VERBOSE
 
-- 
1.7.10.4

[Qemu-devel] [PATCH v2 3/3] rdma: correct newlines in error statements

[mrhines]

From: "Michael R. Hines" <mrhines@us.ibm.com>

Don't print newlines on the error_setg() function,
but still allow newlines on fprintf().

Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>
---
 migration-rdma.c |   68 +++++++++++++++++++++++++++---------------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/migration-rdma.c b/migration-rdma.c
index fe6118d..c958e5f 100644
--- a/migration-rdma.c
+++ b/migration-rdma.c
@@ -60,7 +60,7 @@
  */
 #define ERROR(errp, fmt, ...) \
     do { \
-        fprintf(stderr, "RDMA ERROR: " fmt, ## __VA_ARGS__); \
+        fprintf(stderr, "RDMA ERROR: " fmt "\n", ## __VA_ARGS__); \
         if (errp && (*(errp) == NULL)) { \
             error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \
         } \
@@ -748,21 +748,21 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
     int af = rdma->ipv6 ? PF_INET6 : PF_INET;
 
     if (rdma->host == NULL || !strcmp(rdma->host, "")) {
-        ERROR(errp, "RDMA hostname has not been set\n");
+        ERROR(errp, "RDMA hostname has not been set");
         return -1;
     }
 
     /* create CM channel */
     rdma->channel = rdma_create_event_channel();
     if (!rdma->channel) {
-        ERROR(errp, "could not create CM channel\n");
+        ERROR(errp, "could not create CM channel");
         return -1;
     }
 
     /* create CM id */
     ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP);
     if (ret) {
-        ERROR(errp, "could not create channel id\n");
+        ERROR(errp, "could not create channel id");
         goto err_resolve_create_id;
     }
 
@@ -771,7 +771,7 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
 
     ret = getaddrinfo(rdma->host, port_str, NULL, &res);
     if (ret < 0) {
-        ERROR(errp, "could not getaddrinfo address %s\n", rdma->host);
+        ERROR(errp, "could not getaddrinfo address %s", rdma->host);
         goto err_resolve_get_addr;
     }
 
@@ -783,7 +783,7 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
     ret = rdma_resolve_addr(rdma->cm_id, NULL, res->ai_addr,
             RDMA_RESOLVE_TIMEOUT_MS);
     if (ret) {
-        ERROR(errp, "could not resolve address %s\n", rdma->host);
+        ERROR(errp, "could not resolve address %s", rdma->host);
         goto err_resolve_get_addr;
     }
 
@@ -791,12 +791,12 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
 
     ret = rdma_get_cm_event(rdma->channel, &cm_event);
     if (ret) {
-        ERROR(errp, "could not perform event_addr_resolved\n");
+        ERROR(errp, "could not perform event_addr_resolved");
         goto err_resolve_get_addr;
     }
 
     if (cm_event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
-        ERROR(errp, "result not equal to event_addr_resolved %s\n",
+        ERROR(errp, "result not equal to event_addr_resolved %s",
                 rdma_event_str(cm_event->event));
         perror("rdma_resolve_addr");
         goto err_resolve_get_addr;
@@ -806,17 +806,17 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
     /* resolve route */
     ret = rdma_resolve_route(rdma->cm_id, RDMA_RESOLVE_TIMEOUT_MS);
     if (ret) {
-        ERROR(errp, "could not resolve rdma route\n");
+        ERROR(errp, "could not resolve rdma route");
         goto err_resolve_get_addr;
     }
 
     ret = rdma_get_cm_event(rdma->channel, &cm_event);
     if (ret) {
-        ERROR(errp, "could not perform event_route_resolved\n");
+        ERROR(errp, "could not perform event_route_resolved");
         goto err_resolve_get_addr;
     }
     if (cm_event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
-        ERROR(errp, "result not equal to event_route_resolved: %s\n",
+        ERROR(errp, "result not equal to event_route_resolved: %s",
                         rdma_event_str(cm_event->event));
         rdma_ack_cm_event(cm_event);
         goto err_resolve_get_addr;
@@ -2117,26 +2117,26 @@ static int qemu_rdma_source_init(RDMAContext *rdma, Error **errp, bool pin_all)
     if (ret) {
         ERROR(temp, "rdma migration: error allocating pd and cq! Your mlock()"
                     " limits may be too low. Please check $ ulimit -a # and "
-                    "search for 'ulimit -l' in the output\n");
+                    "search for 'ulimit -l' in the output");
         goto err_rdma_source_init;
     }
 
     ret = qemu_rdma_alloc_qp(rdma);
     if (ret) {
-        ERROR(temp, "rdma migration: error allocating qp!\n");
+        ERROR(temp, "rdma migration: error allocating qp!");
         goto err_rdma_source_init;
     }
 
     ret = qemu_rdma_init_ram_blocks(rdma);
     if (ret) {
-        ERROR(temp, "rdma migration: error initializing ram blocks!\n");
+        ERROR(temp, "rdma migration: error initializing ram blocks!");
         goto err_rdma_source_init;
     }
 
     for (idx = 0; idx <= RDMA_WRID_MAX; idx++) {
         ret = qemu_rdma_reg_control(rdma, idx);
         if (ret) {
-            ERROR(temp, "rdma migration: error registering %d control!\n",
+            ERROR(temp, "rdma migration: error registering %d control!",
                                                             idx);
             goto err_rdma_source_init;
         }
@@ -2178,7 +2178,7 @@ static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
     ret = rdma_connect(rdma->cm_id, &conn_param);
     if (ret) {
         perror("rdma_connect");
-        ERROR(errp, "connecting to destination!\n");
+        ERROR(errp, "connecting to destination!");
         rdma_destroy_id(rdma->cm_id);
         rdma->cm_id = NULL;
         goto err_rdma_source_connect;
@@ -2187,7 +2187,7 @@ static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
     ret = rdma_get_cm_event(rdma->channel, &cm_event);
     if (ret) {
         perror("rdma_get_cm_event after rdma_connect");
-        ERROR(errp, "connecting to destination!\n");
+        ERROR(errp, "connecting to destination!");
         rdma_ack_cm_event(cm_event);
         rdma_destroy_id(rdma->cm_id);
         rdma->cm_id = NULL;
@@ -2196,7 +2196,7 @@ static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
 
     if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
         perror("rdma_get_cm_event != EVENT_ESTABLISHED after rdma_connect");
-        ERROR(errp, "connecting to destination!\n");
+        ERROR(errp, "connecting to destination!");
         rdma_ack_cm_event(cm_event);
         rdma_destroy_id(rdma->cm_id);
         rdma->cm_id = NULL;
@@ -2212,7 +2212,7 @@ static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
      */
     if (rdma->pin_all && !(cap.flags & RDMA_CAPABILITY_PIN_ALL)) {
         ERROR(errp, "Server cannot support pinning all memory. "
-                        "Will register memory dynamically.\n");
+                        "Will register memory dynamically.");
         rdma->pin_all = false;
     }
 
@@ -2222,7 +2222,7 @@ static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
 
     ret = qemu_rdma_post_recv_control(rdma, 0);
     if (ret) {
-        ERROR(errp, "posting second control recv!\n");
+        ERROR(errp, "posting second control recv!");
         goto err_rdma_source_connect;
     }
 
@@ -2251,14 +2251,14 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
     }
 
     if (rdma->host == NULL) {
-        ERROR(errp, "RDMA host is not set!\n");
+        ERROR(errp, "RDMA host is not set!");
         rdma->error_state = -EINVAL;
         return -1;
     }
     /* create CM channel */
     rdma->channel = rdma_create_event_channel();
     if (!rdma->channel) {
-        ERROR(errp, "could not create rdma event channel\n");
+        ERROR(errp, "could not create rdma event channel");
         rdma->error_state = -EINVAL;
         return -1;
     }
@@ -2266,7 +2266,7 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
     /* create CM id */
     ret = rdma_create_id(rdma->channel, &listen_id, NULL, RDMA_PS_TCP);
     if (ret) {
-        ERROR(errp, "could not create cm_id!\n");
+        ERROR(errp, "could not create cm_id!");
         goto err_dest_init_create_listen_id;
     }
 
@@ -2279,7 +2279,7 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
     if (rdma->host && strcmp("", rdma->host)) {
         ret = getaddrinfo(rdma->host, port_str, NULL, &res);
         if (ret < 0) {
-            ERROR(errp, "could not getaddrinfo address %s\n", rdma->host);
+            ERROR(errp, "could not getaddrinfo address %s", rdma->host);
             goto err_dest_init_bind_addr;
         }
 
@@ -2287,7 +2287,7 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
         inet_ntop(af, &((struct sockaddr_in *) res->ai_addr)->sin_addr,
                                     ip, sizeof ip);
     } else {
-        ERROR(errp, "migration host and port not specified!\n");
+        ERROR(errp, "migration host and port not specified!");
         ret = -EINVAL;
         goto err_dest_init_bind_addr;
     }
@@ -2296,7 +2296,7 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
 
     ret = rdma_bind_addr(listen_id, res->ai_addr);
     if (ret) {
-        ERROR(errp, "Error: could not rdma_bind_addr!\n");
+        ERROR(errp, "Error: could not rdma_bind_addr!");
         goto err_dest_init_bind_addr;
     }
 
@@ -3036,7 +3036,7 @@ static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque,
                     &reg_result_idx, rdma->pin_all ?
                     qemu_rdma_reg_whole_ram_blocks : NULL);
         if (ret < 0) {
-            ERROR(errp, "receiving remote info!\n");
+            ERROR(errp, "receiving remote info!");
             return ret;
         }
 
@@ -3061,7 +3061,7 @@ static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque,
         if (local->nb_blocks != nb_remote_blocks) {
             ERROR(errp, "ram blocks mismatch #1! "
                         "Your QEMU command line parameters are probably "
-                        "not identical on both the source and destination.\n");
+                        "not identical on both the source and destination.");
             return -EINVAL;
         }
 
@@ -3077,7 +3077,7 @@ static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque,
                 if (rdma->block[i].length != local->block[j].length) {
                     ERROR(errp, "ram blocks mismatch #2! "
                         "Your QEMU command line parameters are probably "
-                        "not identical on both the source and destination.\n");
+                        "not identical on both the source and destination.");
                     return -EINVAL;
                 }
                 local->block[j].remote_host_addr =
@@ -3089,7 +3089,7 @@ static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque,
             if (j >= local->nb_blocks) {
                 ERROR(errp, "ram blocks mismatch #3! "
                         "Your QEMU command line parameters are probably "
-                        "not identical on both the source and destination.\n");
+                        "not identical on both the source and destination.");
                 return -EINVAL;
             }
         }
@@ -3163,7 +3163,7 @@ static void rdma_accept_incoming_migration(void *opaque)
     ret = qemu_rdma_accept(rdma);
 
     if (ret) {
-        ERROR(errp, "RDMA Migration initialization failed!\n");
+        ERROR(errp, "RDMA Migration initialization failed!");
         return;
     }
 
@@ -3171,7 +3171,7 @@ static void rdma_accept_incoming_migration(void *opaque)
 
     f = qemu_fopen_rdma(rdma, "rb");
     if (f == NULL) {
-        ERROR(errp, "could not qemu_fopen_rdma!\n");
+        ERROR(errp, "could not qemu_fopen_rdma!");
         qemu_rdma_cleanup(rdma);
         return;
     }
@@ -3204,7 +3204,7 @@ void rdma_start_incoming_migration(const char *host_port, Error **errp)
     ret = rdma_listen(rdma->listen_id, 5);
 
     if (ret) {
-        ERROR(errp, "listening on socket!\n");
+        ERROR(errp, "listening on socket!");
         goto err;
     }
 
@@ -3228,7 +3228,7 @@ void rdma_start_outgoing_migration(void *opaque,
     int ret = 0;
 
     if (rdma == NULL) {
-        ERROR(temp, "Failed to initialize RDMA data structures! %d\n", ret);
+        ERROR(temp, "Failed to initialize RDMA data structures! %d", ret);
         goto err;
     }
 
-- 
1.7.10.4

Re: [Qemu-devel] [PATCH v2 0/3] rdma: IPv6 bugfixes and cleanups

[Anthony Liguori]

Applied.  Thanks.

Regards,

Anthony Liguori