[Qemu-devel] [RFC 0/4] QEMU shared-memory over network

[Qemu-devel] [RFC 0/4] QEMU shared-memory over network

[Baptiste Reynal]

This patch series is an extension to "[RFC v2 0/6] QEMU shared-memory backend",
and allows two instances of QEMU to share a memory over the network.

Instead of allocating the memory using hugetlbfs, the
memory is allocated as a simple ram pointer, and when
a slave connects a new mmio region is created in the
master main memory so that each access is trapped and
forwarded over the network.

The slave has direct access to its memory (no performance
loss) while master accesses (which are less frequent) go
over a network socket.

To test those patches modules and test application are
available on https://git.virtualopensystems.com/dev/qemu-het-tools.
- SDM Platform module on branch sdm_test_mod_v1
- Test application on branch sdm_test_app
- Slave files on branch demo_slaves_sdm

QEMU code is available here https://git.virtualopensystems.com/dev/qemu on branch network-extension-v1.

Run QEMU master instance:
./qemu-system-arm -nographic \
        -kernel zImage \
        -M virt -m 1G \
        -initrd busybox.cpio \
        -object multi-socket-backend,id=sock,listen,host="127.0.0.1",port="1234" \
        -object memory-backend-network,id=mem,size=1G,shared-size=512M,socket=sock,master \
        -object sdm-communication-socket,id=scomm,socket=sock \
        -device sdm-platform,comm=scomm,master,num-slaves=2 \
        -append "mem=512M memmap=512M$0x60000000" \
        -numa node,memdev=mem -m 1G

Run QEMU slave instance:
./qemu-system-arm -nographic \
        -M virt,slave \
        -object multi-socket-backend,id=sock,host="127.0.0.1",port="1234" \
        -object memory-backend-network,id=mem,size=512M,socket=sock \
        -object sdm-signal-nboot,id=boot,nhm=mem \
        -object sdm-communication-socket,id=scomm,socket=sock \
        -device sdm-platform,comm=scomm,len-signals=1,signals[0]="boot" \
        -incoming "shared:mem" \
        -numa node,memdev=mem -m 512M

On the master, load the SDM module (insmod sdm_test_mod.ko) and trigger the boot
with the following command:
./sdm-test -b 0 1 zImage slave.dtb

zImage and slave.dtb are available on branch demo_slaves_sdm.

This patch serie depends on:
- [RFC v2 0/6] SDM Interface
- [RFC v2 1/1] backend: multi-client-socket

This work has been sponsored by Huawei Technologies Duesseldorf GmbH.

Baptiste Reynal (4):
  backend: multi-client-socket
  backend: shared memory over network backend
  migration: extend shared migration type
  hw/misc: sdm signal nboot

 backends/Makefile.objs             |   2 +-
 backends/hostmem-network.c         | 301 ++++++++++++++++++++++++++
 backends/multi-socket.c            | 420 +++++++++++++++++++++++++------------
 hw/misc/Makefile.objs              |   1 +
 hw/misc/sdm-signal-nboot.c         |  62 ++++++
 include/hw/misc/sdm-signal-nboot.h |  28 +++
 include/qemu/multi-socket.h        |  57 ++++-
 include/sysemu/hostmem-network.h   |  79 +++++++
 migration/shared.c                 |  20 +-
 9 files changed, 823 insertions(+), 147 deletions(-)
 create mode 100644 backends/hostmem-network.c
 create mode 100644 hw/misc/sdm-signal-nboot.c
 create mode 100644 include/hw/misc/sdm-signal-nboot.h
 create mode 100644 include/sysemu/hostmem-network.h

-- 
2.7.4

[Qemu-devel] [RFC 1/4] backend: multi-client-socket

[Baptiste Reynal]

This patch updates the multi-client-socket, adding the following
functionalities :
- both UNIX and network socket are now supported :
  -object multi-socket-backend,id=<id>,path=<socket_path>,listen=<on/off>
  -object multi-socket-backend,host=<host>,port=<port>,,listen=<on/off>
- the socket now uses NMMessage for communication. The old API is kept
  for backward compatibility
- multi_socket_send_and_block_to function added, to send a message and
  wait for the answer

Signed-off-by: Baptiste Reynal <b.reynal@virtualopensystems.com>
---
 backends/multi-socket.c     | 420 ++++++++++++++++++++++++++++++--------------
 include/qemu/multi-socket.h |  57 +++++-
 2 files changed, 334 insertions(+), 143 deletions(-)

diff --git a/backends/multi-socket.c b/backends/multi-socket.c
index 2cfbb50..daf06b7 100644
--- a/backends/multi-socket.c
+++ b/backends/multi-socket.c
@@ -12,25 +12,128 @@
 #include "qemu/multi-socket.h"
 #include "qemu/error-report.h"
 
-typedef struct MSHandler MSHandler;
-typedef struct MSRegHandler MSRegHandler;
+void multi_socket_add_reg_handler(MSBackend *backend,
+        void (*reg)(MSClient *client, void *opaque),
+        void *opaque)
+{
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(backend);
 
-struct MSHandler {
-    char *name;
-    void (*read)(MSClient *client, const char *message, void *opaque);
-    void *opaque;
+    if (msc->add_reg_handler) {
+        msc->add_reg_handler(backend, reg, opaque);
+    }
+}
 
-    QLIST_ENTRY(MSHandler) next;
-};
+void multi_socket_add_handler(MSBackend *backend, const char *name,
+        void (*read)(MSClient *client, const char *message, void *opaque),
+        void *opaque)
+{
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(backend);
 
-struct MSRegHandler {
-    void (*reg)(MSClient *client, void *opaque);
-    void *opaque;
+    if (strlen(name) > MS_CMD_SIZE) {
+        error_report("Command \"%s\" size is too big.", name);
+        return;
+    }
 
-    QLIST_ENTRY(MSRegHandler) next;
-};
+    if (msc->add_handler) {
+        msc->add_handler(backend, name, read, opaque);
+    }
+}
+
+int multi_socket_send_fds_to(MSClient *client, int *fds, int count,
+        const char *message, int size) {
+    int cmd_len, payload_len;
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+    MSMessage ms_message;
+
+    cmd_len = strlen(message) + 1;
+    payload_len = size - cmd_len;
+
+    if (cmd_len > MS_CMD_SIZE ||
+            payload_len > MS_PAYLOAD_SIZE) {
+        error_report("Command \"%s\" size is too big.", message);
+    }
+
+    memcpy(ms_message.cmd, message, cmd_len);
+    memcpy(ms_message.payload, message + cmd_len, payload_len);
+
+    if (msc->send_fds_to) {
+        return msc->send_fds_to(client, fds, count, &ms_message);
+    } else {
+        return -1;
+    }
+}
+
+int multi_socket_write_to(MSClient *client, const char *message, int size)
+{
+    int cmd_len, payload_len;
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+    MSMessage ms_message;
+
+    cmd_len = strlen(message) + 1;
+    payload_len = size - cmd_len;
+
+    msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+    if (cmd_len > MS_CMD_SIZE ||
+            payload_len > MS_PAYLOAD_SIZE) {
+        error_report("Command \"%s\" size is too big.", message);
+    }
+
+    memcpy(ms_message.cmd, message, cmd_len);
+    memcpy(ms_message.payload, message + cmd_len, payload_len);
+
+    if (msc->write_to) {
+        return msc->write_to(client, &ms_message);
+    } else {
+        return -1;
+    }
+}
+
+int multi_socket_write_message_to(MSClient *client, MSMessage *message)
+{
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+
+    if (msc->write_to) {
+        return msc->write_to(client, message);
+    } else {
+        return -1;
+    }
+}
+
+int multi_socket_get_fds_num_from(MSClient *client)
+{
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+
+    if (msc->get_fds_num_from) {
+        return msc->get_fds_num_from(client);
+    } else {
+        return -1;
+    }
+}
+
+int multi_socket_get_fds_from(MSClient *client, int *fds)
+{
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+
+    if (msc->get_fds_from) {
+        return msc->get_fds_from(client, fds);
+    } else {
+        return -1;
+    }
+}
+
+int multi_socket_write_and_block_to(MSClient *client, MSMessage *message)
+{
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_GET_CLASS(client->backend);
+
+    if (msc->write_and_block_to) {
+        return msc->write_and_block_to(client, message);
+    } else {
+        return -1;
+    }
+}
 
-static void multi_socket_get_fds(MSClient *client, struct msghdr msg)
+typedef struct MSHandler MSHandler;
+static void ms_get_fds(MSClient *client, struct msghdr msg)
 {
     struct cmsghdr *cmsg;
 
@@ -58,51 +161,7 @@ static void multi_socket_get_fds(MSClient *client, struct msghdr msg)
     }
 }
 
-static gboolean
-multi_socket_read_handler(GIOChannel *channel, GIOCondition cond, void *opaque)
-{
-    MSClient *client = (MSClient *) opaque;
-    MSBackend *backend = client->backend;
-
-    char message[BUFFER_SIZE];
-    struct MSHandler *h;
-
-    struct msghdr msg = { NULL, };
-    struct iovec iov[1];
-    union {
-        struct cmsghdr cmsg;
-        char control[CMSG_SPACE(sizeof(int) * MAX_FDS)];
-    } msg_control;
-    int flags = 0;
-    ssize_t ret;
-
-    iov[0].iov_base = message;
-    iov[0].iov_len = BUFFER_SIZE;
-
-    msg.msg_iov = iov;
-    msg.msg_iovlen = 1;
-    msg.msg_control = &msg_control;
-    msg.msg_controllen = sizeof(msg_control);
-
-    ret = recvmsg(client->fd, &msg, flags);
-
-    if (ret > 0) {
-        multi_socket_get_fds(client, msg);
-
-        /* handler callback */
-        QLIST_FOREACH(h, &backend->handlers, next) {
-            if (!strncmp(h->name, message, strlen(h->name))) {
-                h->read(client, message + strlen(h->name) + 1, h->opaque);
-                return TRUE;
-            }
-        }
-        error_report("Unrecognized message: %s", message);
-    }
-
-    return FALSE;
-}
-
-void multi_socket_add_reg_handler(MSBackend *backend,
+static void ms_add_reg_handler(MSBackend *backend,
         void (*reg)(MSClient *client, void *opaque), void *opaque)
 {
     struct MSRegHandler *h;
@@ -115,44 +174,32 @@ void multi_socket_add_reg_handler(MSBackend *backend,
     QLIST_INSERT_HEAD(&backend->reg_handlers, h, next);
 }
 
-void multi_socket_add_handler(MSBackend *backend,
-        const char *name,
-        void (*read)(MSClient *c, const char *message, void *opaque),
+static void ms_add_handler(MSBackend *backend,
+        const char *cmd,
+        void (*read)(MSClient *client, const char *payload, void *opaque),
         void *opaque)
 {
-    struct MSHandler *h;
+    struct MSHandler *handler;
 
-    /* check that the handler name is not taken */
-    QLIST_FOREACH(h, &backend->handlers, next) {
-        if (!strcmp(h->name, name)) {
-            error_report("Handler %s already exists", name);
-            return;
+    /* check the command is available */
+    QLIST_FOREACH(handler, &backend->handlers, next) {
+        if (!strcmp(handler->cmd, cmd)) {
+                error_report("Handler %s already exists", cmd);
+                return;
         }
     }
 
-    h = g_malloc(sizeof(struct MSHandler));
+    handler = g_malloc(sizeof(struct MSHandler));
 
-    h->name = g_strdup(name);
-    h->read = read;
-    h->opaque = opaque;
+    handler->cmd = g_strdup(cmd);
+    handler->read = read;
+    handler->opaque = opaque;
 
-    QLIST_INSERT_HEAD(&backend->handlers, h, next);
+    QLIST_INSERT_HEAD(&backend->handlers, handler, next);
 }
 
-static void multi_socket_init_client(MSBackend *backend,
-        MSClient *client, int fd, GIOFunc handler)
-{
-    client->backend = backend;
-    client->fd = fd;
-    client->chan = g_io_channel_unix_new(fd);
-    client->tag = g_io_add_watch(client->chan, G_IO_IN, handler, client);
-
-    g_io_channel_set_encoding(client->chan, NULL, NULL);
-    g_io_channel_set_buffered(client->chan, FALSE);
-}
-
-int multi_socket_send_fds_to(MSClient *client, int *fds, int count,
-        const char *message, int size)
+static int ms_send_fds_to(MSClient *client, int *fds, int count,
+        MSMessage *message)
 {
     struct msghdr msgh;
     struct iovec iov;
@@ -165,9 +212,8 @@ int multi_socket_send_fds_to(MSClient *client, int *fds, int count,
     memset(&msgh, 0, sizeof(msgh));
     memset(control, 0, sizeof(control));
 
-    /* set the payload */
-    iov.iov_base = (uint8_t *) message;
-    iov.iov_len = size;
+    iov.iov_base = message;
+    iov.iov_len = sizeof(MSMessage);
 
     msgh.msg_iov = &iov;
     msgh.msg_iovlen = 1;
@@ -189,38 +235,92 @@ int multi_socket_send_fds_to(MSClient *client, int *fds, int count,
     return r;
 }
 
-int multi_socket_write_to(MSClient *client, const char *message, int size)
+static int ms_write_to(MSClient *client, MSMessage *message)
 {
-    return multi_socket_send_fds_to(client, 0, 0, message, size);
+    return ms_send_fds_to(client, 0, 0, message);
 }
 
-int multi_socket_get_fds_num_from(MSClient *client)
+static int ms_get_fds_num_from(MSClient *client)
 {
     return client->rcvfds_num;
 }
 
-int multi_socket_get_fds_from(MSClient *client, int *fds)
+static int ms_get_fds_from(MSClient *client, int *fds)
 {
     memcpy(fds, client->rcvfds, client->rcvfds_num * sizeof(int));
+    return ms_get_fds_num_from(client);
+}
 
-    return client->rcvfds_num;
+static gboolean
+ms_read(GIOChannel *channel, GIOCondition cond, void *opaque)
+{
+    MSClient *client = (MSClient *) opaque;
+    MSBackend *backend = client->backend;
+    MSMessage message;
+    MSHandler *handler;
+    int ret;
+
+    struct msghdr msg = { NULL };
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int) * MS_MAX_FDS)];
+    } msg_control;
+    int flags = MSG_WAITALL;
+
+    iov[0].iov_base = &message;
+    iov[0].iov_len = sizeof(MSMessage);
+
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+    msg.msg_control = &msg_control;
+    msg.msg_controllen = sizeof(msg_control);
+
+    ret = recvmsg(client->fd, &msg, flags);
+
+    if (ret > 0) {
+        ms_get_fds(client, msg);
+
+        /* handler callback */
+        QLIST_FOREACH(handler, &backend->handlers, next) {
+            if (!strcmp(handler->cmd, message.cmd)) {
+                handler->read(client, message.payload, handler->opaque);
+                return TRUE;
+            }
+        }
+
+        error_report("Unrecognized command: %s", message.cmd);
+    }
+
+    return FALSE;
 }
 
-static void multi_socket_add_client(MSBackend *backend, int fd)
+static void multi_socket_init_client(MSBackend *backend, MSClient *client,
+        int fd, GIOFunc handler)
 {
-    MSClient *c = g_malloc(sizeof(MSClient));
-    MSRegHandler *h;
+    client->backend = backend;
+    client->fd = fd;
+    client->chan = g_io_channel_unix_new(fd);
+    client->tag = g_io_add_watch(client->chan, G_IO_IN, handler, client);
 
-    multi_socket_init_client(backend, c, fd, multi_socket_read_handler);
-    QLIST_FOREACH(h, &backend->reg_handlers, next) {
-        h->reg(c, h->opaque);
+    g_io_channel_set_encoding(client->chan, NULL, NULL);
+    g_io_channel_set_buffered(client->chan, FALSE);
+}
+
+static void ms_add_client(MSBackend *backend, int fd)
+{
+    MSClient *client = g_malloc(sizeof(MSClient));
+    MSRegHandler *handler;
+
+    multi_socket_init_client(backend, client, fd, ms_read);
+    QLIST_FOREACH(handler, &backend->reg_handlers, next) {
+        handler->reg(client, handler->opaque);
     }
 
-    QLIST_INSERT_HEAD(&backend->clients, c, next);
+    QLIST_INSERT_HEAD(&backend->clients, client, next);
 }
 
-static gboolean
-multi_socket_accept(GIOChannel *channel, GIOCondition cond, void *opaque)
+static gboolean ms_accept(GIOChannel *channel, GIOCondition cond, void *opaque)
 {
     MSClient *client = (MSClient *) opaque;
     MSBackend *backend = client->backend;
@@ -230,11 +330,10 @@ multi_socket_accept(GIOChannel *channel, GIOCondition cond, void *opaque)
     int fd;
 
     len = sizeof(uaddr);
-
     fd = qemu_accept(backend->listener.fd, (struct sockaddr *) &uaddr, &len);
 
     if (fd > 0) {
-        multi_socket_add_client(backend, fd);
+        ms_add_client(backend, fd);
         return true;
     } else {
         perror("Error creating socket.");
@@ -242,54 +341,73 @@ multi_socket_accept(GIOChannel *channel, GIOCondition cond, void *opaque)
     }
 }
 
-static void multi_socket_init_socket(MSBackend *backend)
+static int ms_write_and_block_to(MSClient *client, MSMessage *message)
 {
-    int fd;
+    int ret;
 
-    backend->addr = g_new0(SocketAddress, 1);
-    backend->addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-    backend->addr->q_unix = g_new0(UnixSocketAddress, 1);
-    /* TODO change name with real path */
-    backend->addr->q_unix->path = g_strdup(backend->path);
-
-    if (backend->listen) {
-        fd = socket_listen(backend->addr, NULL);
+    g_source_remove(client->tag);
 
-        if (fd < 0) {
-            perror("Error: Impossible to open socket.");
-            exit(-1);
-        }
-
-        multi_socket_init_client(backend, &backend->listener, fd,
-                multi_socket_accept);
-    } else {
-        fd = socket_connect(backend->addr, NULL, NULL, NULL);
+    ret = ms_write_to(client, message);
+    ms_read(client->chan, G_IO_IN, client);
 
-        if (fd < 0) {
-            perror("Error: Unavailable socket server.");
-            exit(-1);
-        }
+    g_io_add_watch(client->chan, G_IO_IN, ms_read, client);
 
-        multi_socket_init_client(backend, &backend->listener,
-                fd, multi_socket_read_handler);
-    }
+    return ret;
 }
 
 static void multi_socket_backend_complete(UserCreatable *uc, Error **errp)
 {
     MSBackend *backend = MULTI_SOCKET_BACKEND(uc);
+    GIOFunc handler;
 
     QLIST_INIT(&backend->clients);
+    QLIST_INIT(&backend->reg_handlers);
     QLIST_INIT(&backend->handlers);
 
-    multi_socket_init_socket(backend);
+    backend->addr = g_new0(SocketAddress, 1);
+
+    if (backend->path) {
+        backend->addr->kind = SOCKET_ADDRESS_KIND_UNIX;
+        backend->addr->q_unix = g_new0(UnixSocketAddress, 1);
+        backend->addr->q_unix->path = g_strdup(backend->path);
+    } else {
+        backend->addr->kind = SOCKET_ADDRESS_KIND_INET;
+        backend->addr->inet = g_new0(InetSocketAddress, 1);
+        backend->addr->inet->host = g_strdup(backend->host);
+        backend->addr->inet->port = g_strdup(backend->port);
+    }
+
+    if (backend->listen) {
+        backend->fd = socket_listen(backend->addr, NULL);
+        handler = ms_accept;
+    } else {
+        backend->fd = socket_connect(backend->addr, NULL, NULL, NULL);
+        handler = ms_read;
+    }
+
+    if (backend->fd < 0) {
+        perror("Error: Impossible to open socket.");
+        exit(-1);
+    }
+
+    multi_socket_init_client(backend, &backend->listener,
+            backend->fd, handler);
 }
 
 static void multi_socket_class_init(ObjectClass *oc, void *data)
 {
     UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+    MSBackendClass *msc = MULTI_SOCKET_BACKEND_CLASS(oc);
 
     ucc->complete = multi_socket_backend_complete;
+
+    msc->add_reg_handler = ms_add_reg_handler;
+    msc->add_handler = ms_add_handler;
+    msc->write_to = ms_write_to;
+    msc->send_fds_to = ms_send_fds_to;
+    msc->get_fds_num_from = ms_get_fds_num_from;
+    msc->get_fds_from = ms_get_fds_from;
+    msc->write_and_block_to = ms_write_and_block_to;
 }
 
 static bool multi_socket_backend_get_listen(Object *o, Error **errp)
@@ -325,13 +443,45 @@ static void multi_socket_set_path(Object *o, const char *str, Error **errp)
     backend->path = g_strdup(str);
 }
 
+static char *multi_socket_get_host(Object *o, Error **errp)
+{
+    MSBackend *backend = MULTI_SOCKET_BACKEND(o);
+
+    return g_strdup(backend->host);
+}
+
+static void multi_socket_set_host(Object *o, const char *str, Error **errp)
+{
+    MSBackend *backend = MULTI_SOCKET_BACKEND(o);
+
+    backend->host = g_strdup(str);
+}
+
+static char *multi_socket_get_port(Object *o, Error **errp)
+{
+    MSBackend *backend = MULTI_SOCKET_BACKEND(o);
+
+    return g_strdup(backend->port);
+}
+
+static void multi_socket_set_port(Object *o, const char *str, Error **errp)
+{
+    MSBackend *backend = MULTI_SOCKET_BACKEND(o);
+
+    backend->port = g_strdup(str);
+}
+
 static void multi_socket_instance_init(Object *o)
 {
     object_property_add_bool(o, "listen",
-                        multi_socket_backend_get_listen,
-                        multi_socket_backend_set_listen, NULL);
+            multi_socket_backend_get_listen,
+            multi_socket_backend_set_listen, NULL);
     object_property_add_str(o, "path", multi_socket_get_path,
                         multi_socket_set_path, NULL);
+    object_property_add_str(o, "host", multi_socket_get_host,
+                        multi_socket_set_host, NULL);
+    object_property_add_str(o, "port", multi_socket_get_port,
+                        multi_socket_set_port, NULL);
 }
 
 static const TypeInfo multi_socket_backend_info = {
diff --git a/include/qemu/multi-socket.h b/include/qemu/multi-socket.h
index dee866a..8a43e8a 100644
--- a/include/qemu/multi-socket.h
+++ b/include/qemu/multi-socket.h
@@ -1,13 +1,14 @@
 /*
- * QEMU Multi Client socket
+ * QEMU multi client socket
  *
  * Copyright (C) 2015 - Virtual Open Systems
  *
  * Author: Baptiste Reynal <b.reynal@virtualopensystems.com>
  *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * This work is licensed under the terms of the GNU GPL, version 2. See
  * the COPYING file in the top-level directory.
  */
+
 #ifndef QEMU_MS_H
 #define QEMU_MS_H
 
@@ -25,15 +26,23 @@
 #define MULTI_SOCKET_BACKEND_CLASS(klass) \
     OBJECT_CLASS_CHECK(MSBackendClass, (klass), TYPE_MULTI_SOCKET_BACKEND)
 
-#define MAX_FDS     32
-#define BUFFER_SIZE 32
+#define MS_CMD_SIZE     32
+#define MS_PAYLOAD_SIZE 256
+#define MS_MAX_FDS      32
 
 typedef struct MSBackend MSBackend;
 typedef struct MSBackendClass MSBackendClass;
 typedef struct MSClient MSClient;
+typedef struct MSHandler MSHandler;
+typedef struct MSRegHandler MSRegHandler;
+typedef struct MSMessage MSMessage;
+
+struct MSMessage {
+    char cmd[MS_CMD_SIZE];
+    char payload[MS_PAYLOAD_SIZE];
+};
 
 struct MSClient {
-    /* private */
     int fd;
     MSBackend *backend;
     GIOChannel *chan;
@@ -45,17 +54,46 @@ struct MSClient {
     QLIST_ENTRY(MSClient) next;
 };
 
+struct MSHandler {
+    char *cmd;
+    void (*read)(MSClient *client, const char *payload, void *opaque);
+    void *opaque;
+
+    QLIST_ENTRY(MSHandler) next;
+};
+
+struct MSRegHandler {
+    void (*reg)(MSClient *client, void *opaque);
+    void *opaque;
+
+    QLIST_ENTRY(MSRegHandler) next;
+};
+
 struct MSBackendClass {
-    /* private */
     ObjectClass parent_class;
+
+    void (*add_reg_handler)(MSBackend *backend,
+            void (*reg)(MSClient *client, void *opaque),
+            void *opaque);
+    void (*add_handler)(MSBackend *backend, const char *cmd,
+            void (*read)(MSClient *client, const char *payload, void *opaque),
+            void *opaque);
+    int (*write_to)(MSClient *client, MSMessage *message);
+    int (*send_fds_to)(MSClient *client, int *fds, int count,
+            MSMessage *message);
+    int (*get_fds_num_from)(MSClient *client);
+    int (*get_fds_from)(MSClient *client, int *fds);
+    int (*write_and_block_to)(MSClient *client, MSMessage *message);
 };
 
 struct MSBackend {
-    /* private */
     Object parent;
 
-    /* protected */
+    int fd;
+
     char *path;
+    char *host;
+    char *port;
     SocketAddress *addr;
 
     QLIST_HEAD(clients_head, MSClient) clients;
@@ -107,6 +145,7 @@ int multi_socket_send_fds_to(MSClient *client, int *fds, int count,
  * @size: size of the message
  */
 int multi_socket_write_to(MSClient *client, const char *message, int size);
+int multi_socket_write_message_to(MSClient *client, MSMessage *message);
 
 /* Get fds size received with the last message.
  *
@@ -121,4 +160,6 @@ int multi_socket_get_fds_num_from(MSClient *client);
  */
 int multi_socket_get_fds_from(MSClient *client, int *fds);
 
+int multi_socket_write_and_block_to(MSClient *client, MSMessage *message);
+
 #endif
-- 
2.7.4

[Qemu-devel] [RFC 2/4] backend: shared memory over network backend

[Baptiste Reynal]

This patch extends the shared memory backend with the capability to
share a memory region over the network.

On the master side, when a slave connects an mmio region is created and
each access is trapped and sent over the network through the multi-client
socket.

Instantiation on the master:
-object memory-backend-network,id=<id>,size=<memory_sizeK,M,G>,
    socket=<socket>,shared_size=<shared_sizeK,M,G>,master

Instantiation on the slave:
-object memory-backend-network,id=<id>,size=<memory_sizeK,M,H>,
    socket=<socket>

To allocate the memory, the master will start from (size - shared_size),
and search for an area of the slave size (round up to the power of 2).
It is meant to be coherent with the dma_alloc_coherent allocator.

Signed-off-by: Baptiste Reynal <b.reynal@virtualopensystems.com>
---
 backends/Makefile.objs           |   2 +-
 backends/hostmem-network.c       | 301 +++++++++++++++++++++++++++++++++++++++
 include/sysemu/hostmem-network.h |  79 ++++++++++
 3 files changed, 381 insertions(+), 1 deletion(-)
 create mode 100644 backends/hostmem-network.c
 create mode 100644 include/sysemu/hostmem-network.h

diff --git a/backends/Makefile.objs b/backends/Makefile.objs
index de76906..b66c097 100644
--- a/backends/Makefile.objs
+++ b/backends/Makefile.objs
@@ -8,6 +8,6 @@ baum.o-cflags := $(SDL_CFLAGS)
 common-obj-$(CONFIG_TPM) += tpm.o
 
 common-obj-y += hostmem.o hostmem-ram.o
-common-obj-$(CONFIG_LINUX) += hostmem-file.o hostmem-shared.o
+common-obj-$(CONFIG_LINUX) += hostmem-file.o hostmem-shared.o hostmem-network.o
 
 common-obj-y += multi-socket.o
diff --git a/backends/hostmem-network.c b/backends/hostmem-network.c
new file mode 100644
index 0000000..4317d63
--- /dev/null
+++ b/backends/hostmem-network.c
@@ -0,0 +1,301 @@
+/*
+ * QEMU Host Memory Backend over network
+ *
+ * Copyright (C) 2016 - Virtual Open Systems
+ *
+ * Author: Baptiste Reynal <b.reynal@virtualopensystems.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "sysemu/hostmem-network.h"
+#include "migration/vmstate.h"
+#include "qapi-visit.h"
+
+static uint64_t socket_mmio_read(void *opaque, hwaddr addr, unsigned size)
+{
+    NMArea *area = opaque;
+    MSMessage message;
+    NMData nm_data;
+
+    nm_data.addr = addr;
+    nm_data.size = size;
+
+    strcpy(message.cmd, MBN_READ_CMD);
+    memcpy(message.payload, &nm_data, sizeof(NMData));
+
+    multi_socket_write_and_block_to(area->client, &message);
+
+    return area->nhm->last_read.data;
+}
+
+static void
+socket_mmio_write(void *opaque, hwaddr addr, uint64_t data, unsigned size)
+{
+    NMArea *area = opaque;
+    MSMessage message;
+    NMData nm_data;
+
+    nm_data.addr = addr;
+    nm_data.data = data;
+    nm_data.size = size;
+
+    strcpy(message.cmd, MBN_WRITE_CMD);
+    memcpy(message.payload, &nm_data, sizeof(NMData));
+
+    multi_socket_write_message_to(area->client, &message);
+}
+
+static const MemoryRegionOps socket_mmio_ops = {
+    .read = socket_mmio_read,
+    .write = socket_mmio_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void
+network_memory_register(MSClient *c, const char *message, void *opaque)
+{
+    HostMemoryBackend *backend = MEMORY_BACKEND(opaque);
+    HostMemoryBackendNetwork *nhm = MEMORY_BACKEND_NETWORK(opaque);
+    NMArea *area, *new_area;
+    int hw_size = 1, size = *(int *) message;
+
+    /* Search for available space */
+    while (hw_size < size) {
+        hw_size <<= 1;
+    }
+
+    QLIST_FOREACH(area, &nhm->areas, next) {
+        if (!area->client && area->size >= hw_size) {
+            if (area->size > hw_size) {
+                new_area = g_new(NMArea, 1);
+                new_area->size = area->size - hw_size;
+                new_area->addr = area->addr + hw_size;
+                new_area->client = NULL;
+                new_area->nhm = nhm;
+
+                QLIST_INSERT_AFTER(area, new_area, next);
+            }
+
+            area->size = hw_size;
+            area->client = c;
+
+            break;
+        }
+    }
+
+    memory_region_init_io(&area->region,
+            OBJECT(opaque),
+            &socket_mmio_ops,
+            area,
+            "socket_region",
+            size
+    );
+
+    memory_region_add_subregion(&backend->mr, area->addr, &area->region);
+}
+
+static void
+network_memory_read_cb(MSClient *c, const char *message, void *opaque)
+{
+    HostMemoryBackendNetwork *nhm = opaque;
+    NMData *data = (NMData *) message;
+
+    memcpy(&nhm->last_read, message, sizeof(NMData));
+}
+
+static void
+network_memory_read(MSClient *c, const char *message, void *opaque)
+{
+    MSMessage nm_message;
+    NMData *data = (NMData *) message;
+    HostMemoryBackend *backend = MEMORY_BACKEND(opaque);
+
+    memcpy(&data->data, memory_region_get_ram_ptr(&backend->mr) + data->addr,
+            data->size);
+
+    strcpy(nm_message.cmd, MBN_READ_CMD);
+    memcpy(nm_message.payload, data, sizeof(NMData));
+
+    multi_socket_write_message_to(c, &nm_message);
+}
+
+static void
+network_memory_write(MSClient *c, const char *message, void *opaque)
+{
+    NMData *data = (NMData *) message;
+    HostMemoryBackend *backend = MEMORY_BACKEND(opaque);
+
+    memcpy(memory_region_get_ram_ptr(&backend->mr) + data->addr, &data->data,
+            data->size);
+}
+
+static void
+network_backend_memory_alloc(HostMemoryBackend *backend, Error **errp)
+{
+    HostMemoryBackendNetwork *shm = MEMORY_BACKEND_NETWORK(backend);
+    MSMessage message;
+
+    if (!backend->size) {
+        error_setg(errp, "can't create backend with size 0");
+        return;
+    }
+
+    if (!memory_region_size(&backend->mr)) {
+        backend->force_prealloc = mem_prealloc;
+        memory_region_init_ram(&backend->mr,
+                OBJECT(backend),
+                object_get_canonical_path(OBJECT(backend)),
+                backend->size,
+                errp);
+
+        if (shm->master) {
+            multi_socket_add_handler(shm->ms, MBN_REGISTER_CMD,
+                    network_memory_register, shm);
+            multi_socket_add_handler(shm->ms, MBN_READ_CMD,
+                    network_memory_read_cb, shm);
+        } else {
+            multi_socket_add_handler(shm->ms, MBN_READ_CMD,
+                    network_memory_read, shm);
+            multi_socket_add_handler(shm->ms, MBN_WRITE_CMD,
+                    network_memory_write, shm);
+
+            strcpy(message.cmd, MBN_REGISTER_CMD);
+            memcpy(message.payload, &backend->size, sizeof(int));
+
+            multi_socket_write_message_to(&shm->ms->listener, &message);
+        }
+    }
+
+    shm->levent = g_new(EventNotifier, 1);
+    event_notifier_init(shm->levent, 0);
+
+    shm->event = event_notifier_get_fd(shm->levent);
+}
+
+
+static void
+network_memory_backend_complete(UserCreatable *uc, Error **errp)
+{
+    HostMemoryBackend *hm = MEMORY_BACKEND(uc);
+    HostMemoryBackendNetwork *shm = MEMORY_BACKEND_NETWORK(uc);
+    HostMemoryBackendClass *bc = MEMORY_BACKEND_GET_CLASS(uc);
+    HostMemoryBackendNetworkClass *bnc = MEMORY_BACKEND_NETWORK_GET_CLASS(uc);
+
+    QLIST_INIT(&shm->areas);
+
+    NMArea *area = g_new(NMArea, 1);
+    area->size = shm->shared_size;
+    area->addr = hm->size - shm->shared_size;
+    area->client = NULL;
+    area->nhm = shm;
+
+    QLIST_INSERT_HEAD(&shm->areas, area, next);
+
+    if (shm->master) {
+        bnc->parent_complete(uc, errp);
+    } else {
+        bc->alloc(hm, errp);
+    }
+}
+
+static void ms_boot(HostMemoryBackendNetwork *nhm)
+{
+    event_notifier_set(nhm->levent);
+}
+
+static void network_backend_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+    HostMemoryBackendClass *bc = MEMORY_BACKEND_CLASS(oc);
+    HostMemoryBackendNetworkClass *bnc = MEMORY_BACKEND_NETWORK_CLASS(oc);
+
+    bc->alloc = network_backend_memory_alloc;
+    bnc->parent_complete = ucc->complete;
+    bnc->boot = ms_boot;
+    ucc->complete = network_memory_backend_complete;
+}
+
+static bool get_master(Object *o, Error **errp)
+{
+    HostMemoryBackendNetwork *shm = MEMORY_BACKEND_NETWORK(o);
+
+    return shm->master;
+}
+
+static void set_master(Object *o, bool value, Error **errp)
+{
+    HostMemoryBackendNetwork *shm = MEMORY_BACKEND_NETWORK(o);
+
+    shm->master = value;
+}
+
+static void
+host_memory_backend_get_size(Object *obj, Visitor *v, void *opaque,
+                             const char *name, Error **errp)
+{
+    HostMemoryBackendNetwork *backend = MEMORY_BACKEND_NETWORK(obj);
+    uint64_t value = backend->shared_size;
+
+    visit_type_size(v, &value, name, errp);
+}
+
+static void
+host_memory_backend_set_size(Object *obj, Visitor *v, void *opaque,
+                             const char *name, Error **errp)
+{
+    HostMemoryBackend *backend = MEMORY_BACKEND(obj);
+    HostMemoryBackendNetwork *nhm = MEMORY_BACKEND_NETWORK(obj);
+    Error *local_err = NULL;
+    uint64_t value;
+
+    if (memory_region_size(&backend->mr)) {
+        error_setg(&local_err, "cannot change property value");
+        goto out;
+    }
+
+    visit_type_size(v, &value, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (!value) {
+        error_setg(&local_err, "Property '%s.%s' doesn't take value '%"
+                   PRIu64 "'", object_get_typename(obj), name, value);
+        goto out;
+    }
+    nhm->shared_size = value;
+out:
+    error_propagate(errp, local_err);
+}
+
+static void network_backend_instance_init(Object *o)
+{
+    HostMemoryBackendNetwork *shm = MEMORY_BACKEND_NETWORK(o);
+
+    object_property_add_bool(o, "master", get_master, set_master, NULL);
+    object_property_add_link(o, "socket", TYPE_MULTI_SOCKET_BACKEND,
+            (Object **)&shm->ms,
+            object_property_allow_set_link,
+            OBJ_PROP_LINK_UNREF_ON_RELEASE,
+            &error_abort);
+    object_property_add(o, "shared-size", "int",
+            host_memory_backend_get_size,
+            host_memory_backend_set_size, NULL, NULL, NULL);
+}
+
+static const TypeInfo network_backend_info = {
+    .name = TYPE_MEMORY_BACKEND_NETWORK,
+    .parent = TYPE_MEMORY_BACKEND,
+    .class_init = network_backend_class_init,
+    .class_size = sizeof(HostMemoryBackendNetworkClass),
+    .instance_init = network_backend_instance_init,
+    .instance_size = sizeof(HostMemoryBackendNetwork),
+};
+
+static void register_types(void)
+{
+    type_register_static(&network_backend_info);
+}
+
+type_init(register_types);
diff --git a/include/sysemu/hostmem-network.h b/include/sysemu/hostmem-network.h
new file mode 100644
index 0000000..ffadad9
--- /dev/null
+++ b/include/sysemu/hostmem-network.h
@@ -0,0 +1,79 @@
+/*
+ * QEMU Host Memory Backend over network
+ *
+ * Copyright (C) 2016 - Virtual Open Systems
+ *
+ * Author: Baptiste Reynal <b.reynal@virtualopensystems.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_HMN_H
+#define QEMU_HMN_H
+
+#include "qemu-common.h"
+#include "sysemu/hostmem.h"
+#include "qom/object_interfaces.h"
+#include "qemu/multi-socket.h"
+#include "qemu/queue.h"
+
+#define MBN_REGISTER_CMD "nm-register"
+#define MBN_READ_CMD "nm-read"
+#define MBN_WRITE_CMD "nm-write"
+
+#define TYPE_MEMORY_BACKEND_NETWORK "memory-backend-network"
+
+#define MEMORY_BACKEND_NETWORK(obj) \
+    OBJECT_CHECK(HostMemoryBackendNetwork, (obj), TYPE_MEMORY_BACKEND_NETWORK)
+#define MEMORY_BACKEND_NETWORK_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(HostMemoryBackendNetworkClass, (obj), \
+            TYPE_MEMORY_BACKEND_NETWORK)
+#define MEMORY_BACKEND_NETWORK_CLASS(klass) \
+    OBJECT_CLASS_CHECK(HostMemoryBackendNetworkClass, (klass), \
+            TYPE_MEMORY_BACKEND_NETWORK)
+
+typedef struct HostMemoryBackendNetwork HostMemoryBackendNetwork;
+typedef struct HostMemoryBackendNetworkClass HostMemoryBackendNetworkClass;
+typedef struct NMData NMData;
+typedef struct NMArea NMArea;
+
+struct NMData {
+    hwaddr addr;
+    uint64_t data;
+    unsigned size;
+};
+
+struct NMArea {
+    unsigned size;
+    hwaddr addr;
+
+    MemoryRegion region;
+    MSClient *client;
+    HostMemoryBackendNetwork *nhm;
+
+    QLIST_ENTRY(NMArea) next;
+};
+
+struct HostMemoryBackendNetwork {
+    HostMemoryBackend parent_obj;
+
+    bool master;
+    int shared_size;
+    MSBackend *ms;
+
+    int event;
+    EventNotifier *levent;
+
+    NMData last_read;
+
+    QLIST_HEAD(areas_head, NMArea) areas;
+};
+
+struct HostMemoryBackendNetworkClass {
+    HostMemoryBackendClass parent_class;
+
+    void (*parent_complete)(UserCreatable *uc, Error **errp);
+    void (*boot)(HostMemoryBackendNetwork *nhm);
+};
+#endif
-- 
2.7.4

[Qemu-devel] [RFC 3/4] migration: extend shared migration type

[Baptiste Reynal]

shared migration now supports shared or network memory.

Signed-off-by: Baptiste Reynal <b.reynal@virtualopensystems.com>
---
 migration/shared.c | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/migration/shared.c b/migration/shared.c
index 1371a71..eb13eb9 100644
--- a/migration/shared.c
+++ b/migration/shared.c
@@ -6,6 +6,7 @@
 #include "migration/qemu-file.h"
 #include "block/block.h"
 #include "sysemu/hostmem-shared.h"
+#include "sysemu/hostmem-network.h"
 
 static void shared_accept_incoming_migration(void *opaque)
 {
@@ -18,16 +19,29 @@ static void shared_accept_incoming_migration(void *opaque)
 
 void shared_start_incoming_migration(const char *id, Error **errp)
 {
+    HostMemoryBackendNetwork *shmn;
     HostMemoryBackendShared *shm = (HostMemoryBackendShared *)
         object_resolve_path_type(id, TYPE_MEMORY_BACKEND_SHARED, NULL);
     QEMUFile *f;
+    int event = 0;
 
-    if (shm == NULL) {
+    if (!shm) {
+        shmn = (HostMemoryBackendNetwork *) object_resolve_path_type(id,
+                TYPE_MEMORY_BACKEND_NETWORK, NULL);
+
+        if (shmn) {
+            event = shmn->event;
+        }
+    } else {
+        event = shm->event;
+    }
+
+    if (!event) {
         printf("Error: Cannot find shared memory %s\n", id);
         exit(-1);
     }
 
-    f = qemu_fdopen(shm->event, "rb");
+    f = qemu_fdopen(event, "rb");
 
-    qemu_set_fd_handler(shm->event, shared_accept_incoming_migration, NULL, f);
+    qemu_set_fd_handler(event, shared_accept_incoming_migration, NULL, f);
 }
-- 
2.7.4

[Qemu-devel] [RFC 4/4] hw/misc: sdm signal nboot

[Baptiste Reynal]

This patch introduces a new signal for SDM device, which triggers the
boot of a machine using a network memory.

Signed-off-by: Baptiste Reynal <b.reynal@virtualopensystems.com>
---
 hw/misc/Makefile.objs              |  1 +
 hw/misc/sdm-signal-nboot.c         | 62 ++++++++++++++++++++++++++++++++++++++
 include/hw/misc/sdm-signal-nboot.h | 28 +++++++++++++++++
 3 files changed, 91 insertions(+)
 create mode 100644 hw/misc/sdm-signal-nboot.c
 create mode 100644 include/hw/misc/sdm-signal-nboot.h

diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs
index 20a7d82..12d955e 100644
--- a/hw/misc/Makefile.objs
+++ b/hw/misc/Makefile.objs
@@ -28,6 +28,7 @@ obj-$(CONFIG_SDM) += sdm-platform.o
 obj-$(CONFIG_SDM) += sdm-communication-local.o
 obj-$(CONFIG_SDM) += sdm-communication-socket.o
 obj-$(CONFIG_SDM) += sdm-signal-shboot.o
+obj-$(CONFIG_SDM) += sdm-signal-nboot.o
 
 obj-$(CONFIG_REALVIEW) += arm_sysctl.o
 obj-$(CONFIG_NSERIES) += cbus.o
diff --git a/hw/misc/sdm-signal-nboot.c b/hw/misc/sdm-signal-nboot.c
new file mode 100644
index 0000000..c04d1b4
--- /dev/null
+++ b/hw/misc/sdm-signal-nboot.c
@@ -0,0 +1,62 @@
+/*
+ * SDM Signal network boot
+ *
+ * Copyright (C) 2016 - Virtual Open Systems
+ *
+ * Author: Baptiste Reynal <b.reynalb@virtualopensystems.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+#include "hw/misc/sdm-signal-nboot.h"
+
+static int sdm_signal_nboot_hw_ops(SDMSignal *signal, SDMSignalData *data)
+{
+    SDMSignalNBoot *snb = SDM_SIGNAL_NBOOT(signal);
+    HostMemoryBackendNetworkClass *nhmc =
+        MEMORY_BACKEND_NETWORK_GET_CLASS(snb->nhm);
+
+    nhmc->boot(snb->nhm);
+
+    return 0;
+}
+
+static bool sdm_signal_nboot_hw_only(SDMSignal *signal)
+{
+    return true;
+}
+
+static void sdm_signal_nboot_init(Object *obj)
+{
+    SDMSignalNBoot *signal = SDM_SIGNAL_NBOOT(obj);
+
+    object_property_add_link(obj, "nhm",
+            TYPE_MEMORY_BACKEND_NETWORK,
+            (Object **)&signal->nhm,
+            object_property_allow_set_link,
+            OBJ_PROP_LINK_UNREF_ON_RELEASE,
+            &error_abort);
+}
+
+static void sdm_signal_nboot_class_init(ObjectClass *oc, void *data)
+{
+    SDMSignalClass *signalc = SDM_SIGNAL_CLASS(oc);
+
+    signalc->hw_ops = sdm_signal_nboot_hw_ops;
+    signalc->hw_only = sdm_signal_nboot_hw_only;
+}
+
+static const TypeInfo sdm_signal_nboot_info = {
+    .name = TYPE_SDM_SIGNAL_NBOOT,
+    .parent = TYPE_SDM_SIGNAL,
+    .class_init = sdm_signal_nboot_class_init,
+    .instance_size = sizeof(SDMSignalNBoot),
+    .instance_init = sdm_signal_nboot_init,
+};
+
+static void register_types(void)
+{
+    type_register_static(&sdm_signal_nboot_info);
+}
+
+type_init(register_types);
diff --git a/include/hw/misc/sdm-signal-nboot.h b/include/hw/misc/sdm-signal-nboot.h
new file mode 100644
index 0000000..a56ce44
--- /dev/null
+++ b/include/hw/misc/sdm-signal-nboot.h
@@ -0,0 +1,28 @@
+/*
+ * SDM Signal network boot
+ *
+ * Copyright (C) 2016 - Virtual Open Systems
+ *
+ * Author: Baptiste Reynal <b.reynalb@virtualopensystems.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+#ifndef HW_SDM_SIGNAL_NBOOT_H
+#define HW_SDM_SIGNAL_NBOOT_H
+
+#include "hw/misc/sdm-signal.h"
+#include "sysemu/hostmem-network.h"
+
+#define TYPE_SDM_SIGNAL_NBOOT "sdm-signal-nboot"
+#define SDM_SIGNAL_NBOOT(obj) \
+    OBJECT_CHECK(SDMSignalNBoot, (obj), TYPE_SDM_SIGNAL)
+
+typedef struct SDMSignalNBoot SDMSignalNBoot;
+
+struct SDMSignalNBoot {
+    SDMSignal parent;
+
+    HostMemoryBackendNetwork *nhm;
+};
+#endif
-- 
2.7.4