[Qemu-devel] [PATCH v5 0/2] vhost user: Add live migration

[Qemu-devel] [PATCH v5 0/2] vhost user: Add live migration

[Thibaut Collet]

v4->v5
1. The first patch is unchanged
2. The second patch is based on top of "vhost-user: protocol updates" series
   proposed earlier by Michael S. Tsirkin and "vhost-user: add migration log
   support" series proposed earlier by Marc Andre Lureau.

The first patch provides limited live migration:
- guest without GUEST_ANNOUNCE capabilities does not announce migration ending
and peers talking to the migrated guest can suffer important network outage.
- Some packets sent by remote peers to the guest can be lost during migration.

The second patch fixes limitation for guest without GUEST_ANNOUNCE capabilities
and patches from Marc Andre Lureau fix potential packet's lost during migration.

Thibaut Collet (2):
  vhost user: add support of live migration
  vhost user: add rarp sending after live migration for legacy guest

 docs/specs/vhost-user.txt         |   15 +++++++++++++
 hw/net/vhost_net.c                |   18 ++++++++++++++++
 hw/virtio/vhost-user.c            |   32 +++++++++++++++++++++++++--
 include/hw/virtio/vhost-backend.h |    2 ++
 include/net/vhost_net.h           |    1 +
 net/vhost-user.c                  |   43 +++++++++++++++++++++++++++++++++++--
 6 files changed, 107 insertions(+), 4 deletions(-)

-- 
1.7.10.4

[Qemu-devel] [PATCH v5 1/2] vhost user: add support of live migration

[Thibaut Collet]

Some vhost user backends are able to support live migration.
To provide this service the following features must be added:
1. Add the VIRTIO_NET_F_GUEST_ANNOUNCE capability to vhost-net when netdev
   backend is vhost-user.
2. Provide a nop receive callback to vhost-user.
   This callback is used only by qemu_announce_self after a migration to send
   fake RARP to avoid network outage for peers talking to the migrated guest.
   - For guest with GUEST_ANNOUNCE capabilities these packets must be discarded,
     GARP are sent by the guest thanks the GUEST_ANNOUNCE.
   - For guest without GUEST_ANNOUNCE capabilities these packets are discarded
     too, migration termination is notified when the guest sends packets.

Signed-off-by: Thibaut Collet <thibaut.collet@6wind.com>
---
 hw/net/vhost_net.c |    2 ++
 net/vhost-user.c   |   25 +++++++++++++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index c864237..9850520 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -85,6 +85,8 @@ static const int user_feature_bits[] = {
     VIRTIO_NET_F_CTRL_MAC_ADDR,
     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
 
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+
     VIRTIO_NET_F_MQ,
 
     VHOST_INVALID_FEATURE_BIT
diff --git a/net/vhost-user.c b/net/vhost-user.c
index 93dcecd..2290271 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -65,6 +65,28 @@ static void vhost_user_stop(VhostUserState *s)
     s->vhost_net = 0;
 }
 
+static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
+{
+    /* A live migration is done. Display an error if the packet is not a RARP.
+     * RARP are just discarded: guest is already notified about live migration
+     * by the virtio-net NIC or by the vhost-user backend.
+     */
+    if (size != 60) {
+        static int display_trace = 1;
+
+        if (display_trace) {
+            fprintf(stderr,
+                    "Vhost user expects only RARP (size 60)."
+                    "Receives unexpected packets with size %lu\n",
+                    size);
+            fflush(stderr);
+            display_trace = 0;
+        }
+    }
+    return size;
+}
+
 static void vhost_user_cleanup(NetClientState *nc)
 {
     VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
@@ -90,6 +112,7 @@ static bool vhost_user_has_ufo(NetClientState *nc)
 static NetClientInfo net_vhost_user_info = {
         .type = NET_CLIENT_OPTIONS_KIND_VHOST_USER,
         .size = sizeof(VhostUserState),
+        .receive = vhost_user_receive,
         .cleanup = vhost_user_cleanup,
         .has_vnet_hdr = vhost_user_has_vnet_hdr,
         .has_ufo = vhost_user_has_ufo,
@@ -143,8 +166,6 @@ static int net_vhost_user_init(NetClientState *peer, const char *device,
 
     s = DO_UPCAST(VhostUserState, nc, nc);
 
-    /* We don't provide a receive callback */
-    s->nc.receive_disabled = 1;
     s->chr = chr;
 
     qemu_chr_add_handlers(s->chr, NULL, NULL, net_vhost_user_event, s);
-- 
1.7.10.4

Re: [Qemu-devel] [PATCH v5 1/2] vhost user: add support of live migration

[Marc-André Lureau]

Hi Thibaut

On Mon, Aug 3, 2015 at 11:22 AM, Thibaut Collet
<thibaut.collet@6wind.com> wrote:
> Some vhost user backends are able to support live migration.
> To provide this service the following features must be added:
> 1. Add the VIRTIO_NET_F_GUEST_ANNOUNCE capability to vhost-net when netdev
>    backend is vhost-user.
> 2. Provide a nop receive callback to vhost-user.
>    This callback is used only by qemu_announce_self after a migration to send
>    fake RARP to avoid network outage for peers talking to the migrated guest.
>    - For guest with GUEST_ANNOUNCE capabilities these packets must be discarded,
>      GARP are sent by the guest thanks the GUEST_ANNOUNCE.
>    - For guest without GUEST_ANNOUNCE capabilities these packets are discarded
>      too, migration termination is notified when the guest sends packets.
>
> Signed-off-by: Thibaut Collet <thibaut.collet@6wind.com>
> ---
>  hw/net/vhost_net.c |    2 ++
>  net/vhost-user.c   |   25 +++++++++++++++++++++++--
>  2 files changed, 25 insertions(+), 2 deletions(-)
>
> diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
> index c864237..9850520 100644
> --- a/hw/net/vhost_net.c
> +++ b/hw/net/vhost_net.c
> @@ -85,6 +85,8 @@ static const int user_feature_bits[] = {
>      VIRTIO_NET_F_CTRL_MAC_ADDR,
>      VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
>
> +    VIRTIO_NET_F_GUEST_ANNOUNCE,
> +
>      VIRTIO_NET_F_MQ,
>
>      VHOST_INVALID_FEATURE_BIT
> diff --git a/net/vhost-user.c b/net/vhost-user.c
> index 93dcecd..2290271 100644
> --- a/net/vhost-user.c
> +++ b/net/vhost-user.c
> @@ -65,6 +65,28 @@ static void vhost_user_stop(VhostUserState *s)
>      s->vhost_net = 0;
>  }
>
> +static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
> +                                  size_t size)
> +{
> +    /* A live migration is done. Display an error if the packet is not a RARP.
> +     * RARP are just discarded: guest is already notified about live migration
> +     * by the virtio-net NIC or by the vhost-user backend.
> +     */
> +    if (size != 60) {
> +        static int display_trace = 1;
> +
> +        if (display_trace) {
> +            fprintf(stderr,
> +                    "Vhost user expects only RARP (size 60)."
> +                    "Receives unexpected packets with size %lu\n",
> +                    size);
> +            fflush(stderr);
> +            display_trace = 0;
> +        }
> +    }
> +    return size;
> +}

This warning appears during a dummy boot with vapp: qemu-system-x86_64
-m 512 -object memory-backend-file,id=mem,size=512M,mem-path=/hugetlbfs/,share=on
-numa node,memdev=mem -netdev
vhost-user,id=net0,chardev=chr0,vhostforce -device
virtio-net-pci,netdev=net0 -chardev
socket,id=chr0,path=/tmp/vapp.sock.

I think silentely returning 0 (without warning) is the right way to
keep the packets queued in the ring instead. But I must say I am quite
confused as this is triggered by a VIRTIO_PCI_QUEUE_NOTIFY ioport
write, and I fail to see how vhost-user handles it then without
eventfd kick...

>  static void vhost_user_cleanup(NetClientState *nc)
>  {
>      VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
> @@ -90,6 +112,7 @@ static bool vhost_user_has_ufo(NetClientState *nc)
>  static NetClientInfo net_vhost_user_info = {
>          .type = NET_CLIENT_OPTIONS_KIND_VHOST_USER,
>          .size = sizeof(VhostUserState),
> +        .receive = vhost_user_receive,
>          .cleanup = vhost_user_cleanup,
>          .has_vnet_hdr = vhost_user_has_vnet_hdr,
>          .has_ufo = vhost_user_has_ufo,
> @@ -143,8 +166,6 @@ static int net_vhost_user_init(NetClientState *peer, const char *device,
>
>      s = DO_UPCAST(VhostUserState, nc, nc);
>
> -    /* We don't provide a receive callback */
> -    s->nc.receive_disabled = 1;
>      s->chr = chr;
>
>      qemu_chr_add_handlers(s->chr, NULL, NULL, net_vhost_user_event, s);
> --
> 1.7.10.4
>



-- 
Marc-André Lureau

Re: [Qemu-devel] [PATCH v5 1/2] vhost user: add support of live migration

[Thibaut Collet]

On Tue, Aug 4, 2015 at 7:50 PM, Marc-André Lureau
<marcandre.lureau@gmail.com> wrote:
> Hi Thibaut
>
> On Mon, Aug 3, 2015 at 11:22 AM, Thibaut Collet
> <thibaut.collet@6wind.com> wrote:
>> Some vhost user backends are able to support live migration.
>> To provide this service the following features must be added:
>> 1. Add the VIRTIO_NET_F_GUEST_ANNOUNCE capability to vhost-net when netdev
>>    backend is vhost-user.
>> 2. Provide a nop receive callback to vhost-user.
>>    This callback is used only by qemu_announce_self after a migration to send
>>    fake RARP to avoid network outage for peers talking to the migrated guest.
>>    - For guest with GUEST_ANNOUNCE capabilities these packets must be discarded,
>>      GARP are sent by the guest thanks the GUEST_ANNOUNCE.
>>    - For guest without GUEST_ANNOUNCE capabilities these packets are discarded
>>      too, migration termination is notified when the guest sends packets.
>>
>> Signed-off-by: Thibaut Collet <thibaut.collet@6wind.com>
>> ---
>>  hw/net/vhost_net.c |    2 ++
>>  net/vhost-user.c   |   25 +++++++++++++++++++++++--
>>  2 files changed, 25 insertions(+), 2 deletions(-)
>>
>> diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
>> index c864237..9850520 100644
>> --- a/hw/net/vhost_net.c
>> +++ b/hw/net/vhost_net.c
>> @@ -85,6 +85,8 @@ static const int user_feature_bits[] = {
>>      VIRTIO_NET_F_CTRL_MAC_ADDR,
>>      VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
>>
>> +    VIRTIO_NET_F_GUEST_ANNOUNCE,
>> +
>>      VIRTIO_NET_F_MQ,
>>
>>      VHOST_INVALID_FEATURE_BIT
>> diff --git a/net/vhost-user.c b/net/vhost-user.c
>> index 93dcecd..2290271 100644
>> --- a/net/vhost-user.c
>> +++ b/net/vhost-user.c
>> @@ -65,6 +65,28 @@ static void vhost_user_stop(VhostUserState *s)
>>      s->vhost_net = 0;
>>  }
>>
>> +static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
>> +                                  size_t size)
>> +{
>> +    /* A live migration is done. Display an error if the packet is not a RARP.
>> +     * RARP are just discarded: guest is already notified about live migration
>> +     * by the virtio-net NIC or by the vhost-user backend.
>> +     */
>> +    if (size != 60) {
>> +        static int display_trace = 1;
>> +
>> +        if (display_trace) {
>> +            fprintf(stderr,
>> +                    "Vhost user expects only RARP (size 60)."
>> +                    "Receives unexpected packets with size %lu\n",
>> +                    size);
>> +            fflush(stderr);
>> +            display_trace = 0;
>> +        }
>> +    }
>> +    return size;
>> +}
>
> This warning appears during a dummy boot with vapp: qemu-system-x86_64
> -m 512 -object memory-backend-file,id=mem,size=512M,mem-path=/hugetlbfs/,share=on
> -numa node,memdev=mem -netdev
> vhost-user,id=net0,chardev=chr0,vhostforce -device
> virtio-net-pci,netdev=net0 -chardev
> socket,id=chr0,path=/tmp/vapp.sock.
>
> I think silentely returning 0 (without warning) is the right way to
> keep the packets queued in the ring instead. But I must say I am quite
> confused as this is triggered by a VIRTIO_PCI_QUEUE_NOTIFY ioport
> write, and I fail to see how vhost-user handles it then without
> eventfd kick...
>
>>  static void vhost_user_cleanup(NetClientState *nc)
>>  {
>>      VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
>> @@ -90,6 +112,7 @@ static bool vhost_user_has_ufo(NetClientState *nc)
>>  static NetClientInfo net_vhost_user_info = {
>>          .type = NET_CLIENT_OPTIONS_KIND_VHOST_USER,
>>          .size = sizeof(VhostUserState),
>> +        .receive = vhost_user_receive,
>>          .cleanup = vhost_user_cleanup,
>>          .has_vnet_hdr = vhost_user_has_vnet_hdr,
>>          .has_ufo = vhost_user_has_ufo,
>> @@ -143,8 +166,6 @@ static int net_vhost_user_init(NetClientState *peer, const char *device,
>>
>>      s = DO_UPCAST(VhostUserState, nc, nc);
>>
>> -    /* We don't provide a receive callback */
>> -    s->nc.receive_disabled = 1;
>>      s->chr = chr;
>>
>>      qemu_chr_add_handlers(s->chr, NULL, NULL, net_vhost_user_event, s);
>> --
>> 1.7.10.4
>>
>
>
>
> --
> Marc-André Lureau

Hi,

I have reproduced the issue and the message receive by vhost-user is
quite strange: this message is a virtio header plus a bootp/dhcp
request (I do not understand why there are the virtio header)
With a dummy boot, vhost user backend already receives 4 bootp/dhcp
requests form the bios guest so I do not see any interest to transmit
this request to vhostuser backend.

I suggest to remove the warning and discard the dhcp/bootp request (it
is useless to duplicate bootp/dhcp request)


Thibaut.

[Qemu-devel] [PATCH v5 2/2] vhost user: add rarp sending after live migration for legacy guest

[Thibaut Collet]

A new vhost user message is added to allow QEMU to ask to vhost user backend to
broadcast a fake RARP after live migration for guest without GUEST_ANNOUNCE
capability.

This new message is sent only if the backend supports the new
VHOST_USER_PROTOCOL_F_RARP protocol feature.
The payload of this new message is the MAC address of the guest (not known by
the backend). The MAC address is copied in the first 6 bytes of a u64 to avoid
to create a new payload message type.

This new message has no equivalent ioctl so a new callback is added in the
userOps structure to send the request.

Upon reception of this new message the vhost user backend must generate and
broadcast a fake RARP request to notify the migration is terminated.

Signed-off-by: Thibaut Collet <thibaut.collet@6wind.com>
---
 docs/specs/vhost-user.txt         |   15 +++++++++++++++
 hw/net/vhost_net.c                |   16 ++++++++++++++++
 hw/virtio/vhost-user.c            |   32 ++++++++++++++++++++++++++++++--
 include/hw/virtio/vhost-backend.h |    2 ++
 include/net/vhost_net.h           |    1 +
 net/vhost-user.c                  |   18 ++++++++++++++++++
 6 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/docs/specs/vhost-user.txt b/docs/specs/vhost-user.txt
index c2d2e2a..8c05301 100644
--- a/docs/specs/vhost-user.txt
+++ b/docs/specs/vhost-user.txt
@@ -140,6 +140,7 @@ Protocol features
 -----------------
 
 #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 0
+#define VHOST_USER_PROTOCOL_F_RARP      1
 
 Message types
 -------------
@@ -308,6 +309,20 @@ Message types
       invalid FD flag. This flag is set when there is no file descriptor
       in the ancillary data.
 
+ * VHOST_USER_SEND_RARP
+
+      Id: 17
+      Equivalent ioctl: N/A
+      Master payload: u64
+
+      Ask vhost user backend to broadcast a fake RARP to notify the migration
+      is terminated for guest that does not support GUEST_ANNOUNCE.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES and protocol feature bit VHOST_USER_PROTOCOL_F_RARP
+      is present in VHOST_USER_GET_PROTOCOL_FEATURES.
+      The first 6 bytes of the payload contain the mac address of the guest to
+      allow the vhost user backend to construct and broadcast the fake RARP.
+
 Migration
 ---------
 
diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index 9850520..c3b9664 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -383,6 +383,17 @@ void vhost_net_cleanup(struct vhost_net *net)
     g_free(net);
 }
 
+int vhost_net_migrate(struct vhost_net *net, char* mac_addr)
+{
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+    int r = -1;
+
+    if (vhost_ops->vhost_backend_migrate)
+        r = vhost_ops->vhost_backend_migrate(&net->dev, mac_addr);
+
+    return r;
+}
+
 bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
 {
     return vhost_virtqueue_pending(&net->dev, idx);
@@ -456,6 +467,11 @@ void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
 {
 }
 
+int vhost_net_migrate(struct vhost_net *net)
+{
+    return -1;
+}
+
 VHostNetState *get_vhost_net(NetClientState *nc)
 {
     return 0;
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index fe75618..693840f 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -10,6 +10,7 @@
 
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
 #include "sysemu/char.h"
 #include "sysemu/kvm.h"
 #include "qemu/error-report.h"
@@ -27,8 +28,9 @@
 
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
-#define VHOST_USER_PROTOCOL_FEATURE_MASK 0x1ULL
+#define VHOST_USER_PROTOCOL_FEATURE_MASK 0x3ULL
 #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 0
+#define VHOST_USER_PROTOCOL_F_RARP      1
 
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
@@ -48,6 +50,7 @@ typedef enum VhostUserRequest {
     VHOST_USER_SET_VRING_ERR = 14,
     VHOST_USER_GET_PROTOCOL_FEATURES = 15,
     VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_SEND_RARP = 17,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -409,9 +412,34 @@ static int vhost_user_cleanup(struct vhost_dev *dev)
     return 0;
 }
 
+static int vhost_user_migrate(struct vhost_dev *dev, char* mac_addr)
+{
+    VhostUserMsg msg = { 0 };
+    int err;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    /* If guest supports GUEST_ANNOUNCE do nothing */
+    if (__virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE))
+        return 0;
+
+    /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
+    if (__virtio_has_feature(dev->protocol_features, VHOST_USER_PROTOCOL_F_RARP)) {
+        msg.request = VHOST_USER_SEND_RARP;
+        msg.flags = VHOST_USER_VERSION;
+        memcpy((char*) &msg.u64, mac_addr, 6);
+        msg.size = sizeof(m.u64);
+
+        err = vhost_user_write(dev, &msg, NULL, 0);
+        return err;
+    }
+    return -1;
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
         .vhost_call = vhost_user_call,
         .vhost_backend_init = vhost_user_init,
-        .vhost_backend_cleanup = vhost_user_cleanup
+        .vhost_backend_cleanup = vhost_user_cleanup,
+        .vhost_backend_migrate = vhost_user_migrate
         };
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index e472f29..cafc7ec 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -24,12 +24,14 @@ typedef int (*vhost_call)(struct vhost_dev *dev, unsigned long int request,
              void *arg);
 typedef int (*vhost_backend_init)(struct vhost_dev *dev, void *opaque);
 typedef int (*vhost_backend_cleanup)(struct vhost_dev *dev);
+typedef int (*vhost_backend_migrate)(struct vhost_dev *dev, char* mac_addr);
 
 typedef struct VhostOps {
     VhostBackendType backend_type;
     vhost_call vhost_call;
     vhost_backend_init vhost_backend_init;
     vhost_backend_cleanup vhost_backend_cleanup;
+    vhost_backend_migrate vhost_backend_migrate;
 } VhostOps;
 
 extern const VhostOps user_ops;
diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h
index 840d4b1..e5700fb 100644
--- a/include/net/vhost_net.h
+++ b/include/net/vhost_net.h
@@ -26,5 +26,6 @@ void vhost_net_ack_features(VHostNetState *net, uint64_t features);
 bool vhost_net_virtqueue_pending(VHostNetState *net, int n);
 void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
                               int idx, bool mask);
+int vhost_net_migrate(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
 #endif
diff --git a/net/vhost-user.c b/net/vhost-user.c
index 2290271..3cdfd35 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -84,6 +84,24 @@ static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
             display_trace = 0;
         }
     }
+    else {
+        VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
+        int r;
+        static int display_rarp_failure = 1;
+        char mac_addr[6];
+
+        // extract guest mac address from the RARP message
+        memcpy(mac_addr, &buf[6], 6);
+
+        r = vhost_net_migrate(s->vhost_net, mac_addr);
+
+        if ((r != 0) && (display_rarp_failure)) {
+            fprintf(stderr,
+                    "Vhost user backend fails to broadcast fake RARP\n");
+            fflush(stderr);
+            display_rarp_failure = 0;
+        }
+    }
     return size;
 }
 
-- 
1.7.10.4

Re: [Qemu-devel] [PATCH v5 2/2] vhost user: add rarp sending after live migration for legacy guest

[Jason Wang]

On 08/03/2015 05:22 PM, Thibaut Collet wrote:
> A new vhost user message is added to allow QEMU to ask to vhost user backend to
> broadcast a fake RARP after live migration for guest without GUEST_ANNOUNCE
> capability.
>
> This new message is sent only if the backend supports the new
> VHOST_USER_PROTOCOL_F_RARP protocol feature.
> The payload of this new message is the MAC address of the guest (not known by
> the backend). The MAC address is copied in the first 6 bytes of a u64 to avoid
> to create a new payload message type.
>
> This new message has no equivalent ioctl so a new callback is added in the
> userOps structure to send the request.
>
> Upon reception of this new message the vhost user backend must generate and
> broadcast a fake RARP request to notify the migration is terminated.
>
> Signed-off-by: Thibaut Collet <thibaut.collet@6wind.com>
> ---
>  docs/specs/vhost-user.txt         |   15 +++++++++++++++
>  hw/net/vhost_net.c                |   16 ++++++++++++++++
>  hw/virtio/vhost-user.c            |   32 ++++++++++++++++++++++++++++++--
>  include/hw/virtio/vhost-backend.h |    2 ++
>  include/net/vhost_net.h           |    1 +
>  net/vhost-user.c                  |   18 ++++++++++++++++++
>  6 files changed, 82 insertions(+), 2 deletions(-)
>
> diff --git a/docs/specs/vhost-user.txt b/docs/specs/vhost-user.txt
> index c2d2e2a..8c05301 100644
> --- a/docs/specs/vhost-user.txt
> +++ b/docs/specs/vhost-user.txt
> @@ -140,6 +140,7 @@ Protocol features
>  -----------------
>  
>  #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 0
> +#define VHOST_USER_PROTOCOL_F_RARP      1
>  
>  Message types
>  -------------
> @@ -308,6 +309,20 @@ Message types
>        invalid FD flag. This flag is set when there is no file descriptor
>        in the ancillary data.
>  
> + * VHOST_USER_SEND_RARP
> +
> +      Id: 17
> +      Equivalent ioctl: N/A
> +      Master payload: u64
> +
> +      Ask vhost user backend to broadcast a fake RARP to notify the migration
> +      is terminated for guest that does not support GUEST_ANNOUNCE.
> +      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
> +      VHOST_USER_GET_FEATURES and protocol feature bit VHOST_USER_PROTOCOL_F_RARP
> +      is present in VHOST_USER_GET_PROTOCOL_FEATURES.
> +      The first 6 bytes of the payload contain the mac address of the guest to
> +      allow the vhost user backend to construct and broadcast the fake RARP.
> +
>  Migration

If for some reason the announce packet was changed in the future, we may
then need another kind of message type. Just wonder whether allowing
qemu to inject packet directly to vhost-user is better.

>  ---------
>  
> diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
> index 9850520..c3b9664 100644
> --- a/hw/net/vhost_net.c
> +++ b/hw/net/vhost_net.c
> @@ -383,6 +383,17 @@ void vhost_net_cleanup(struct vhost_net *net)
>      g_free(net);
>  }
>  
> +int vhost_net_migrate(struct vhost_net *net, char* mac_addr)
> +{
> +    const VhostOps *vhost_ops = net->dev.vhost_ops;
> +    int r = -1;
> +
> +    if (vhost_ops->vhost_backend_migrate)
> +        r = vhost_ops->vhost_backend_migrate(&net->dev, mac_addr);
> +
> +    return r;
> +}
> +
>  bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
>  {
>      return vhost_virtqueue_pending(&net->dev, idx);
> @@ -456,6 +467,11 @@ void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
>  {
>  }
>  
> +int vhost_net_migrate(struct vhost_net *net)
> +{
> +    return -1;
> +}
> +
>  VHostNetState *get_vhost_net(NetClientState *nc)
>  {
>      return 0;
> diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
> index fe75618..693840f 100644
> --- a/hw/virtio/vhost-user.c
> +++ b/hw/virtio/vhost-user.c
> @@ -10,6 +10,7 @@
>  
>  #include "hw/virtio/vhost.h"
>  #include "hw/virtio/vhost-backend.h"
> +#include "hw/virtio/virtio-net.h"
>  #include "sysemu/char.h"
>  #include "sysemu/kvm.h"
>  #include "qemu/error-report.h"
> @@ -27,8 +28,9 @@
>  
>  #define VHOST_USER_F_PROTOCOL_FEATURES 30
>  
> -#define VHOST_USER_PROTOCOL_FEATURE_MASK 0x1ULL
> +#define VHOST_USER_PROTOCOL_FEATURE_MASK 0x3ULL
>  #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 0
> +#define VHOST_USER_PROTOCOL_F_RARP      1
>  
>  typedef enum VhostUserRequest {
>      VHOST_USER_NONE = 0,
> @@ -48,6 +50,7 @@ typedef enum VhostUserRequest {
>      VHOST_USER_SET_VRING_ERR = 14,
>      VHOST_USER_GET_PROTOCOL_FEATURES = 15,
>      VHOST_USER_SET_PROTOCOL_FEATURES = 16,
> +    VHOST_USER_SEND_RARP = 17,
>      VHOST_USER_MAX
>  } VhostUserRequest;
>  
> @@ -409,9 +412,34 @@ static int vhost_user_cleanup(struct vhost_dev *dev)
>      return 0;
>  }
>  
> +static int vhost_user_migrate(struct vhost_dev *dev, char* mac_addr)
> +{

Not sure migrate is a good name since migration was not done in this
function and it was called in vhost_user_receive().

Re: [Qemu-devel] [PATCH v5 2/2] vhost user: add rarp sending after live migration for legacy guest

[Thibaut Collet]

On Tue, Aug 4, 2015 at 8:22 AM, Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 08/03/2015 05:22 PM, Thibaut Collet wrote:
>> A new vhost user message is added to allow QEMU to ask to vhost user backend to
>> broadcast a fake RARP after live migration for guest without GUEST_ANNOUNCE
>> capability.
>>
>> This new message is sent only if the backend supports the new
>> VHOST_USER_PROTOCOL_F_RARP protocol feature.
>> The payload of this new message is the MAC address of the guest (not known by
>> the backend). The MAC address is copied in the first 6 bytes of a u64 to avoid
>> to create a new payload message type.
>>
>> This new message has no equivalent ioctl so a new callback is added in the
>> userOps structure to send the request.
>>
>> Upon reception of this new message the vhost user backend must generate and
>> broadcast a fake RARP request to notify the migration is terminated.
>>
>> Signed-off-by: Thibaut Collet <thibaut.collet@6wind.com>
>> ---
>>  docs/specs/vhost-user.txt         |   15 +++++++++++++++
>>  hw/net/vhost_net.c                |   16 ++++++++++++++++
>>  hw/virtio/vhost-user.c            |   32 ++++++++++++++++++++++++++++++--
>>  include/hw/virtio/vhost-backend.h |    2 ++
>>  include/net/vhost_net.h           |    1 +
>>  net/vhost-user.c                  |   18 ++++++++++++++++++
>>  6 files changed, 82 insertions(+), 2 deletions(-)
>>
>> diff --git a/docs/specs/vhost-user.txt b/docs/specs/vhost-user.txt
>> index c2d2e2a..8c05301 100644
>> --- a/docs/specs/vhost-user.txt
>> +++ b/docs/specs/vhost-user.txt
>> @@ -140,6 +140,7 @@ Protocol features
>>  -----------------
>>
>>  #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 0
>> +#define VHOST_USER_PROTOCOL_F_RARP      1
>>
>>  Message types
>>  -------------
>> @@ -308,6 +309,20 @@ Message types
>>        invalid FD flag. This flag is set when there is no file descriptor
>>        in the ancillary data.
>>
>> + * VHOST_USER_SEND_RARP
>> +
>> +      Id: 17
>> +      Equivalent ioctl: N/A
>> +      Master payload: u64
>> +
>> +      Ask vhost user backend to broadcast a fake RARP to notify the migration
>> +      is terminated for guest that does not support GUEST_ANNOUNCE.
>> +      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
>> +      VHOST_USER_GET_FEATURES and protocol feature bit VHOST_USER_PROTOCOL_F_RARP
>> +      is present in VHOST_USER_GET_PROTOCOL_FEATURES.
>> +      The first 6 bytes of the payload contain the mac address of the guest to
>> +      allow the vhost user backend to construct and broadcast the fake RARP.
>> +
>>  Migration
>
> If for some reason the announce packet was changed in the future, we may
> then need another kind of message type. Just wonder whether allowing
> qemu to inject packet directly to vhost-user is better.
>

Purpose of the announce packet is to notified switches that MAC
address of the guest has moved to avoid network outage.
So only knowledge of the MAC address is needed to construct the
announce packet (whatever is format)
The new message provides the MAC address.  Vhost user backend can
create an announce packet that is not necessary the same of QEMU

Providing the announce packet directly to vhost user is possible but
needs a lot of additional change:
 - The announce packet can not be copy in the master payload (in the
future if the announce packet changed its size can changed too)
The master payload can be a u64 that gives the size of the message.
The message will be stored in a shared memory between QEMU and backend
and a fd to this shared memory must be provided too.
That can be interesting if QEMU has several type of packets to inject
to vhost user but it is not the case and I prefer to keep my solution
(less complex and risk of bugs or memory leakage is less)

>>  ---------
>>
>> diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
>> index 9850520..c3b9664 100644
>> --- a/hw/net/vhost_net.c
>> +++ b/hw/net/vhost_net.c
>> @@ -383,6 +383,17 @@ void vhost_net_cleanup(struct vhost_net *net)
>>      g_free(net);
>>  }
>>
>> +int vhost_net_migrate(struct vhost_net *net, char* mac_addr)
>> +{
>> +    const VhostOps *vhost_ops = net->dev.vhost_ops;
>> +    int r = -1;
>> +
>> +    if (vhost_ops->vhost_backend_migrate)
>> +        r = vhost_ops->vhost_backend_migrate(&net->dev, mac_addr);
>> +
>> +    return r;
>> +}
>> +
>>  bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
>>  {
>>      return vhost_virtqueue_pending(&net->dev, idx);
>> @@ -456,6 +467,11 @@ void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
>>  {
>>  }
>>
>> +int vhost_net_migrate(struct vhost_net *net)
>> +{
>> +    return -1;
>> +}
>> +
>>  VHostNetState *get_vhost_net(NetClientState *nc)
>>  {
>>      return 0;
>> diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
>> index fe75618..693840f 100644
>> --- a/hw/virtio/vhost-user.c
>> +++ b/hw/virtio/vhost-user.c
>> @@ -10,6 +10,7 @@
>>
>>  #include "hw/virtio/vhost.h"
>>  #include "hw/virtio/vhost-backend.h"
>> +#include "hw/virtio/virtio-net.h"
>>  #include "sysemu/char.h"
>>  #include "sysemu/kvm.h"
>>  #include "qemu/error-report.h"
>> @@ -27,8 +28,9 @@
>>
>>  #define VHOST_USER_F_PROTOCOL_FEATURES 30
>>
>> -#define VHOST_USER_PROTOCOL_FEATURE_MASK 0x1ULL
>> +#define VHOST_USER_PROTOCOL_FEATURE_MASK 0x3ULL
>>  #define VHOST_USER_PROTOCOL_F_LOG_SHMFD 0
>> +#define VHOST_USER_PROTOCOL_F_RARP      1
>>
>>  typedef enum VhostUserRequest {
>>      VHOST_USER_NONE = 0,
>> @@ -48,6 +50,7 @@ typedef enum VhostUserRequest {
>>      VHOST_USER_SET_VRING_ERR = 14,
>>      VHOST_USER_GET_PROTOCOL_FEATURES = 15,
>>      VHOST_USER_SET_PROTOCOL_FEATURES = 16,
>> +    VHOST_USER_SEND_RARP = 17,
>>      VHOST_USER_MAX
>>  } VhostUserRequest;
>>
>> @@ -409,9 +412,34 @@ static int vhost_user_cleanup(struct vhost_dev *dev)
>>      return 0;
>>  }
>>
>> +static int vhost_user_migrate(struct vhost_dev *dev, char* mac_addr)
>> +{
>
> Not sure migrate is a good name since migration was not done in this
> function and it was called in vhost_user_receive().
>

OK. I will rename it to vhost_user_notify_migration_done that is the
purpose of this function