[PATCH] vhost-user-scsi: support reconnect to backend

[PATCH] vhost-user-scsi: support reconnect to backend

[Li Feng]

If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/block/vhost-user-blk.c           |   2 -
 hw/scsi/vhost-scsi-common.c         |  27 ++---
 hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
 include/hw/virtio/vhost-user-scsi.h |   3 +
 include/hw/virtio/vhost.h           |   2 +
 5 files changed, 165 insertions(+), 32 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index eecf3f7a81..f250c740b5 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -32,8 +32,6 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
-
 static const int user_feature_bits[] = {
     VIRTIO_BLK_F_SIZE_MAX,
     VIRTIO_BLK_F_SEG_MAX,
diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a06f01af26..08801886b8 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     vsc->dev.acked_features = vdev->guest_features;
 
-    assert(vsc->inflight == NULL);
-    vsc->inflight = g_new0(struct vhost_inflight, 1);
-    ret = vhost_dev_get_inflight(&vsc->dev,
-                                 vs->conf.virtqueue_size,
-                                 vsc->inflight);
+    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error get inflight: %d", -ret);
+        error_report("Error setting inflight format: %d", -ret);
         goto err_guest_notifiers;
     }
 
+    if (!vsc->inflight->addr) {
+        ret = vhost_dev_get_inflight(&vsc->dev,
+                                    vs->conf.virtqueue_size,
+                                    vsc->inflight);
+        if (ret < 0) {
+            error_report("Error get inflight: %d", -ret);
+            goto err_guest_notifiers;
+        }
+    }
+
     ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
     if (ret < 0) {
         error_report("Error set inflight: %d", -ret);
@@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     return ret;
 
 err_guest_notifiers:
-    g_free(vsc->inflight);
-    vsc->inflight = NULL;
-
     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
 err_host_notifiers:
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
@@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     }
     assert(ret >= 0);
 
-    if (vsc->inflight) {
-        vhost_dev_free_inflight(vsc->inflight);
-        g_free(vsc->inflight);
-        vsc->inflight = NULL;
-    }
-
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
 }
 
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index ee99b19e7a..e0e88b0c42 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
 }
 
+static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret = 0;
+
+    if (s->connected) {
+        return 0;
+    }
+    s->connected = true;
+
+    vsc->dev.num_queues = vs->conf.num_queues;
+    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
+    vsc->dev.vqs = s->vhost_vqs;
+    vsc->dev.vq_index = 0;
+    vsc->dev.backend_features = 0;
+
+    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
+                         errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* restore vhost state */
+    if (virtio_device_started(vdev, vdev->status)) {
+        ret = vhost_scsi_common_start(vsc);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
+
+static void vhost_user_scsi_disconnect(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    if (!s->connected) {
+        return;
+    }
+    s->connected = false;
+
+    vhost_scsi_common_stop(vsc);
+
+    vhost_dev_cleanup(&vsc->dev);
+
+    /* Re-instate the event handler for new connections */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
+                             vhost_user_scsi_event, NULL, dev, NULL, true);
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
+{
+    DeviceState *dev = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    Error *local_err = NULL;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
+            error_report_err(local_err);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
+            return;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        /* defer close until later to avoid circular close */
+        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
+                               vhost_user_scsi_disconnect);
+        break;
+    case CHR_EVENT_BREAK:
+    case CHR_EVENT_MUX_IN:
+    case CHR_EVENT_MUX_OUT:
+        /* Ignore */
+        break;
+    }
+}
+
+static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
+{
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret;
+
+    s->connected = false;
+
+    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_user_scsi_connect(dev, errp);
+    if (ret < 0) {
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return ret;
+    }
+    assert(s->connected);
+
+    return 0;
+}
+
 static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
 {
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = NULL;
     Error *err = NULL;
     int ret;
+    int retries = REALIZE_CONNECTION_RETRIES;
 
     if (!vs->conf.chardev.chr) {
         error_setg(errp, "vhost-user-scsi: missing chardev");
@@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     }
 
     if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {
-        goto free_virtio;
+        goto free_vhost;
     }
 
-    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
-    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
-    vsc->dev.vq_index = 0;
-    vsc->dev.backend_features = 0;
-    vqs = vsc->dev.vqs;
+    vsc->inflight = g_new0(struct vhost_inflight, 1);
+    s->vhost_vqs = g_new0(struct vhost_virtqueue,
+                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
+
+    assert(!*errp);
+    do {
+        if (*errp) {
+            error_prepend(errp, "Reconnecting after error: ");
+            error_report_err(*errp);
+            *errp = NULL;
+        }
+        ret = vhost_user_scsi_realize_connect(s, errp);
+    } while (ret < 0 && retries--);
 
-    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
-                         VHOST_BACKEND_TYPE_USER, 0, errp);
     if (ret < 0) {
-        goto free_vhost;
+        goto free_vqs;
     }
 
+    /* we're fully initialized, now we can operate, so add the handler */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
+                             vhost_user_scsi_event, NULL, (void *)dev,
+                             NULL, true);
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
 
     return;
 
+free_vqs:
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
+
 free_vhost:
     vhost_user_cleanup(&s->vhost_user);
-    g_free(vqs);
-free_virtio:
+
     virtio_scsi_common_unrealize(dev);
 }
 
@@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = vsc->dev.vqs;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
 
     /* This will stop the vhost backend. */
     vhost_user_scsi_set_status(vdev, 0);
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
+                             NULL, false);
 
     vhost_dev_cleanup(&vsc->dev);
-    g_free(vqs);
+    vhost_dev_free_inflight(vsc->inflight);
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
 
-    virtio_scsi_common_unrealize(dev);
     vhost_user_cleanup(&s->vhost_user);
+    virtio_scsi_common_unrealize(dev);
 }
 
 static Property vhost_user_scsi_properties[] = {
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 521b08e559..c66acc68b7 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
 struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     VhostUserState vhost_user;
+    bool connected;
+
+    struct vhost_virtqueue *vhost_vqs;
 };
 
 #endif /* VHOST_USER_SCSI_H */
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 6a173cb9fa..b904346fe1 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -8,6 +8,8 @@
 #define VHOST_F_DEVICE_IOTLB 63
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+#define REALIZE_CONNECTION_RETRIES 3
+
 /* Generic structures common for any vhost based device. */
 
 struct vhost_inflight {
-- 
2.41.0

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Raphael Norwitz]

Very excited to see this. High level looks good modulo a few small things.

My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.

The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.

> On Jul 21, 2023, at 6:51 AM, Li Feng <fengli@smartx.com> wrote:
> 
> If the backend crashes and restarts, the device is broken.
> This patch adds reconnect for vhost-user-scsi.
> 
> Tested with spdk backend.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/block/vhost-user-blk.c           |   2 -
> hw/scsi/vhost-scsi-common.c         |  27 ++---
> hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
> include/hw/virtio/vhost-user-scsi.h |   3 +
> include/hw/virtio/vhost.h           |   2 +
> 5 files changed, 165 insertions(+), 32 deletions(-)
> 
> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
> index eecf3f7a81..f250c740b5 100644
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -32,8 +32,6 @@
> #include "sysemu/sysemu.h"
> #include "sysemu/runstate.h"
> 
> -#define REALIZE_CONNECTION_RETRIES 3
> -
> static const int user_feature_bits[] = {
>     VIRTIO_BLK_F_SIZE_MAX,
>     VIRTIO_BLK_F_SEG_MAX,
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c

Why can’t all the vhost-scsi-common stuff be moved to a separate change?

Especially the stuff introduced for vhost-user-blk in 1b0063b3048af65dfaae6422a572c87db8575a92 should be moved out.

> index a06f01af26..08801886b8 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> 
>     vsc->dev.acked_features = vdev->guest_features;
> 
> -    assert(vsc->inflight == NULL);
> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> -    ret = vhost_dev_get_inflight(&vsc->dev,
> -                                 vs->conf.virtqueue_size,
> -                                 vsc->inflight);
> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>     if (ret < 0) {
> -        error_report("Error get inflight: %d", -ret);
> +        error_report("Error setting inflight format: %d", -ret);
>         goto err_guest_notifiers;
>     }
> 
> +    if (!vsc->inflight->addr) {
> +        ret = vhost_dev_get_inflight(&vsc->dev,
> +                                    vs->conf.virtqueue_size,
> +                                    vsc->inflight);
> +        if (ret < 0) {
> +            error_report("Error get inflight: %d", -ret);
> +            goto err_guest_notifiers;
> +        }
> +    }
> +
>     ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>     if (ret < 0) {
>         error_report("Error set inflight: %d", -ret);
> @@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>     return ret;
> 
> err_guest_notifiers:
> -    g_free(vsc->inflight);
> -    vsc->inflight = NULL;
> -
>     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
> err_host_notifiers:
>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> @@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>     }
>     assert(ret >= 0);
> 

In the vhost-scsi (kernel backend) path, what will cleanup vsc->inflight now?

> -    if (vsc->inflight) {
> -        vhost_dev_free_inflight(vsc->inflight);
> -        g_free(vsc->inflight);
> -        vsc->inflight = NULL;
> -    }
> -
>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> }
> 
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index ee99b19e7a..e0e88b0c42 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> }
> 
> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret = 0;
> +
> +    if (s->connected) {
> +        return 0;
> +    }
> +    s->connected = true;
> +
> +    vsc->dev.num_queues = vs->conf.num_queues;
> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> +    vsc->dev.vqs = s->vhost_vqs;
> +    vsc->dev.vq_index = 0;
> +    vsc->dev.backend_features = 0;
> +
> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> +                         errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    /* restore vhost state */

Should this use virtio_device_should_start like vhost_user_blk?

> +    if (virtio_device_started(vdev, vdev->status)) {
> +        ret = vhost_scsi_common_start(vsc);
> +        if (ret < 0) {
> +            return ret;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> +
> +static void vhost_user_scsi_disconnect(DeviceState *dev)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +

I don’t think we want to execute vhost_scsi_common_stop() if the device hasn’t been started. I remember that caused a number of races with the vhost_user_blk connecting/disconnecting on startup.

Let’s add a similar started_vu check?

> +    if (!s->connected) {
> +        return;
> +    }
> +    s->connected = false;
> +
> +    vhost_scsi_common_stop(vsc);
> +
> +    vhost_dev_cleanup(&vsc->dev);
> +
> +    /* Re-instate the event handler for new connections */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> +{
> +    DeviceState *dev = opaque;
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    Error *local_err = NULL;
> +
> +    switch (event) {
> +    case CHR_EVENT_OPENED:
> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> +            error_report_err(local_err);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> +            return;
> +        }
> +        break;
> +    case CHR_EVENT_CLOSED:
> +        /* defer close until later to avoid circular close */
> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> +                               vhost_user_scsi_disconnect);
> +        break;
> +    case CHR_EVENT_BREAK:
> +    case CHR_EVENT_MUX_IN:
> +    case CHR_EVENT_MUX_OUT:
> +        /* Ignore */
> +        break;
> +    }
> +}
> +
> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> +{
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret;
> +
> +    s->connected = false;
> +
> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    ret = vhost_user_scsi_connect(dev, errp);
> +    if (ret < 0) {
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return ret;
> +    }
> +    assert(s->connected);
> +
> +    return 0;
> +}
> +
> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> {
>     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = NULL;
>     Error *err = NULL;
>     int ret;
> +    int retries = REALIZE_CONNECTION_RETRIES;
> 
>     if (!vs->conf.chardev.chr) {
>         error_setg(errp, "vhost-user-scsi: missing chardev");
> @@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     }
> 
>     if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {

Why execute vhost_user_cleanup() if vhost_user_init() fails?

> -        goto free_virtio;
> +        goto free_vhost;
>     }
> 
> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> -    vsc->dev.vq_index = 0;
> -    vsc->dev.backend_features = 0;
> -    vqs = vsc->dev.vqs;
> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> +
> +    assert(!*errp);
> +    do {
> +        if (*errp) {
> +            error_prepend(errp, "Reconnecting after error: ");
> +            error_report_err(*errp);
> +            *errp = NULL;
> +        }
> +        ret = vhost_user_scsi_realize_connect(s, errp);
> +    } while (ret < 0 && retries--);
> 
> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>     if (ret < 0) {
> -        goto free_vhost;
> +        goto free_vqs;
>     }
> 
> +    /* we're fully initialized, now we can operate, so add the handler */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> +                             vhost_user_scsi_event, NULL, (void *)dev,
> +                             NULL, true);
>     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>     vsc->channel = 0;
>     vsc->lun = 0;
> @@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> 
>     return;
> 
> +free_vqs:
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
> +
> free_vhost:
>     vhost_user_cleanup(&s->vhost_user);
> -    g_free(vqs);
> -free_virtio:
> +
>     virtio_scsi_common_unrealize(dev);
> }
> 
> @@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> 
>     /* This will stop the vhost backend. */
>     vhost_user_scsi_set_status(vdev, 0);
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> +                             NULL, false);
> 
>     vhost_dev_cleanup(&vsc->dev);
> -    g_free(vqs);

Nit: Why not put vhost_dev_free_inflight next to the remaining inflight cleanup?

> +    vhost_dev_free_inflight(vsc->inflight);
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
> 

Curiosity - why reorder here? Is something in vhost_user_cleanup() dependent on state freed in virtio_scsi_common_unrealize()?

If so, should that go as a standalone fix?

> -    virtio_scsi_common_unrealize(dev);
>     vhost_user_cleanup(&s->vhost_user);
> +    virtio_scsi_common_unrealize(dev);
> }
> 
> static Property vhost_user_scsi_properties[] = {
> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> index 521b08e559..c66acc68b7 100644
> --- a/include/hw/virtio/vhost-user-scsi.h
> +++ b/include/hw/virtio/vhost-user-scsi.h
> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
> struct VHostUserSCSI {
>     VHostSCSICommon parent_obj;
>     VhostUserState vhost_user;

See above - we should probably have started_vu here/

Maybe we should have some shared struct with vhost_user_blk for connectivity params?

> +    bool connected;
> +
> +    struct vhost_virtqueue *vhost_vqs;
> };
> 
> #endif /* VHOST_USER_SCSI_H */
> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
> index 6a173cb9fa..b904346fe1 100644
> --- a/include/hw/virtio/vhost.h
> +++ b/include/hw/virtio/vhost.h
> @@ -8,6 +8,8 @@
> #define VHOST_F_DEVICE_IOTLB 63
> #define VHOST_USER_F_PROTOCOL_FEATURES 30
> 

Should the macro name indicate that this is for vhost-user?

VU_REALIZE_CONN_RETRIES? 

> +#define REALIZE_CONNECTION_RETRIES 3
> +
> /* Generic structures common for any vhost based device. */
> 
> struct vhost_inflight {
> -- 
> 2.41.0
> 

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Michael S. Tsirkin]

On Mon, Jul 24, 2023 at 05:21:37PM +0000, Raphael Norwitz wrote:
> Very excited to see this. High level looks good modulo a few small things.
> 
> My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.
> 
> The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.

No, device itself can store the state somewhere. And if it wants to,
it can check VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD and fail reconnect.

-- 
MST

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 14353 bytes --]

Thanks for your comments.

> 2023年7月25日 上午1:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
> 
> Very excited to see this. High level looks good modulo a few small things.
> 
> My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.
> 
> The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.

Based on what MST said, is there anything else I need to do?
> 
>> On Jul 21, 2023, at 6:51 AM, Li Feng <fengli@smartx.com> wrote:
>> 
>> If the backend crashes and restarts, the device is broken.
>> This patch adds reconnect for vhost-user-scsi.
>> 
>> Tested with spdk backend.
>> 
>> Signed-off-by: Li Feng <fengli@smartx.com>
>> ---
>> hw/block/vhost-user-blk.c           |   2 -
>> hw/scsi/vhost-scsi-common.c         |  27 ++---
>> hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
>> include/hw/virtio/vhost-user-scsi.h |   3 +
>> include/hw/virtio/vhost.h           |   2 +
>> 5 files changed, 165 insertions(+), 32 deletions(-)
>> 
>> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
>> index eecf3f7a81..f250c740b5 100644
>> --- a/hw/block/vhost-user-blk.c
>> +++ b/hw/block/vhost-user-blk.c
>> @@ -32,8 +32,6 @@
>> #include "sysemu/sysemu.h"
>> #include "sysemu/runstate.h"
>> 
>> -#define REALIZE_CONNECTION_RETRIES 3
>> -
>> static const int user_feature_bits[] = {
>>    VIRTIO_BLK_F_SIZE_MAX,
>>    VIRTIO_BLK_F_SEG_MAX,
>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> 
> Why can’t all the vhost-scsi-common stuff be moved to a separate change?

I will move this code to separate patch.
> 
> Especially the stuff introduced for vhost-user-blk in 1b0063b3048af65dfaae6422a572c87db8575a92 should be moved out.
OK.

> 
>> index a06f01af26..08801886b8 100644
>> --- a/hw/scsi/vhost-scsi-common.c
>> +++ b/hw/scsi/vhost-scsi-common.c
>> @@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>> 
>>    vsc->dev.acked_features = vdev->guest_features;
>> 
>> -    assert(vsc->inflight == NULL);
>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>> -                                 vs->conf.virtqueue_size,
>> -                                 vsc->inflight);
>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>    if (ret < 0) {
>> -        error_report("Error get inflight: %d", -ret);
>> +        error_report("Error setting inflight format: %d", -ret);
>>        goto err_guest_notifiers;
>>    }
>> 
>> +    if (!vsc->inflight->addr) {
>> +        ret = vhost_dev_get_inflight(&vsc->dev,
>> +                                    vs->conf.virtqueue_size,
>> +                                    vsc->inflight);
>> +        if (ret < 0) {
>> +            error_report("Error get inflight: %d", -ret);
>> +            goto err_guest_notifiers;
>> +        }
>> +    }
>> +
>>    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>    if (ret < 0) {
>>        error_report("Error set inflight: %d", -ret);
>> @@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>    return ret;
>> 
>> err_guest_notifiers:
>> -    g_free(vsc->inflight);
>> -    vsc->inflight = NULL;
>> -
>>    k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>> err_host_notifiers:
>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>> @@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>    }
>>    assert(ret >= 0);
>> 
> 
> In the vhost-scsi (kernel backend) path, what will cleanup vsc->inflight now?
OK, we should check the vsc->inflight if it is null, the vhost-scsi doesn’t allocate the
inflight object memory.

> 
>> -    if (vsc->inflight) {
>> -        vhost_dev_free_inflight(vsc->inflight);
>> -        g_free(vsc->inflight);
>> -        vsc->inflight = NULL;
>> -    }
>> -
>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>> }
>> 
>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>> index ee99b19e7a..e0e88b0c42 100644
>> --- a/hw/scsi/vhost-user-scsi.c
>> +++ b/hw/scsi/vhost-user-scsi.c
>> @@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>> {
>> }
>> 
>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>> +{
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    int ret = 0;
>> +
>> +    if (s->connected) {
>> +        return 0;
>> +    }
>> +    s->connected = true;
>> +
>> +    vsc->dev.num_queues = vs->conf.num_queues;
>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>> +    vsc->dev.vqs = s->vhost_vqs;
>> +    vsc->dev.vq_index = 0;
>> +    vsc->dev.backend_features = 0;
>> +
>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>> +                         errp);
>> +    if (ret < 0) {
>> +        return ret;
>> +    }
>> +
>> +    /* restore vhost state */
> 
> Should this use virtio_device_should_start like vhost_user_blk?
I will change this.
> 
>> +    if (virtio_device_started(vdev, vdev->status)) {
>> +        ret = vhost_scsi_common_start(vsc);
>> +        if (ret < 0) {
>> +            return ret;
>> +        }
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>> +
>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>> +{
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +
> 
> I don’t think we want to execute vhost_scsi_common_stop() if the device hasn’t been started. I remember that caused a number of races with the vhost_user_blk connecting/disconnecting on startup.
> 
> Let’s add a similar started_vu check?
I will add it.
> 
>> +    if (!s->connected) {
>> +        return;
>> +    }
>> +    s->connected = false;
>> +
>> +    vhost_scsi_common_stop(vsc);
>> +
>> +    vhost_dev_cleanup(&vsc->dev);
>> +
>> +    /* Re-instate the event handler for new connections */
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>> +}
>> +
>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>> +{
>> +    DeviceState *dev = opaque;
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    Error *local_err = NULL;
>> +
>> +    switch (event) {
>> +    case CHR_EVENT_OPENED:
>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>> +            error_report_err(local_err);
>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +            return;
>> +        }
>> +        break;
>> +    case CHR_EVENT_CLOSED:
>> +        /* defer close until later to avoid circular close */
>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>> +                               vhost_user_scsi_disconnect);
>> +        break;
>> +    case CHR_EVENT_BREAK:
>> +    case CHR_EVENT_MUX_IN:
>> +    case CHR_EVENT_MUX_OUT:
>> +        /* Ignore */
>> +        break;
>> +    }
>> +}
>> +
>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>> +{
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    int ret;
>> +
>> +    s->connected = false;
>> +
>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>> +    if (ret < 0) {
>> +        return ret;
>> +    }
>> +
>> +    ret = vhost_user_scsi_connect(dev, errp);
>> +    if (ret < 0) {
>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +        return ret;
>> +    }
>> +    assert(s->connected);
>> +
>> +    return 0;
>> +}
>> +
>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>> {
>>    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>    VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    struct vhost_virtqueue *vqs = NULL;
>>    Error *err = NULL;
>>    int ret;
>> +    int retries = REALIZE_CONNECTION_RETRIES;
>> 
>>    if (!vs->conf.chardev.chr) {
>>        error_setg(errp, "vhost-user-scsi: missing chardev");
>> @@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>    }
>> 
>>    if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {
> 
> Why execute vhost_user_cleanup() if vhost_user_init() fails?
OK, move this line up in v2.

> 
>> -        goto free_virtio;
>> +        goto free_vhost;
>>    }
>> 
>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>> -    vsc->dev.vq_index = 0;
>> -    vsc->dev.backend_features = 0;
>> -    vqs = vsc->dev.vqs;
>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>> +
>> +    assert(!*errp);
>> +    do {
>> +        if (*errp) {
>> +            error_prepend(errp, "Reconnecting after error: ");
>> +            error_report_err(*errp);
>> +            *errp = NULL;
>> +        }
>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>> +    } while (ret < 0 && retries--);
>> 
>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>    if (ret < 0) {
>> -        goto free_vhost;
>> +        goto free_vqs;
>>    }
>> 
>> +    /* we're fully initialized, now we can operate, so add the handler */
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>> +                             NULL, true);
>>    /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>    vsc->channel = 0;
>>    vsc->lun = 0;
>> @@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>> 
>>    return;
>> 
>> +free_vqs:
>> +    g_free(s->vhost_vqs);
>> +    s->vhost_vqs = NULL;
>> +    g_free(vsc->inflight);
>> +    vsc->inflight = NULL;
>> +
>> free_vhost:
>>    vhost_user_cleanup(&s->vhost_user);
>> -    g_free(vqs);
>> -free_virtio:
>> +
>>    virtio_scsi_common_unrealize(dev);
>> }
>> 
>> @@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>    VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> 
>>    /* This will stop the vhost backend. */
>>    vhost_user_scsi_set_status(vdev, 0);
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>> +                             NULL, false);
>> 
>>    vhost_dev_cleanup(&vsc->dev);
>> -    g_free(vqs);
> 
> Nit: Why not put vhost_dev_free_inflight next to the remaining inflight cleanup?
OK.
> 
>> +    vhost_dev_free_inflight(vsc->inflight);
>> +    g_free(s->vhost_vqs);
>> +    s->vhost_vqs = NULL;
>> +    g_free(vsc->inflight);
>> +    vsc->inflight = NULL;
>> 
> 
> Curiosity - why reorder here? Is something in vhost_user_cleanup() dependent on state freed in virtio_scsi_common_unrealize()?
> 
> If so, should that go as a standalone fix?

Because in vhost_user_scsi_realize, we initialize in order:
virtio_scsi_common_realize
vhost_user_init

And in the error handler of vhost_user_scsi_realize, the uninitialize in order:
vhost_user_cleanup
virtio_scsi_common_unrealize

I think in vhost_user_scsi_unrealize we should keep it the same order, right?

> 
>> -    virtio_scsi_common_unrealize(dev);
>>    vhost_user_cleanup(&s->vhost_user);
>> +    virtio_scsi_common_unrealize(dev);
>> }
>> 
>> static Property vhost_user_scsi_properties[] = {
>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>> index 521b08e559..c66acc68b7 100644
>> --- a/include/hw/virtio/vhost-user-scsi.h
>> +++ b/include/hw/virtio/vhost-user-scsi.h
>> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>> struct VHostUserSCSI {
>>    VHostSCSICommon parent_obj;
>>    VhostUserState vhost_user;
> 
> See above - we should probably have started_vu here/
I will add it.
> 
> Maybe we should have some shared struct with vhost_user_blk for connectivity params?

In the future vhost-user-blk/scsi can be refactored to share the same code.
> 
>> +    bool connected;
>> +
>> +    struct vhost_virtqueue *vhost_vqs;
>> };
>> 
>> #endif /* VHOST_USER_SCSI_H */
>> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
>> index 6a173cb9fa..b904346fe1 100644
>> --- a/include/hw/virtio/vhost.h
>> +++ b/include/hw/virtio/vhost.h
>> @@ -8,6 +8,8 @@
>> #define VHOST_F_DEVICE_IOTLB 63
>> #define VHOST_USER_F_PROTOCOL_FEATURES 30
>> 
> 
> Should the macro name indicate that this is for vhost-user?
> 
> VU_REALIZE_CONN_RETRIES? 
I will rename it in v2.

> 
>> +#define REALIZE_CONNECTION_RETRIES 3
>> +
>> /* Generic structures common for any vhost based device. */
>> 
>> struct vhost_inflight {
>> -- 
>> 2.41.0


[-- Attachment #2: Type: text/html, Size: 39755 bytes --]

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Raphael Norwitz]

> On Jul 25, 2023, at 6:19 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Thanks for your comments.
> 
>> 2023年7月25日 上午1:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>> 
>> Very excited to see this. High level looks good modulo a few small things.
>> 
>> My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.
>> 
>> The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.
> 
> Based on what MST said, is there anything else I need to do?

I don’t think so.

>> 
>>> On Jul 21, 2023, at 6:51 AM, Li Feng <fengli@smartx.com> wrote:
>>> 
>>> If the backend crashes and restarts, the device is broken.
>>> This patch adds reconnect for vhost-user-scsi.
>>> 
>>> Tested with spdk backend.
>>> 
>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>> ---
>>> hw/block/vhost-user-blk.c           |   2 -
>>> hw/scsi/vhost-scsi-common.c         |  27 ++---
>>> hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
>>> include/hw/virtio/vhost-user-scsi.h |   3 +
>>> include/hw/virtio/vhost.h           |   2 +
>>> 5 files changed, 165 insertions(+), 32 deletions(-)
>>> 
>>> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
>>> index eecf3f7a81..f250c740b5 100644
>>> --- a/hw/block/vhost-user-blk.c
>>> +++ b/hw/block/vhost-user-blk.c
>>> @@ -32,8 +32,6 @@
>>> #include "sysemu/sysemu.h"
>>> #include "sysemu/runstate.h"
>>> 
>>> -#define REALIZE_CONNECTION_RETRIES 3
>>> -
>>> static const int user_feature_bits[] = {
>>>    VIRTIO_BLK_F_SIZE_MAX,
>>>    VIRTIO_BLK_F_SEG_MAX,
>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>> 
>> Why can’t all the vhost-scsi-common stuff be moved to a separate change?
> 
> I will move this code to separate patch.
>> 
>> Especially the stuff introduced for vhost-user-blk in 1b0063b3048af65dfaae6422a572c87db8575a92 should be moved out.
> OK.
> 
>> 
>>> index a06f01af26..08801886b8 100644
>>> --- a/hw/scsi/vhost-scsi-common.c
>>> +++ b/hw/scsi/vhost-scsi-common.c
>>> @@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>> 
>>>    vsc->dev.acked_features = vdev->guest_features;
>>> 
>>> -    assert(vsc->inflight == NULL);
>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>> -                                 vs->conf.virtqueue_size,
>>> -                                 vsc->inflight);
>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>    if (ret < 0) {
>>> -        error_report("Error get inflight: %d", -ret);
>>> +        error_report("Error setting inflight format: %d", -ret);
>>>        goto err_guest_notifiers;
>>>    }
>>> 
>>> +    if (!vsc->inflight->addr) {
>>> +        ret = vhost_dev_get_inflight(&vsc->dev,
>>> +                                    vs->conf.virtqueue_size,
>>> +                                    vsc->inflight);
>>> +        if (ret < 0) {
>>> +            error_report("Error get inflight: %d", -ret);
>>> +            goto err_guest_notifiers;
>>> +        }
>>> +    }
>>> +
>>>    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>    if (ret < 0) {
>>>        error_report("Error set inflight: %d", -ret);
>>> @@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>    return ret;
>>> 
>>> err_guest_notifiers:
>>> -    g_free(vsc->inflight);
>>> -    vsc->inflight = NULL;
>>> -
>>>    k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>> err_host_notifiers:
>>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>> @@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>    }
>>>    assert(ret >= 0);
>>> 
>> 
>> In the vhost-scsi (kernel backend) path, what will cleanup vsc->inflight now?
> OK, we should check the vsc->inflight if it is null, the vhost-scsi doesn’t allocate the
> inflight object memory.

Are you saying vhost-scsi never allocates inflight so we don’t need to check for it?

> 
>> 
>>> -    if (vsc->inflight) {
>>> -        vhost_dev_free_inflight(vsc->inflight);
>>> -        g_free(vsc->inflight);
>>> -        vsc->inflight = NULL;
>>> -    }
>>> -
>>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>> }
>>> 
>>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>>> index ee99b19e7a..e0e88b0c42 100644
>>> --- a/hw/scsi/vhost-user-scsi.c
>>> +++ b/hw/scsi/vhost-user-scsi.c
>>> @@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>>> {
>>> }
>>> 
>>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>>> +{
>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>> +    int ret = 0;
>>> +
>>> +    if (s->connected) {
>>> +        return 0;
>>> +    }
>>> +    s->connected = true;
>>> +
>>> +    vsc->dev.num_queues = vs->conf.num_queues;
>>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>> +    vsc->dev.vqs = s->vhost_vqs;
>>> +    vsc->dev.vq_index = 0;
>>> +    vsc->dev.backend_features = 0;
>>> +
>>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>>> +                         errp);
>>> +    if (ret < 0) {
>>> +        return ret;
>>> +    }
>>> +
>>> +    /* restore vhost state */
>> 
>> Should this use virtio_device_should_start like vhost_user_blk?
> I will change this.
>> 
>>> +    if (virtio_device_started(vdev, vdev->status)) {
>>> +        ret = vhost_scsi_common_start(vsc);
>>> +        if (ret < 0) {
>>> +            return ret;
>>> +        }
>>> +    }
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>>> +
>>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>>> +{
>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>> +
>> 
>> I don’t think we want to execute vhost_scsi_common_stop() if the device hasn’t been started. I remember that caused a number of races with the vhost_user_blk connecting/disconnecting on startup.
>> 
>> Let’s add a similar started_vu check?
> I will add it.
>> 
>>> +    if (!s->connected) {
>>> +        return;
>>> +    }
>>> +    s->connected = false;
>>> +
>>> +    vhost_scsi_common_stop(vsc);
>>> +
>>> +    vhost_dev_cleanup(&vsc->dev);
>>> +
>>> +    /* Re-instate the event handler for new connections */
>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>>> +}
>>> +
>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>>> +{
>>> +    DeviceState *dev = opaque;
>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>> +    Error *local_err = NULL;
>>> +
>>> +    switch (event) {
>>> +    case CHR_EVENT_OPENED:
>>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>>> +            error_report_err(local_err);
>>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>>> +            return;
>>> +        }
>>> +        break;
>>> +    case CHR_EVENT_CLOSED:
>>> +        /* defer close until later to avoid circular close */
>>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>>> +                               vhost_user_scsi_disconnect);
>>> +        break;
>>> +    case CHR_EVENT_BREAK:
>>> +    case CHR_EVENT_MUX_IN:
>>> +    case CHR_EVENT_MUX_OUT:
>>> +        /* Ignore */
>>> +        break;
>>> +    }
>>> +}
>>> +
>>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>>> +{
>>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>> +    int ret;
>>> +
>>> +    s->connected = false;
>>> +
>>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>>> +    if (ret < 0) {
>>> +        return ret;
>>> +    }
>>> +
>>> +    ret = vhost_user_scsi_connect(dev, errp);
>>> +    if (ret < 0) {
>>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>>> +        return ret;
>>> +    }
>>> +    assert(s->connected);
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>> {
>>>    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>    VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>> -    struct vhost_virtqueue *vqs = NULL;
>>>    Error *err = NULL;
>>>    int ret;
>>> +    int retries = REALIZE_CONNECTION_RETRIES;
>>> 
>>>    if (!vs->conf.chardev.chr) {
>>>        error_setg(errp, "vhost-user-scsi: missing chardev");
>>> @@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>    }
>>> 
>>>    if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {
>> 
>> Why execute vhost_user_cleanup() if vhost_user_init() fails?
> OK, move this line up in v2.
> 
>> 
>>> -        goto free_virtio;
>>> +        goto free_vhost;
>>>    }
>>> 
>>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>>> -    vsc->dev.vq_index = 0;
>>> -    vsc->dev.backend_features = 0;
>>> -    vqs = vsc->dev.vqs;
>>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>>> +
>>> +    assert(!*errp);
>>> +    do {
>>> +        if (*errp) {
>>> +            error_prepend(errp, "Reconnecting after error: ");
>>> +            error_report_err(*errp);
>>> +            *errp = NULL;
>>> +        }
>>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>>> +    } while (ret < 0 && retries--);
>>> 
>>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>>    if (ret < 0) {
>>> -        goto free_vhost;
>>> +        goto free_vqs;
>>>    }
>>> 
>>> +    /* we're fully initialized, now we can operate, so add the handler */
>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>>> +                             NULL, true);
>>>    /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>>    vsc->channel = 0;
>>>    vsc->lun = 0;
>>> @@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>> 
>>>    return;
>>> 
>>> +free_vqs:
>>> +    g_free(s->vhost_vqs);
>>> +    s->vhost_vqs = NULL;
>>> +    g_free(vsc->inflight);
>>> +    vsc->inflight = NULL;
>>> +
>>> free_vhost:
>>>    vhost_user_cleanup(&s->vhost_user);
>>> -    g_free(vqs);
>>> -free_virtio:
>>> +
>>>    virtio_scsi_common_unrealize(dev);
>>> }
>>> 
>>> @@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>>    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>    VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>> 
>>>    /* This will stop the vhost backend. */
>>>    vhost_user_scsi_set_status(vdev, 0);
>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>>> +                             NULL, false);
>>> 
>>>    vhost_dev_cleanup(&vsc->dev);
>>> -    g_free(vqs);
>> 
>> Nit: Why not put vhost_dev_free_inflight next to the remaining inflight cleanup?
> OK.
>> 
>>> +    vhost_dev_free_inflight(vsc->inflight);
>>> +    g_free(s->vhost_vqs);
>>> +    s->vhost_vqs = NULL;
>>> +    g_free(vsc->inflight);
>>> +    vsc->inflight = NULL;
>>> 
>> 
>> Curiosity - why reorder here? Is something in vhost_user_cleanup() dependent on state freed in virtio_scsi_common_unrealize()?
>> 
>> If so, should that go as a standalone fix?
> 
> Because in vhost_user_scsi_realize, we initialize in order:
> virtio_scsi_common_realize
> vhost_user_init
> 
> And in the error handler of vhost_user_scsi_realize, the uninitialize in order:
> vhost_user_cleanup
> virtio_scsi_common_unrealize
> 
> I think in vhost_user_scsi_unrealize we should keep it the same order, right?

I’m not saying it’s wrong. If there’s no dependency (i.e. this is not fixing a bug, just a stylistic improvement) it can stay in the same change.

> 
>> 
>>> -    virtio_scsi_common_unrealize(dev);
>>>    vhost_user_cleanup(&s->vhost_user);
>>> +    virtio_scsi_common_unrealize(dev);
>>> }
>>> 
>>> static Property vhost_user_scsi_properties[] = {
>>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>>> index 521b08e559..c66acc68b7 100644
>>> --- a/include/hw/virtio/vhost-user-scsi.h
>>> +++ b/include/hw/virtio/vhost-user-scsi.h
>>> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>>> struct VHostUserSCSI {
>>>    VHostSCSICommon parent_obj;
>>>    VhostUserState vhost_user;
>> 
>> See above - we should probably have started_vu here/
> I will add it.
>> 
>> Maybe we should have some shared struct with vhost_user_blk for connectivity params?
> 
> In the future vhost-user-blk/scsi can be refactored to share the same code.

Sure - this can be done at some point in the future.

>> 
>>> +    bool connected;
>>> +
>>> +    struct vhost_virtqueue *vhost_vqs;
>>> };
>>> 
>>> #endif /* VHOST_USER_SCSI_H */
>>> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
>>> index 6a173cb9fa..b904346fe1 100644
>>> --- a/include/hw/virtio/vhost.h
>>> +++ b/include/hw/virtio/vhost.h
>>> @@ -8,6 +8,8 @@
>>> #define VHOST_F_DEVICE_IOTLB 63
>>> #define VHOST_USER_F_PROTOCOL_FEATURES 30
>>> 
>> 
>> Should the macro name indicate that this is for vhost-user?
>> 
>> VU_REALIZE_CONN_RETRIES? 
> I will rename it in v2.
> 
>> 
>>> +#define REALIZE_CONNECTION_RETRIES 3
>>> +
>>> /* Generic structures common for any vhost based device. */
>>> 
>>> struct vhost_inflight {
>>> -- 
>>> 2.41.0

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 15899 bytes --]

Thanks for your reply.

> 2023年7月28日 上午5:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
> 
> 
> 
>> On Jul 25, 2023, at 6:19 AM, Li Feng <fengli@smartx.com> wrote:
>> 
>> Thanks for your comments.
>> 
>>> 2023年7月25日 上午1:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>>> 
>>> Very excited to see this. High level looks good modulo a few small things.
>>> 
>>> My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.
>>> 
>>> The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.
>> 
>> Based on what MST said, is there anything else I need to do?
> 
> I don’t think so.
> 
>>> 
>>>> On Jul 21, 2023, at 6:51 AM, Li Feng <fengli@smartx.com> wrote:
>>>> 
>>>> If the backend crashes and restarts, the device is broken.
>>>> This patch adds reconnect for vhost-user-scsi.
>>>> 
>>>> Tested with spdk backend.
>>>> 
>>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>>> ---
>>>> hw/block/vhost-user-blk.c           |   2 -
>>>> hw/scsi/vhost-scsi-common.c         |  27 ++---
>>>> hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
>>>> include/hw/virtio/vhost-user-scsi.h |   3 +
>>>> include/hw/virtio/vhost.h           |   2 +
>>>> 5 files changed, 165 insertions(+), 32 deletions(-)
>>>> 
>>>> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
>>>> index eecf3f7a81..f250c740b5 100644
>>>> --- a/hw/block/vhost-user-blk.c
>>>> +++ b/hw/block/vhost-user-blk.c
>>>> @@ -32,8 +32,6 @@
>>>> #include "sysemu/sysemu.h"
>>>> #include "sysemu/runstate.h"
>>>> 
>>>> -#define REALIZE_CONNECTION_RETRIES 3
>>>> -
>>>> static const int user_feature_bits[] = {
>>>>   VIRTIO_BLK_F_SIZE_MAX,
>>>>   VIRTIO_BLK_F_SEG_MAX,
>>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>>> 
>>> Why can’t all the vhost-scsi-common stuff be moved to a separate change?
>> 
>> I will move this code to separate patch.
>>> 
>>> Especially the stuff introduced for vhost-user-blk in 1b0063b3048af65dfaae6422a572c87db8575a92 should be moved out.
>> OK.
>> 
>>> 
>>>> index a06f01af26..08801886b8 100644
>>>> --- a/hw/scsi/vhost-scsi-common.c
>>>> +++ b/hw/scsi/vhost-scsi-common.c
>>>> @@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>> 
>>>>   vsc->dev.acked_features = vdev->guest_features;
>>>> 
>>>> -    assert(vsc->inflight == NULL);
>>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>>> -                                 vs->conf.virtqueue_size,
>>>> -                                 vsc->inflight);
>>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>>   if (ret < 0) {
>>>> -        error_report("Error get inflight: %d", -ret);
>>>> +        error_report("Error setting inflight format: %d", -ret);
>>>>       goto err_guest_notifiers;
>>>>   }
>>>> 
>>>> +    if (!vsc->inflight->addr) {
>>>> +        ret = vhost_dev_get_inflight(&vsc->dev,
>>>> +                                    vs->conf.virtqueue_size,
>>>> +                                    vsc->inflight);
>>>> +        if (ret < 0) {
>>>> +            error_report("Error get inflight: %d", -ret);
>>>> +            goto err_guest_notifiers;
>>>> +        }
>>>> +    }
>>>> +
>>>>   ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>>   if (ret < 0) {
>>>>       error_report("Error set inflight: %d", -ret);
>>>> @@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>   return ret;
>>>> 
>>>> err_guest_notifiers:
>>>> -    g_free(vsc->inflight);
>>>> -    vsc->inflight = NULL;
>>>> -
>>>>   k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>>> err_host_notifiers:
>>>>   vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>> @@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>>   }
>>>>   assert(ret >= 0);
>>>> 
>>> 
>>> In the vhost-scsi (kernel backend) path, what will cleanup vsc->inflight now?
>> OK, we should check the vsc->inflight if it is null, the vhost-scsi doesn’t allocate the
>> inflight object memory.
> 
> Are you saying vhost-scsi never allocates inflight so we don’t need to check for it?
We have checked the vsc->inflight, and only if allocated, we send the get/set_inflight_fd.
This works with vhost-user-scsi/vhost-scsi both.
> 
>> 
>>> 
>>>> -    if (vsc->inflight) {
>>>> -        vhost_dev_free_inflight(vsc->inflight);
>>>> -        g_free(vsc->inflight);
>>>> -        vsc->inflight = NULL;
>>>> -    }
>>>> -
>>>>   vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>> }
>>>> 
>>>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>>>> index ee99b19e7a..e0e88b0c42 100644
>>>> --- a/hw/scsi/vhost-user-scsi.c
>>>> +++ b/hw/scsi/vhost-user-scsi.c
>>>> @@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>>>> {
>>>> }
>>>> 
>>>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>>>> +{
>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>> +    int ret = 0;
>>>> +
>>>> +    if (s->connected) {
>>>> +        return 0;
>>>> +    }
>>>> +    s->connected = true;
>>>> +
>>>> +    vsc->dev.num_queues = vs->conf.num_queues;
>>>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>>> +    vsc->dev.vqs = s->vhost_vqs;
>>>> +    vsc->dev.vq_index = 0;
>>>> +    vsc->dev.backend_features = 0;
>>>> +
>>>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>>>> +                         errp);
>>>> +    if (ret < 0) {
>>>> +        return ret;
>>>> +    }
>>>> +
>>>> +    /* restore vhost state */
>>> 
>>> Should this use virtio_device_should_start like vhost_user_blk?
>> I will change this.
>>> 
>>>> +    if (virtio_device_started(vdev, vdev->status)) {
>>>> +        ret = vhost_scsi_common_start(vsc);
>>>> +        if (ret < 0) {
>>>> +            return ret;
>>>> +        }
>>>> +    }
>>>> +
>>>> +    return 0;
>>>> +}
>>>> +
>>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>>>> +
>>>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>>>> +{
>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>> +
>>> 
>>> I don’t think we want to execute vhost_scsi_common_stop() if the device hasn’t been started. I remember that caused a number of races with the vhost_user_blk connecting/disconnecting on startup.
>>> 
>>> Let’s add a similar started_vu check?
>> I will add it.
>>> 
>>>> +    if (!s->connected) {
>>>> +        return;
>>>> +    }
>>>> +    s->connected = false;
>>>> +
>>>> +    vhost_scsi_common_stop(vsc);
>>>> +
>>>> +    vhost_dev_cleanup(&vsc->dev);
>>>> +
>>>> +    /* Re-instate the event handler for new connections */
>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>>>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>>>> +}
>>>> +
>>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>>>> +{
>>>> +    DeviceState *dev = opaque;
>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>> +    Error *local_err = NULL;
>>>> +
>>>> +    switch (event) {
>>>> +    case CHR_EVENT_OPENED:
>>>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>>>> +            error_report_err(local_err);
>>>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>>>> +            return;
>>>> +        }
>>>> +        break;
>>>> +    case CHR_EVENT_CLOSED:
>>>> +        /* defer close until later to avoid circular close */
>>>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>>>> +                               vhost_user_scsi_disconnect);
>>>> +        break;
>>>> +    case CHR_EVENT_BREAK:
>>>> +    case CHR_EVENT_MUX_IN:
>>>> +    case CHR_EVENT_MUX_OUT:
>>>> +        /* Ignore */
>>>> +        break;
>>>> +    }
>>>> +}
>>>> +
>>>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>>>> +{
>>>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>> +    int ret;
>>>> +
>>>> +    s->connected = false;
>>>> +
>>>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>>>> +    if (ret < 0) {
>>>> +        return ret;
>>>> +    }
>>>> +
>>>> +    ret = vhost_user_scsi_connect(dev, errp);
>>>> +    if (ret < 0) {
>>>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>>>> +        return ret;
>>>> +    }
>>>> +    assert(s->connected);
>>>> +
>>>> +    return 0;
>>>> +}
>>>> +
>>>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>> {
>>>>   VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>   VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>>   VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>> -    struct vhost_virtqueue *vqs = NULL;
>>>>   Error *err = NULL;
>>>>   int ret;
>>>> +    int retries = REALIZE_CONNECTION_RETRIES;
>>>> 
>>>>   if (!vs->conf.chardev.chr) {
>>>>       error_setg(errp, "vhost-user-scsi: missing chardev");
>>>> @@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>   }
>>>> 
>>>>   if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {
>>> 
>>> Why execute vhost_user_cleanup() if vhost_user_init() fails?
>> OK, move this line up in v2.
>> 
>>> 
>>>> -        goto free_virtio;
>>>> +        goto free_vhost;
>>>>   }
>>>> 
>>>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>>>> -    vsc->dev.vq_index = 0;
>>>> -    vsc->dev.backend_features = 0;
>>>> -    vqs = vsc->dev.vqs;
>>>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>>>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>>>> +
>>>> +    assert(!*errp);
>>>> +    do {
>>>> +        if (*errp) {
>>>> +            error_prepend(errp, "Reconnecting after error: ");
>>>> +            error_report_err(*errp);
>>>> +            *errp = NULL;
>>>> +        }
>>>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>>>> +    } while (ret < 0 && retries--);
>>>> 
>>>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>>>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>>>   if (ret < 0) {
>>>> -        goto free_vhost;
>>>> +        goto free_vqs;
>>>>   }
>>>> 
>>>> +    /* we're fully initialized, now we can operate, so add the handler */
>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>>>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>>>> +                             NULL, true);
>>>>   /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>>>   vsc->channel = 0;
>>>>   vsc->lun = 0;
>>>> @@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>> 
>>>>   return;
>>>> 
>>>> +free_vqs:
>>>> +    g_free(s->vhost_vqs);
>>>> +    s->vhost_vqs = NULL;
>>>> +    g_free(vsc->inflight);
>>>> +    vsc->inflight = NULL;
>>>> +
>>>> free_vhost:
>>>>   vhost_user_cleanup(&s->vhost_user);
>>>> -    g_free(vqs);
>>>> -free_virtio:
>>>> +
>>>>   virtio_scsi_common_unrealize(dev);
>>>> }
>>>> 
>>>> @@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>>>   VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>   VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>>   VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>> 
>>>>   /* This will stop the vhost backend. */
>>>>   vhost_user_scsi_set_status(vdev, 0);
>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>>>> +                             NULL, false);
>>>> 
>>>>   vhost_dev_cleanup(&vsc->dev);
>>>> -    g_free(vqs);
>>> 
>>> Nit: Why not put vhost_dev_free_inflight next to the remaining inflight cleanup?
>> OK.
>>> 
>>>> +    vhost_dev_free_inflight(vsc->inflight);
>>>> +    g_free(s->vhost_vqs);
>>>> +    s->vhost_vqs = NULL;
>>>> +    g_free(vsc->inflight);
>>>> +    vsc->inflight = NULL;
>>>> 
>>> 
>>> Curiosity - why reorder here? Is something in vhost_user_cleanup() dependent on state freed in virtio_scsi_common_unrealize()?
>>> 
>>> If so, should that go as a standalone fix?
>> 
>> Because in vhost_user_scsi_realize, we initialize in order:
>> virtio_scsi_common_realize
>> vhost_user_init
>> 
>> And in the error handler of vhost_user_scsi_realize, the uninitialize in order:
>> vhost_user_cleanup
>> virtio_scsi_common_unrealize
>> 
>> I think in vhost_user_scsi_unrealize we should keep it the same order, right?
> 
> I’m not saying it’s wrong. If there’s no dependency (i.e. this is not fixing a bug, just a stylistic improvement) it can stay in the same change.
OK.
> 
>> 
>>> 
>>>> -    virtio_scsi_common_unrealize(dev);
>>>>   vhost_user_cleanup(&s->vhost_user);
>>>> +    virtio_scsi_common_unrealize(dev);
>>>> }
>>>> 
>>>> static Property vhost_user_scsi_properties[] = {
>>>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>>>> index 521b08e559..c66acc68b7 100644
>>>> --- a/include/hw/virtio/vhost-user-scsi.h
>>>> +++ b/include/hw/virtio/vhost-user-scsi.h
>>>> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>>>> struct VHostUserSCSI {
>>>>   VHostSCSICommon parent_obj;
>>>>   VhostUserState vhost_user;
>>> 
>>> See above - we should probably have started_vu here/
>> I will add it.
>>> 
>>> Maybe we should have some shared struct with vhost_user_blk for connectivity params?
>> 
>> In the future vhost-user-blk/scsi can be refactored to share the same code.
> 
> Sure - this can be done at some point in the future.
> 
>>> 
>>>> +    bool connected;
>>>> +
>>>> +    struct vhost_virtqueue *vhost_vqs;
>>>> };
>>>> 
>>>> #endif /* VHOST_USER_SCSI_H */
>>>> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
>>>> index 6a173cb9fa..b904346fe1 100644
>>>> --- a/include/hw/virtio/vhost.h
>>>> +++ b/include/hw/virtio/vhost.h
>>>> @@ -8,6 +8,8 @@
>>>> #define VHOST_F_DEVICE_IOTLB 63
>>>> #define VHOST_USER_F_PROTOCOL_FEATURES 30
>>>> 
>>> 
>>> Should the macro name indicate that this is for vhost-user?
>>> 
>>> VU_REALIZE_CONN_RETRIES? 
>> I will rename it in v2.
>> 
>>> 
>>>> +#define REALIZE_CONNECTION_RETRIES 3
>>>> +
>>>> /* Generic structures common for any vhost based device. */
>>>> 
>>>> struct vhost_inflight {
>>>> -- 
>>>> 2.41.0

Any comments about other patches?


[-- Attachment #2: Type: text/html, Size: 30005 bytes --]

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Raphael Norwitz]

> On Jul 28, 2023, at 3:48 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Thanks for your reply.
> 
>> 2023年7月28日 上午5:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>> 
>> 
>> 
>>> On Jul 25, 2023, at 6:19 AM, Li Feng <fengli@smartx.com> wrote:
>>> 
>>> Thanks for your comments.
>>> 
>>>> 2023年7月25日 上午1:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>>>> 
>>>> Very excited to see this. High level looks good modulo a few small things.
>>>> 
>>>> My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.
>>>> 
>>>> The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.
>>> 
>>> Based on what MST said, is there anything else I need to do?
>> 
>> I don’t think so.
>> 
>>>> 
>>>>> On Jul 21, 2023, at 6:51 AM, Li Feng <fengli@smartx.com> wrote:
>>>>> 
>>>>> If the backend crashes and restarts, the device is broken.
>>>>> This patch adds reconnect for vhost-user-scsi.
>>>>> 
>>>>> Tested with spdk backend.
>>>>> 
>>>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>>>> ---
>>>>> hw/block/vhost-user-blk.c           |   2 -
>>>>> hw/scsi/vhost-scsi-common.c         |  27 ++---
>>>>> hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
>>>>> include/hw/virtio/vhost-user-scsi.h |   3 +
>>>>> include/hw/virtio/vhost.h           |   2 +
>>>>> 5 files changed, 165 insertions(+), 32 deletions(-)
>>>>> 
>>>>> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
>>>>> index eecf3f7a81..f250c740b5 100644
>>>>> --- a/hw/block/vhost-user-blk.c
>>>>> +++ b/hw/block/vhost-user-blk.c
>>>>> @@ -32,8 +32,6 @@
>>>>> #include "sysemu/sysemu.h"
>>>>> #include "sysemu/runstate.h"
>>>>> 
>>>>> -#define REALIZE_CONNECTION_RETRIES 3
>>>>> -
>>>>> static const int user_feature_bits[] = {
>>>>>   VIRTIO_BLK_F_SIZE_MAX,
>>>>>   VIRTIO_BLK_F_SEG_MAX,
>>>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>>>> 
>>>> Why can’t all the vhost-scsi-common stuff be moved to a separate change?
>>> 
>>> I will move this code to separate patch.
>>>> 
>>>> Especially the stuff introduced for vhost-user-blk in 1b0063b3048af65dfaae6422a572c87db8575a92 should be moved out.
>>> OK.
>>> 
>>>> 
>>>>> index a06f01af26..08801886b8 100644
>>>>> --- a/hw/scsi/vhost-scsi-common.c
>>>>> +++ b/hw/scsi/vhost-scsi-common.c
>>>>> @@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>> 
>>>>>   vsc->dev.acked_features = vdev->guest_features;
>>>>> 
>>>>> -    assert(vsc->inflight == NULL);
>>>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>>>> -                                 vs->conf.virtqueue_size,
>>>>> -                                 vsc->inflight);
>>>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>>>   if (ret < 0) {
>>>>> -        error_report("Error get inflight: %d", -ret);
>>>>> +        error_report("Error setting inflight format: %d", -ret);
>>>>>       goto err_guest_notifiers;
>>>>>   }
>>>>> 
>>>>> +    if (!vsc->inflight->addr) {
>>>>> +        ret = vhost_dev_get_inflight(&vsc->dev,
>>>>> +                                    vs->conf.virtqueue_size,
>>>>> +                                    vsc->inflight);
>>>>> +        if (ret < 0) {
>>>>> +            error_report("Error get inflight: %d", -ret);
>>>>> +            goto err_guest_notifiers;
>>>>> +        }
>>>>> +    }
>>>>> +
>>>>>   ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>>>   if (ret < 0) {
>>>>>       error_report("Error set inflight: %d", -ret);
>>>>> @@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>>   return ret;
>>>>> 
>>>>> err_guest_notifiers:
>>>>> -    g_free(vsc->inflight);
>>>>> -    vsc->inflight = NULL;
>>>>> -
>>>>>   k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>>>> err_host_notifiers:
>>>>>   vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>>> @@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>>>   }
>>>>>   assert(ret >= 0);
>>>>> 
>>>> 
>>>> In the vhost-scsi (kernel backend) path, what will cleanup vsc->inflight now?
>>> OK, we should check the vsc->inflight if it is null, the vhost-scsi doesn’t allocate the
>>> inflight object memory.
>> 
>> Are you saying vhost-scsi never allocates inflight so we don’t need to check for it?
> We have checked the vsc->inflight, and only if allocated, we send the get/set_inflight_fd.
> This works with vhost-user-scsi/vhost-scsi both.

So then it sounds like this code introduces a resource leak. g_free(vsc->inflight) should be added to the vhost-scsi code in vhost_scsi_stop().

>> 
>>> 
>>>> 
>>>>> -    if (vsc->inflight) {
>>>>> -        vhost_dev_free_inflight(vsc->inflight);
>>>>> -        g_free(vsc->inflight);
>>>>> -        vsc->inflight = NULL;
>>>>> -    }
>>>>> -
>>>>>   vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>>> }
>>>>> 
>>>>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>>>>> index ee99b19e7a..e0e88b0c42 100644
>>>>> --- a/hw/scsi/vhost-user-scsi.c
>>>>> +++ b/hw/scsi/vhost-user-scsi.c
>>>>> @@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>>>>> {
>>>>> }
>>>>> 
>>>>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>>>>> +{
>>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>> +    int ret = 0;
>>>>> +
>>>>> +    if (s->connected) {
>>>>> +        return 0;
>>>>> +    }
>>>>> +    s->connected = true;
>>>>> +
>>>>> +    vsc->dev.num_queues = vs->conf.num_queues;
>>>>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>>>> +    vsc->dev.vqs = s->vhost_vqs;
>>>>> +    vsc->dev.vq_index = 0;
>>>>> +    vsc->dev.backend_features = 0;
>>>>> +
>>>>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>>>>> +                         errp);
>>>>> +    if (ret < 0) {
>>>>> +        return ret;
>>>>> +    }
>>>>> +
>>>>> +    /* restore vhost state */
>>>> 
>>>> Should this use virtio_device_should_start like vhost_user_blk?
>>> I will change this.
>>>> 
>>>>> +    if (virtio_device_started(vdev, vdev->status)) {
>>>>> +        ret = vhost_scsi_common_start(vsc);
>>>>> +        if (ret < 0) {
>>>>> +            return ret;
>>>>> +        }
>>>>> +    }
>>>>> +
>>>>> +    return 0;
>>>>> +}
>>>>> +
>>>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>>>>> +
>>>>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>>>>> +{
>>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>> +
>>>> 
>>>> I don’t think we want to execute vhost_scsi_common_stop() if the device hasn’t been started. I remember that caused a number of races with the vhost_user_blk connecting/disconnecting on startup.
>>>> 
>>>> Let’s add a similar started_vu check?
>>> I will add it.
>>>> 
>>>>> +    if (!s->connected) {
>>>>> +        return;
>>>>> +    }
>>>>> +    s->connected = false;
>>>>> +
>>>>> +    vhost_scsi_common_stop(vsc);
>>>>> +
>>>>> +    vhost_dev_cleanup(&vsc->dev);
>>>>> +
>>>>> +    /* Re-instate the event handler for new connections */
>>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>>>>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>>>>> +}
>>>>> +
>>>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>>>>> +{
>>>>> +    DeviceState *dev = opaque;
>>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>> +    Error *local_err = NULL;
>>>>> +
>>>>> +    switch (event) {
>>>>> +    case CHR_EVENT_OPENED:
>>>>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>>>>> +            error_report_err(local_err);
>>>>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>>>>> +            return;
>>>>> +        }
>>>>> +        break;
>>>>> +    case CHR_EVENT_CLOSED:
>>>>> +        /* defer close until later to avoid circular close */
>>>>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>>>>> +                               vhost_user_scsi_disconnect);
>>>>> +        break;
>>>>> +    case CHR_EVENT_BREAK:
>>>>> +    case CHR_EVENT_MUX_IN:
>>>>> +    case CHR_EVENT_MUX_OUT:
>>>>> +        /* Ignore */
>>>>> +        break;
>>>>> +    }
>>>>> +}
>>>>> +
>>>>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>>>>> +{
>>>>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>> +    int ret;
>>>>> +
>>>>> +    s->connected = false;
>>>>> +
>>>>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>>>>> +    if (ret < 0) {
>>>>> +        return ret;
>>>>> +    }
>>>>> +
>>>>> +    ret = vhost_user_scsi_connect(dev, errp);
>>>>> +    if (ret < 0) {
>>>>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>>>>> +        return ret;
>>>>> +    }
>>>>> +    assert(s->connected);
>>>>> +
>>>>> +    return 0;
>>>>> +}
>>>>> +
>>>>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>> {
>>>>>   VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>   VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>>>   VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>> -    struct vhost_virtqueue *vqs = NULL;
>>>>>   Error *err = NULL;
>>>>>   int ret;
>>>>> +    int retries = REALIZE_CONNECTION_RETRIES;
>>>>> 
>>>>>   if (!vs->conf.chardev.chr) {
>>>>>       error_setg(errp, "vhost-user-scsi: missing chardev");
>>>>> @@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>>   }
>>>>> 
>>>>>   if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {
>>>> 
>>>> Why execute vhost_user_cleanup() if vhost_user_init() fails?
>>> OK, move this line up in v2.
>>> 
>>>> 
>>>>> -        goto free_virtio;
>>>>> +        goto free_vhost;
>>>>>   }
>>>>> 
>>>>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>>>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>>>>> -    vsc->dev.vq_index = 0;
>>>>> -    vsc->dev.backend_features = 0;
>>>>> -    vqs = vsc->dev.vqs;
>>>>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>>>>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>>>>> +
>>>>> +    assert(!*errp);
>>>>> +    do {
>>>>> +        if (*errp) {
>>>>> +            error_prepend(errp, "Reconnecting after error: ");
>>>>> +            error_report_err(*errp);
>>>>> +            *errp = NULL;
>>>>> +        }
>>>>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>>>>> +    } while (ret < 0 && retries--);
>>>>> 
>>>>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>>>>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>>>>   if (ret < 0) {
>>>>> -        goto free_vhost;
>>>>> +        goto free_vqs;
>>>>>   }
>>>>> 
>>>>> +    /* we're fully initialized, now we can operate, so add the handler */
>>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>>>>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>>>>> +                             NULL, true);
>>>>>   /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>>>>   vsc->channel = 0;
>>>>>   vsc->lun = 0;
>>>>> @@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>> 
>>>>>   return;
>>>>> 
>>>>> +free_vqs:
>>>>> +    g_free(s->vhost_vqs);
>>>>> +    s->vhost_vqs = NULL;
>>>>> +    g_free(vsc->inflight);
>>>>> +    vsc->inflight = NULL;
>>>>> +
>>>>> free_vhost:
>>>>>   vhost_user_cleanup(&s->vhost_user);
>>>>> -    g_free(vqs);
>>>>> -free_virtio:
>>>>> +
>>>>>   virtio_scsi_common_unrealize(dev);
>>>>> }
>>>>> 
>>>>> @@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>>>>   VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>>   VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>>>   VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>> 
>>>>>   /* This will stop the vhost backend. */
>>>>>   vhost_user_scsi_set_status(vdev, 0);
>>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>>>>> +                             NULL, false);
>>>>> 
>>>>>   vhost_dev_cleanup(&vsc->dev);
>>>>> -    g_free(vqs);
>>>> 
>>>> Nit: Why not put vhost_dev_free_inflight next to the remaining inflight cleanup?
>>> OK.
>>>> 
>>>>> +    vhost_dev_free_inflight(vsc->inflight);
>>>>> +    g_free(s->vhost_vqs);
>>>>> +    s->vhost_vqs = NULL;
>>>>> +    g_free(vsc->inflight);
>>>>> +    vsc->inflight = NULL;
>>>>> 
>>>> 
>>>> Curiosity - why reorder here? Is something in vhost_user_cleanup() dependent on state freed in virtio_scsi_common_unrealize()?
>>>> 
>>>> If so, should that go as a standalone fix?
>>> 
>>> Because in vhost_user_scsi_realize, we initialize in order:
>>> virtio_scsi_common_realize
>>> vhost_user_init
>>> 
>>> And in the error handler of vhost_user_scsi_realize, the uninitialize in order:
>>> vhost_user_cleanup
>>> virtio_scsi_common_unrealize
>>> 
>>> I think in vhost_user_scsi_unrealize we should keep it the same order, right?
>> 
>> I’m not saying it’s wrong. If there’s no dependency (i.e. this is not fixing a bug, just a stylistic improvement) it can stay in the same change.
> OK.
>> 
>>> 
>>>> 
>>>>> -    virtio_scsi_common_unrealize(dev);
>>>>>   vhost_user_cleanup(&s->vhost_user);
>>>>> +    virtio_scsi_common_unrealize(dev);
>>>>> }
>>>>> 
>>>>> static Property vhost_user_scsi_properties[] = {
>>>>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>>>>> index 521b08e559..c66acc68b7 100644
>>>>> --- a/include/hw/virtio/vhost-user-scsi.h
>>>>> +++ b/include/hw/virtio/vhost-user-scsi.h
>>>>> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>>>>> struct VHostUserSCSI {
>>>>>   VHostSCSICommon parent_obj;
>>>>>   VhostUserState vhost_user;
>>>> 
>>>> See above - we should probably have started_vu here/
>>> I will add it.
>>>> 
>>>> Maybe we should have some shared struct with vhost_user_blk for connectivity params?
>>> 
>>> In the future vhost-user-blk/scsi can be refactored to share the same code.
>> 
>> Sure - this can be done at some point in the future.
>> 
>>>> 
>>>>> +    bool connected;
>>>>> +
>>>>> +    struct vhost_virtqueue *vhost_vqs;
>>>>> };
>>>>> 
>>>>> #endif /* VHOST_USER_SCSI_H */
>>>>> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
>>>>> index 6a173cb9fa..b904346fe1 100644
>>>>> --- a/include/hw/virtio/vhost.h
>>>>> +++ b/include/hw/virtio/vhost.h
>>>>> @@ -8,6 +8,8 @@
>>>>> #define VHOST_F_DEVICE_IOTLB 63
>>>>> #define VHOST_USER_F_PROTOCOL_FEATURES 30
>>>>> 
>>>> 
>>>> Should the macro name indicate that this is for vhost-user?
>>>> 
>>>> VU_REALIZE_CONN_RETRIES? 
>>> I will rename it in v2.
>>> 
>>>> 
>>>>> +#define REALIZE_CONNECTION_RETRIES 3
>>>>> +
>>>>> /* Generic structures common for any vhost based device. */
>>>>> 
>>>>> struct vhost_inflight {
>>>>> -- 
>>>>> 2.41.0
> 
> Any comments about other patches?

I’ll send shortly.

Re: [PATCH] vhost-user-scsi: support reconnect to backend

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 17231 bytes --]



> 2023年7月31日 06:09，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
> 
> 
> 
>> On Jul 28, 2023, at 3:48 AM, Li Feng <fengli@smartx.com> wrote:
>> 
>> Thanks for your reply.
>> 
>>> 2023年7月28日 上午5:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>>> 
>>> 
>>> 
>>>> On Jul 25, 2023, at 6:19 AM, Li Feng <fengli@smartx.com> wrote:
>>>> 
>>>> Thanks for your comments.
>>>> 
>>>>> 2023年7月25日 上午1:21，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>>>>> 
>>>>> Very excited to see this. High level looks good modulo a few small things.
>>>>> 
>>>>> My major concern is around existing vhost-user-scsi backends which don’t support VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD. IMO we should hide the reconnect behavior behind a VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD check. We may want to do the same for vhost-user-blk.
>>>>> 
>>>>> The question is then what happens if the check is false. IIUC without an inflight FD, if a device processes requests out of order, it’s not safe to continue execution on reconnect, as there’s no way for the backend to know how to replay IO. Should we permanently wedge the device or have QEMU fail out? May be nice to have a toggle for this.
>>>> 
>>>> Based on what MST said, is there anything else I need to do?
>>> 
>>> I don’t think so.
>>> 
>>>>> 
>>>>>> On Jul 21, 2023, at 6:51 AM, Li Feng <fengli@smartx.com> wrote:
>>>>>> 
>>>>>> If the backend crashes and restarts, the device is broken.
>>>>>> This patch adds reconnect for vhost-user-scsi.
>>>>>> 
>>>>>> Tested with spdk backend.
>>>>>> 
>>>>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>>>>> ---
>>>>>> hw/block/vhost-user-blk.c           |   2 -
>>>>>> hw/scsi/vhost-scsi-common.c         |  27 ++---
>>>>>> hw/scsi/vhost-user-scsi.c           | 163 +++++++++++++++++++++++++---
>>>>>> include/hw/virtio/vhost-user-scsi.h |   3 +
>>>>>> include/hw/virtio/vhost.h           |   2 +
>>>>>> 5 files changed, 165 insertions(+), 32 deletions(-)
>>>>>> 
>>>>>> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
>>>>>> index eecf3f7a81..f250c740b5 100644
>>>>>> --- a/hw/block/vhost-user-blk.c
>>>>>> +++ b/hw/block/vhost-user-blk.c
>>>>>> @@ -32,8 +32,6 @@
>>>>>> #include "sysemu/sysemu.h"
>>>>>> #include "sysemu/runstate.h"
>>>>>> 
>>>>>> -#define REALIZE_CONNECTION_RETRIES 3
>>>>>> -
>>>>>> static const int user_feature_bits[] = {
>>>>>>  VIRTIO_BLK_F_SIZE_MAX,
>>>>>>  VIRTIO_BLK_F_SEG_MAX,
>>>>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>>>>> 
>>>>> Why can’t all the vhost-scsi-common stuff be moved to a separate change?
>>>> 
>>>> I will move this code to separate patch.
>>>>> 
>>>>> Especially the stuff introduced for vhost-user-blk in 1b0063b3048af65dfaae6422a572c87db8575a92 should be moved out.
>>>> OK.
>>>> 
>>>>> 
>>>>>> index a06f01af26..08801886b8 100644
>>>>>> --- a/hw/scsi/vhost-scsi-common.c
>>>>>> +++ b/hw/scsi/vhost-scsi-common.c
>>>>>> @@ -52,16 +52,22 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>>> 
>>>>>>  vsc->dev.acked_features = vdev->guest_features;
>>>>>> 
>>>>>> -    assert(vsc->inflight == NULL);
>>>>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>>>>> -                                 vs->conf.virtqueue_size,
>>>>>> -                                 vsc->inflight);
>>>>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>>>>  if (ret < 0) {
>>>>>> -        error_report("Error get inflight: %d", -ret);
>>>>>> +        error_report("Error setting inflight format: %d", -ret);
>>>>>>      goto err_guest_notifiers;
>>>>>>  }
>>>>>> 
>>>>>> +    if (!vsc->inflight->addr) {
>>>>>> +        ret = vhost_dev_get_inflight(&vsc->dev,
>>>>>> +                                    vs->conf.virtqueue_size,
>>>>>> +                                    vsc->inflight);
>>>>>> +        if (ret < 0) {
>>>>>> +            error_report("Error get inflight: %d", -ret);
>>>>>> +            goto err_guest_notifiers;
>>>>>> +        }
>>>>>> +    }
>>>>>> +
>>>>>>  ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>>>>  if (ret < 0) {
>>>>>>      error_report("Error set inflight: %d", -ret);
>>>>>> @@ -85,9 +91,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>>>  return ret;
>>>>>> 
>>>>>> err_guest_notifiers:
>>>>>> -    g_free(vsc->inflight);
>>>>>> -    vsc->inflight = NULL;
>>>>>> -
>>>>>>  k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>>>>> err_host_notifiers:
>>>>>>  vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>>>> @@ -111,12 +114,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>>>>  }
>>>>>>  assert(ret >= 0);
>>>>>> 
>>>>> 
>>>>> In the vhost-scsi (kernel backend) path, what will cleanup vsc->inflight now?
>>>> OK, we should check the vsc->inflight if it is null, the vhost-scsi doesn’t allocate the
>>>> inflight object memory.
>>> 
>>> Are you saying vhost-scsi never allocates inflight so we don’t need to check for it?
>> We have checked the vsc->inflight, and only if allocated, we send the get/set_inflight_fd.
>> This works with vhost-user-scsi/vhost-scsi both.
> 
> So then it sounds like this code introduces a resource leak. g_free(vsc->inflight) should be added to the vhost-scsi code in vhost_scsi_stop().

No, the vhost-scsi doesn’t need ‘inflight', it doesn’t allocate the inflight memory.

The rule is ‘who allocates, who free it’.

> 
>>> 
>>>> 
>>>>> 
>>>>>> -    if (vsc->inflight) {
>>>>>> -        vhost_dev_free_inflight(vsc->inflight);
>>>>>> -        g_free(vsc->inflight);
>>>>>> -        vsc->inflight = NULL;
>>>>>> -    }
>>>>>> -
>>>>>>  vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>>>> }
>>>>>> 
>>>>>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>>>>>> index ee99b19e7a..e0e88b0c42 100644
>>>>>> --- a/hw/scsi/vhost-user-scsi.c
>>>>>> +++ b/hw/scsi/vhost-user-scsi.c
>>>>>> @@ -89,14 +89,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>>>>>> {
>>>>>> }
>>>>>> 
>>>>>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>>>>>> +{
>>>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>> +    int ret = 0;
>>>>>> +
>>>>>> +    if (s->connected) {
>>>>>> +        return 0;
>>>>>> +    }
>>>>>> +    s->connected = true;
>>>>>> +
>>>>>> +    vsc->dev.num_queues = vs->conf.num_queues;
>>>>>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>>>>> +    vsc->dev.vqs = s->vhost_vqs;
>>>>>> +    vsc->dev.vq_index = 0;
>>>>>> +    vsc->dev.backend_features = 0;
>>>>>> +
>>>>>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>>>>>> +                         errp);
>>>>>> +    if (ret < 0) {
>>>>>> +        return ret;
>>>>>> +    }
>>>>>> +
>>>>>> +    /* restore vhost state */
>>>>> 
>>>>> Should this use virtio_device_should_start like vhost_user_blk?
>>>> I will change this.
>>>>> 
>>>>>> +    if (virtio_device_started(vdev, vdev->status)) {
>>>>>> +        ret = vhost_scsi_common_start(vsc);
>>>>>> +        if (ret < 0) {
>>>>>> +            return ret;
>>>>>> +        }
>>>>>> +    }
>>>>>> +
>>>>>> +    return 0;
>>>>>> +}
>>>>>> +
>>>>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>>>>>> +
>>>>>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>>>>>> +{
>>>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>> +
>>>>> 
>>>>> I don’t think we want to execute vhost_scsi_common_stop() if the device hasn’t been started. I remember that caused a number of races with the vhost_user_blk connecting/disconnecting on startup.
>>>>> 
>>>>> Let’s add a similar started_vu check?
>>>> I will add it.
>>>>> 
>>>>>> +    if (!s->connected) {
>>>>>> +        return;
>>>>>> +    }
>>>>>> +    s->connected = false;
>>>>>> +
>>>>>> +    vhost_scsi_common_stop(vsc);
>>>>>> +
>>>>>> +    vhost_dev_cleanup(&vsc->dev);
>>>>>> +
>>>>>> +    /* Re-instate the event handler for new connections */
>>>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>>>>>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>>>>>> +}
>>>>>> +
>>>>>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>>>>>> +{
>>>>>> +    DeviceState *dev = opaque;
>>>>>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>>>>>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>> +    Error *local_err = NULL;
>>>>>> +
>>>>>> +    switch (event) {
>>>>>> +    case CHR_EVENT_OPENED:
>>>>>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>>>>>> +            error_report_err(local_err);
>>>>>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>>>>>> +            return;
>>>>>> +        }
>>>>>> +        break;
>>>>>> +    case CHR_EVENT_CLOSED:
>>>>>> +        /* defer close until later to avoid circular close */
>>>>>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>>>>>> +                               vhost_user_scsi_disconnect);
>>>>>> +        break;
>>>>>> +    case CHR_EVENT_BREAK:
>>>>>> +    case CHR_EVENT_MUX_IN:
>>>>>> +    case CHR_EVENT_MUX_OUT:
>>>>>> +        /* Ignore */
>>>>>> +        break;
>>>>>> +    }
>>>>>> +}
>>>>>> +
>>>>>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>>>>>> +{
>>>>>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>> +    int ret;
>>>>>> +
>>>>>> +    s->connected = false;
>>>>>> +
>>>>>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>>>>>> +    if (ret < 0) {
>>>>>> +        return ret;
>>>>>> +    }
>>>>>> +
>>>>>> +    ret = vhost_user_scsi_connect(dev, errp);
>>>>>> +    if (ret < 0) {
>>>>>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>>>>>> +        return ret;
>>>>>> +    }
>>>>>> +    assert(s->connected);
>>>>>> +
>>>>>> +    return 0;
>>>>>> +}
>>>>>> +
>>>>>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>>> {
>>>>>>  VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>>  VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>>>>  VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>>> -    struct vhost_virtqueue *vqs = NULL;
>>>>>>  Error *err = NULL;
>>>>>>  int ret;
>>>>>> +    int retries = REALIZE_CONNECTION_RETRIES;
>>>>>> 
>>>>>>  if (!vs->conf.chardev.chr) {
>>>>>>      error_setg(errp, "vhost-user-scsi: missing chardev");
>>>>>> @@ -112,21 +224,31 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>>>  }
>>>>>> 
>>>>>>  if (!vhost_user_init(&s->vhost_user, &vs->conf.chardev, errp)) {
>>>>> 
>>>>> Why execute vhost_user_cleanup() if vhost_user_init() fails?
>>>> OK, move this line up in v2.
>>>> 
>>>>> 
>>>>>> -        goto free_virtio;
>>>>>> +        goto free_vhost;
>>>>>>  }
>>>>>> 
>>>>>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>>>>>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>>>>>> -    vsc->dev.vq_index = 0;
>>>>>> -    vsc->dev.backend_features = 0;
>>>>>> -    vqs = vsc->dev.vqs;
>>>>>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>>>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>>>>>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>>>>>> +
>>>>>> +    assert(!*errp);
>>>>>> +    do {
>>>>>> +        if (*errp) {
>>>>>> +            error_prepend(errp, "Reconnecting after error: ");
>>>>>> +            error_report_err(*errp);
>>>>>> +            *errp = NULL;
>>>>>> +        }
>>>>>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>>>>>> +    } while (ret < 0 && retries--);
>>>>>> 
>>>>>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>>>>>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>>>>>  if (ret < 0) {
>>>>>> -        goto free_vhost;
>>>>>> +        goto free_vqs;
>>>>>>  }
>>>>>> 
>>>>>> +    /* we're fully initialized, now we can operate, so add the handler */
>>>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>>>>>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>>>>>> +                             NULL, true);
>>>>>>  /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>>>>>  vsc->channel = 0;
>>>>>>  vsc->lun = 0;
>>>>>> @@ -134,10 +256,15 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>>>>> 
>>>>>>  return;
>>>>>> 
>>>>>> +free_vqs:
>>>>>> +    g_free(s->vhost_vqs);
>>>>>> +    s->vhost_vqs = NULL;
>>>>>> +    g_free(vsc->inflight);
>>>>>> +    vsc->inflight = NULL;
>>>>>> +
>>>>>> free_vhost:
>>>>>>  vhost_user_cleanup(&s->vhost_user);
>>>>>> -    g_free(vqs);
>>>>>> -free_virtio:
>>>>>> +
>>>>>>  virtio_scsi_common_unrealize(dev);
>>>>>> }
>>>>>> 
>>>>>> @@ -146,16 +273,22 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>>>>>  VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>>>>>  VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>>>>>  VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>>>>>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>>>>>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>>>>> 
>>>>>>  /* This will stop the vhost backend. */
>>>>>>  vhost_user_scsi_set_status(vdev, 0);
>>>>>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>>>>>> +                             NULL, false);
>>>>>> 
>>>>>>  vhost_dev_cleanup(&vsc->dev);
>>>>>> -    g_free(vqs);
>>>>> 
>>>>> Nit: Why not put vhost_dev_free_inflight next to the remaining inflight cleanup?
>>>> OK.
>>>>> 
>>>>>> +    vhost_dev_free_inflight(vsc->inflight);
>>>>>> +    g_free(s->vhost_vqs);
>>>>>> +    s->vhost_vqs = NULL;
>>>>>> +    g_free(vsc->inflight);
>>>>>> +    vsc->inflight = NULL;
>>>>>> 
>>>>> 
>>>>> Curiosity - why reorder here? Is something in vhost_user_cleanup() dependent on state freed in virtio_scsi_common_unrealize()?
>>>>> 
>>>>> If so, should that go as a standalone fix?
>>>> 
>>>> Because in vhost_user_scsi_realize, we initialize in order:
>>>> virtio_scsi_common_realize
>>>> vhost_user_init
>>>> 
>>>> And in the error handler of vhost_user_scsi_realize, the uninitialize in order:
>>>> vhost_user_cleanup
>>>> virtio_scsi_common_unrealize
>>>> 
>>>> I think in vhost_user_scsi_unrealize we should keep it the same order, right?
>>> 
>>> I’m not saying it’s wrong. If there’s no dependency (i.e. this is not fixing a bug, just a stylistic improvement) it can stay in the same change.
>> OK.
>>> 
>>>> 
>>>>> 
>>>>>> -    virtio_scsi_common_unrealize(dev);
>>>>>>  vhost_user_cleanup(&s->vhost_user);
>>>>>> +    virtio_scsi_common_unrealize(dev);
>>>>>> }
>>>>>> 
>>>>>> static Property vhost_user_scsi_properties[] = {
>>>>>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>>>>>> index 521b08e559..c66acc68b7 100644
>>>>>> --- a/include/hw/virtio/vhost-user-scsi.h
>>>>>> +++ b/include/hw/virtio/vhost-user-scsi.h
>>>>>> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>>>>>> struct VHostUserSCSI {
>>>>>>  VHostSCSICommon parent_obj;
>>>>>>  VhostUserState vhost_user;
>>>>> 
>>>>> See above - we should probably have started_vu here/
>>>> I will add it.
>>>>> 
>>>>> Maybe we should have some shared struct with vhost_user_blk for connectivity params?
>>>> 
>>>> In the future vhost-user-blk/scsi can be refactored to share the same code.
>>> 
>>> Sure - this can be done at some point in the future.
>>> 
>>>>> 
>>>>>> +    bool connected;
>>>>>> +
>>>>>> +    struct vhost_virtqueue *vhost_vqs;
>>>>>> };
>>>>>> 
>>>>>> #endif /* VHOST_USER_SCSI_H */
>>>>>> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
>>>>>> index 6a173cb9fa..b904346fe1 100644
>>>>>> --- a/include/hw/virtio/vhost.h
>>>>>> +++ b/include/hw/virtio/vhost.h
>>>>>> @@ -8,6 +8,8 @@
>>>>>> #define VHOST_F_DEVICE_IOTLB 63
>>>>>> #define VHOST_USER_F_PROTOCOL_FEATURES 30
>>>>>> 
>>>>> 
>>>>> Should the macro name indicate that this is for vhost-user?
>>>>> 
>>>>> VU_REALIZE_CONN_RETRIES? 
>>>> I will rename it in v2.
>>>> 
>>>>> 
>>>>>> +#define REALIZE_CONNECTION_RETRIES 3
>>>>>> +
>>>>>> /* Generic structures common for any vhost based device. */
>>>>>> 
>>>>>> struct vhost_inflight {
>>>>>> -- 
>>>>>> 2.41.0
>> 
>> Any comments about other patches?
> 
> I’ll send shortly.


[-- Attachment #2: Type: text/html, Size: 26145 bytes --]

[PATCH v2 0/4] Implement reconnect for vhost-user-scsi

[Li Feng]

Hi,

This patchset adds reconnect support for vhost-user-scsi. At the same
times, fix vhost fd leak and refactor some code.

Changes for v2:
- Split the v1 patch to small separate patchset;
- New patch for fixing fd leak, which has sent to reviewers in another
  mail;
- Implement the `vhost_user_scsi_handle_output`;
- Add the started_vu safe check;
- Fix error handler;
- Check the inflight before set/get inflight fd.

Li Feng (4):
  vhost: fix the fd leak
  vhost-user-common: send get_inflight_fd once
  vhost: move and rename the conn retry times
  vhost-user-scsi: support reconnect to backend

 hw/block/vhost-user-blk.c             |   4 +-
 hw/scsi/vhost-scsi-common.c           |  43 ++---
 hw/scsi/vhost-user-scsi.c             | 220 +++++++++++++++++++++++---
 hw/virtio/vhost-user-gpio.c           |   3 +-
 hw/virtio/vhost.c                     |   2 +
 include/hw/virtio/vhost-scsi-common.h |   3 +
 include/hw/virtio/vhost-user-scsi.h   |   3 +
 include/hw/virtio/vhost.h             |   2 +
 8 files changed, 235 insertions(+), 45 deletions(-)

-- 
2.41.0

[PATCH v2 1/4] vhost: fix the fd leak

[Li Feng]

When the vhost-user reconnect to the backend, the notifer should be
cleanup. Otherwise, the fd resource will be exhausted.

Fixes: f9a09ca3ea ("vhost: add support for configure interrupt")

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/virtio/vhost.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index abf0d03c8d..e2f6ffb446 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -2044,6 +2044,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings)
     event_notifier_test_and_clear(
         &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
     event_notifier_test_and_clear(&vdev->config_notifier);
+    event_notifier_cleanup(
+        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
 
     trace_vhost_dev_stop(hdev, vdev->name, vrings);
 
-- 
2.41.0

Re: [PATCH v2 1/4] vhost: fix the fd leak

[Raphael Norwitz]

> On Jul 25, 2023, at 6:42 AM, Li Feng <fengli@smartx.com> wrote:
> 
> When the vhost-user reconnect to the backend, the notifer should be
> cleanup. Otherwise, the fd resource will be exhausted.
> 
> Fixes: f9a09ca3ea ("vhost: add support for configure interrupt")
> 
> Signed-off-by: Li Feng <fengli@smartx.com>

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> ---
> hw/virtio/vhost.c | 2 ++
> 1 file changed, 2 insertions(+)
> 
> diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
> index abf0d03c8d..e2f6ffb446 100644
> --- a/hw/virtio/vhost.c
> +++ b/hw/virtio/vhost.c
> @@ -2044,6 +2044,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings)
>     event_notifier_test_and_clear(
>         &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
>     event_notifier_test_and_clear(&vdev->config_notifier);
> +    event_notifier_cleanup(
> +        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
> 
>     trace_vhost_dev_stop(hdev, vdev->name, vrings);
> 
> -- 
> 2.41.0
> 

[PATCH v2 2/4] vhost-user-common: send get_inflight_fd once

[Li Feng]

Get_inflight_fd is sent only once. When reconnecting to the backend,
qemu sent set_inflight_fd to the backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a06f01af26..664adb15b4 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     vsc->dev.acked_features = vdev->guest_features;
 
-    assert(vsc->inflight == NULL);
-    vsc->inflight = g_new0(struct vhost_inflight, 1);
-    ret = vhost_dev_get_inflight(&vsc->dev,
-                                 vs->conf.virtqueue_size,
-                                 vsc->inflight);
+    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error get inflight: %d", -ret);
+        error_report("Error setting inflight format: %d", -ret);
         goto err_guest_notifiers;
     }
 
-    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
-    if (ret < 0) {
-        error_report("Error set inflight: %d", -ret);
-        goto err_guest_notifiers;
+    if (vsc->inflight) {
+        if (!vsc->inflight->addr) {
+            ret = vhost_dev_get_inflight(&vsc->dev,
+                                        vs->conf.virtqueue_size,
+                                        vsc->inflight);
+            if (ret < 0) {
+                error_report("Error get inflight: %d", -ret);
+                goto err_guest_notifiers;
+            }
+        }
+
+        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
+        if (ret < 0) {
+            error_report("Error set inflight: %d", -ret);
+            goto err_guest_notifiers;
+        }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
@@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     return ret;
 
 err_guest_notifiers:
-    g_free(vsc->inflight);
-    vsc->inflight = NULL;
-
     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
 err_host_notifiers:
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
@@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     }
     assert(ret >= 0);
 
-    if (vsc->inflight) {
-        vhost_dev_free_inflight(vsc->inflight);
-        g_free(vsc->inflight);
-        vsc->inflight = NULL;
-    }
-
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
 }
 
-- 
2.41.0

Re: [PATCH v2 2/4] vhost-user-common: send get_inflight_fd once

[Michael S. Tsirkin]

On Tue, Jul 25, 2023 at 06:42:45PM +0800, Li Feng wrote:
> Get_inflight_fd is sent only once. When reconnecting to the backend,
> qemu sent set_inflight_fd to the backend.

I don't understand what you are trying to say here.
Should be:
Currently ABCD. This is wrong/unnecessary because EFG. This patch HIJ.

> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
>  hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
>  1 file changed, 18 insertions(+), 19 deletions(-)
> 
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> index a06f01af26..664adb15b4 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>  
>      vsc->dev.acked_features = vdev->guest_features;
>  
> -    assert(vsc->inflight == NULL);
> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> -    ret = vhost_dev_get_inflight(&vsc->dev,
> -                                 vs->conf.virtqueue_size,
> -                                 vsc->inflight);
> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>      if (ret < 0) {
> -        error_report("Error get inflight: %d", -ret);
> +        error_report("Error setting inflight format: %d", -ret);
>          goto err_guest_notifiers;
>      }
>  
> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> -    if (ret < 0) {
> -        error_report("Error set inflight: %d", -ret);
> -        goto err_guest_notifiers;
> +    if (vsc->inflight) {
> +        if (!vsc->inflight->addr) {
> +            ret = vhost_dev_get_inflight(&vsc->dev,
> +                                        vs->conf.virtqueue_size,
> +                                        vsc->inflight);
> +            if (ret < 0) {
> +                error_report("Error get inflight: %d", -ret);

As long as you are fixing this - should be "getting inflight".

> +                goto err_guest_notifiers;
> +            }
> +        }
> +
> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> +        if (ret < 0) {
> +            error_report("Error set inflight: %d", -ret);
> +            goto err_guest_notifiers;
> +        }
>      }
>  
>      ret = vhost_dev_start(&vsc->dev, vdev, true);
> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>      return ret;
>  
>  err_guest_notifiers:
> -    g_free(vsc->inflight);
> -    vsc->inflight = NULL;
> -
>      k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>  err_host_notifiers:
>      vhost_dev_disable_notifiers(&vsc->dev, vdev);
> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>      }
>      assert(ret >= 0);
>  
> -    if (vsc->inflight) {
> -        vhost_dev_free_inflight(vsc->inflight);
> -        g_free(vsc->inflight);
> -        vsc->inflight = NULL;
> -    }
> -
>      vhost_dev_disable_notifiers(&vsc->dev, vdev);
>  }
>  
> -- 
> 2.41.0

Re: [PATCH v2 2/4] vhost-user-common: send get_inflight_fd once

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 3249 bytes --]



> 2023年7月28日 下午2:04，Michael S. Tsirkin <mst@redhat.com> 写道：
> 
> On Tue, Jul 25, 2023 at 06:42:45PM +0800, Li Feng wrote:
>> Get_inflight_fd is sent only once. When reconnecting to the backend,
>> qemu sent set_inflight_fd to the backend.
> 
> I don't understand what you are trying to say here.
> Should be:
> Currently ABCD. This is wrong/unnecessary because EFG. This patch HIJ.

Thanks, I will reorganize the commit message in v3.
> 
>> Signed-off-by: Li Feng <fengli@smartx.com>
>> ---
>> hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
>> 1 file changed, 18 insertions(+), 19 deletions(-)
>> 
>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>> index a06f01af26..664adb15b4 100644
>> --- a/hw/scsi/vhost-scsi-common.c
>> +++ b/hw/scsi/vhost-scsi-common.c
>> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>> 
>>     vsc->dev.acked_features = vdev->guest_features;
>> 
>> -    assert(vsc->inflight == NULL);
>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>> -                                 vs->conf.virtqueue_size,
>> -                                 vsc->inflight);
>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>     if (ret < 0) {
>> -        error_report("Error get inflight: %d", -ret);
>> +        error_report("Error setting inflight format: %d", -ret);
>>         goto err_guest_notifiers;
>>     }
>> 
>> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>> -    if (ret < 0) {
>> -        error_report("Error set inflight: %d", -ret);
>> -        goto err_guest_notifiers;
>> +    if (vsc->inflight) {
>> +        if (!vsc->inflight->addr) {
>> +            ret = vhost_dev_get_inflight(&vsc->dev,
>> +                                        vs->conf.virtqueue_size,
>> +                                        vsc->inflight);
>> +            if (ret < 0) {
>> +                error_report("Error get inflight: %d", -ret);
> 
> As long as you are fixing this - should be "getting inflight”.
I will fix it in v3.
> 
>> +                goto err_guest_notifiers;
>> +            }
>> +        }
>> +
>> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>> +        if (ret < 0) {
>> +            error_report("Error set inflight: %d", -ret);
>> +            goto err_guest_notifiers;
>> +        }
>>     }
>> 
>>     ret = vhost_dev_start(&vsc->dev, vdev, true);
>> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>     return ret;
>> 
>> err_guest_notifiers:
>> -    g_free(vsc->inflight);
>> -    vsc->inflight = NULL;
>> -
>>     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>> err_host_notifiers:
>>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
>> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>     }
>>     assert(ret >= 0);
>> 
>> -    if (vsc->inflight) {
>> -        vhost_dev_free_inflight(vsc->inflight);
>> -        g_free(vsc->inflight);
>> -        vsc->inflight = NULL;
>> -    }
>> -
>>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
>> }
>> 
>> -- 
>> 2.41.0


[-- Attachment #2: Type: text/html, Size: 11641 bytes --]

Re: [PATCH v2 2/4] vhost-user-common: send get_inflight_fd once

[Raphael Norwitz]

> On Jul 28, 2023, at 3:49 AM, Li Feng <fengli@smartx.com> wrote:
> 
> 
> 
>> 2023年7月28日 下午2:04，Michael S. Tsirkin <mst@redhat.com> 写道：
>> 
>> On Tue, Jul 25, 2023 at 06:42:45PM +0800, Li Feng wrote:
>>> Get_inflight_fd is sent only once. When reconnecting to the backend,
>>> qemu sent set_inflight_fd to the backend.
>> 
>> I don't understand what you are trying to say here.
>> Should be:
>> Currently ABCD. This is wrong/unnecessary because EFG. This patch HIJ.
> 
> Thanks, I will reorganize the commit message in v3.
>> 
>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>> ---
>>> hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
>>> 1 file changed, 18 insertions(+), 19 deletions(-)
>>> 
>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>>> index a06f01af26..664adb15b4 100644
>>> --- a/hw/scsi/vhost-scsi-common.c
>>> +++ b/hw/scsi/vhost-scsi-common.c
>>> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>> 
>>>     vsc->dev.acked_features = vdev->guest_features;
>>> 
>>> -    assert(vsc->inflight == NULL);
>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>> -                                 vs->conf.virtqueue_size,
>>> -                                 vsc->inflight);
>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>     if (ret < 0) {
>>> -        error_report("Error get inflight: %d", -ret);
>>> +        error_report("Error setting inflight format: %d", -ret);
>>>         goto err_guest_notifiers;
>>>     }
>>> 
>>> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>> -    if (ret < 0) {
>>> -        error_report("Error set inflight: %d", -ret);
>>> -        goto err_guest_notifiers;
>>> +    if (vsc->inflight) {
>>> +        if (!vsc->inflight->addr) {
>>> +            ret = vhost_dev_get_inflight(&vsc->dev,
>>> +                                        vs->conf.virtqueue_size,
>>> +                                        vsc->inflight);
>>> +            if (ret < 0) {
>>> +                error_report("Error get inflight: %d", -ret);
>> 
>> As long as you are fixing this - should be "getting inflight”.
> I will fix it in v3.
>> 
>>> +                goto err_guest_notifiers;
>>> +            }
>>> +        }
>>> +

Looks like you reworked this a bit so to avoid a potential crash if vsc->inflight is NULL

Should we fix it for vhost-user-blk too?

>>> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>> +        if (ret < 0) {
>>> +            error_report("Error set inflight: %d", -ret);
>>> +            goto err_guest_notifiers;
>>> +        }
>>>     }
>>> 
>>>     ret = vhost_dev_start(&vsc->dev, vdev, true);
>>> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>     return ret;
>>> 
>>> err_guest_notifiers:
>>> -    g_free(vsc->inflight);
>>> -    vsc->inflight = NULL;
>>> -
>>>     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>> err_host_notifiers:
>>>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>     }
>>>     assert(ret >= 0);
>>> 

As I said before, I think this introduces a leak.

>>> -    if (vsc->inflight) {
>>> -        vhost_dev_free_inflight(vsc->inflight);
>>> -        g_free(vsc->inflight);
>>> -        vsc->inflight = NULL;
>>> -    }
>>> -
>>>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>> }
>>> 
>>> -- 
>>> 2.41.0

Re: [PATCH v2 2/4] vhost-user-common: send get_inflight_fd once

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 4053 bytes --]



> 2023年7月31日 06:13，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
> 
>> 
>> On Jul 28, 2023, at 3:49 AM, Li Feng <fengli@smartx.com> wrote:
>> 
>> 
>> 
>>> 2023年7月28日 下午2:04，Michael S. Tsirkin <mst@redhat.com> 写道：
>>> 
>>> On Tue, Jul 25, 2023 at 06:42:45PM +0800, Li Feng wrote:
>>>> Get_inflight_fd is sent only once. When reconnecting to the backend,
>>>> qemu sent set_inflight_fd to the backend.
>>> 
>>> I don't understand what you are trying to say here.
>>> Should be:
>>> Currently ABCD. This is wrong/unnecessary because EFG. This patch HIJ.
>> 
>> Thanks, I will reorganize the commit message in v3.
>>> 
>>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>>> ---
>>>> hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
>>>> 1 file changed, 18 insertions(+), 19 deletions(-)
>>>> 
>>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>>>> index a06f01af26..664adb15b4 100644
>>>> --- a/hw/scsi/vhost-scsi-common.c
>>>> +++ b/hw/scsi/vhost-scsi-common.c
>>>> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>> 
>>>>    vsc->dev.acked_features = vdev->guest_features;
>>>> 
>>>> -    assert(vsc->inflight == NULL);
>>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>>> -                                 vs->conf.virtqueue_size,
>>>> -                                 vsc->inflight);
>>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>>    if (ret < 0) {
>>>> -        error_report("Error get inflight: %d", -ret);
>>>> +        error_report("Error setting inflight format: %d", -ret);
>>>>        goto err_guest_notifiers;
>>>>    }
>>>> 
>>>> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>> -    if (ret < 0) {
>>>> -        error_report("Error set inflight: %d", -ret);
>>>> -        goto err_guest_notifiers;
>>>> +    if (vsc->inflight) {
>>>> +        if (!vsc->inflight->addr) {
>>>> +            ret = vhost_dev_get_inflight(&vsc->dev,
>>>> +                                        vs->conf.virtqueue_size,
>>>> +                                        vsc->inflight);
>>>> +            if (ret < 0) {
>>>> +                error_report("Error get inflight: %d", -ret);
>>> 
>>> As long as you are fixing this - should be "getting inflight”.
>> I will fix it in v3.
>>> 
>>>> +                goto err_guest_notifiers;
>>>> +            }
>>>> +        }
>>>> +
> 
> Looks like you reworked this a bit so to avoid a potential crash if vsc->inflight is NULL
> 
> Should we fix it for vhost-user-blk too?
> 
This check is mainly for the vhost-scsi code, that doesn’t need allocate the inflight memory.

The vhost-user-blk doesn’t need this check, because there isn't a vhost-blk device that reuse the code.

>>>> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>> +        if (ret < 0) {
>>>> +            error_report("Error set inflight: %d", -ret);
>>>> +            goto err_guest_notifiers;
>>>> +        }
>>>>    }
>>>> 
>>>>    ret = vhost_dev_start(&vsc->dev, vdev, true);
>>>> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>    return ret;
>>>> 
>>>> err_guest_notifiers:
>>>> -    g_free(vsc->inflight);
>>>> -    vsc->inflight = NULL;
>>>> -
>>>>    k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>>> err_host_notifiers:
>>>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>>    }
>>>>    assert(ret >= 0);
>>>> 
> 
> As I said before, I think this introduces a leak.
I have answered in the previous mail.

> 
>>>> -    if (vsc->inflight) {
>>>> -        vhost_dev_free_inflight(vsc->inflight);
>>>> -        g_free(vsc->inflight);
>>>> -        vsc->inflight = NULL;
>>>> -    }
>>>> -
>>>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>> }
>>>> 
>>>> -- 
>>>> 2.41.0


[-- Attachment #2: Type: text/html, Size: 11314 bytes --]

Re: [PATCH v2 2/4] vhost-user-common: send get_inflight_fd once

[Raphael Norwitz]

> On Jul 31, 2023, at 7:38 AM, Li Feng <fengli@smartx.com> wrote:
> 
> 
> 
>> 2023年7月31日 06:13，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
>> 
>>> 
>>> On Jul 28, 2023, at 3:49 AM, Li Feng <fengli@smartx.com> wrote:
>>> 
>>> 
>>> 
>>>> 2023年7月28日 下午2:04，Michael S. Tsirkin <mst@redhat.com> 写道：
>>>> 
>>>> On Tue, Jul 25, 2023 at 06:42:45PM +0800, Li Feng wrote:
>>>>> Get_inflight_fd is sent only once. When reconnecting to the backend,
>>>>> qemu sent set_inflight_fd to the backend.
>>>> 
>>>> I don't understand what you are trying to say here.
>>>> Should be:
>>>> Currently ABCD. This is wrong/unnecessary because EFG. This patch HIJ.
>>> 
>>> Thanks, I will reorganize the commit message in v3.
>>>> 
>>>>> Signed-off-by: Li Feng <fengli@smartx.com>
>>>>> ---
>>>>> hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
>>>>> 1 file changed, 18 insertions(+), 19 deletions(-)
>>>>> 
>>>>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>>>>> index a06f01af26..664adb15b4 100644
>>>>> --- a/hw/scsi/vhost-scsi-common.c
>>>>> +++ b/hw/scsi/vhost-scsi-common.c
>>>>> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>> 
>>>>>    vsc->dev.acked_features = vdev->guest_features;
>>>>> 
>>>>> -    assert(vsc->inflight == NULL);
>>>>> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
>>>>> -    ret = vhost_dev_get_inflight(&vsc->dev,
>>>>> -                                 vs->conf.virtqueue_size,
>>>>> -                                 vsc->inflight);
>>>>> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>>>>>    if (ret < 0) {
>>>>> -        error_report("Error get inflight: %d", -ret);
>>>>> +        error_report("Error setting inflight format: %d", -ret);
>>>>>        goto err_guest_notifiers;
>>>>>    }
>>>>> 
>>>>> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>>> -    if (ret < 0) {
>>>>> -        error_report("Error set inflight: %d", -ret);
>>>>> -        goto err_guest_notifiers;
>>>>> +    if (vsc->inflight) {
>>>>> +        if (!vsc->inflight->addr) {
>>>>> +            ret = vhost_dev_get_inflight(&vsc->dev,
>>>>> +                                        vs->conf.virtqueue_size,
>>>>> +                                        vsc->inflight);
>>>>> +            if (ret < 0) {
>>>>> +                error_report("Error get inflight: %d", -ret);
>>>> 
>>>> As long as you are fixing this - should be "getting inflight”.
>>> I will fix it in v3.
>>>> 
>>>>> +                goto err_guest_notifiers;
>>>>> +            }
>>>>> +        }
>>>>> +
>> 
>> Looks like you reworked this a bit so to avoid a potential crash if vsc->inflight is NULL
>> 
>> Should we fix it for vhost-user-blk too?
>> 
> This check is mainly for the vhost-scsi code, that doesn’t need allocate the inflight memory.
> 
> The vhost-user-blk doesn’t need this check, because there isn't a vhost-blk device that reuse the code.
> 

Makes sense.

>>>>> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>>>>> +        if (ret < 0) {
>>>>> +            error_report("Error set inflight: %d", -ret);
>>>>> +            goto err_guest_notifiers;
>>>>> +        }
>>>>>    }
>>>>> 
>>>>>    ret = vhost_dev_start(&vsc->dev, vdev, true);
>>>>> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>>>>    return ret;
>>>>> 
>>>>> err_guest_notifiers:
>>>>> -    g_free(vsc->inflight);
>>>>> -    vsc->inflight = NULL;
>>>>> -
>>>>>    k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
>>>>> err_host_notifiers:
>>>>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>>> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>>>>    }
>>>>>    assert(ret >= 0);
>>>>> 
>> 
>> As I said before, I think this introduces a leak.
> I have answered in the previous mail.
> 

On re-review I agree it’s fine since vac-inflight isn’t set.

>> 
>>>>> -    if (vsc->inflight) {
>>>>> -        vhost_dev_free_inflight(vsc->inflight);
>>>>> -        g_free(vsc->inflight);
>>>>> -        vsc->inflight = NULL;
>>>>> -    }
>>>>> -
>>>>>    vhost_dev_disable_notifiers(&vsc->dev, vdev);
>>>>> }
>>>>> 
>>>>> -- 
>>>>> 2.41.0

[PATCH v2 3/4] vhost: move and rename the conn retry times

[Li Feng]

Multile devices need this macro, move it to a common header.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/block/vhost-user-blk.c   | 4 +---
 hw/virtio/vhost-user-gpio.c | 3 +--
 include/hw/virtio/vhost.h   | 2 ++
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index eecf3f7a81..3c69fa47d5 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -32,8 +32,6 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
-
 static const int user_feature_bits[] = {
     VIRTIO_BLK_F_SIZE_MAX,
     VIRTIO_BLK_F_SEG_MAX,
@@ -482,7 +480,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
     s->inflight = g_new0(struct vhost_inflight, 1);
     s->vhost_vqs = g_new0(struct vhost_virtqueue, s->num_queues);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     assert(!*errp);
     do {
         if (*errp) {
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index 3b013f2d0f..d9979aa5db 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -15,7 +15,6 @@
 #include "standard-headers/linux/virtio_ids.h"
 #include "trace.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
 #define VHOST_NVQS 2
 
 /* Features required from VirtIO */
@@ -359,7 +358,7 @@ static void vu_gpio_device_realize(DeviceState *dev, Error **errp)
     qemu_chr_fe_set_handlers(&gpio->chardev, NULL, NULL, vu_gpio_event, NULL,
                              dev, NULL, true);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     g_assert(!*errp);
     do {
         if (*errp) {
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 6a173cb9fa..ca3131b1af 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -8,6 +8,8 @@
 #define VHOST_F_DEVICE_IOTLB 63
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+#define VU_REALIZE_CONN_RETRIES 3
+
 /* Generic structures common for any vhost based device. */
 
 struct vhost_inflight {
-- 
2.41.0

Re: [PATCH v2 3/4] vhost: move and rename the conn retry times

[Raphael Norwitz]

> On Jul 25, 2023, at 6:42 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Multile devices need this macro, move it to a common header.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> ---
> hw/block/vhost-user-blk.c   | 4 +---
> hw/virtio/vhost-user-gpio.c | 3 +--
> include/hw/virtio/vhost.h   | 2 ++
> 3 files changed, 4 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
> index eecf3f7a81..3c69fa47d5 100644
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -32,8 +32,6 @@
> #include "sysemu/sysemu.h"
> #include "sysemu/runstate.h"
> 
> -#define REALIZE_CONNECTION_RETRIES 3
> -
> static const int user_feature_bits[] = {
>     VIRTIO_BLK_F_SIZE_MAX,
>     VIRTIO_BLK_F_SEG_MAX,
> @@ -482,7 +480,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
>     s->inflight = g_new0(struct vhost_inflight, 1);
>     s->vhost_vqs = g_new0(struct vhost_virtqueue, s->num_queues);
> 
> -    retries = REALIZE_CONNECTION_RETRIES;
> +    retries = VU_REALIZE_CONN_RETRIES;
>     assert(!*errp);
>     do {
>         if (*errp) {
> diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
> index 3b013f2d0f..d9979aa5db 100644
> --- a/hw/virtio/vhost-user-gpio.c
> +++ b/hw/virtio/vhost-user-gpio.c
> @@ -15,7 +15,6 @@
> #include "standard-headers/linux/virtio_ids.h"
> #include "trace.h"
> 
> -#define REALIZE_CONNECTION_RETRIES 3
> #define VHOST_NVQS 2
> 
> /* Features required from VirtIO */
> @@ -359,7 +358,7 @@ static void vu_gpio_device_realize(DeviceState *dev, Error **errp)
>     qemu_chr_fe_set_handlers(&gpio->chardev, NULL, NULL, vu_gpio_event, NULL,
>                              dev, NULL, true);
> 
> -    retries = REALIZE_CONNECTION_RETRIES;
> +    retries = VU_REALIZE_CONN_RETRIES;
>     g_assert(!*errp);
>     do {
>         if (*errp) {
> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
> index 6a173cb9fa..ca3131b1af 100644
> --- a/include/hw/virtio/vhost.h
> +++ b/include/hw/virtio/vhost.h
> @@ -8,6 +8,8 @@
> #define VHOST_F_DEVICE_IOTLB 63
> #define VHOST_USER_F_PROTOCOL_FEATURES 30
> 
> +#define VU_REALIZE_CONN_RETRIES 3
> +
> /* Generic structures common for any vhost based device. */
> 
> struct vhost_inflight {
> -- 
> 2.41.0
> 

[PATCH v2 4/4] vhost-user-scsi: support reconnect to backend

[Li Feng]

If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c           |   6 +
 hw/scsi/vhost-user-scsi.c             | 220 +++++++++++++++++++++++---
 include/hw/virtio/vhost-scsi-common.h |   3 +
 include/hw/virtio/vhost-user-scsi.h   |   3 +
 4 files changed, 211 insertions(+), 21 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index 664adb15b4..3fde477eee 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -81,6 +81,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
         error_report("Error start vhost dev");
         goto err_guest_notifiers;
     }
+    vsc->started_vu = true;
 
     /* guest_notifier_mask/pending not used yet, so just unmask
      * everything here.  virtio-pci will do the right thing by
@@ -106,6 +107,11 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     int ret = 0;
 
+    if (!vsc->started_vu) {
+        return;
+    }
+    vsc->started_vu = false;
+
     vhost_dev_stop(&vsc->dev, vdev, true);
 
     if (k->set_guest_notifiers) {
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index ee99b19e7a..bd32dcf999 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -46,20 +46,25 @@ enum VhostUserProtocolFeature {
 static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    bool should_start = virtio_device_should_start(vdev, status);
+    int ret;
 
-    if (vhost_dev_is_started(&vsc->dev) == start) {
+    if (!s->connected) {
         return;
     }
 
-    if (start) {
-        int ret;
+    if (vhost_dev_is_started(&vsc->dev) == should_start) {
+        return;
+    }
 
+    if (should_start) {
         ret = vhost_scsi_common_start(vsc);
         if (ret < 0) {
             error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
-            exit(1);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
         }
     } else {
         vhost_scsi_common_stop(vsc);
@@ -85,8 +90,160 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
     }
 }
 
-static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
+    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    Error *local_err = NULL;
+    int i, ret;
+
+    if (!vdev->start_on_kick) {
+        return;
+    }
+
+    if (!s->connected) {
+        return;
+    }
+
+    if (vhost_dev_is_started(&vsc->dev)) {
+        return;
+    }
+
+    /*
+     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * vhost here instead of waiting for .set_status().
+     */
+    ret = vhost_scsi_common_start(vsc);
+    if (ret < 0) {
+        error_reportf_err(local_err, "vhost-user-blk: vhost start failed: ");
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return;
+    }
+
+    /* Kick right away to begin processing requests already in vring */
+    for (i = 0; i < vsc->dev.nvqs; i++) {
+        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
+
+        if (!virtio_queue_get_desc_addr(vdev, i)) {
+            continue;
+        }
+        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
+    }
+}
+
+static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret = 0;
+
+    if (s->connected) {
+        return 0;
+    }
+    s->connected = true;
+
+    vsc->dev.num_queues = vs->conf.num_queues;
+    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
+    vsc->dev.vqs = s->vhost_vqs;
+    vsc->dev.vq_index = 0;
+    vsc->dev.backend_features = 0;
+
+    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
+                         errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* restore vhost state */
+    if (virtio_device_started(vdev, vdev->status)) {
+        ret = vhost_scsi_common_start(vsc);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
+
+static void vhost_user_scsi_disconnect(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    if (!s->connected) {
+        return;
+    }
+    s->connected = false;
+
+    vhost_scsi_common_stop(vsc);
+
+    vhost_dev_cleanup(&vsc->dev);
+
+    /* Re-instate the event handler for new connections */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
+                             vhost_user_scsi_event, NULL, dev, NULL, true);
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
+{
+    DeviceState *dev = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    Error *local_err = NULL;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
+            error_report_err(local_err);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
+            return;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        /* defer close until later to avoid circular close */
+        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
+                               vhost_user_scsi_disconnect);
+        break;
+    case CHR_EVENT_BREAK:
+    case CHR_EVENT_MUX_IN:
+    case CHR_EVENT_MUX_OUT:
+        /* Ignore */
+        break;
+    }
+}
+
+static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
+{
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret;
+
+    s->connected = false;
+
+    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_user_scsi_connect(dev, errp);
+    if (ret < 0) {
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return ret;
+    }
+    assert(s->connected);
+
+    return 0;
 }
 
 static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
@@ -94,18 +251,18 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = NULL;
     Error *err = NULL;
     int ret;
+    int retries = VU_REALIZE_CONN_RETRIES;
 
     if (!vs->conf.chardev.chr) {
         error_setg(errp, "vhost-user-scsi: missing chardev");
         return;
     }
 
-    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
-                               vhost_dummy_handle_output,
-                               vhost_dummy_handle_output, &err);
+    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         return;
@@ -115,18 +272,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         goto free_virtio;
     }
 
-    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
-    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
-    vsc->dev.vq_index = 0;
-    vsc->dev.backend_features = 0;
-    vqs = vsc->dev.vqs;
+    vsc->inflight = g_new0(struct vhost_inflight, 1);
+    s->vhost_vqs = g_new0(struct vhost_virtqueue,
+                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
+
+    assert(!*errp);
+    do {
+        if (*errp) {
+            error_prepend(errp, "Reconnecting after error: ");
+            error_report_err(*errp);
+            *errp = NULL;
+        }
+        ret = vhost_user_scsi_realize_connect(s, errp);
+    } while (ret < 0 && retries--);
 
-    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
-                         VHOST_BACKEND_TYPE_USER, 0, errp);
     if (ret < 0) {
         goto free_vhost;
     }
 
+    /* we're fully initialized, now we can operate, so add the handler */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
+                             vhost_user_scsi_event, NULL, (void *)dev,
+                             NULL, true);
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -135,8 +302,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     return;
 
 free_vhost:
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
     vhost_user_cleanup(&s->vhost_user);
-    g_free(vqs);
+
 free_virtio:
     virtio_scsi_common_unrealize(dev);
 }
@@ -146,16 +317,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = vsc->dev.vqs;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
 
     /* This will stop the vhost backend. */
     vhost_user_scsi_set_status(vdev, 0);
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
+                             NULL, false);
 
     vhost_dev_cleanup(&vsc->dev);
-    g_free(vqs);
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+
+    vhost_dev_free_inflight(vsc->inflight);
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
 
-    virtio_scsi_common_unrealize(dev);
     vhost_user_cleanup(&s->vhost_user);
+    virtio_scsi_common_unrealize(dev);
 }
 
 static Property vhost_user_scsi_properties[] = {
diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
index 18f115527c..daa183018c 100644
--- a/include/hw/virtio/vhost-scsi-common.h
+++ b/include/hw/virtio/vhost-scsi-common.h
@@ -37,6 +37,9 @@ struct VHostSCSICommon {
     bool migratable;
 
     struct vhost_inflight *inflight;
+
+    /* vhost_scsi_common_start/vhost_scsi_common_stop */
+    bool started_vu;
 };
 
 int vhost_scsi_common_start(VHostSCSICommon *vsc);
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 521b08e559..c66acc68b7 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
 struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     VhostUserState vhost_user;
+    bool connected;
+
+    struct vhost_virtqueue *vhost_vqs;
 };
 
 #endif /* VHOST_USER_SCSI_H */
-- 
2.41.0

Re: [PATCH v2 4/4] vhost-user-scsi: support reconnect to backend

[Li Feng]

> 2023年7月25日 下午6:42，Li Feng <fengli@smartx.com> 写道：
> 
> If the backend crashes and restarts, the device is broken.
> This patch adds reconnect for vhost-user-scsi.
> 
> Tested with spdk backend.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/scsi/vhost-scsi-common.c           |   6 +
> hw/scsi/vhost-user-scsi.c             | 220 +++++++++++++++++++++++---
> include/hw/virtio/vhost-scsi-common.h |   3 +
> include/hw/virtio/vhost-user-scsi.h   |   3 +
> 4 files changed, 211 insertions(+), 21 deletions(-)
> 
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> index 664adb15b4..3fde477eee 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -81,6 +81,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>         error_report("Error start vhost dev");
>         goto err_guest_notifiers;
>     }
> +    vsc->started_vu = true;
> 
>     /* guest_notifier_mask/pending not used yet, so just unmask
>      * everything here.  virtio-pci will do the right thing by
> @@ -106,6 +107,11 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
>     int ret = 0;
> 
> +    if (!vsc->started_vu) {
> +        return;
> +    }
> +    vsc->started_vu = false;
> +
>     vhost_dev_stop(&vsc->dev, vdev, true);
> 
>     if (k->set_guest_notifiers) {
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index ee99b19e7a..bd32dcf999 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -46,20 +46,25 @@ enum VhostUserProtocolFeature {
> static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
> {
>     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    bool should_start = virtio_device_should_start(vdev, status);
> +    int ret;
> 
> -    if (vhost_dev_is_started(&vsc->dev) == start) {
> +    if (!s->connected) {
>         return;
>     }
> 
> -    if (start) {
> -        int ret;
> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
> +        return;
> +    }
> 
> +    if (should_start) {
>         ret = vhost_scsi_common_start(vsc);
>         if (ret < 0) {
>             error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
> -            exit(1);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>         }
>     } else {
>         vhost_scsi_common_stop(vsc);
> @@ -85,8 +90,160 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>     }
> }
> 
> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    Error *local_err = NULL;
> +    int i, ret;
> +
> +    if (!vdev->start_on_kick) {
> +        return;
> +    }
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +
> +    if (vhost_dev_is_started(&vsc->dev)) {
> +        return;
> +    }
> +
> +    /*
> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
> +     * vhost here instead of waiting for .set_status().
> +     */
> +    ret = vhost_scsi_common_start(vsc);
> +    if (ret < 0) {
> +        error_reportf_err(local_err, "vhost-user-blk: vhost start failed: “);

Need fix typo in v3. s/vhost-user-blk/vhost-user-scsi/g

> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return;
> +    }
> +
> +    /* Kick right away to begin processing requests already in vring */
> +    for (i = 0; i < vsc->dev.nvqs; i++) {
> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
> +
> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
> +            continue;
> +        }
> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
> +    }
> +}
> +
> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret = 0;
> +
> +    if (s->connected) {
> +        return 0;
> +    }
> +    s->connected = true;
> +
> +    vsc->dev.num_queues = vs->conf.num_queues;
> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> +    vsc->dev.vqs = s->vhost_vqs;
> +    vsc->dev.vq_index = 0;
> +    vsc->dev.backend_features = 0;
> +
> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> +                         errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    /* restore vhost state */
> +    if (virtio_device_started(vdev, vdev->status)) {
> +        ret = vhost_scsi_common_start(vsc);
> +        if (ret < 0) {
> +            return ret;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> +
> +static void vhost_user_scsi_disconnect(DeviceState *dev)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +    s->connected = false;
> +
> +    vhost_scsi_common_stop(vsc);
> +
> +    vhost_dev_cleanup(&vsc->dev);
> +
> +    /* Re-instate the event handler for new connections */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> +{
> +    DeviceState *dev = opaque;
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    Error *local_err = NULL;
> +
> +    switch (event) {
> +    case CHR_EVENT_OPENED:
> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> +            error_report_err(local_err);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> +            return;
> +        }
> +        break;
> +    case CHR_EVENT_CLOSED:
> +        /* defer close until later to avoid circular close */
> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> +                               vhost_user_scsi_disconnect);
> +        break;
> +    case CHR_EVENT_BREAK:
> +    case CHR_EVENT_MUX_IN:
> +    case CHR_EVENT_MUX_OUT:
> +        /* Ignore */
> +        break;
> +    }
> +}
> +
> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> +{
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret;
> +
> +    s->connected = false;
> +
> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    ret = vhost_user_scsi_connect(dev, errp);
> +    if (ret < 0) {
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return ret;
> +    }
> +    assert(s->connected);
> +
> +    return 0;
> }
> 
> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> @@ -94,18 +251,18 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = NULL;
>     Error *err = NULL;
>     int ret;
> +    int retries = VU_REALIZE_CONN_RETRIES;
> 
>     if (!vs->conf.chardev.chr) {
>         error_setg(errp, "vhost-user-scsi: missing chardev");
>         return;
>     }
> 
> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output, &err);
> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output, &err);
>     if (err != NULL) {
>         error_propagate(errp, err);
>         return;
> @@ -115,18 +272,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>         goto free_virtio;
>     }
> 
> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> -    vsc->dev.vq_index = 0;
> -    vsc->dev.backend_features = 0;
> -    vqs = vsc->dev.vqs;
> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> +
> +    assert(!*errp);
> +    do {
> +        if (*errp) {
> +            error_prepend(errp, "Reconnecting after error: ");
> +            error_report_err(*errp);
> +            *errp = NULL;
> +        }
> +        ret = vhost_user_scsi_realize_connect(s, errp);
> +    } while (ret < 0 && retries--);
> 
> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>     if (ret < 0) {
>         goto free_vhost;
>     }
> 
> +    /* we're fully initialized, now we can operate, so add the handler */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> +                             vhost_user_scsi_event, NULL, (void *)dev,
> +                             NULL, true);
>     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>     vsc->channel = 0;
>     vsc->lun = 0;
> @@ -135,8 +302,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     return;
> 
> free_vhost:
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
>     vhost_user_cleanup(&s->vhost_user);
> -    g_free(vqs);
> +
> free_virtio:
>     virtio_scsi_common_unrealize(dev);
> }
> @@ -146,16 +317,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> 
>     /* This will stop the vhost backend. */
>     vhost_user_scsi_set_status(vdev, 0);
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> +                             NULL, false);
> 
>     vhost_dev_cleanup(&vsc->dev);
> -    g_free(vqs);
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +
> +    vhost_dev_free_inflight(vsc->inflight);
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
> 
> -    virtio_scsi_common_unrealize(dev);
>     vhost_user_cleanup(&s->vhost_user);
> +    virtio_scsi_common_unrealize(dev);
> }
> 
> static Property vhost_user_scsi_properties[] = {
> diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
> index 18f115527c..daa183018c 100644
> --- a/include/hw/virtio/vhost-scsi-common.h
> +++ b/include/hw/virtio/vhost-scsi-common.h
> @@ -37,6 +37,9 @@ struct VHostSCSICommon {
>     bool migratable;
> 
>     struct vhost_inflight *inflight;
> +
> +    /* vhost_scsi_common_start/vhost_scsi_common_stop */
> +    bool started_vu;
> };
> 
> int vhost_scsi_common_start(VHostSCSICommon *vsc);
> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> index 521b08e559..c66acc68b7 100644
> --- a/include/hw/virtio/vhost-user-scsi.h
> +++ b/include/hw/virtio/vhost-user-scsi.h
> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
> struct VHostUserSCSI {
>     VHostSCSICommon parent_obj;
>     VhostUserState vhost_user;
> +    bool connected;
> +
> +    struct vhost_virtqueue *vhost_vqs;
> };
> 
> #endif /* VHOST_USER_SCSI_H */
> -- 
> 2.41.0
> 

Re: [PATCH v2 4/4] vhost-user-scsi: support reconnect to backend

[Raphael Norwitz]

I don’t think we should be changing any vhost-scsi-common code here. I’d rather implement wrappers around vhost_user_scsi_start/stop() around vhost_user_scsi_common_start/stop() and check started_vu there.

Otherwise I think this is looking good. 

Glad to see you caught the vhost_user_scsi_handle_ouptut and implemented it like vhost-user-blk. Can it go in a separate change?

> On Jul 25, 2023, at 6:42 AM, Li Feng <fengli@smartx.com> wrote:
> 
> If the backend crashes and restarts, the device is broken.
> This patch adds reconnect for vhost-user-scsi.
> 
> Tested with spdk backend.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/scsi/vhost-scsi-common.c           |   6 +
> hw/scsi/vhost-user-scsi.c             | 220 +++++++++++++++++++++++---
> include/hw/virtio/vhost-scsi-common.h |   3 +
> include/hw/virtio/vhost-user-scsi.h   |   3 +
> 4 files changed, 211 insertions(+), 21 deletions(-)
> 
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> index 664adb15b4..3fde477eee 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -81,6 +81,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>         error_report("Error start vhost dev");
>         goto err_guest_notifiers;
>     }
> +    vsc->started_vu = true;
> 
>     /* guest_notifier_mask/pending not used yet, so just unmask
>      * everything here.  virtio-pci will do the right thing by
> @@ -106,6 +107,11 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
>     int ret = 0;
> 
> +    if (!vsc->started_vu) {
> +        return;
> +    }
> +    vsc->started_vu = false;
> +
>     vhost_dev_stop(&vsc->dev, vdev, true);
> 
>     if (k->set_guest_notifiers) {
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index ee99b19e7a..bd32dcf999 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -46,20 +46,25 @@ enum VhostUserProtocolFeature {
> static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
> {
>     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    bool should_start = virtio_device_should_start(vdev, status);
> +    int ret;
> 
> -    if (vhost_dev_is_started(&vsc->dev) == start) {
> +    if (!s->connected) {
>         return;
>     }
> 
> -    if (start) {
> -        int ret;
> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
> +        return;
> +    }
> 
> +    if (should_start) {
>         ret = vhost_scsi_common_start(vsc);
>         if (ret < 0) {
>             error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
> -            exit(1);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>         }
>     } else {
>         vhost_scsi_common_stop(vsc);
> @@ -85,8 +90,160 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>     }
> }
> 
> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    Error *local_err = NULL;
> +    int i, ret;
> +
> +    if (!vdev->start_on_kick) {
> +        return;
> +    }
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +
> +    if (vhost_dev_is_started(&vsc->dev)) {
> +        return;
> +    }
> +
> +    /*
> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
> +     * vhost here instead of waiting for .set_status().
> +     */
> +    ret = vhost_scsi_common_start(vsc);
> +    if (ret < 0) {
> +        error_reportf_err(local_err, "vhost-user-blk: vhost start failed: ");
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return;
> +    }
> +
> +    /* Kick right away to begin processing requests already in vring */
> +    for (i = 0; i < vsc->dev.nvqs; i++) {
> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
> +
> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
> +            continue;
> +        }
> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
> +    }
> +}
> +
> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret = 0;
> +
> +    if (s->connected) {
> +        return 0;
> +    }
> +    s->connected = true;
> +
> +    vsc->dev.num_queues = vs->conf.num_queues;
> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> +    vsc->dev.vqs = s->vhost_vqs;
> +    vsc->dev.vq_index = 0;
> +    vsc->dev.backend_features = 0;
> +
> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> +                         errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    /* restore vhost state */
> +    if (virtio_device_started(vdev, vdev->status)) {
> +        ret = vhost_scsi_common_start(vsc);
> +        if (ret < 0) {
> +            return ret;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> +
> +static void vhost_user_scsi_disconnect(DeviceState *dev)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +    s->connected = false;
> +
> +    vhost_scsi_common_stop(vsc);
> +
> +    vhost_dev_cleanup(&vsc->dev);
> +
> +    /* Re-instate the event handler for new connections */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> +{
> +    DeviceState *dev = opaque;
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    Error *local_err = NULL;
> +
> +    switch (event) {
> +    case CHR_EVENT_OPENED:
> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> +            error_report_err(local_err);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> +            return;
> +        }
> +        break;
> +    case CHR_EVENT_CLOSED:
> +        /* defer close until later to avoid circular close */
> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> +                               vhost_user_scsi_disconnect);
> +        break;
> +    case CHR_EVENT_BREAK:
> +    case CHR_EVENT_MUX_IN:
> +    case CHR_EVENT_MUX_OUT:
> +        /* Ignore */
> +        break;
> +    }
> +}
> +
> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> +{
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret;
> +
> +    s->connected = false;
> +
> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    ret = vhost_user_scsi_connect(dev, errp);
> +    if (ret < 0) {
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return ret;
> +    }
> +    assert(s->connected);
> +
> +    return 0;
> }
> 
> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> @@ -94,18 +251,18 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = NULL;
>     Error *err = NULL;
>     int ret;
> +    int retries = VU_REALIZE_CONN_RETRIES;
> 
>     if (!vs->conf.chardev.chr) {
>         error_setg(errp, "vhost-user-scsi: missing chardev");
>         return;
>     }
> 
> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output, &err);
> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output, &err);
>     if (err != NULL) {
>         error_propagate(errp, err);
>         return;
> @@ -115,18 +272,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>         goto free_virtio;
>     }
> 
> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> -    vsc->dev.vq_index = 0;
> -    vsc->dev.backend_features = 0;
> -    vqs = vsc->dev.vqs;
> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> +
> +    assert(!*errp);
> +    do {
> +        if (*errp) {
> +            error_prepend(errp, "Reconnecting after error: ");
> +            error_report_err(*errp);
> +            *errp = NULL;
> +        }
> +        ret = vhost_user_scsi_realize_connect(s, errp);
> +    } while (ret < 0 && retries--);
> 
> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>     if (ret < 0) {
>         goto free_vhost;
>     }
> 
> +    /* we're fully initialized, now we can operate, so add the handler */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> +                             vhost_user_scsi_event, NULL, (void *)dev,
> +                             NULL, true);
>     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>     vsc->channel = 0;
>     vsc->lun = 0;
> @@ -135,8 +302,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     return;
> 
> free_vhost:
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
>     vhost_user_cleanup(&s->vhost_user);
> -    g_free(vqs);
> +
> free_virtio:
>     virtio_scsi_common_unrealize(dev);
> }
> @@ -146,16 +317,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> 
>     /* This will stop the vhost backend. */
>     vhost_user_scsi_set_status(vdev, 0);
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> +                             NULL, false);
> 
>     vhost_dev_cleanup(&vsc->dev);
> -    g_free(vqs);
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +
> +    vhost_dev_free_inflight(vsc->inflight);
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
> 
> -    virtio_scsi_common_unrealize(dev);
>     vhost_user_cleanup(&s->vhost_user);
> +    virtio_scsi_common_unrealize(dev);
> }
> 
> static Property vhost_user_scsi_properties[] = {
> diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
> index 18f115527c..daa183018c 100644
> --- a/include/hw/virtio/vhost-scsi-common.h
> +++ b/include/hw/virtio/vhost-scsi-common.h
> @@ -37,6 +37,9 @@ struct VHostSCSICommon {
>     bool migratable;
> 
>     struct vhost_inflight *inflight;
> +
> +    /* vhost_scsi_common_start/vhost_scsi_common_stop */
> +    bool started_vu;

Move to include/hw/virtio/vhost-user-scsi.h

> };
> 
> int vhost_scsi_common_start(VHostSCSICommon *vsc);
> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> index 521b08e559..c66acc68b7 100644
> --- a/include/hw/virtio/vhost-user-scsi.h
> +++ b/include/hw/virtio/vhost-user-scsi.h
> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
> struct VHostUserSCSI {
>     VHostSCSICommon parent_obj;
>     VhostUserState vhost_user;
> +    bool connected;
> +
> +    struct vhost_virtqueue *vhost_vqs;
> };
> 
> #endif /* VHOST_USER_SCSI_H */
> -- 
> 2.41.0
> 

Re: [PATCH v2 4/4] vhost-user-scsi: support reconnect to backend

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 13625 bytes --]



> 2023年7月31日 06:14，Raphael Norwitz <raphael.norwitz@nutanix.com> 写道：
> 
> I don’t think we should be changing any vhost-scsi-common code here. I’d rather implement wrappers around vhost_user_scsi_start/stop() around vhost_user_scsi_common_start/stop() and check started_vu there.
> 
> Otherwise I think this is looking good. 
> 
> Glad to see you caught the vhost_user_scsi_handle_ouptut and implemented it like vhost-user-blk. Can it go in a separate change?

I will fix it in v3.

> 
>> On Jul 25, 2023, at 6:42 AM, Li Feng <fengli@smartx.com> wrote:
>> 
>> If the backend crashes and restarts, the device is broken.
>> This patch adds reconnect for vhost-user-scsi.
>> 
>> Tested with spdk backend.
>> 
>> Signed-off-by: Li Feng <fengli@smartx.com>
>> ---
>> hw/scsi/vhost-scsi-common.c           |   6 +
>> hw/scsi/vhost-user-scsi.c             | 220 +++++++++++++++++++++++---
>> include/hw/virtio/vhost-scsi-common.h |   3 +
>> include/hw/virtio/vhost-user-scsi.h   |   3 +
>> 4 files changed, 211 insertions(+), 21 deletions(-)
>> 
>> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
>> index 664adb15b4..3fde477eee 100644
>> --- a/hw/scsi/vhost-scsi-common.c
>> +++ b/hw/scsi/vhost-scsi-common.c
>> @@ -81,6 +81,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>>        error_report("Error start vhost dev");
>>        goto err_guest_notifiers;
>>    }
>> +    vsc->started_vu = true;
>> 
>>    /* guest_notifier_mask/pending not used yet, so just unmask
>>     * everything here.  virtio-pci will do the right thing by
>> @@ -106,6 +107,11 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>>    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
>>    int ret = 0;
>> 
>> +    if (!vsc->started_vu) {
>> +        return;
>> +    }
>> +    vsc->started_vu = false;
>> +
>>    vhost_dev_stop(&vsc->dev, vdev, true);
>> 
>>    if (k->set_guest_notifiers) {
>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>> index ee99b19e7a..bd32dcf999 100644
>> --- a/hw/scsi/vhost-user-scsi.c
>> +++ b/hw/scsi/vhost-user-scsi.c
>> @@ -46,20 +46,25 @@ enum VhostUserProtocolFeature {
>> static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
>> {
>>    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    bool should_start = virtio_device_should_start(vdev, status);
>> +    int ret;
>> 
>> -    if (vhost_dev_is_started(&vsc->dev) == start) {
>> +    if (!s->connected) {
>>        return;
>>    }
>> 
>> -    if (start) {
>> -        int ret;
>> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
>> +        return;
>> +    }
>> 
>> +    if (should_start) {
>>        ret = vhost_scsi_common_start(vsc);
>>        if (ret < 0) {
>>            error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
>> -            exit(1);
>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>>        }
>>    } else {
>>        vhost_scsi_common_stop(vsc);
>> @@ -85,8 +90,160 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>>    }
>> }
>> 
>> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>> {
>> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +
>> +    Error *local_err = NULL;
>> +    int i, ret;
>> +
>> +    if (!vdev->start_on_kick) {
>> +        return;
>> +    }
>> +
>> +    if (!s->connected) {
>> +        return;
>> +    }
>> +
>> +    if (vhost_dev_is_started(&vsc->dev)) {
>> +        return;
>> +    }
>> +
>> +    /*
>> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
>> +     * vhost here instead of waiting for .set_status().
>> +     */
>> +    ret = vhost_scsi_common_start(vsc);
>> +    if (ret < 0) {
>> +        error_reportf_err(local_err, "vhost-user-blk: vhost start failed: ");
>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +        return;
>> +    }
>> +
>> +    /* Kick right away to begin processing requests already in vring */
>> +    for (i = 0; i < vsc->dev.nvqs; i++) {
>> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
>> +
>> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
>> +            continue;
>> +        }
>> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
>> +    }
>> +}
>> +
>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>> +{
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    int ret = 0;
>> +
>> +    if (s->connected) {
>> +        return 0;
>> +    }
>> +    s->connected = true;
>> +
>> +    vsc->dev.num_queues = vs->conf.num_queues;
>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>> +    vsc->dev.vqs = s->vhost_vqs;
>> +    vsc->dev.vq_index = 0;
>> +    vsc->dev.backend_features = 0;
>> +
>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>> +                         errp);
>> +    if (ret < 0) {
>> +        return ret;
>> +    }
>> +
>> +    /* restore vhost state */
>> +    if (virtio_device_started(vdev, vdev->status)) {
>> +        ret = vhost_scsi_common_start(vsc);
>> +        if (ret < 0) {
>> +            return ret;
>> +        }
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>> +
>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>> +{
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +
>> +    if (!s->connected) {
>> +        return;
>> +    }
>> +    s->connected = false;
>> +
>> +    vhost_scsi_common_stop(vsc);
>> +
>> +    vhost_dev_cleanup(&vsc->dev);
>> +
>> +    /* Re-instate the event handler for new connections */
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>> +}
>> +
>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>> +{
>> +    DeviceState *dev = opaque;
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    Error *local_err = NULL;
>> +
>> +    switch (event) {
>> +    case CHR_EVENT_OPENED:
>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>> +            error_report_err(local_err);
>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +            return;
>> +        }
>> +        break;
>> +    case CHR_EVENT_CLOSED:
>> +        /* defer close until later to avoid circular close */
>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>> +                               vhost_user_scsi_disconnect);
>> +        break;
>> +    case CHR_EVENT_BREAK:
>> +    case CHR_EVENT_MUX_IN:
>> +    case CHR_EVENT_MUX_OUT:
>> +        /* Ignore */
>> +        break;
>> +    }
>> +}
>> +
>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>> +{
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    int ret;
>> +
>> +    s->connected = false;
>> +
>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>> +    if (ret < 0) {
>> +        return ret;
>> +    }
>> +
>> +    ret = vhost_user_scsi_connect(dev, errp);
>> +    if (ret < 0) {
>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +        return ret;
>> +    }
>> +    assert(s->connected);
>> +
>> +    return 0;
>> }
>> 
>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>> @@ -94,18 +251,18 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>    VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    struct vhost_virtqueue *vqs = NULL;
>>    Error *err = NULL;
>>    int ret;
>> +    int retries = VU_REALIZE_CONN_RETRIES;
>> 
>>    if (!vs->conf.chardev.chr) {
>>        error_setg(errp, "vhost-user-scsi: missing chardev");
>>        return;
>>    }
>> 
>> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
>> -                               vhost_dummy_handle_output,
>> -                               vhost_dummy_handle_output, &err);
>> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
>> +                               vhost_user_scsi_handle_output,
>> +                               vhost_user_scsi_handle_output, &err);
>>    if (err != NULL) {
>>        error_propagate(errp, err);
>>        return;
>> @@ -115,18 +272,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>        goto free_virtio;
>>    }
>> 
>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>> -    vsc->dev.vq_index = 0;
>> -    vsc->dev.backend_features = 0;
>> -    vqs = vsc->dev.vqs;
>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>> +
>> +    assert(!*errp);
>> +    do {
>> +        if (*errp) {
>> +            error_prepend(errp, "Reconnecting after error: ");
>> +            error_report_err(*errp);
>> +            *errp = NULL;
>> +        }
>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>> +    } while (ret < 0 && retries--);
>> 
>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>    if (ret < 0) {
>>        goto free_vhost;
>>    }
>> 
>> +    /* we're fully initialized, now we can operate, so add the handler */
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>> +                             NULL, true);
>>    /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>    vsc->channel = 0;
>>    vsc->lun = 0;
>> @@ -135,8 +302,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>    return;
>> 
>> free_vhost:
>> +    g_free(s->vhost_vqs);
>> +    s->vhost_vqs = NULL;
>> +    g_free(vsc->inflight);
>> +    vsc->inflight = NULL;
>>    vhost_user_cleanup(&s->vhost_user);
>> -    g_free(vqs);
>> +
>> free_virtio:
>>    virtio_scsi_common_unrealize(dev);
>> }
>> @@ -146,16 +317,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>    VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> 
>>    /* This will stop the vhost backend. */
>>    vhost_user_scsi_set_status(vdev, 0);
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>> +                             NULL, false);
>> 
>>    vhost_dev_cleanup(&vsc->dev);
>> -    g_free(vqs);
>> +    g_free(s->vhost_vqs);
>> +    s->vhost_vqs = NULL;
>> +
>> +    vhost_dev_free_inflight(vsc->inflight);
>> +    g_free(vsc->inflight);
>> +    vsc->inflight = NULL;
>> 
>> -    virtio_scsi_common_unrealize(dev);
>>    vhost_user_cleanup(&s->vhost_user);
>> +    virtio_scsi_common_unrealize(dev);
>> }
>> 
>> static Property vhost_user_scsi_properties[] = {
>> diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
>> index 18f115527c..daa183018c 100644
>> --- a/include/hw/virtio/vhost-scsi-common.h
>> +++ b/include/hw/virtio/vhost-scsi-common.h
>> @@ -37,6 +37,9 @@ struct VHostSCSICommon {
>>    bool migratable;
>> 
>>    struct vhost_inflight *inflight;
>> +
>> +    /* vhost_scsi_common_start/vhost_scsi_common_stop */
>> +    bool started_vu;
> 
> Move to include/hw/virtio/vhost-user-scsi.h

I will fix it in v3.

> 
>> };
>> 
>> int vhost_scsi_common_start(VHostSCSICommon *vsc);
>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>> index 521b08e559..c66acc68b7 100644
>> --- a/include/hw/virtio/vhost-user-scsi.h
>> +++ b/include/hw/virtio/vhost-user-scsi.h
>> @@ -29,6 +29,9 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>> struct VHostUserSCSI {
>>    VHostSCSICommon parent_obj;
>>    VhostUserState vhost_user;
>> +    bool connected;
>> +
>> +    struct vhost_virtqueue *vhost_vqs;
>> };
>> 
>> #endif /* VHOST_USER_SCSI_H */
>> -- 
>> 2.41.0


[-- Attachment #2: Type: text/html, Size: 26272 bytes --]

[PATCH v3 0/5] Implement reconnect for vhost-user-scsi

[Li Feng]

This patchset adds reconnect support for vhost-user-scsi. At the same
times, fix vhost fd leak and refactor some code.

Changes for v3:
- Split the vhost_user_scsi_handle_output to a separate patch;
- Move the started_vu from vhost scsi common header to vhost-user-scsi header;
- Fix a log print error;

Changes for v2:
- Split the v1 patch to small separate patchset;
- New patch for fixing fd leak, which has sent to reviewers in another
  mail;
- Implement the `vhost_user_scsi_handle_output`;
- Add the started_vu safe check;
- Fix error handler;
- Check the inflight before set/get inflight fd.

Li Feng (5):
  vhost: fix the fd leak
  vhost-user-common: send get_inflight_fd once
  vhost: move and rename the conn retry times
  vhost-user-scsi: support reconnect to backend
  vhost-user-scsi: start vhost when guest kicks

 hw/block/vhost-user-blk.c           |   4 +-
 hw/scsi/vhost-scsi-common.c         |  37 ++---
 hw/scsi/vhost-user-scsi.c           | 247 +++++++++++++++++++++++++---
 hw/virtio/vhost-user-gpio.c         |   3 +-
 hw/virtio/vhost.c                   |   2 +
 include/hw/virtio/vhost-user-scsi.h |   4 +
 include/hw/virtio/vhost.h           |   2 +
 7 files changed, 252 insertions(+), 47 deletions(-)

-- 
2.41.0

[PATCH v3 1/5] vhost: fix the fd leak

[Li Feng]

When the vhost-user reconnect to the backend, the notifer should be
cleanup. Otherwise, the fd resource will be exhausted.

Fixes: f9a09ca3ea ("vhost: add support for configure interrupt")

Signed-off-by: Li Feng <fengli@smartx.com>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
---
 hw/virtio/vhost.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index abf0d03c8d..e2f6ffb446 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -2044,6 +2044,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings)
     event_notifier_test_and_clear(
         &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
     event_notifier_test_and_clear(&vdev->config_notifier);
+    event_notifier_cleanup(
+        &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier);
 
     trace_vhost_dev_stop(hdev, vdev->name, vrings);
 
-- 
2.41.0

Re: [PATCH v3 1/5] vhost: fix the fd leak

[Fiona Ebner]

Am 31.07.23 um 14:10 schrieb Li Feng:
> When the vhost-user reconnect to the backend, the notifer should be
> cleanup. Otherwise, the fd resource will be exhausted.
> 
> Fixes: f9a09ca3ea ("vhost: add support for configure interrupt")
> 
> Signed-off-by: Li Feng <fengli@smartx.com>
> Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

Tested-by: Fiona Ebner <f.ebner@proxmox.com>

Thank you for the fix! CC-ing stable, because the issue is present since
8.0.0.

[PATCH v3 2/5] vhost-user-common: send get_inflight_fd once

[Li Feng]

Currently the get_inflight_fd will be sent every time the device is started, and
the backend will allocate shared memory to save the inflight state. If the
backend finds that it receives the second get_inflight_fd, it will release the
previous shared memory, which breaks inflight working logic.

This patch is a preparation for the following patches.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a06f01af26..a61cd0e907 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     vsc->dev.acked_features = vdev->guest_features;
 
-    assert(vsc->inflight == NULL);
-    vsc->inflight = g_new0(struct vhost_inflight, 1);
-    ret = vhost_dev_get_inflight(&vsc->dev,
-                                 vs->conf.virtqueue_size,
-                                 vsc->inflight);
+    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error get inflight: %d", -ret);
+        error_report("Error setting inflight format: %d", -ret);
         goto err_guest_notifiers;
     }
 
-    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
-    if (ret < 0) {
-        error_report("Error set inflight: %d", -ret);
-        goto err_guest_notifiers;
+    if (vsc->inflight) {
+        if (!vsc->inflight->addr) {
+            ret = vhost_dev_get_inflight(&vsc->dev,
+                                        vs->conf.virtqueue_size,
+                                        vsc->inflight);
+            if (ret < 0) {
+                error_report("Error getting inflight: %d", -ret);
+                goto err_guest_notifiers;
+            }
+        }
+
+        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
+        if (ret < 0) {
+            error_report("Error setting inflight: %d", -ret);
+            goto err_guest_notifiers;
+        }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
@@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     return ret;
 
 err_guest_notifiers:
-    g_free(vsc->inflight);
-    vsc->inflight = NULL;
-
     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
 err_host_notifiers:
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
@@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     }
     assert(ret >= 0);
 
-    if (vsc->inflight) {
-        vhost_dev_free_inflight(vsc->inflight);
-        g_free(vsc->inflight);
-        vsc->inflight = NULL;
-    }
-
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
 }
 
-- 
2.41.0

Re: [PATCH v3 2/5] vhost-user-common: send get_inflight_fd once

[Raphael Norwitz]

> On Jul 31, 2023, at 8:10 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Currently the get_inflight_fd will be sent every time the device is started, and
> the backend will allocate shared memory to save the inflight state. If the
> backend finds that it receives the second get_inflight_fd, it will release the
> previous shared memory, which breaks inflight working logic.
> 
> This patch is a preparation for the following patches.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> ---
> hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
> 1 file changed, 18 insertions(+), 19 deletions(-)
> 
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> index a06f01af26..a61cd0e907 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> 
>     vsc->dev.acked_features = vdev->guest_features;
> 
> -    assert(vsc->inflight == NULL);
> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> -    ret = vhost_dev_get_inflight(&vsc->dev,
> -                                 vs->conf.virtqueue_size,
> -                                 vsc->inflight);
> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>     if (ret < 0) {
> -        error_report("Error get inflight: %d", -ret);
> +        error_report("Error setting inflight format: %d", -ret);
>         goto err_guest_notifiers;
>     }
> 
> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> -    if (ret < 0) {
> -        error_report("Error set inflight: %d", -ret);
> -        goto err_guest_notifiers;
> +    if (vsc->inflight) {
> +        if (!vsc->inflight->addr) {
> +            ret = vhost_dev_get_inflight(&vsc->dev,
> +                                        vs->conf.virtqueue_size,
> +                                        vsc->inflight);
> +            if (ret < 0) {
> +                error_report("Error getting inflight: %d", -ret);
> +                goto err_guest_notifiers;
> +            }
> +        }
> +
> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> +        if (ret < 0) {
> +            error_report("Error setting inflight: %d", -ret);
> +            goto err_guest_notifiers;
> +        }
>     }
> 
>     ret = vhost_dev_start(&vsc->dev, vdev, true);
> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>     return ret;
> 
> err_guest_notifiers:
> -    g_free(vsc->inflight);
> -    vsc->inflight = NULL;
> -
>     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
> err_host_notifiers:
>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>     }
>     assert(ret >= 0);
> 
> -    if (vsc->inflight) {
> -        vhost_dev_free_inflight(vsc->inflight);
> -        g_free(vsc->inflight);
> -        vsc->inflight = NULL;
> -    }
> -
>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> }
> 
> -- 
> 2.41.0
> 

[PATCH v3 3/5] vhost: move and rename the conn retry times

[Li Feng]

Multiple devices need this macro, move it to a common header.

Signed-off-by: Li Feng <fengli@smartx.com>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
---
 hw/block/vhost-user-blk.c   | 4 +---
 hw/virtio/vhost-user-gpio.c | 3 +--
 include/hw/virtio/vhost.h   | 2 ++
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index eecf3f7a81..3c69fa47d5 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -32,8 +32,6 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
-
 static const int user_feature_bits[] = {
     VIRTIO_BLK_F_SIZE_MAX,
     VIRTIO_BLK_F_SEG_MAX,
@@ -482,7 +480,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
     s->inflight = g_new0(struct vhost_inflight, 1);
     s->vhost_vqs = g_new0(struct vhost_virtqueue, s->num_queues);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     assert(!*errp);
     do {
         if (*errp) {
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index 3b013f2d0f..d9979aa5db 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -15,7 +15,6 @@
 #include "standard-headers/linux/virtio_ids.h"
 #include "trace.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
 #define VHOST_NVQS 2
 
 /* Features required from VirtIO */
@@ -359,7 +358,7 @@ static void vu_gpio_device_realize(DeviceState *dev, Error **errp)
     qemu_chr_fe_set_handlers(&gpio->chardev, NULL, NULL, vu_gpio_event, NULL,
                              dev, NULL, true);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     g_assert(!*errp);
     do {
         if (*errp) {
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 6a173cb9fa..ca3131b1af 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -8,6 +8,8 @@
 #define VHOST_F_DEVICE_IOTLB 63
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+#define VU_REALIZE_CONN_RETRIES 3
+
 /* Generic structures common for any vhost based device. */
 
 struct vhost_inflight {
-- 
2.41.0

[PATCH v3 4/5] vhost-user-scsi: support reconnect to backend

[Li Feng]

If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-user-scsi.c           | 199 +++++++++++++++++++++++++---
 include/hw/virtio/vhost-user-scsi.h |   4 +
 2 files changed, 184 insertions(+), 19 deletions(-)

diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index ee99b19e7a..5bf012461b 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -43,26 +43,54 @@ enum VhostUserProtocolFeature {
     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
 };
 
+static int vhost_user_scsi_start(VHostUserSCSI *s)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    int ret;
+
+    ret = vhost_scsi_common_start(vsc);
+    s->started_vu = (ret < 0 ? false : true);
+
+    return ret;
+}
+
+static void vhost_user_scsi_stop(VHostUserSCSI *s)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+
+    if (!s->started_vu) {
+        return;
+    }
+    s->started_vu = false;
+
+    vhost_scsi_common_stop(vsc);
+}
+
 static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    bool should_start = virtio_device_should_start(vdev, status);
+    int ret;
 
-    if (vhost_dev_is_started(&vsc->dev) == start) {
+    if (!s->connected) {
         return;
     }
 
-    if (start) {
-        int ret;
+    if (vhost_dev_is_started(&vsc->dev) == should_start) {
+        return;
+    }
 
-        ret = vhost_scsi_common_start(vsc);
+    if (should_start) {
+        ret = vhost_user_scsi_start(s);
         if (ret < 0) {
             error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
-            exit(1);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
         }
     } else {
-        vhost_scsi_common_stop(vsc);
+        vhost_user_scsi_stop(s);
     }
 }
 
@@ -89,14 +117,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
 }
 
+static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret = 0;
+
+    if (s->connected) {
+        return 0;
+    }
+    s->connected = true;
+
+    vsc->dev.num_queues = vs->conf.num_queues;
+    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
+    vsc->dev.vqs = s->vhost_vqs;
+    vsc->dev.vq_index = 0;
+    vsc->dev.backend_features = 0;
+
+    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
+                         errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* restore vhost state */
+    if (virtio_device_started(vdev, vdev->status)) {
+        ret = vhost_user_scsi_start(s);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
+
+static void vhost_user_scsi_disconnect(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    if (!s->connected) {
+        return;
+    }
+    s->connected = false;
+
+    vhost_user_scsi_stop(s);
+
+    vhost_dev_cleanup(&vsc->dev);
+
+    /* Re-instate the event handler for new connections */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
+                             vhost_user_scsi_event, NULL, dev, NULL, true);
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
+{
+    DeviceState *dev = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    Error *local_err = NULL;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
+            error_report_err(local_err);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
+            return;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        /* defer close until later to avoid circular close */
+        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
+                               vhost_user_scsi_disconnect);
+        break;
+    case CHR_EVENT_BREAK:
+    case CHR_EVENT_MUX_IN:
+    case CHR_EVENT_MUX_OUT:
+        /* Ignore */
+        break;
+    }
+}
+
+static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
+{
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret;
+
+    s->connected = false;
+
+    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_user_scsi_connect(dev, errp);
+    if (ret < 0) {
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return ret;
+    }
+    assert(s->connected);
+
+    return 0;
+}
+
 static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
 {
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = NULL;
     Error *err = NULL;
     int ret;
+    int retries = VU_REALIZE_CONN_RETRIES;
 
     if (!vs->conf.chardev.chr) {
         error_setg(errp, "vhost-user-scsi: missing chardev");
@@ -115,18 +255,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         goto free_virtio;
     }
 
-    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
-    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
-    vsc->dev.vq_index = 0;
-    vsc->dev.backend_features = 0;
-    vqs = vsc->dev.vqs;
+    vsc->inflight = g_new0(struct vhost_inflight, 1);
+    s->vhost_vqs = g_new0(struct vhost_virtqueue,
+                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
+
+    assert(!*errp);
+    do {
+        if (*errp) {
+            error_prepend(errp, "Reconnecting after error: ");
+            error_report_err(*errp);
+            *errp = NULL;
+        }
+        ret = vhost_user_scsi_realize_connect(s, errp);
+    } while (ret < 0 && retries--);
 
-    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
-                         VHOST_BACKEND_TYPE_USER, 0, errp);
     if (ret < 0) {
         goto free_vhost;
     }
 
+    /* we're fully initialized, now we can operate, so add the handler */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
+                             vhost_user_scsi_event, NULL, (void *)dev,
+                             NULL, true);
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -135,8 +285,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     return;
 
 free_vhost:
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
     vhost_user_cleanup(&s->vhost_user);
-    g_free(vqs);
+
 free_virtio:
     virtio_scsi_common_unrealize(dev);
 }
@@ -146,16 +300,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = vsc->dev.vqs;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
 
     /* This will stop the vhost backend. */
     vhost_user_scsi_set_status(vdev, 0);
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
+                             NULL, false);
 
     vhost_dev_cleanup(&vsc->dev);
-    g_free(vqs);
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+
+    vhost_dev_free_inflight(vsc->inflight);
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
 
-    virtio_scsi_common_unrealize(dev);
     vhost_user_cleanup(&s->vhost_user);
+    virtio_scsi_common_unrealize(dev);
 }
 
 static Property vhost_user_scsi_properties[] = {
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 521b08e559..b405ec952a 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
 struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     VhostUserState vhost_user;
+    bool connected;
+    bool started_vu;
+
+    struct vhost_virtqueue *vhost_vqs;
 };
 
 #endif /* VHOST_USER_SCSI_H */
-- 
2.41.0

Re: [PATCH v3 4/5] vhost-user-scsi: support reconnect to backend

[Raphael Norwitz]

> On Jul 31, 2023, at 8:10 AM, Li Feng <fengli@smartx.com> wrote:
> 
> If the backend crashes and restarts, the device is broken.
> This patch adds reconnect for vhost-user-scsi.
> 
> Tested with spdk backend.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> ---
> hw/scsi/vhost-user-scsi.c           | 199 +++++++++++++++++++++++++---
> include/hw/virtio/vhost-user-scsi.h |   4 +
> 2 files changed, 184 insertions(+), 19 deletions(-)
> 
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index ee99b19e7a..5bf012461b 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -43,26 +43,54 @@ enum VhostUserProtocolFeature {
>     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
> };
> 
> +static int vhost_user_scsi_start(VHostUserSCSI *s)
> +{
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    int ret;
> +
> +    ret = vhost_scsi_common_start(vsc);
> +    s->started_vu = (ret < 0 ? false : true);
> +
> +    return ret;
> +}
> +
> +static void vhost_user_scsi_stop(VHostUserSCSI *s)
> +{
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +
> +    if (!s->started_vu) {
> +        return;
> +    }
> +    s->started_vu = false;
> +
> +    vhost_scsi_common_stop(vsc);
> +}
> +
> static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
> {
>     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    bool should_start = virtio_device_should_start(vdev, status);
> +    int ret;
> 
> -    if (vhost_dev_is_started(&vsc->dev) == start) {
> +    if (!s->connected) {
>         return;
>     }
> 
> -    if (start) {
> -        int ret;
> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
> +        return;
> +    }
> 
> -        ret = vhost_scsi_common_start(vsc);
> +    if (should_start) {
> +        ret = vhost_user_scsi_start(s);
>         if (ret < 0) {
>             error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
> -            exit(1);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>         }
>     } else {
> -        vhost_scsi_common_stop(vsc);
> +        vhost_user_scsi_stop(s);
>     }
> }
> 
> @@ -89,14 +117,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> }
> 
> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret = 0;
> +
> +    if (s->connected) {
> +        return 0;
> +    }
> +    s->connected = true;
> +
> +    vsc->dev.num_queues = vs->conf.num_queues;
> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> +    vsc->dev.vqs = s->vhost_vqs;
> +    vsc->dev.vq_index = 0;
> +    vsc->dev.backend_features = 0;
> +
> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> +                         errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    /* restore vhost state */
> +    if (virtio_device_started(vdev, vdev->status)) {
> +        ret = vhost_user_scsi_start(s);
> +        if (ret < 0) {
> +            return ret;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> +
> +static void vhost_user_scsi_disconnect(DeviceState *dev)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +    s->connected = false;
> +
> +    vhost_user_scsi_stop(s);
> +
> +    vhost_dev_cleanup(&vsc->dev);
> +
> +    /* Re-instate the event handler for new connections */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> +{
> +    DeviceState *dev = opaque;
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    Error *local_err = NULL;
> +
> +    switch (event) {
> +    case CHR_EVENT_OPENED:
> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> +            error_report_err(local_err);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> +            return;
> +        }
> +        break;
> +    case CHR_EVENT_CLOSED:
> +        /* defer close until later to avoid circular close */
> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> +                               vhost_user_scsi_disconnect);
> +        break;
> +    case CHR_EVENT_BREAK:
> +    case CHR_EVENT_MUX_IN:
> +    case CHR_EVENT_MUX_OUT:
> +        /* Ignore */
> +        break;
> +    }
> +}
> +
> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> +{
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret;
> +
> +    s->connected = false;
> +
> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    ret = vhost_user_scsi_connect(dev, errp);
> +    if (ret < 0) {
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return ret;
> +    }
> +    assert(s->connected);
> +
> +    return 0;
> +}
> +
> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> {
>     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = NULL;
>     Error *err = NULL;
>     int ret;
> +    int retries = VU_REALIZE_CONN_RETRIES;
> 
>     if (!vs->conf.chardev.chr) {
>         error_setg(errp, "vhost-user-scsi: missing chardev");
> @@ -115,18 +255,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>         goto free_virtio;
>     }
> 
> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> -    vsc->dev.vq_index = 0;
> -    vsc->dev.backend_features = 0;
> -    vqs = vsc->dev.vqs;
> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> +
> +    assert(!*errp);
> +    do {
> +        if (*errp) {
> +            error_prepend(errp, "Reconnecting after error: ");
> +            error_report_err(*errp);
> +            *errp = NULL;
> +        }
> +        ret = vhost_user_scsi_realize_connect(s, errp);
> +    } while (ret < 0 && retries--);
> 
> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>     if (ret < 0) {
>         goto free_vhost;
>     }
> 
> +    /* we're fully initialized, now we can operate, so add the handler */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> +                             vhost_user_scsi_event, NULL, (void *)dev,
> +                             NULL, true);
>     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>     vsc->channel = 0;
>     vsc->lun = 0;
> @@ -135,8 +285,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     return;
> 
> free_vhost:
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
>     vhost_user_cleanup(&s->vhost_user);
> -    g_free(vqs);
> +
> free_virtio:
>     virtio_scsi_common_unrealize(dev);
> }
> @@ -146,16 +300,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> 
>     /* This will stop the vhost backend. */
>     vhost_user_scsi_set_status(vdev, 0);
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> +                             NULL, false);
> 
>     vhost_dev_cleanup(&vsc->dev);
> -    g_free(vqs);
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +
> +    vhost_dev_free_inflight(vsc->inflight);
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
> 
> -    virtio_scsi_common_unrealize(dev);
>     vhost_user_cleanup(&s->vhost_user);
> +    virtio_scsi_common_unrealize(dev);
> }
> 
> static Property vhost_user_scsi_properties[] = {
> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> index 521b08e559..b405ec952a 100644
> --- a/include/hw/virtio/vhost-user-scsi.h
> +++ b/include/hw/virtio/vhost-user-scsi.h
> @@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
> struct VHostUserSCSI {
>     VHostSCSICommon parent_obj;
>     VhostUserState vhost_user;
> +    bool connected;
> +    bool started_vu;
> +
> +    struct vhost_virtqueue *vhost_vqs;
> };
> 
> #endif /* VHOST_USER_SCSI_H */
> -- 
> 2.41.0
> 

Re: [PATCH v3 4/5] vhost-user-scsi: support reconnect to backend

[Markus Armbruster]

Li Feng <fengli@smartx.com> writes:

> If the backend crashes and restarts, the device is broken.
> This patch adds reconnect for vhost-user-scsi.
>
> Tested with spdk backend.
>
> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
>  hw/scsi/vhost-user-scsi.c           | 199 +++++++++++++++++++++++++---
>  include/hw/virtio/vhost-user-scsi.h |   4 +
>  2 files changed, 184 insertions(+), 19 deletions(-)
>
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index ee99b19e7a..5bf012461b 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -43,26 +43,54 @@ enum VhostUserProtocolFeature {
>      VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
>  };
>  
> +static int vhost_user_scsi_start(VHostUserSCSI *s)
> +{
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    int ret;
> +
> +    ret = vhost_scsi_common_start(vsc);
> +    s->started_vu = (ret < 0 ? false : true);
> +
> +    return ret;
> +}
> +
> +static void vhost_user_scsi_stop(VHostUserSCSI *s)
> +{
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +
> +    if (!s->started_vu) {
> +        return;
> +    }
> +    s->started_vu = false;
> +
> +    vhost_scsi_common_stop(vsc);
> +}
> +
>  static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
>  {
>      VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>      VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    bool should_start = virtio_device_should_start(vdev, status);
> +    int ret;
>  
> -    if (vhost_dev_is_started(&vsc->dev) == start) {
> +    if (!s->connected) {
>          return;
>      }
>  
> -    if (start) {
> -        int ret;
> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
> +        return;
> +    }
>  
> -        ret = vhost_scsi_common_start(vsc);
> +    if (should_start) {
> +        ret = vhost_user_scsi_start(s);
>          if (ret < 0) {
>              error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
> -            exit(1);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>          }
>      } else {
> -        vhost_scsi_common_stop(vsc);
> +        vhost_user_scsi_stop(s);
>      }
>  }
>  
> @@ -89,14 +117,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>  {
>  }
>  
> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret = 0;
> +
> +    if (s->connected) {
> +        return 0;
> +    }
> +    s->connected = true;
> +
> +    vsc->dev.num_queues = vs->conf.num_queues;
> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> +    vsc->dev.vqs = s->vhost_vqs;
> +    vsc->dev.vq_index = 0;
> +    vsc->dev.backend_features = 0;
> +
> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> +                         errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    /* restore vhost state */
> +    if (virtio_device_started(vdev, vdev->status)) {
> +        ret = vhost_user_scsi_start(s);
> +        if (ret < 0) {
> +            return ret;
> +        }

Fails without setting an error, violating the function's (tacit)
postcondition.  Callers:

* vhost_user_scsi_event()

  Dereferences the null error and crashes.

* vhost_user_scsi_realize_connect()

  Also fails without setting an error.  Caller:

  - vhost_user_scsi_realize()

    1. Doesn't emit the "Reconnecting after error: " message.  See
       also below.

    2. Succeeds instead of failing!

> +    }
> +
> +    return 0;
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> +
> +static void vhost_user_scsi_disconnect(DeviceState *dev)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +    s->connected = false;
> +
> +    vhost_user_scsi_stop(s);
> +
> +    vhost_dev_cleanup(&vsc->dev);
> +
> +    /* Re-instate the event handler for new connections */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> +{
> +    DeviceState *dev = opaque;
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    Error *local_err = NULL;
> +
> +    switch (event) {
> +    case CHR_EVENT_OPENED:
> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> +            error_report_err(local_err);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> +            return;
> +        }
> +        break;
> +    case CHR_EVENT_CLOSED:
> +        /* defer close until later to avoid circular close */
> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> +                               vhost_user_scsi_disconnect);
> +        break;
> +    case CHR_EVENT_BREAK:
> +    case CHR_EVENT_MUX_IN:
> +    case CHR_EVENT_MUX_OUT:
> +        /* Ignore */
> +        break;
> +    }
> +}
> +
> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> +{
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret;
> +
> +    s->connected = false;
> +
> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    ret = vhost_user_scsi_connect(dev, errp);
> +    if (ret < 0) {
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return ret;
> +    }
> +    assert(s->connected);
> +
> +    return 0;
> +}
> +
>  static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>  {
>      VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>      VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>      VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = NULL;
>      Error *err = NULL;
>      int ret;
> +    int retries = VU_REALIZE_CONN_RETRIES;
>  
>      if (!vs->conf.chardev.chr) {
>          error_setg(errp, "vhost-user-scsi: missing chardev");
> @@ -115,18 +255,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>          goto free_virtio;
>      }
>  
> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> -    vsc->dev.vq_index = 0;
> -    vsc->dev.backend_features = 0;
> -    vqs = vsc->dev.vqs;
> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> +
> +    assert(!*errp);

Dereferencing *errp is wrong.  Quoting qapi/error.h's big comment:

 * = Why, when and how to use ERRP_GUARD() =
 *
 * Without ERRP_GUARD(), use of the @errp parameter is restricted:
 * - It must not be dereferenced, because it may be null.
 * - It should not be passed to error_prepend() or
 *   error_append_hint(), because that doesn't work with &error_fatal.
 * ERRP_GUARD() lifts these restrictions.

See there for how to use ERRP_GUARD().

> +    do {
> +        if (*errp) {
> +            error_prepend(errp, "Reconnecting after error: ");
> +            error_report_err(*errp);
> +            *errp = NULL;
> +        }
> +        ret = vhost_user_scsi_realize_connect(s, errp);
> +    } while (ret < 0 && retries--);

Reports "Reconnecting ..." to the HMP monitor when in HMP context, else
to stderr.  I.e. reports to stderr when we're in QMP context.  Is this
really what we want?

>  
> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>      if (ret < 0) {
>          goto free_vhost;
>      }
>  
> +    /* we're fully initialized, now we can operate, so add the handler */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> +                             vhost_user_scsi_event, NULL, (void *)dev,
> +                             NULL, true);
>      /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>      vsc->channel = 0;
>      vsc->lun = 0;
> @@ -135,8 +285,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>      return;
>  
>  free_vhost:
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
>      vhost_user_cleanup(&s->vhost_user);
> -    g_free(vqs);
> +
>  free_virtio:
>      virtio_scsi_common_unrealize(dev);
>  }
> @@ -146,16 +300,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>      VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>      VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>      VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>  
>      /* This will stop the vhost backend. */
>      vhost_user_scsi_set_status(vdev, 0);
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> +                             NULL, false);
>  
>      vhost_dev_cleanup(&vsc->dev);
> -    g_free(vqs);
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +
> +    vhost_dev_free_inflight(vsc->inflight);
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
>  
> -    virtio_scsi_common_unrealize(dev);
>      vhost_user_cleanup(&s->vhost_user);
> +    virtio_scsi_common_unrealize(dev);
>  }
>  
>  static Property vhost_user_scsi_properties[] = {
> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> index 521b08e559..b405ec952a 100644
> --- a/include/hw/virtio/vhost-user-scsi.h
> +++ b/include/hw/virtio/vhost-user-scsi.h
> @@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>  struct VHostUserSCSI {
>      VHostSCSICommon parent_obj;
>      VhostUserState vhost_user;
> +    bool connected;
> +    bool started_vu;
> +
> +    struct vhost_virtqueue *vhost_vqs;
>  };
>  
>  #endif /* VHOST_USER_SCSI_H */

Re: [PATCH v3 4/5] vhost-user-scsi: support reconnect to backend

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 11543 bytes --]



> On 1 Sep 2023, at 8:00 PM, Markus Armbruster <armbru@redhat.com> wrote:
> 
> Li Feng <fengli@smartx.com <mailto:fengli@smartx.com>> writes:
> 
>> If the backend crashes and restarts, the device is broken.
>> This patch adds reconnect for vhost-user-scsi.
>> 
>> Tested with spdk backend.
>> 
>> Signed-off-by: Li Feng <fengli@smartx.com>
>> ---
>> hw/scsi/vhost-user-scsi.c           | 199 +++++++++++++++++++++++++---
>> include/hw/virtio/vhost-user-scsi.h |   4 +
>> 2 files changed, 184 insertions(+), 19 deletions(-)
>> 
>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>> index ee99b19e7a..5bf012461b 100644
>> --- a/hw/scsi/vhost-user-scsi.c
>> +++ b/hw/scsi/vhost-user-scsi.c
>> @@ -43,26 +43,54 @@ enum VhostUserProtocolFeature {
>>     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
>> };
>> 
>> +static int vhost_user_scsi_start(VHostUserSCSI *s)
>> +{
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    int ret;
>> +
>> +    ret = vhost_scsi_common_start(vsc);
>> +    s->started_vu = (ret < 0 ? false : true);
>> +
>> +    return ret;
>> +}
>> +
>> +static void vhost_user_scsi_stop(VHostUserSCSI *s)
>> +{
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +
>> +    if (!s->started_vu) {
>> +        return;
>> +    }
>> +    s->started_vu = false;
>> +
>> +    vhost_scsi_common_stop(vsc);
>> +}
>> +
>> static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
>> {
>>     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    bool should_start = virtio_device_should_start(vdev, status);
>> +    int ret;
>> 
>> -    if (vhost_dev_is_started(&vsc->dev) == start) {
>> +    if (!s->connected) {
>>         return;
>>     }
>> 
>> -    if (start) {
>> -        int ret;
>> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
>> +        return;
>> +    }
>> 
>> -        ret = vhost_scsi_common_start(vsc);
>> +    if (should_start) {
>> +        ret = vhost_user_scsi_start(s);
>>         if (ret < 0) {
>>             error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
>> -            exit(1);
>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>>         }
>>     } else {
>> -        vhost_scsi_common_stop(vsc);
>> +        vhost_user_scsi_stop(s);
>>     }
>> }
>> 
>> @@ -89,14 +117,126 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>> {
>> }
>> 
>> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>> +{
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    int ret = 0;
>> +
>> +    if (s->connected) {
>> +        return 0;
>> +    }
>> +    s->connected = true;
>> +
>> +    vsc->dev.num_queues = vs->conf.num_queues;
>> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>> +    vsc->dev.vqs = s->vhost_vqs;
>> +    vsc->dev.vq_index = 0;
>> +    vsc->dev.backend_features = 0;
>> +
>> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>> +                         errp);
>> +    if (ret < 0) {
>> +        return ret;
>> +    }
>> +
>> +    /* restore vhost state */
>> +    if (virtio_device_started(vdev, vdev->status)) {
>> +        ret = vhost_user_scsi_start(s);
>> +        if (ret < 0) {
>> +            return ret;
>> +        }
> 
> Fails without setting an error, violating the function's (tacit)
> postcondition.  Callers:
> 
> * vhost_user_scsi_event()
> 
>  Dereferences the null error and crashes.
> 
> * vhost_user_scsi_realize_connect()
> 
>  Also fails without setting an error.  Caller:
> 
>  - vhost_user_scsi_realize()
> 
>    1. Doesn't emit the "Reconnecting after error: " message.  See
>       also below.
> 
>    2. Succeeds instead of failing!
Ack.

> 
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
>> +
>> +static void vhost_user_scsi_disconnect(DeviceState *dev)
>> +{
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +
>> +    if (!s->connected) {
>> +        return;
>> +    }
>> +    s->connected = false;
>> +
>> +    vhost_user_scsi_stop(s);
>> +
>> +    vhost_dev_cleanup(&vsc->dev);
>> +
>> +    /* Re-instate the event handler for new connections */
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
>> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
>> +}
>> +
>> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>> +{
>> +    DeviceState *dev = opaque;
>> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    Error *local_err = NULL;
>> +
>> +    switch (event) {
>> +    case CHR_EVENT_OPENED:
>> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
>> +            error_report_err(local_err);
>> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +            return;
>> +        }
>> +        break;
>> +    case CHR_EVENT_CLOSED:
>> +        /* defer close until later to avoid circular close */
>> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
>> +                               vhost_user_scsi_disconnect);
>> +        break;
>> +    case CHR_EVENT_BREAK:
>> +    case CHR_EVENT_MUX_IN:
>> +    case CHR_EVENT_MUX_OUT:
>> +        /* Ignore */
>> +        break;
>> +    }
>> +}
>> +
>> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
>> +{
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +    int ret;
>> +
>> +    s->connected = false;
>> +
>> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
>> +    if (ret < 0) {
>> +        return ret;
>> +    }
>> +
>> +    ret = vhost_user_scsi_connect(dev, errp);
>> +    if (ret < 0) {
>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +        return ret;
>> +    }
>> +    assert(s->connected);
>> +
>> +    return 0;
>> +}
>> +
>> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>> {
>>     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    struct vhost_virtqueue *vqs = NULL;
>>     Error *err = NULL;
>>     int ret;
>> +    int retries = VU_REALIZE_CONN_RETRIES;
>> 
>>     if (!vs->conf.chardev.chr) {
>>         error_setg(errp, "vhost-user-scsi: missing chardev");
>> @@ -115,18 +255,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>         goto free_virtio;
>>     }
>> 
>> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
>> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
>> -    vsc->dev.vq_index = 0;
>> -    vsc->dev.backend_features = 0;
>> -    vqs = vsc->dev.vqs;
>> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
>> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
>> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
>> +
>> +    assert(!*errp);
> 
> Dereferencing *errp is wrong.  Quoting qapi/error.h's big comment:
> 
> * = Why, when and how to use ERRP_GUARD() =
> *
> * Without ERRP_GUARD(), use of the @errp parameter is restricted:
> * - It must not be dereferenced, because it may be null.
> * - It should not be passed to error_prepend() or
> *   error_append_hint(), because that doesn't work with &error_fatal.
> * ERRP_GUARD() lifts these restrictions.
> 
> See there for how to use ERRP_GUARD().

Thanks, good catch. I have understood this knowledge.
> 
>> +    do {
>> +        if (*errp) {
>> +            error_prepend(errp, "Reconnecting after error: ");
>> +            error_report_err(*errp);
>> +            *errp = NULL;
>> +        }
>> +        ret = vhost_user_scsi_realize_connect(s, errp);
>> +    } while (ret < 0 && retries--);
> 
> Reports "Reconnecting ..." to the HMP monitor when in HMP context, else
> to stderr.  I.e. reports to stderr when we're in QMP context.  Is this
> really what we want?
The vhost-user-blk has the same code, so just keep it the same here is a
good idea?

> 
>> 
>> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
>> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>>     if (ret < 0) {
>>         goto free_vhost;
>>     }
>> 
>> +    /* we're fully initialized, now we can operate, so add the handler */
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
>> +                             vhost_user_scsi_event, NULL, (void *)dev,
>> +                             NULL, true);
>>     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>>     vsc->channel = 0;
>>     vsc->lun = 0;
>> @@ -135,8 +285,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>     return;
>> 
>> free_vhost:
>> +    g_free(s->vhost_vqs);
>> +    s->vhost_vqs = NULL;
>> +    g_free(vsc->inflight);
>> +    vsc->inflight = NULL;
>>     vhost_user_cleanup(&s->vhost_user);
>> -    g_free(vqs);
>> +
>> free_virtio:
>>     virtio_scsi_common_unrealize(dev);
>> }
>> @@ -146,16 +300,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>>     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> 
>>     /* This will stop the vhost backend. */
>>     vhost_user_scsi_set_status(vdev, 0);
>> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
>> +                             NULL, false);
>> 
>>     vhost_dev_cleanup(&vsc->dev);
>> -    g_free(vqs);
>> +    g_free(s->vhost_vqs);
>> +    s->vhost_vqs = NULL;
>> +
>> +    vhost_dev_free_inflight(vsc->inflight);
>> +    g_free(vsc->inflight);
>> +    vsc->inflight = NULL;
>> 
>> -    virtio_scsi_common_unrealize(dev);
>>     vhost_user_cleanup(&s->vhost_user);
>> +    virtio_scsi_common_unrealize(dev);
>> }
>> 
>> static Property vhost_user_scsi_properties[] = {
>> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
>> index 521b08e559..b405ec952a 100644
>> --- a/include/hw/virtio/vhost-user-scsi.h
>> +++ b/include/hw/virtio/vhost-user-scsi.h
>> @@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
>> struct VHostUserSCSI {
>>     VHostSCSICommon parent_obj;
>>     VhostUserState vhost_user;
>> +    bool connected;
>> +    bool started_vu;
>> +
>> +    struct vhost_virtqueue *vhost_vqs;
>> };
>> 
>> #endif /* VHOST_USER_SCSI_H */


[-- Attachment #2: Type: text/html, Size: 40301 bytes --]

[PATCH v3 5/5] vhost-user-scsi: start vhost when guest kicks

[Li Feng]

Let's keep the same behavior as vhost-user-blk.

Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
 1 file changed, 44 insertions(+), 4 deletions(-)

diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index 5bf012461b..a7fa8e8df2 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -113,8 +113,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
     }
 }
 
-static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
+    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    Error *local_err = NULL;
+    int i, ret;
+
+    if (!vdev->start_on_kick) {
+        return;
+    }
+
+    if (!s->connected) {
+        return;
+    }
+
+    if (vhost_dev_is_started(&vsc->dev)) {
+        return;
+    }
+
+    /*
+     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * vhost here instead of waiting for .set_status().
+     */
+    ret = vhost_user_scsi_start(s);
+    if (ret < 0) {
+        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return;
+    }
+
+    /* Kick right away to begin processing requests already in vring */
+    for (i = 0; i < vsc->dev.nvqs; i++) {
+        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
+
+        if (!virtio_queue_get_desc_addr(vdev, i)) {
+            continue;
+        }
+        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
+    }
 }
 
 static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
@@ -243,9 +283,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
-                               vhost_dummy_handle_output,
-                               vhost_dummy_handle_output, &err);
+    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         return;
-- 
2.41.0

Re: [PATCH v3 5/5] vhost-user-scsi: start vhost when guest kicks

[Raphael Norwitz]

> On Jul 31, 2023, at 8:10 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Let's keep the same behavior as vhost-user-blk.
> 
> Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> ---
> hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
> 1 file changed, 44 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index 5bf012461b..a7fa8e8df2 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -113,8 +113,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>     }
> }
> 
> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    Error *local_err = NULL;
> +    int i, ret;
> +
> +    if (!vdev->start_on_kick) {
> +        return;
> +    }
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +
> +    if (vhost_dev_is_started(&vsc->dev)) {
> +        return;
> +    }
> +
> +    /*
> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
> +     * vhost here instead of waiting for .set_status().
> +     */
> +    ret = vhost_user_scsi_start(s);
> +    if (ret < 0) {
> +        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return;
> +    }
> +
> +    /* Kick right away to begin processing requests already in vring */
> +    for (i = 0; i < vsc->dev.nvqs; i++) {
> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
> +
> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
> +            continue;
> +        }
> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
> +    }
> }
> 
> static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> @@ -243,9 +283,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>         return;
>     }
> 
> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output, &err);
> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output, &err);
>     if (err != NULL) {
>         error_propagate(errp, err);
>         return;
> -- 
> 2.41.0
> 

Re: [PATCH v3 5/5] vhost-user-scsi: start vhost when guest kicks

[Markus Armbruster]

Li Feng <fengli@smartx.com> writes:

> Let's keep the same behavior as vhost-user-blk.
>
> Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.
>
> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
>  hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
>  1 file changed, 44 insertions(+), 4 deletions(-)
>
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index 5bf012461b..a7fa8e8df2 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -113,8 +113,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>      }
>  }
>  
> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>  {
> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    Error *local_err = NULL;
> +    int i, ret;
> +
> +    if (!vdev->start_on_kick) {
> +        return;
> +    }
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +
> +    if (vhost_dev_is_started(&vsc->dev)) {
> +        return;
> +    }
> +
> +    /*
> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
> +     * vhost here instead of waiting for .set_status().
> +     */
> +    ret = vhost_user_scsi_start(s);
> +    if (ret < 0) {
> +        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");

Crashes, since @local_err is null.  Please test your error paths.

Obvious fix: drop this call.

> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return;
> +    }
> +
> +    /* Kick right away to begin processing requests already in vring */
> +    for (i = 0; i < vsc->dev.nvqs; i++) {
> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
> +
> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
> +            continue;
> +        }
> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
> +    }
>  }
>  
>  static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> @@ -243,9 +283,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>          return;
>      }
>  
> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output, &err);
> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output, &err);
>      if (err != NULL) {
>          error_propagate(errp, err);
>          return;

Re: [PATCH v3 5/5] vhost-user-scsi: start vhost when guest kicks

[Li Feng]

[-- Attachment #1: Type: text/plain, Size: 3238 bytes --]



> On 1 Sep 2023, at 7:44 PM, Markus Armbruster <armbru@redhat.com> wrote:
> 
> Li Feng <fengli@smartx.com <mailto:fengli@smartx.com>> writes:
> 
>> Let's keep the same behavior as vhost-user-blk.
>> 
>> Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.
>> 
>> Signed-off-by: Li Feng <fengli@smartx.com>
>> ---
>> hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
>> 1 file changed, 44 insertions(+), 4 deletions(-)
>> 
>> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
>> index 5bf012461b..a7fa8e8df2 100644
>> --- a/hw/scsi/vhost-user-scsi.c
>> +++ b/hw/scsi/vhost-user-scsi.c
>> @@ -113,8 +113,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>>     }
>> }
>> 
>> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
>> {
>> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
>> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>> +
>> +    Error *local_err = NULL;
>> +    int i, ret;
>> +
>> +    if (!vdev->start_on_kick) {
>> +        return;
>> +    }
>> +
>> +    if (!s->connected) {
>> +        return;
>> +    }
>> +
>> +    if (vhost_dev_is_started(&vsc->dev)) {
>> +        return;
>> +    }
>> +
>> +    /*
>> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
>> +     * vhost here instead of waiting for .set_status().
>> +     */
>> +    ret = vhost_user_scsi_start(s);
>> +    if (ret < 0) {
>> +        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
> 
> Crashes, since @local_err is null.  Please test your error paths.
> 
> Obvious fix: drop this call.
Emmm, actually I have tested the error path, so I find some issues that I have fixed
in the following patches.
I will merge the later series into this series.


> 
>> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
>> +        return;
>> +    }
>> +
>> +    /* Kick right away to begin processing requests already in vring */
>> +    for (i = 0; i < vsc->dev.nvqs; i++) {
>> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
>> +
>> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
>> +            continue;
>> +        }
>> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
>> +    }
>> }
>> 
>> static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
>> @@ -243,9 +283,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>>         return;
>>     }
>> 
>> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
>> -                               vhost_dummy_handle_output,
>> -                               vhost_dummy_handle_output, &err);
>> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
>> +                               vhost_user_scsi_handle_output,
>> +                               vhost_user_scsi_handle_output, &err);
>>     if (err != NULL) {
>>         error_propagate(errp, err);
>>         return;


[-- Attachment #2: Type: text/html, Size: 9924 bytes --]

[PATCH v4 0/5] Implement reconnect for vhost-user-scsi

[Li Feng]

This patchset adds reconnect support for vhost-user-scsi. 
At the same time, improve the error messages and silent errors are now reported.
And fix a lost reconnect issue for all vhost-user backend.

Changes for v4:
- Merge
  https://lore.kernel.org/all/20230830045722.611224-1-fengli@smartx.com/ to
  this series.
- Add ERRP_GUARD in vhost_user_scsi_realize;
- Reword the commit messages.

Changes for v3:
- Split the vhost_user_scsi_handle_output to a separate patch;
- Move the started_vu from vhost scsi common header to vhost-user-scsi header;
- Fix a log print error;

Changes for v2:
- Split the v1 patch to small separate patchset;
- New patch for fixing fd leak, which has sent to reviewers in another
  mail;
- Implement the `vhost_user_scsi_handle_output`;
- Add the started_vu safe check;
- Fix error handler;
- Check the inflight before set/get inflight fd.

Li Feng (5):
  vhost-user-common: send get_inflight_fd once
  vhost: move and rename the conn retry times
  vhost-user-scsi: support reconnect to backend
  vhost-user-scsi: start vhost when guest kicks
  vhost-user: fix lost reconnect

 hw/block/vhost-user-blk.c             |   6 +-
 hw/scsi/vhost-scsi-common.c           |  47 ++---
 hw/scsi/vhost-scsi.c                  |   5 +-
 hw/scsi/vhost-user-scsi.c             | 253 +++++++++++++++++++++++---
 hw/virtio/vhost-user-gpio.c           |   5 +-
 hw/virtio/vhost-user.c                |   9 +-
 include/hw/virtio/vhost-scsi-common.h |   2 +-
 include/hw/virtio/vhost-user-scsi.h   |   4 +
 include/hw/virtio/vhost-user.h        |   3 +-
 include/hw/virtio/vhost.h             |   2 +
 10 files changed, 276 insertions(+), 60 deletions(-)

-- 
2.41.0

[PATCH v4 1/5] vhost-user-common: send get_inflight_fd once

[Li Feng]

Currently the get_inflight_fd will be sent every time the device is started, and
the backend will allocate shared memory to save the inflight state. If the
backend finds that it receives the second get_inflight_fd, it will release the
previous shared memory, which breaks inflight working logic.

This patch is a preparation for the following patches.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a06f01af26..a61cd0e907 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     vsc->dev.acked_features = vdev->guest_features;
 
-    assert(vsc->inflight == NULL);
-    vsc->inflight = g_new0(struct vhost_inflight, 1);
-    ret = vhost_dev_get_inflight(&vsc->dev,
-                                 vs->conf.virtqueue_size,
-                                 vsc->inflight);
+    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error get inflight: %d", -ret);
+        error_report("Error setting inflight format: %d", -ret);
         goto err_guest_notifiers;
     }
 
-    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
-    if (ret < 0) {
-        error_report("Error set inflight: %d", -ret);
-        goto err_guest_notifiers;
+    if (vsc->inflight) {
+        if (!vsc->inflight->addr) {
+            ret = vhost_dev_get_inflight(&vsc->dev,
+                                        vs->conf.virtqueue_size,
+                                        vsc->inflight);
+            if (ret < 0) {
+                error_report("Error getting inflight: %d", -ret);
+                goto err_guest_notifiers;
+            }
+        }
+
+        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
+        if (ret < 0) {
+            error_report("Error setting inflight: %d", -ret);
+            goto err_guest_notifiers;
+        }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
@@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     return ret;
 
 err_guest_notifiers:
-    g_free(vsc->inflight);
-    vsc->inflight = NULL;
-
     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
 err_host_notifiers:
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
@@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     }
     assert(ret >= 0);
 
-    if (vsc->inflight) {
-        vhost_dev_free_inflight(vsc->inflight);
-        g_free(vsc->inflight);
-        vsc->inflight = NULL;
-    }
-
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
 }
 
-- 
2.41.0

[PATCH v4 2/5] vhost: move and rename the conn retry times

[Li Feng]

Multiple devices need this macro, move it to a common header.

Signed-off-by: Li Feng <fengli@smartx.com>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
---
 hw/block/vhost-user-blk.c   | 4 +---
 hw/virtio/vhost-user-gpio.c | 3 +--
 include/hw/virtio/vhost.h   | 2 ++
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index eecf3f7a81..3c69fa47d5 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -32,8 +32,6 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
-
 static const int user_feature_bits[] = {
     VIRTIO_BLK_F_SIZE_MAX,
     VIRTIO_BLK_F_SEG_MAX,
@@ -482,7 +480,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
     s->inflight = g_new0(struct vhost_inflight, 1);
     s->vhost_vqs = g_new0(struct vhost_virtqueue, s->num_queues);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     assert(!*errp);
     do {
         if (*errp) {
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index 3b013f2d0f..d9979aa5db 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -15,7 +15,6 @@
 #include "standard-headers/linux/virtio_ids.h"
 #include "trace.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
 #define VHOST_NVQS 2
 
 /* Features required from VirtIO */
@@ -359,7 +358,7 @@ static void vu_gpio_device_realize(DeviceState *dev, Error **errp)
     qemu_chr_fe_set_handlers(&gpio->chardev, NULL, NULL, vu_gpio_event, NULL,
                              dev, NULL, true);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     g_assert(!*errp);
     do {
         if (*errp) {
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 6a173cb9fa..ca3131b1af 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -8,6 +8,8 @@
 #define VHOST_F_DEVICE_IOTLB 63
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+#define VU_REALIZE_CONN_RETRIES 3
+
 /* Generic structures common for any vhost based device. */
 
 struct vhost_inflight {
-- 
2.41.0

[PATCH v4 3/5] vhost-user-scsi: support reconnect to backend

[Li Feng]

If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

This patch also improves the error messages, and reports some silent errors.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c           |  16 +-
 hw/scsi/vhost-scsi.c                  |   5 +-
 hw/scsi/vhost-user-scsi.c             | 204 +++++++++++++++++++++++---
 include/hw/virtio/vhost-scsi-common.h |   2 +-
 include/hw/virtio/vhost-user-scsi.h   |   4 +
 5 files changed, 201 insertions(+), 30 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a61cd0e907..4c8637045d 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -16,6 +16,7 @@
  */
 
 #include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "hw/virtio/vhost.h"
@@ -25,7 +26,7 @@
 #include "hw/virtio/virtio-access.h"
 #include "hw/fw-path-provider.h"
 
-int vhost_scsi_common_start(VHostSCSICommon *vsc)
+int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp)
 {
     int ret, i;
     VirtIODevice *vdev = VIRTIO_DEVICE(vsc);
@@ -35,18 +36,19 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     VirtIOSCSICommon *vs = (VirtIOSCSICommon *)vsc;
 
     if (!k->set_guest_notifiers) {
-        error_report("binding does not support guest notifiers");
+        error_setg(errp, "binding does not support guest notifiers");
         return -ENOSYS;
     }
 
     ret = vhost_dev_enable_notifiers(&vsc->dev, vdev);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Error enabling host notifiers");
         return ret;
     }
 
     ret = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, true);
     if (ret < 0) {
-        error_report("Error binding guest notifier");
+        error_setg_errno(errp, -ret, "Error binding guest notifier");
         goto err_host_notifiers;
     }
 
@@ -54,7 +56,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error setting inflight format: %d", -ret);
+        error_setg_errno(errp, -ret, "Error setting inflight format");
         goto err_guest_notifiers;
     }
 
@@ -64,21 +66,21 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
                                         vs->conf.virtqueue_size,
                                         vsc->inflight);
             if (ret < 0) {
-                error_report("Error getting inflight: %d", -ret);
+                error_setg_errno(errp, -ret, "Error getting inflight");
                 goto err_guest_notifiers;
             }
         }
 
         ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
         if (ret < 0) {
-            error_report("Error setting inflight: %d", -ret);
+            error_setg_errno(errp, -ret, "Error setting inflight");
             goto err_guest_notifiers;
         }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
     if (ret < 0) {
-        error_report("Error start vhost dev");
+        error_setg_errno(errp, -ret, "Error starting vhost dev");
         goto err_guest_notifiers;
     }
 
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 443f67daa4..01a3ab4277 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -75,6 +75,7 @@ static int vhost_scsi_start(VHostSCSI *s)
     int ret, abi_version;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
     const VhostOps *vhost_ops = vsc->dev.vhost_ops;
+    Error *local_err = NULL;
 
     ret = vhost_ops->vhost_scsi_get_abi_version(&vsc->dev, &abi_version);
     if (ret < 0) {
@@ -88,14 +89,14 @@ static int vhost_scsi_start(VHostSCSI *s)
         return -ENOSYS;
     }
 
-    ret = vhost_scsi_common_start(vsc);
+    ret = vhost_scsi_common_start(vsc, &local_err);
     if (ret < 0) {
         return ret;
     }
 
     ret = vhost_scsi_set_endpoint(s);
     if (ret < 0) {
-        error_report("Error setting vhost-scsi endpoint");
+        error_reportf_err(local_err, "Error setting vhost-scsi endpoint");
         vhost_scsi_common_stop(vsc);
     }
 
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index ee99b19e7a..dc109154ad 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -43,26 +43,56 @@ enum VhostUserProtocolFeature {
     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
 };
 
+static int vhost_user_scsi_start(VHostUserSCSI *s, Error **errp)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    int ret;
+
+    ret = vhost_scsi_common_start(vsc, errp);
+    s->started_vu = (ret < 0 ? false : true);
+
+    return ret;
+}
+
+static void vhost_user_scsi_stop(VHostUserSCSI *s)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+
+    if (!s->started_vu) {
+        return;
+    }
+    s->started_vu = false;
+
+    vhost_scsi_common_stop(vsc);
+}
+
 static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    bool should_start = virtio_device_should_start(vdev, status);
+    Error *local_err = NULL;
+    int ret;
 
-    if (vhost_dev_is_started(&vsc->dev) == start) {
+    if (!s->connected) {
         return;
     }
 
-    if (start) {
-        int ret;
+    if (vhost_dev_is_started(&vsc->dev) == should_start) {
+        return;
+    }
 
-        ret = vhost_scsi_common_start(vsc);
+    if (should_start) {
+        ret = vhost_user_scsi_start(s, &local_err);
         if (ret < 0) {
-            error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
-            exit(1);
+            error_reportf_err(local_err, "unable to start vhost-user-scsi: %s",
+                              strerror(-ret));
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
         }
     } else {
-        vhost_scsi_common_stop(vsc);
+        vhost_user_scsi_stop(s);
     }
 }
 
@@ -89,14 +119,127 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
 }
 
+static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret = 0;
+
+    if (s->connected) {
+        return 0;
+    }
+    s->connected = true;
+
+    vsc->dev.num_queues = vs->conf.num_queues;
+    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
+    vsc->dev.vqs = s->vhost_vqs;
+    vsc->dev.vq_index = 0;
+    vsc->dev.backend_features = 0;
+
+    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
+                         errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* restore vhost state */
+    if (virtio_device_started(vdev, vdev->status)) {
+        ret = vhost_user_scsi_start(s, errp);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
+
+static void vhost_user_scsi_disconnect(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    if (!s->connected) {
+        return;
+    }
+    s->connected = false;
+
+    vhost_user_scsi_stop(s);
+
+    vhost_dev_cleanup(&vsc->dev);
+
+    /* Re-instate the event handler for new connections */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
+                             vhost_user_scsi_event, NULL, dev, NULL, true);
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
+{
+    DeviceState *dev = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    Error *local_err = NULL;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
+            error_report_err(local_err);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
+            return;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        /* defer close until later to avoid circular close */
+        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
+                               vhost_user_scsi_disconnect);
+        break;
+    case CHR_EVENT_BREAK:
+    case CHR_EVENT_MUX_IN:
+    case CHR_EVENT_MUX_OUT:
+        /* Ignore */
+        break;
+    }
+}
+
+static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
+{
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret;
+
+    s->connected = false;
+
+    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_user_scsi_connect(dev, errp);
+    if (ret < 0) {
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return ret;
+    }
+    assert(s->connected);
+
+    return 0;
+}
+
 static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
 {
+    ERRP_GUARD();
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = NULL;
     Error *err = NULL;
     int ret;
+    int retries = VU_REALIZE_CONN_RETRIES;
 
     if (!vs->conf.chardev.chr) {
         error_setg(errp, "vhost-user-scsi: missing chardev");
@@ -115,18 +258,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         goto free_virtio;
     }
 
-    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
-    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
-    vsc->dev.vq_index = 0;
-    vsc->dev.backend_features = 0;
-    vqs = vsc->dev.vqs;
+    vsc->inflight = g_new0(struct vhost_inflight, 1);
+    s->vhost_vqs = g_new0(struct vhost_virtqueue,
+                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
+
+    assert(!*errp);
+    do {
+        if (*errp) {
+            error_prepend(errp, "Reconnecting after error: ");
+            error_report_err(*errp);
+            *errp = NULL;
+        }
+        ret = vhost_user_scsi_realize_connect(s, errp);
+    } while (ret < 0 && retries--);
 
-    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
-                         VHOST_BACKEND_TYPE_USER, 0, errp);
     if (ret < 0) {
         goto free_vhost;
     }
 
+    /* we're fully initialized, now we can operate, so add the handler */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
+                             vhost_user_scsi_event, NULL, (void *)dev,
+                             NULL, true);
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -135,8 +288,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     return;
 
 free_vhost:
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
     vhost_user_cleanup(&s->vhost_user);
-    g_free(vqs);
+
 free_virtio:
     virtio_scsi_common_unrealize(dev);
 }
@@ -146,16 +303,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = vsc->dev.vqs;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
 
     /* This will stop the vhost backend. */
     vhost_user_scsi_set_status(vdev, 0);
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
+                             NULL, false);
 
     vhost_dev_cleanup(&vsc->dev);
-    g_free(vqs);
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+
+    vhost_dev_free_inflight(vsc->inflight);
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
 
-    virtio_scsi_common_unrealize(dev);
     vhost_user_cleanup(&s->vhost_user);
+    virtio_scsi_common_unrealize(dev);
 }
 
 static Property vhost_user_scsi_properties[] = {
diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
index 18f115527c..c5d2c09455 100644
--- a/include/hw/virtio/vhost-scsi-common.h
+++ b/include/hw/virtio/vhost-scsi-common.h
@@ -39,7 +39,7 @@ struct VHostSCSICommon {
     struct vhost_inflight *inflight;
 };
 
-int vhost_scsi_common_start(VHostSCSICommon *vsc);
+int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp);
 void vhost_scsi_common_stop(VHostSCSICommon *vsc);
 char *vhost_scsi_common_get_fw_dev_path(FWPathProvider *p, BusState *bus,
                                         DeviceState *dev);
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 521b08e559..b405ec952a 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
 struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     VhostUserState vhost_user;
+    bool connected;
+    bool started_vu;
+
+    struct vhost_virtqueue *vhost_vqs;
 };
 
 #endif /* VHOST_USER_SCSI_H */
-- 
2.41.0

[PATCH v4 4/5] vhost-user-scsi: start vhost when guest kicks

[Li Feng]

Let's keep the same behavior as vhost-user-blk.

Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-user-scsi.c | 51 +++++++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index dc109154ad..0effbb4787 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -115,8 +115,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
     }
 }
 
-static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
+    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    Error *local_err = NULL;
+    int i, ret;
+
+    if (!vdev->start_on_kick) {
+        return;
+    }
+
+    if (!s->connected) {
+        return;
+    }
+
+    if (vhost_dev_is_started(&vsc->dev)) {
+        return;
+    }
+
+    /*
+     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * vhost here instead of waiting for .set_status().
+     */
+    ret = vhost_user_scsi_start(s, &local_err);
+    if (ret < 0) {
+        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return;
+    }
+
+    /* Kick right away to begin processing requests already in vring */
+    for (i = 0; i < vsc->dev.nvqs; i++) {
+        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
+
+        if (!virtio_queue_get_desc_addr(vdev, i)) {
+            continue;
+        }
+        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
+    }
 }
 
 static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
@@ -198,7 +238,8 @@ static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
-                               vhost_user_scsi_disconnect);
+                               vhost_user_scsi_disconnect,
+                               vhost_user_scsi_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
@@ -246,9 +287,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
-                               vhost_dummy_handle_output,
-                               vhost_dummy_handle_output, &err);
+    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         return;
-- 
2.41.0

[PATCH v4 5/5] vhost-user: fix lost reconnect

[Li Feng]

When the vhost-user is reconnecting to the backend, and if the vhost-user fails
at the get_features in vhost_dev_init(), then the reconnect will fail
and it will not be retriggered forever.

The reason is:
When the vhost-user fails at get_features, the vhost_dev_cleanup will be called
immediately.

vhost_dev_cleanup calls 'memset(hdev, 0, sizeof(struct vhost_dev))'.

The reconnect path is:
vhost_user_blk_event
   vhost_user_async_close(.. vhost_user_blk_disconnect ..)
     qemu_chr_fe_set_handlers <----- clear the notifier callback
       schedule vhost_user_async_close_bh

The vhost->vdev is null, so the vhost_user_blk_disconnect will not be
called, then the event fd callback will not be reinstalled.

All vhost-user devices have this issue, including vhost-user-blk/scsi.

With this patch, if the vdev->vdev is null, the fd callback will still
be reinstalled.

Fixes: 71e076a07d ("hw/virtio: generalise CHR_EVENT_CLOSED handling")

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/block/vhost-user-blk.c      | 2 +-
 hw/virtio/vhost-user-gpio.c    | 2 +-
 hw/virtio/vhost-user.c         | 9 +++++++--
 include/hw/virtio/vhost-user.h | 3 ++-
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index 3c69fa47d5..95c758200d 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -391,7 +391,7 @@ static void vhost_user_blk_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &s->chardev, &s->dev,
-                               vhost_user_blk_disconnect);
+                               vhost_user_blk_disconnect, vhost_user_blk_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index d9979aa5db..04c2cc79f4 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -283,7 +283,7 @@ static void vu_gpio_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &gpio->chardev, &gpio->vhost_dev,
-                               vu_gpio_disconnect);
+                               vu_gpio_disconnect, vu_gpio_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 8dcf049d42..12c4a41f30 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -2643,6 +2643,7 @@ typedef struct {
     DeviceState *dev;
     CharBackend *cd;
     struct vhost_dev *vhost;
+    IOEventHandler *event_cb;
 } VhostAsyncCallback;
 
 static void vhost_user_async_close_bh(void *opaque)
@@ -2657,7 +2658,10 @@ static void vhost_user_async_close_bh(void *opaque)
      */
     if (vhost->vdev) {
         data->cb(data->dev);
-    }
+    } else if (data->event_cb) {
+        qemu_chr_fe_set_handlers(data->cd, NULL, NULL, data->event_cb,
+                                 NULL, data->dev, NULL, true);
+   }
 
     g_free(data);
 }
@@ -2669,7 +2673,8 @@ static void vhost_user_async_close_bh(void *opaque)
  */
 void vhost_user_async_close(DeviceState *d,
                             CharBackend *chardev, struct vhost_dev *vhost,
-                            vu_async_close_fn cb)
+                            vu_async_close_fn cb,
+                            IOEventHandler *event_cb)
 {
     if (!runstate_check(RUN_STATE_SHUTDOWN)) {
         /*
diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
index 191216a74f..649e9dd54f 100644
--- a/include/hw/virtio/vhost-user.h
+++ b/include/hw/virtio/vhost-user.h
@@ -84,6 +84,7 @@ typedef void (*vu_async_close_fn)(DeviceState *cb);
 
 void vhost_user_async_close(DeviceState *d,
                             CharBackend *chardev, struct vhost_dev *vhost,
-                            vu_async_close_fn cb);
+                            vu_async_close_fn cb,
+                            IOEventHandler *event_cb);
 
 #endif
-- 
2.41.0

[PATCH v5 0/5] Implement reconnect for vhost-user-scsi

[Li Feng]

Ping ...
Could anyone review this series and merge them?

Changes for v5:
- No logic has been changed, just move part of the code from patch 4 to patch 5.

Changes for v4:
- Merge
  https://lore.kernel.org/all/20230830045722.611224-1-fengli@smartx.com/ to
  this series.
- Add ERRP_GUARD in vhost_user_scsi_realize;
- Reword the commit messages.

Changes for v3:
- Split the vhost_user_scsi_handle_output to a separate patch;
- Move the started_vu from vhost scsi common header to vhost-user-scsi header;
- Fix a log print error;

Changes for v2:
- Split the v1 patch to small separate patchset;
- New patch for fixing fd leak, which has sent to reviewers in another
  mail;
- Implement the `vhost_user_scsi_handle_output`;
- Add the started_vu safe check;
- Fix error handler;
- Check the inflight before set/get inflight fd.

Li Feng (5):
  vhost-user-common: send get_inflight_fd once
  vhost: move and rename the conn retry times
  vhost-user-scsi: support reconnect to backend
  vhost-user-scsi: start vhost when guest kicks
  vhost-user: fix lost reconnect

 hw/block/vhost-user-blk.c             |   6 +-
 hw/scsi/vhost-scsi-common.c           |  47 ++---
 hw/scsi/vhost-scsi.c                  |   5 +-
 hw/scsi/vhost-user-scsi.c             | 253 +++++++++++++++++++++++---
 hw/virtio/vhost-user-gpio.c           |   5 +-
 hw/virtio/vhost-user.c                |   9 +-
 include/hw/virtio/vhost-scsi-common.h |   2 +-
 include/hw/virtio/vhost-user-scsi.h   |   4 +
 include/hw/virtio/vhost-user.h        |   3 +-
 include/hw/virtio/vhost.h             |   2 +
 10 files changed, 276 insertions(+), 60 deletions(-)

-- 
2.41.0

[PATCH v5 1/5] vhost-user-common: send get_inflight_fd once

[Li Feng]

Currently the get_inflight_fd will be sent every time the device is started, and
the backend will allocate shared memory to save the inflight state. If the
backend finds that it receives the second get_inflight_fd, it will release the
previous shared memory, which breaks inflight working logic.

This patch is a preparation for the following patches.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a06f01af26..a61cd0e907 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     vsc->dev.acked_features = vdev->guest_features;
 
-    assert(vsc->inflight == NULL);
-    vsc->inflight = g_new0(struct vhost_inflight, 1);
-    ret = vhost_dev_get_inflight(&vsc->dev,
-                                 vs->conf.virtqueue_size,
-                                 vsc->inflight);
+    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error get inflight: %d", -ret);
+        error_report("Error setting inflight format: %d", -ret);
         goto err_guest_notifiers;
     }
 
-    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
-    if (ret < 0) {
-        error_report("Error set inflight: %d", -ret);
-        goto err_guest_notifiers;
+    if (vsc->inflight) {
+        if (!vsc->inflight->addr) {
+            ret = vhost_dev_get_inflight(&vsc->dev,
+                                        vs->conf.virtqueue_size,
+                                        vsc->inflight);
+            if (ret < 0) {
+                error_report("Error getting inflight: %d", -ret);
+                goto err_guest_notifiers;
+            }
+        }
+
+        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
+        if (ret < 0) {
+            error_report("Error setting inflight: %d", -ret);
+            goto err_guest_notifiers;
+        }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
@@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     return ret;
 
 err_guest_notifiers:
-    g_free(vsc->inflight);
-    vsc->inflight = NULL;
-
     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
 err_host_notifiers:
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
@@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     }
     assert(ret >= 0);
 
-    if (vsc->inflight) {
-        vhost_dev_free_inflight(vsc->inflight);
-        g_free(vsc->inflight);
-        vsc->inflight = NULL;
-    }
-
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
 }
 
-- 
2.41.0

[PATCH v5 2/5] vhost: move and rename the conn retry times

[Li Feng]

Multiple devices need this macro, move it to a common header.

Signed-off-by: Li Feng <fengli@smartx.com>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
---
 hw/block/vhost-user-blk.c   | 4 +---
 hw/virtio/vhost-user-gpio.c | 3 +--
 include/hw/virtio/vhost.h   | 2 ++
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index eecf3f7a81..3c69fa47d5 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -32,8 +32,6 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
-
 static const int user_feature_bits[] = {
     VIRTIO_BLK_F_SIZE_MAX,
     VIRTIO_BLK_F_SEG_MAX,
@@ -482,7 +480,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
     s->inflight = g_new0(struct vhost_inflight, 1);
     s->vhost_vqs = g_new0(struct vhost_virtqueue, s->num_queues);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     assert(!*errp);
     do {
         if (*errp) {
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index 3b013f2d0f..d9979aa5db 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -15,7 +15,6 @@
 #include "standard-headers/linux/virtio_ids.h"
 #include "trace.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
 #define VHOST_NVQS 2
 
 /* Features required from VirtIO */
@@ -359,7 +358,7 @@ static void vu_gpio_device_realize(DeviceState *dev, Error **errp)
     qemu_chr_fe_set_handlers(&gpio->chardev, NULL, NULL, vu_gpio_event, NULL,
                              dev, NULL, true);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     g_assert(!*errp);
     do {
         if (*errp) {
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 6a173cb9fa..ca3131b1af 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -8,6 +8,8 @@
 #define VHOST_F_DEVICE_IOTLB 63
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+#define VU_REALIZE_CONN_RETRIES 3
+
 /* Generic structures common for any vhost based device. */
 
 struct vhost_inflight {
-- 
2.41.0

[PATCH v5 3/5] vhost-user-scsi: support reconnect to backend

[Li Feng]

If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

This patch also improves the error messages, and reports some silent errors.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c           |  16 +-
 hw/scsi/vhost-scsi.c                  |   5 +-
 hw/scsi/vhost-user-scsi.c             | 204 +++++++++++++++++++++++---
 include/hw/virtio/vhost-scsi-common.h |   2 +-
 include/hw/virtio/vhost-user-scsi.h   |   4 +
 5 files changed, 201 insertions(+), 30 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a61cd0e907..4c8637045d 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -16,6 +16,7 @@
  */
 
 #include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "hw/virtio/vhost.h"
@@ -25,7 +26,7 @@
 #include "hw/virtio/virtio-access.h"
 #include "hw/fw-path-provider.h"
 
-int vhost_scsi_common_start(VHostSCSICommon *vsc)
+int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp)
 {
     int ret, i;
     VirtIODevice *vdev = VIRTIO_DEVICE(vsc);
@@ -35,18 +36,19 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     VirtIOSCSICommon *vs = (VirtIOSCSICommon *)vsc;
 
     if (!k->set_guest_notifiers) {
-        error_report("binding does not support guest notifiers");
+        error_setg(errp, "binding does not support guest notifiers");
         return -ENOSYS;
     }
 
     ret = vhost_dev_enable_notifiers(&vsc->dev, vdev);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Error enabling host notifiers");
         return ret;
     }
 
     ret = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, true);
     if (ret < 0) {
-        error_report("Error binding guest notifier");
+        error_setg_errno(errp, -ret, "Error binding guest notifier");
         goto err_host_notifiers;
     }
 
@@ -54,7 +56,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error setting inflight format: %d", -ret);
+        error_setg_errno(errp, -ret, "Error setting inflight format");
         goto err_guest_notifiers;
     }
 
@@ -64,21 +66,21 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
                                         vs->conf.virtqueue_size,
                                         vsc->inflight);
             if (ret < 0) {
-                error_report("Error getting inflight: %d", -ret);
+                error_setg_errno(errp, -ret, "Error getting inflight");
                 goto err_guest_notifiers;
             }
         }
 
         ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
         if (ret < 0) {
-            error_report("Error setting inflight: %d", -ret);
+            error_setg_errno(errp, -ret, "Error setting inflight");
             goto err_guest_notifiers;
         }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
     if (ret < 0) {
-        error_report("Error start vhost dev");
+        error_setg_errno(errp, -ret, "Error starting vhost dev");
         goto err_guest_notifiers;
     }
 
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 443f67daa4..01a3ab4277 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -75,6 +75,7 @@ static int vhost_scsi_start(VHostSCSI *s)
     int ret, abi_version;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
     const VhostOps *vhost_ops = vsc->dev.vhost_ops;
+    Error *local_err = NULL;
 
     ret = vhost_ops->vhost_scsi_get_abi_version(&vsc->dev, &abi_version);
     if (ret < 0) {
@@ -88,14 +89,14 @@ static int vhost_scsi_start(VHostSCSI *s)
         return -ENOSYS;
     }
 
-    ret = vhost_scsi_common_start(vsc);
+    ret = vhost_scsi_common_start(vsc, &local_err);
     if (ret < 0) {
         return ret;
     }
 
     ret = vhost_scsi_set_endpoint(s);
     if (ret < 0) {
-        error_report("Error setting vhost-scsi endpoint");
+        error_reportf_err(local_err, "Error setting vhost-scsi endpoint");
         vhost_scsi_common_stop(vsc);
     }
 
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index ee99b19e7a..dc109154ad 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -43,26 +43,56 @@ enum VhostUserProtocolFeature {
     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
 };
 
+static int vhost_user_scsi_start(VHostUserSCSI *s, Error **errp)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    int ret;
+
+    ret = vhost_scsi_common_start(vsc, errp);
+    s->started_vu = (ret < 0 ? false : true);
+
+    return ret;
+}
+
+static void vhost_user_scsi_stop(VHostUserSCSI *s)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+
+    if (!s->started_vu) {
+        return;
+    }
+    s->started_vu = false;
+
+    vhost_scsi_common_stop(vsc);
+}
+
 static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    bool should_start = virtio_device_should_start(vdev, status);
+    Error *local_err = NULL;
+    int ret;
 
-    if (vhost_dev_is_started(&vsc->dev) == start) {
+    if (!s->connected) {
         return;
     }
 
-    if (start) {
-        int ret;
+    if (vhost_dev_is_started(&vsc->dev) == should_start) {
+        return;
+    }
 
-        ret = vhost_scsi_common_start(vsc);
+    if (should_start) {
+        ret = vhost_user_scsi_start(s, &local_err);
         if (ret < 0) {
-            error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
-            exit(1);
+            error_reportf_err(local_err, "unable to start vhost-user-scsi: %s",
+                              strerror(-ret));
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
         }
     } else {
-        vhost_scsi_common_stop(vsc);
+        vhost_user_scsi_stop(s);
     }
 }
 
@@ -89,14 +119,127 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
 }
 
+static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret = 0;
+
+    if (s->connected) {
+        return 0;
+    }
+    s->connected = true;
+
+    vsc->dev.num_queues = vs->conf.num_queues;
+    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
+    vsc->dev.vqs = s->vhost_vqs;
+    vsc->dev.vq_index = 0;
+    vsc->dev.backend_features = 0;
+
+    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
+                         errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* restore vhost state */
+    if (virtio_device_started(vdev, vdev->status)) {
+        ret = vhost_user_scsi_start(s, errp);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
+
+static void vhost_user_scsi_disconnect(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    if (!s->connected) {
+        return;
+    }
+    s->connected = false;
+
+    vhost_user_scsi_stop(s);
+
+    vhost_dev_cleanup(&vsc->dev);
+
+    /* Re-instate the event handler for new connections */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
+                             vhost_user_scsi_event, NULL, dev, NULL, true);
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
+{
+    DeviceState *dev = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    Error *local_err = NULL;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
+            error_report_err(local_err);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
+            return;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        /* defer close until later to avoid circular close */
+        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
+                               vhost_user_scsi_disconnect);
+        break;
+    case CHR_EVENT_BREAK:
+    case CHR_EVENT_MUX_IN:
+    case CHR_EVENT_MUX_OUT:
+        /* Ignore */
+        break;
+    }
+}
+
+static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
+{
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret;
+
+    s->connected = false;
+
+    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_user_scsi_connect(dev, errp);
+    if (ret < 0) {
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return ret;
+    }
+    assert(s->connected);
+
+    return 0;
+}
+
 static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
 {
+    ERRP_GUARD();
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = NULL;
     Error *err = NULL;
     int ret;
+    int retries = VU_REALIZE_CONN_RETRIES;
 
     if (!vs->conf.chardev.chr) {
         error_setg(errp, "vhost-user-scsi: missing chardev");
@@ -115,18 +258,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         goto free_virtio;
     }
 
-    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
-    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
-    vsc->dev.vq_index = 0;
-    vsc->dev.backend_features = 0;
-    vqs = vsc->dev.vqs;
+    vsc->inflight = g_new0(struct vhost_inflight, 1);
+    s->vhost_vqs = g_new0(struct vhost_virtqueue,
+                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
+
+    assert(!*errp);
+    do {
+        if (*errp) {
+            error_prepend(errp, "Reconnecting after error: ");
+            error_report_err(*errp);
+            *errp = NULL;
+        }
+        ret = vhost_user_scsi_realize_connect(s, errp);
+    } while (ret < 0 && retries--);
 
-    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
-                         VHOST_BACKEND_TYPE_USER, 0, errp);
     if (ret < 0) {
         goto free_vhost;
     }
 
+    /* we're fully initialized, now we can operate, so add the handler */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
+                             vhost_user_scsi_event, NULL, (void *)dev,
+                             NULL, true);
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -135,8 +288,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     return;
 
 free_vhost:
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
     vhost_user_cleanup(&s->vhost_user);
-    g_free(vqs);
+
 free_virtio:
     virtio_scsi_common_unrealize(dev);
 }
@@ -146,16 +303,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = vsc->dev.vqs;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
 
     /* This will stop the vhost backend. */
     vhost_user_scsi_set_status(vdev, 0);
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
+                             NULL, false);
 
     vhost_dev_cleanup(&vsc->dev);
-    g_free(vqs);
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+
+    vhost_dev_free_inflight(vsc->inflight);
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
 
-    virtio_scsi_common_unrealize(dev);
     vhost_user_cleanup(&s->vhost_user);
+    virtio_scsi_common_unrealize(dev);
 }
 
 static Property vhost_user_scsi_properties[] = {
diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
index 18f115527c..c5d2c09455 100644
--- a/include/hw/virtio/vhost-scsi-common.h
+++ b/include/hw/virtio/vhost-scsi-common.h
@@ -39,7 +39,7 @@ struct VHostSCSICommon {
     struct vhost_inflight *inflight;
 };
 
-int vhost_scsi_common_start(VHostSCSICommon *vsc);
+int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp);
 void vhost_scsi_common_stop(VHostSCSICommon *vsc);
 char *vhost_scsi_common_get_fw_dev_path(FWPathProvider *p, BusState *bus,
                                         DeviceState *dev);
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 521b08e559..b405ec952a 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
 struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     VhostUserState vhost_user;
+    bool connected;
+    bool started_vu;
+
+    struct vhost_virtqueue *vhost_vqs;
 };
 
 #endif /* VHOST_USER_SCSI_H */
-- 
2.41.0

[PATCH v5 4/5] vhost-user-scsi: start vhost when guest kicks

[Li Feng]

Let's keep the same behavior as vhost-user-blk.

Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
 1 file changed, 44 insertions(+), 4 deletions(-)

diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index dc109154ad..53a62c3170 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -115,8 +115,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
     }
 }
 
-static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
+    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    Error *local_err = NULL;
+    int i, ret;
+
+    if (!vdev->start_on_kick) {
+        return;
+    }
+
+    if (!s->connected) {
+        return;
+    }
+
+    if (vhost_dev_is_started(&vsc->dev)) {
+        return;
+    }
+
+    /*
+     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * vhost here instead of waiting for .set_status().
+     */
+    ret = vhost_user_scsi_start(s, &local_err);
+    if (ret < 0) {
+        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return;
+    }
+
+    /* Kick right away to begin processing requests already in vring */
+    for (i = 0; i < vsc->dev.nvqs; i++) {
+        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
+
+        if (!virtio_queue_get_desc_addr(vdev, i)) {
+            continue;
+        }
+        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
+    }
 }
 
 static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
@@ -246,9 +286,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
-                               vhost_dummy_handle_output,
-                               vhost_dummy_handle_output, &err);
+    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         return;
-- 
2.41.0

[PATCH v5 5/5] vhost-user: fix lost reconnect

[Li Feng]

When the vhost-user is reconnecting to the backend, and if the vhost-user fails
at the get_features in vhost_dev_init(), then the reconnect will fail
and it will not be retriggered forever.

The reason is:
When the vhost-user fails at get_features, the vhost_dev_cleanup will be called
immediately.

vhost_dev_cleanup calls 'memset(hdev, 0, sizeof(struct vhost_dev))'.

The reconnect path is:
vhost_user_blk_event
   vhost_user_async_close(.. vhost_user_blk_disconnect ..)
     qemu_chr_fe_set_handlers <----- clear the notifier callback
       schedule vhost_user_async_close_bh

The vhost->vdev is null, so the vhost_user_blk_disconnect will not be
called, then the event fd callback will not be reinstalled.

All vhost-user devices have this issue, including vhost-user-blk/scsi.

With this patch, if the vdev->vdev is null, the fd callback will still
be reinstalled.

Fixes: 71e076a07d ("hw/virtio: generalise CHR_EVENT_CLOSED handling")

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/block/vhost-user-blk.c      | 2 +-
 hw/scsi/vhost-user-scsi.c      | 3 ++-
 hw/virtio/vhost-user-gpio.c    | 2 +-
 hw/virtio/vhost-user.c         | 9 +++++++--
 include/hw/virtio/vhost-user.h | 3 ++-
 5 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index 3c69fa47d5..95c758200d 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -391,7 +391,7 @@ static void vhost_user_blk_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &s->chardev, &s->dev,
-                               vhost_user_blk_disconnect);
+                               vhost_user_blk_disconnect, vhost_user_blk_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index 53a62c3170..0effbb4787 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -238,7 +238,8 @@ static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
-                               vhost_user_scsi_disconnect);
+                               vhost_user_scsi_disconnect,
+                               vhost_user_scsi_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index d9979aa5db..04c2cc79f4 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -283,7 +283,7 @@ static void vu_gpio_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &gpio->chardev, &gpio->vhost_dev,
-                               vu_gpio_disconnect);
+                               vu_gpio_disconnect, vu_gpio_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 8dcf049d42..12c4a41f30 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -2643,6 +2643,7 @@ typedef struct {
     DeviceState *dev;
     CharBackend *cd;
     struct vhost_dev *vhost;
+    IOEventHandler *event_cb;
 } VhostAsyncCallback;
 
 static void vhost_user_async_close_bh(void *opaque)
@@ -2657,7 +2658,10 @@ static void vhost_user_async_close_bh(void *opaque)
      */
     if (vhost->vdev) {
         data->cb(data->dev);
-    }
+    } else if (data->event_cb) {
+        qemu_chr_fe_set_handlers(data->cd, NULL, NULL, data->event_cb,
+                                 NULL, data->dev, NULL, true);
+   }
 
     g_free(data);
 }
@@ -2669,7 +2673,8 @@ static void vhost_user_async_close_bh(void *opaque)
  */
 void vhost_user_async_close(DeviceState *d,
                             CharBackend *chardev, struct vhost_dev *vhost,
-                            vu_async_close_fn cb)
+                            vu_async_close_fn cb,
+                            IOEventHandler *event_cb)
 {
     if (!runstate_check(RUN_STATE_SHUTDOWN)) {
         /*
diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
index 191216a74f..649e9dd54f 100644
--- a/include/hw/virtio/vhost-user.h
+++ b/include/hw/virtio/vhost-user.h
@@ -84,6 +84,7 @@ typedef void (*vu_async_close_fn)(DeviceState *cb);
 
 void vhost_user_async_close(DeviceState *d,
                             CharBackend *chardev, struct vhost_dev *vhost,
-                            vu_async_close_fn cb);
+                            vu_async_close_fn cb,
+                            IOEventHandler *event_cb);
 
 #endif
-- 
2.41.0

[PATCH v6 0/5] Implement reconnect for vhost-user-scsi

[Li Feng]

Changes for v6:
- [PATCH] vhost-user: fix lost reconnect
  - Fix missing assign event_cb.

Changes for v5:
- No logic has been changed, just move part of the code from patch 4 to patch 5.

Changes for v4:
- Merge
  https://lore.kernel.org/all/20230830045722.611224-1-fengli@smartx.com/ to
  this series.
- Add ERRP_GUARD in vhost_user_scsi_realize;
- Reword the commit messages.

Changes for v3:
- Split the vhost_user_scsi_handle_output to a separate patch;
- Move the started_vu from vhost scsi common header to vhost-user-scsi header;
- Fix a log print error;

Changes for v2:
- Split the v1 patch to small separate patchset;
- New patch for fixing fd leak, which has sent to reviewers in another
  mail;
- Implement the `vhost_user_scsi_handle_output`;
- Add the started_vu safe check;
- Fix error handler;
- Check the inflight before set/get inflight fd.

Li Feng (5):
  vhost-user-common: send get_inflight_fd once
  vhost: move and rename the conn retry times
  vhost-user-scsi: support reconnect to backend
  vhost-user-scsi: start vhost when guest kicks
  vhost-user: fix lost reconnect

 hw/block/vhost-user-blk.c             |   6 +-
 hw/scsi/vhost-scsi-common.c           |  47 ++---
 hw/scsi/vhost-scsi.c                  |   5 +-
 hw/scsi/vhost-user-scsi.c             | 253 +++++++++++++++++++++++---
 hw/virtio/vhost-user-gpio.c           |   5 +-
 hw/virtio/vhost-user.c                |  10 +-
 include/hw/virtio/vhost-scsi-common.h |   2 +-
 include/hw/virtio/vhost-user-scsi.h   |   4 +
 include/hw/virtio/vhost-user.h        |   3 +-
 include/hw/virtio/vhost.h             |   2 +
 10 files changed, 277 insertions(+), 60 deletions(-)

-- 
2.41.0

[PATCH v6 1/5] vhost-user-common: send get_inflight_fd once

[Li Feng]

Currently the get_inflight_fd will be sent every time the device is started, and
the backend will allocate shared memory to save the inflight state. If the
backend finds that it receives the second get_inflight_fd, it will release the
previous shared memory, which breaks inflight working logic.

This patch is a preparation for the following patches.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a06f01af26..a61cd0e907 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     vsc->dev.acked_features = vdev->guest_features;
 
-    assert(vsc->inflight == NULL);
-    vsc->inflight = g_new0(struct vhost_inflight, 1);
-    ret = vhost_dev_get_inflight(&vsc->dev,
-                                 vs->conf.virtqueue_size,
-                                 vsc->inflight);
+    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error get inflight: %d", -ret);
+        error_report("Error setting inflight format: %d", -ret);
         goto err_guest_notifiers;
     }
 
-    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
-    if (ret < 0) {
-        error_report("Error set inflight: %d", -ret);
-        goto err_guest_notifiers;
+    if (vsc->inflight) {
+        if (!vsc->inflight->addr) {
+            ret = vhost_dev_get_inflight(&vsc->dev,
+                                        vs->conf.virtqueue_size,
+                                        vsc->inflight);
+            if (ret < 0) {
+                error_report("Error getting inflight: %d", -ret);
+                goto err_guest_notifiers;
+            }
+        }
+
+        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
+        if (ret < 0) {
+            error_report("Error setting inflight: %d", -ret);
+            goto err_guest_notifiers;
+        }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
@@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     return ret;
 
 err_guest_notifiers:
-    g_free(vsc->inflight);
-    vsc->inflight = NULL;
-
     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
 err_host_notifiers:
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
@@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
     }
     assert(ret >= 0);
 
-    if (vsc->inflight) {
-        vhost_dev_free_inflight(vsc->inflight);
-        g_free(vsc->inflight);
-        vsc->inflight = NULL;
-    }
-
     vhost_dev_disable_notifiers(&vsc->dev, vdev);
 }
 
-- 
2.41.0

Re: [PATCH v6 1/5] vhost-user-common: send get_inflight_fd once

[Raphael Norwitz]

> On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Currently the get_inflight_fd will be sent every time the device is started, and
> the backend will allocate shared memory to save the inflight state. If the
> backend finds that it receives the second get_inflight_fd, it will release the
> previous shared memory, which breaks inflight working logic.
> 
> This patch is a preparation for the following patches.

This looks identical to the v3 patch I reviewed? If I’ve missed something can you please point it out?


> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
> 1 file changed, 18 insertions(+), 19 deletions(-)
> 
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> index a06f01af26..a61cd0e907 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> 
>     vsc->dev.acked_features = vdev->guest_features;
> 
> -    assert(vsc->inflight == NULL);
> -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> -    ret = vhost_dev_get_inflight(&vsc->dev,
> -                                 vs->conf.virtqueue_size,
> -                                 vsc->inflight);
> +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>     if (ret < 0) {
> -        error_report("Error get inflight: %d", -ret);
> +        error_report("Error setting inflight format: %d", -ret);
>         goto err_guest_notifiers;
>     }
> 
> -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> -    if (ret < 0) {
> -        error_report("Error set inflight: %d", -ret);
> -        goto err_guest_notifiers;
> +    if (vsc->inflight) {
> +        if (!vsc->inflight->addr) {
> +            ret = vhost_dev_get_inflight(&vsc->dev,
> +                                        vs->conf.virtqueue_size,
> +                                        vsc->inflight);
> +            if (ret < 0) {
> +                error_report("Error getting inflight: %d", -ret);
> +                goto err_guest_notifiers;
> +            }
> +        }
> +
> +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> +        if (ret < 0) {
> +            error_report("Error setting inflight: %d", -ret);
> +            goto err_guest_notifiers;
> +        }
>     }
> 
>     ret = vhost_dev_start(&vsc->dev, vdev, true);
> @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>     return ret;
> 
> err_guest_notifiers:
> -    g_free(vsc->inflight);
> -    vsc->inflight = NULL;
> -
>     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
> err_host_notifiers:
>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
>     }
>     assert(ret >= 0);
> 
> -    if (vsc->inflight) {
> -        vhost_dev_free_inflight(vsc->inflight);
> -        g_free(vsc->inflight);
> -        vsc->inflight = NULL;
> -    }
> -
>     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> }
> 
> -- 
> 2.41.0
> 

Re: [PATCH v6 1/5] vhost-user-common: send get_inflight_fd once

[Li Feng]

On Fri, Sep 29, 2023 at 8:55 AM Raphael Norwitz
<raphael.norwitz@nutanix.com> wrote:
>
>
>
> > On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> >
> > Currently the get_inflight_fd will be sent every time the device is started, and
> > the backend will allocate shared memory to save the inflight state. If the
> > backend finds that it receives the second get_inflight_fd, it will release the
> > previous shared memory, which breaks inflight working logic.
> >
> > This patch is a preparation for the following patches.
>
> This looks identical to the v3 patch I reviewed? If I’ve missed something can you please point it out?
Yes, nothing changed in this patch.

>
>
> > Signed-off-by: Li Feng <fengli@smartx.com>
> > ---
> > hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
> > 1 file changed, 18 insertions(+), 19 deletions(-)
> >
> > diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> > index a06f01af26..a61cd0e907 100644
> > --- a/hw/scsi/vhost-scsi-common.c
> > +++ b/hw/scsi/vhost-scsi-common.c
> > @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> >
> >     vsc->dev.acked_features = vdev->guest_features;
> >
> > -    assert(vsc->inflight == NULL);
> > -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> > -    ret = vhost_dev_get_inflight(&vsc->dev,
> > -                                 vs->conf.virtqueue_size,
> > -                                 vsc->inflight);
> > +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
> >     if (ret < 0) {
> > -        error_report("Error get inflight: %d", -ret);
> > +        error_report("Error setting inflight format: %d", -ret);
> >         goto err_guest_notifiers;
> >     }
> >
> > -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> > -    if (ret < 0) {
> > -        error_report("Error set inflight: %d", -ret);
> > -        goto err_guest_notifiers;
> > +    if (vsc->inflight) {
> > +        if (!vsc->inflight->addr) {
> > +            ret = vhost_dev_get_inflight(&vsc->dev,
> > +                                        vs->conf.virtqueue_size,
> > +                                        vsc->inflight);
> > +            if (ret < 0) {
> > +                error_report("Error getting inflight: %d", -ret);
> > +                goto err_guest_notifiers;
> > +            }
> > +        }
> > +
> > +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> > +        if (ret < 0) {
> > +            error_report("Error setting inflight: %d", -ret);
> > +            goto err_guest_notifiers;
> > +        }
> >     }
> >
> >     ret = vhost_dev_start(&vsc->dev, vdev, true);
> > @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> >     return ret;
> >
> > err_guest_notifiers:
> > -    g_free(vsc->inflight);
> > -    vsc->inflight = NULL;
> > -
> >     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
> > err_host_notifiers:
> >     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> > @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
> >     }
> >     assert(ret >= 0);
> >
> > -    if (vsc->inflight) {
> > -        vhost_dev_free_inflight(vsc->inflight);
> > -        g_free(vsc->inflight);
> > -        vsc->inflight = NULL;
> > -    }
> > -
> >     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> > }
> >
> > --
> > 2.41.0
> >
>

Re: [PATCH v6 1/5] vhost-user-common: send get_inflight_fd once

[Michael S. Tsirkin]

On Sun, Oct 08, 2023 at 04:49:05PM +0800, Li Feng wrote:
> On Fri, Sep 29, 2023 at 8:55 AM Raphael Norwitz
> <raphael.norwitz@nutanix.com> wrote:
> >
> >
> >
> > > On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> > >
> > > Currently the get_inflight_fd will be sent every time the device is started, and
> > > the backend will allocate shared memory to save the inflight state. If the
> > > backend finds that it receives the second get_inflight_fd, it will release the
> > > previous shared memory, which breaks inflight working logic.
> > >
> > > This patch is a preparation for the following patches.
> >
> > This looks identical to the v3 patch I reviewed? If I’ve missed something can you please point it out?
> Yes, nothing changed in this patch.


Then you should include tags such as reviewed/acked by for previous
version. if you drop tags you indicate to people they have to
re-review.
also, mentioning which patches changed in the cover letter is
a courtesy to reviewers.

> >
> >
> > > Signed-off-by: Li Feng <fengli@smartx.com>
> > > ---
> > > hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
> > > 1 file changed, 18 insertions(+), 19 deletions(-)
> > >
> > > diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> > > index a06f01af26..a61cd0e907 100644
> > > --- a/hw/scsi/vhost-scsi-common.c
> > > +++ b/hw/scsi/vhost-scsi-common.c
> > > @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> > >
> > >     vsc->dev.acked_features = vdev->guest_features;
> > >
> > > -    assert(vsc->inflight == NULL);
> > > -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> > > -    ret = vhost_dev_get_inflight(&vsc->dev,
> > > -                                 vs->conf.virtqueue_size,
> > > -                                 vsc->inflight);
> > > +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
> > >     if (ret < 0) {
> > > -        error_report("Error get inflight: %d", -ret);
> > > +        error_report("Error setting inflight format: %d", -ret);
> > >         goto err_guest_notifiers;
> > >     }
> > >
> > > -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> > > -    if (ret < 0) {
> > > -        error_report("Error set inflight: %d", -ret);
> > > -        goto err_guest_notifiers;
> > > +    if (vsc->inflight) {
> > > +        if (!vsc->inflight->addr) {
> > > +            ret = vhost_dev_get_inflight(&vsc->dev,
> > > +                                        vs->conf.virtqueue_size,
> > > +                                        vsc->inflight);
> > > +            if (ret < 0) {
> > > +                error_report("Error getting inflight: %d", -ret);
> > > +                goto err_guest_notifiers;
> > > +            }
> > > +        }
> > > +
> > > +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> > > +        if (ret < 0) {
> > > +            error_report("Error setting inflight: %d", -ret);
> > > +            goto err_guest_notifiers;
> > > +        }
> > >     }
> > >
> > >     ret = vhost_dev_start(&vsc->dev, vdev, true);
> > > @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> > >     return ret;
> > >
> > > err_guest_notifiers:
> > > -    g_free(vsc->inflight);
> > > -    vsc->inflight = NULL;
> > > -
> > >     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
> > > err_host_notifiers:
> > >     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> > > @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
> > >     }
> > >     assert(ret >= 0);
> > >
> > > -    if (vsc->inflight) {
> > > -        vhost_dev_free_inflight(vsc->inflight);
> > > -        g_free(vsc->inflight);
> > > -        vsc->inflight = NULL;
> > > -    }
> > > -
> > >     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> > > }
> > >
> > > --
> > > 2.41.0
> > >
> >

Re: [PATCH v6 1/5] vhost-user-common: send get_inflight_fd once

[Li Feng]

On Sun, Oct 8, 2023 at 4:51 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Sun, Oct 08, 2023 at 04:49:05PM +0800, Li Feng wrote:
> > On Fri, Sep 29, 2023 at 8:55 AM Raphael Norwitz
> > <raphael.norwitz@nutanix.com> wrote:
> > >
> > >
> > >
> > > > On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> > > >
> > > > Currently the get_inflight_fd will be sent every time the device is started, and
> > > > the backend will allocate shared memory to save the inflight state. If the
> > > > backend finds that it receives the second get_inflight_fd, it will release the
> > > > previous shared memory, which breaks inflight working logic.
> > > >
> > > > This patch is a preparation for the following patches.
> > >
> > > This looks identical to the v3 patch I reviewed? If I’ve missed something can you please point it out?
> > Yes, nothing changed in this patch.
>
>
> Then you should include tags such as reviewed/acked by for previous
> version. if you drop tags you indicate to people they have to
> re-review.
> also, mentioning which patches changed in the cover letter is
> a courtesy to reviewers.
OK.

>
> > >
> > >
> > > > Signed-off-by: Li Feng <fengli@smartx.com>
> > > > ---
> > > > hw/scsi/vhost-scsi-common.c | 37 ++++++++++++++++++-------------------
> > > > 1 file changed, 18 insertions(+), 19 deletions(-)
> > > >
> > > > diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> > > > index a06f01af26..a61cd0e907 100644
> > > > --- a/hw/scsi/vhost-scsi-common.c
> > > > +++ b/hw/scsi/vhost-scsi-common.c
> > > > @@ -52,20 +52,28 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> > > >
> > > >     vsc->dev.acked_features = vdev->guest_features;
> > > >
> > > > -    assert(vsc->inflight == NULL);
> > > > -    vsc->inflight = g_new0(struct vhost_inflight, 1);
> > > > -    ret = vhost_dev_get_inflight(&vsc->dev,
> > > > -                                 vs->conf.virtqueue_size,
> > > > -                                 vsc->inflight);
> > > > +    ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
> > > >     if (ret < 0) {
> > > > -        error_report("Error get inflight: %d", -ret);
> > > > +        error_report("Error setting inflight format: %d", -ret);
> > > >         goto err_guest_notifiers;
> > > >     }
> > > >
> > > > -    ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> > > > -    if (ret < 0) {
> > > > -        error_report("Error set inflight: %d", -ret);
> > > > -        goto err_guest_notifiers;
> > > > +    if (vsc->inflight) {
> > > > +        if (!vsc->inflight->addr) {
> > > > +            ret = vhost_dev_get_inflight(&vsc->dev,
> > > > +                                        vs->conf.virtqueue_size,
> > > > +                                        vsc->inflight);
> > > > +            if (ret < 0) {
> > > > +                error_report("Error getting inflight: %d", -ret);
> > > > +                goto err_guest_notifiers;
> > > > +            }
> > > > +        }
> > > > +
> > > > +        ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> > > > +        if (ret < 0) {
> > > > +            error_report("Error setting inflight: %d", -ret);
> > > > +            goto err_guest_notifiers;
> > > > +        }
> > > >     }
> > > >
> > > >     ret = vhost_dev_start(&vsc->dev, vdev, true);
> > > > @@ -85,9 +93,6 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> > > >     return ret;
> > > >
> > > > err_guest_notifiers:
> > > > -    g_free(vsc->inflight);
> > > > -    vsc->inflight = NULL;
> > > > -
> > > >     k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false);
> > > > err_host_notifiers:
> > > >     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> > > > @@ -111,12 +116,6 @@ void vhost_scsi_common_stop(VHostSCSICommon *vsc)
> > > >     }
> > > >     assert(ret >= 0);
> > > >
> > > > -    if (vsc->inflight) {
> > > > -        vhost_dev_free_inflight(vsc->inflight);
> > > > -        g_free(vsc->inflight);
> > > > -        vsc->inflight = NULL;
> > > > -    }
> > > > -
> > > >     vhost_dev_disable_notifiers(&vsc->dev, vdev);
> > > > }
> > > >
> > > > --
> > > > 2.41.0
> > > >
> > >
>

[PATCH v6 2/5] vhost: move and rename the conn retry times

[Li Feng]

Multiple devices need this macro, move it to a common header.

Signed-off-by: Li Feng <fengli@smartx.com>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
---
 hw/block/vhost-user-blk.c   | 4 +---
 hw/virtio/vhost-user-gpio.c | 3 +--
 include/hw/virtio/vhost.h   | 2 ++
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index eecf3f7a81..3c69fa47d5 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -32,8 +32,6 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/runstate.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
-
 static const int user_feature_bits[] = {
     VIRTIO_BLK_F_SIZE_MAX,
     VIRTIO_BLK_F_SEG_MAX,
@@ -482,7 +480,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
     s->inflight = g_new0(struct vhost_inflight, 1);
     s->vhost_vqs = g_new0(struct vhost_virtqueue, s->num_queues);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     assert(!*errp);
     do {
         if (*errp) {
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index 3b013f2d0f..d9979aa5db 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -15,7 +15,6 @@
 #include "standard-headers/linux/virtio_ids.h"
 #include "trace.h"
 
-#define REALIZE_CONNECTION_RETRIES 3
 #define VHOST_NVQS 2
 
 /* Features required from VirtIO */
@@ -359,7 +358,7 @@ static void vu_gpio_device_realize(DeviceState *dev, Error **errp)
     qemu_chr_fe_set_handlers(&gpio->chardev, NULL, NULL, vu_gpio_event, NULL,
                              dev, NULL, true);
 
-    retries = REALIZE_CONNECTION_RETRIES;
+    retries = VU_REALIZE_CONN_RETRIES;
     g_assert(!*errp);
     do {
         if (*errp) {
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 6a173cb9fa..ca3131b1af 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -8,6 +8,8 @@
 #define VHOST_F_DEVICE_IOTLB 63
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+#define VU_REALIZE_CONN_RETRIES 3
+
 /* Generic structures common for any vhost based device. */
 
 struct vhost_inflight {
-- 
2.41.0

[PATCH v6 3/5] vhost-user-scsi: support reconnect to backend

[Li Feng]

If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

This patch also improves the error messages, and reports some silent errors.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-scsi-common.c           |  16 +-
 hw/scsi/vhost-scsi.c                  |   5 +-
 hw/scsi/vhost-user-scsi.c             | 204 +++++++++++++++++++++++---
 include/hw/virtio/vhost-scsi-common.h |   2 +-
 include/hw/virtio/vhost-user-scsi.h   |   4 +
 5 files changed, 201 insertions(+), 30 deletions(-)

diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
index a61cd0e907..4c8637045d 100644
--- a/hw/scsi/vhost-scsi-common.c
+++ b/hw/scsi/vhost-scsi-common.c
@@ -16,6 +16,7 @@
  */
 
 #include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "hw/virtio/vhost.h"
@@ -25,7 +26,7 @@
 #include "hw/virtio/virtio-access.h"
 #include "hw/fw-path-provider.h"
 
-int vhost_scsi_common_start(VHostSCSICommon *vsc)
+int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp)
 {
     int ret, i;
     VirtIODevice *vdev = VIRTIO_DEVICE(vsc);
@@ -35,18 +36,19 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
     VirtIOSCSICommon *vs = (VirtIOSCSICommon *)vsc;
 
     if (!k->set_guest_notifiers) {
-        error_report("binding does not support guest notifiers");
+        error_setg(errp, "binding does not support guest notifiers");
         return -ENOSYS;
     }
 
     ret = vhost_dev_enable_notifiers(&vsc->dev, vdev);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Error enabling host notifiers");
         return ret;
     }
 
     ret = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, true);
     if (ret < 0) {
-        error_report("Error binding guest notifier");
+        error_setg_errno(errp, -ret, "Error binding guest notifier");
         goto err_host_notifiers;
     }
 
@@ -54,7 +56,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
 
     ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
     if (ret < 0) {
-        error_report("Error setting inflight format: %d", -ret);
+        error_setg_errno(errp, -ret, "Error setting inflight format");
         goto err_guest_notifiers;
     }
 
@@ -64,21 +66,21 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
                                         vs->conf.virtqueue_size,
                                         vsc->inflight);
             if (ret < 0) {
-                error_report("Error getting inflight: %d", -ret);
+                error_setg_errno(errp, -ret, "Error getting inflight");
                 goto err_guest_notifiers;
             }
         }
 
         ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
         if (ret < 0) {
-            error_report("Error setting inflight: %d", -ret);
+            error_setg_errno(errp, -ret, "Error setting inflight");
             goto err_guest_notifiers;
         }
     }
 
     ret = vhost_dev_start(&vsc->dev, vdev, true);
     if (ret < 0) {
-        error_report("Error start vhost dev");
+        error_setg_errno(errp, -ret, "Error starting vhost dev");
         goto err_guest_notifiers;
     }
 
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 443f67daa4..01a3ab4277 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -75,6 +75,7 @@ static int vhost_scsi_start(VHostSCSI *s)
     int ret, abi_version;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
     const VhostOps *vhost_ops = vsc->dev.vhost_ops;
+    Error *local_err = NULL;
 
     ret = vhost_ops->vhost_scsi_get_abi_version(&vsc->dev, &abi_version);
     if (ret < 0) {
@@ -88,14 +89,14 @@ static int vhost_scsi_start(VHostSCSI *s)
         return -ENOSYS;
     }
 
-    ret = vhost_scsi_common_start(vsc);
+    ret = vhost_scsi_common_start(vsc, &local_err);
     if (ret < 0) {
         return ret;
     }
 
     ret = vhost_scsi_set_endpoint(s);
     if (ret < 0) {
-        error_report("Error setting vhost-scsi endpoint");
+        error_reportf_err(local_err, "Error setting vhost-scsi endpoint");
         vhost_scsi_common_stop(vsc);
     }
 
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index ee99b19e7a..dc109154ad 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -43,26 +43,56 @@ enum VhostUserProtocolFeature {
     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
 };
 
+static int vhost_user_scsi_start(VHostUserSCSI *s, Error **errp)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    int ret;
+
+    ret = vhost_scsi_common_start(vsc, errp);
+    s->started_vu = (ret < 0 ? false : true);
+
+    return ret;
+}
+
+static void vhost_user_scsi_stop(VHostUserSCSI *s)
+{
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+
+    if (!s->started_vu) {
+        return;
+    }
+    s->started_vu = false;
+
+    vhost_scsi_common_stop(vsc);
+}
+
 static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    bool should_start = virtio_device_should_start(vdev, status);
+    Error *local_err = NULL;
+    int ret;
 
-    if (vhost_dev_is_started(&vsc->dev) == start) {
+    if (!s->connected) {
         return;
     }
 
-    if (start) {
-        int ret;
+    if (vhost_dev_is_started(&vsc->dev) == should_start) {
+        return;
+    }
 
-        ret = vhost_scsi_common_start(vsc);
+    if (should_start) {
+        ret = vhost_user_scsi_start(s, &local_err);
         if (ret < 0) {
-            error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
-            exit(1);
+            error_reportf_err(local_err, "unable to start vhost-user-scsi: %s",
+                              strerror(-ret));
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
         }
     } else {
-        vhost_scsi_common_stop(vsc);
+        vhost_user_scsi_stop(s);
     }
 }
 
@@ -89,14 +119,127 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
 }
 
+static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret = 0;
+
+    if (s->connected) {
+        return 0;
+    }
+    s->connected = true;
+
+    vsc->dev.num_queues = vs->conf.num_queues;
+    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
+    vsc->dev.vqs = s->vhost_vqs;
+    vsc->dev.vq_index = 0;
+    vsc->dev.backend_features = 0;
+
+    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
+                         errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* restore vhost state */
+    if (virtio_device_started(vdev, vdev->status)) {
+        ret = vhost_user_scsi_start(s, errp);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
+
+static void vhost_user_scsi_disconnect(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    if (!s->connected) {
+        return;
+    }
+    s->connected = false;
+
+    vhost_user_scsi_stop(s);
+
+    vhost_dev_cleanup(&vsc->dev);
+
+    /* Re-instate the event handler for new connections */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
+                             vhost_user_scsi_event, NULL, dev, NULL, true);
+}
+
+static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
+{
+    DeviceState *dev = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    Error *local_err = NULL;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
+            error_report_err(local_err);
+            qemu_chr_fe_disconnect(&vs->conf.chardev);
+            return;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        /* defer close until later to avoid circular close */
+        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
+                               vhost_user_scsi_disconnect);
+        break;
+    case CHR_EVENT_BREAK:
+    case CHR_EVENT_MUX_IN:
+    case CHR_EVENT_MUX_OUT:
+        /* Ignore */
+        break;
+    }
+}
+
+static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
+{
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+    int ret;
+
+    s->connected = false;
+
+    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = vhost_user_scsi_connect(dev, errp);
+    if (ret < 0) {
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return ret;
+    }
+    assert(s->connected);
+
+    return 0;
+}
+
 static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
 {
+    ERRP_GUARD();
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = NULL;
     Error *err = NULL;
     int ret;
+    int retries = VU_REALIZE_CONN_RETRIES;
 
     if (!vs->conf.chardev.chr) {
         error_setg(errp, "vhost-user-scsi: missing chardev");
@@ -115,18 +258,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         goto free_virtio;
     }
 
-    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
-    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
-    vsc->dev.vq_index = 0;
-    vsc->dev.backend_features = 0;
-    vqs = vsc->dev.vqs;
+    vsc->inflight = g_new0(struct vhost_inflight, 1);
+    s->vhost_vqs = g_new0(struct vhost_virtqueue,
+                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
+
+    assert(!*errp);
+    do {
+        if (*errp) {
+            error_prepend(errp, "Reconnecting after error: ");
+            error_report_err(*errp);
+            *errp = NULL;
+        }
+        ret = vhost_user_scsi_realize_connect(s, errp);
+    } while (ret < 0 && retries--);
 
-    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
-                         VHOST_BACKEND_TYPE_USER, 0, errp);
     if (ret < 0) {
         goto free_vhost;
     }
 
+    /* we're fully initialized, now we can operate, so add the handler */
+    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
+                             vhost_user_scsi_event, NULL, (void *)dev,
+                             NULL, true);
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -135,8 +288,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     return;
 
 free_vhost:
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
     vhost_user_cleanup(&s->vhost_user);
-    g_free(vqs);
+
 free_virtio:
     virtio_scsi_common_unrealize(dev);
 }
@@ -146,16 +303,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
-    struct vhost_virtqueue *vqs = vsc->dev.vqs;
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
 
     /* This will stop the vhost backend. */
     vhost_user_scsi_set_status(vdev, 0);
+    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
+                             NULL, false);
 
     vhost_dev_cleanup(&vsc->dev);
-    g_free(vqs);
+    g_free(s->vhost_vqs);
+    s->vhost_vqs = NULL;
+
+    vhost_dev_free_inflight(vsc->inflight);
+    g_free(vsc->inflight);
+    vsc->inflight = NULL;
 
-    virtio_scsi_common_unrealize(dev);
     vhost_user_cleanup(&s->vhost_user);
+    virtio_scsi_common_unrealize(dev);
 }
 
 static Property vhost_user_scsi_properties[] = {
diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
index 18f115527c..c5d2c09455 100644
--- a/include/hw/virtio/vhost-scsi-common.h
+++ b/include/hw/virtio/vhost-scsi-common.h
@@ -39,7 +39,7 @@ struct VHostSCSICommon {
     struct vhost_inflight *inflight;
 };
 
-int vhost_scsi_common_start(VHostSCSICommon *vsc);
+int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp);
 void vhost_scsi_common_stop(VHostSCSICommon *vsc);
 char *vhost_scsi_common_get_fw_dev_path(FWPathProvider *p, BusState *bus,
                                         DeviceState *dev);
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 521b08e559..b405ec952a 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
 struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     VhostUserState vhost_user;
+    bool connected;
+    bool started_vu;
+
+    struct vhost_virtqueue *vhost_vqs;
 };
 
 #endif /* VHOST_USER_SCSI_H */
-- 
2.41.0

Re: [PATCH v6 3/5] vhost-user-scsi: support reconnect to backend

[Raphael Norwitz]

One comment on the logging stuff in vhost-scsi. As far as I can tell the logging in vhost-user-scsi looks good.

Markus - does this look better to you? Otherwise do you think we should also fix up the vhost-user-blk realize function?

> On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> 
> If the backend crashes and restarts, the device is broken.
> This patch adds reconnect for vhost-user-scsi.
> 
> This patch also improves the error messages, and reports some silent errors.
> 
> Tested with spdk backend.
> 
> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/scsi/vhost-scsi-common.c           |  16 +-
> hw/scsi/vhost-scsi.c                  |   5 +-
> hw/scsi/vhost-user-scsi.c             | 204 +++++++++++++++++++++++---
> include/hw/virtio/vhost-scsi-common.h |   2 +-
> include/hw/virtio/vhost-user-scsi.h   |   4 +
> 5 files changed, 201 insertions(+), 30 deletions(-)
> 
> diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> index a61cd0e907..4c8637045d 100644
> --- a/hw/scsi/vhost-scsi-common.c
> +++ b/hw/scsi/vhost-scsi-common.c
> @@ -16,6 +16,7 @@
>  */
> 
> #include "qemu/osdep.h"
> +#include "qapi/error.h"
> #include "qemu/error-report.h"
> #include "qemu/module.h"
> #include "hw/virtio/vhost.h"
> @@ -25,7 +26,7 @@
> #include "hw/virtio/virtio-access.h"
> #include "hw/fw-path-provider.h"
> 
> -int vhost_scsi_common_start(VHostSCSICommon *vsc)
> +int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp)
> {
>     int ret, i;
>     VirtIODevice *vdev = VIRTIO_DEVICE(vsc);
> @@ -35,18 +36,19 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>     VirtIOSCSICommon *vs = (VirtIOSCSICommon *)vsc;
> 
>     if (!k->set_guest_notifiers) {
> -        error_report("binding does not support guest notifiers");
> +        error_setg(errp, "binding does not support guest notifiers");
>         return -ENOSYS;
>     }
> 
>     ret = vhost_dev_enable_notifiers(&vsc->dev, vdev);
>     if (ret < 0) {
> +        error_setg_errno(errp, -ret, "Error enabling host notifiers");
>         return ret;
>     }
> 
>     ret = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, true);
>     if (ret < 0) {
> -        error_report("Error binding guest notifier");
> +        error_setg_errno(errp, -ret, "Error binding guest notifier");
>         goto err_host_notifiers;
>     }
> 
> @@ -54,7 +56,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> 
>     ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
>     if (ret < 0) {
> -        error_report("Error setting inflight format: %d", -ret);
> +        error_setg_errno(errp, -ret, "Error setting inflight format");
>         goto err_guest_notifiers;
>     }
> 
> @@ -64,21 +66,21 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
>                                         vs->conf.virtqueue_size,
>                                         vsc->inflight);
>             if (ret < 0) {
> -                error_report("Error getting inflight: %d", -ret);
> +                error_setg_errno(errp, -ret, "Error getting inflight");
>                 goto err_guest_notifiers;
>             }
>         }
> 
>         ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
>         if (ret < 0) {
> -            error_report("Error setting inflight: %d", -ret);
> +            error_setg_errno(errp, -ret, "Error setting inflight");
>             goto err_guest_notifiers;
>         }
>     }
> 
>     ret = vhost_dev_start(&vsc->dev, vdev, true);
>     if (ret < 0) {
> -        error_report("Error start vhost dev");
> +        error_setg_errno(errp, -ret, "Error starting vhost dev");
>         goto err_guest_notifiers;
>     }
> 
> diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
> index 443f67daa4..01a3ab4277 100644
> --- a/hw/scsi/vhost-scsi.c
> +++ b/hw/scsi/vhost-scsi.c
> @@ -75,6 +75,7 @@ static int vhost_scsi_start(VHostSCSI *s)
>     int ret, abi_version;
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
>     const VhostOps *vhost_ops = vsc->dev.vhost_ops;
> +    Error *local_err = NULL;
> 
>     ret = vhost_ops->vhost_scsi_get_abi_version(&vsc->dev, &abi_version);
>     if (ret < 0) {
> @@ -88,14 +89,14 @@ static int vhost_scsi_start(VHostSCSI *s)
>         return -ENOSYS;
>     }
> 
> -    ret = vhost_scsi_common_start(vsc);
> +    ret = vhost_scsi_common_start(vsc, &local_err);
>     if (ret < 0) {

Why aren’t you reporting the error here?

>         return ret;
>     }
> 
>     ret = vhost_scsi_set_endpoint(s);
>     if (ret < 0) {
> -        error_report("Error setting vhost-scsi endpoint");
> +        error_reportf_err(local_err, "Error setting vhost-scsi endpoint");
>         vhost_scsi_common_stop(vsc);
>     }
> 
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index ee99b19e7a..dc109154ad 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -43,26 +43,56 @@ enum VhostUserProtocolFeature {
>     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
> };
> 
> +static int vhost_user_scsi_start(VHostUserSCSI *s, Error **errp)
> +{
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    int ret;
> +
> +    ret = vhost_scsi_common_start(vsc, errp);
> +    s->started_vu = (ret < 0 ? false : true);
> +
> +    return ret;
> +}
> +
> +static void vhost_user_scsi_stop(VHostUserSCSI *s)
> +{
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +
> +    if (!s->started_vu) {
> +        return;
> +    }
> +    s->started_vu = false;
> +
> +    vhost_scsi_common_stop(vsc);
> +}
> +
> static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
> {
>     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    bool should_start = virtio_device_should_start(vdev, status);
> +    Error *local_err = NULL;
> +    int ret;
> 
> -    if (vhost_dev_is_started(&vsc->dev) == start) {
> +    if (!s->connected) {
>         return;
>     }
> 
> -    if (start) {
> -        int ret;
> +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
> +        return;
> +    }
> 
> -        ret = vhost_scsi_common_start(vsc);
> +    if (should_start) {
> +        ret = vhost_user_scsi_start(s, &local_err);
>         if (ret < 0) {
> -            error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
> -            exit(1);
> +            error_reportf_err(local_err, "unable to start vhost-user-scsi: %s",
> +                              strerror(-ret));
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
>         }
>     } else {
> -        vhost_scsi_common_stop(vsc);
> +        vhost_user_scsi_stop(s);
>     }
> }
> 
> @@ -89,14 +119,127 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> }
> 
> +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret = 0;
> +
> +    if (s->connected) {
> +        return 0;
> +    }
> +    s->connected = true;
> +
> +    vsc->dev.num_queues = vs->conf.num_queues;
> +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> +    vsc->dev.vqs = s->vhost_vqs;
> +    vsc->dev.vq_index = 0;
> +    vsc->dev.backend_features = 0;
> +
> +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> +                         errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    /* restore vhost state */
> +    if (virtio_device_started(vdev, vdev->status)) {
> +        ret = vhost_user_scsi_start(s, errp);
> +        if (ret < 0) {
> +            return ret;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> +
> +static void vhost_user_scsi_disconnect(DeviceState *dev)
> +{
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +    s->connected = false;
> +
> +    vhost_user_scsi_stop(s);
> +
> +    vhost_dev_cleanup(&vsc->dev);
> +
> +    /* Re-instate the event handler for new connections */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> +}
> +
> +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> +{
> +    DeviceState *dev = opaque;
> +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    Error *local_err = NULL;
> +
> +    switch (event) {
> +    case CHR_EVENT_OPENED:
> +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> +            error_report_err(local_err);
> +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> +            return;
> +        }
> +        break;
> +    case CHR_EVENT_CLOSED:
> +        /* defer close until later to avoid circular close */
> +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> +                               vhost_user_scsi_disconnect);
> +        break;
> +    case CHR_EVENT_BREAK:
> +    case CHR_EVENT_MUX_IN:
> +    case CHR_EVENT_MUX_OUT:
> +        /* Ignore */
> +        break;
> +    }
> +}
> +
> +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> +{
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +    int ret;
> +
> +    s->connected = false;
> +
> +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    ret = vhost_user_scsi_connect(dev, errp);
> +    if (ret < 0) {
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return ret;
> +    }
> +    assert(s->connected);
> +
> +    return 0;
> +}
> +
> static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> {
> +    ERRP_GUARD();
>     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = NULL;
>     Error *err = NULL;
>     int ret;
> +    int retries = VU_REALIZE_CONN_RETRIES;
> 
>     if (!vs->conf.chardev.chr) {
>         error_setg(errp, "vhost-user-scsi: missing chardev");
> @@ -115,18 +258,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>         goto free_virtio;
>     }
> 
> -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> -    vsc->dev.vq_index = 0;
> -    vsc->dev.backend_features = 0;
> -    vqs = vsc->dev.vqs;
> +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> +
> +    assert(!*errp);
> +    do {
> +        if (*errp) {
> +            error_prepend(errp, "Reconnecting after error: ");
> +            error_report_err(*errp);
> +            *errp = NULL;
> +        }
> +        ret = vhost_user_scsi_realize_connect(s, errp);
> +    } while (ret < 0 && retries--);
> 
> -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> -                         VHOST_BACKEND_TYPE_USER, 0, errp);
>     if (ret < 0) {
>         goto free_vhost;
>     }
> 
> +    /* we're fully initialized, now we can operate, so add the handler */
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> +                             vhost_user_scsi_event, NULL, (void *)dev,
> +                             NULL, true);
>     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
>     vsc->channel = 0;
>     vsc->lun = 0;
> @@ -135,8 +288,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>     return;
> 
> free_vhost:
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
>     vhost_user_cleanup(&s->vhost_user);
> -    g_free(vqs);
> +
> free_virtio:
>     virtio_scsi_common_unrealize(dev);
> }
> @@ -146,16 +303,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
>     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
>     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
>     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> 
>     /* This will stop the vhost backend. */
>     vhost_user_scsi_set_status(vdev, 0);
> +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> +                             NULL, false);
> 
>     vhost_dev_cleanup(&vsc->dev);
> -    g_free(vqs);
> +    g_free(s->vhost_vqs);
> +    s->vhost_vqs = NULL;
> +
> +    vhost_dev_free_inflight(vsc->inflight);
> +    g_free(vsc->inflight);
> +    vsc->inflight = NULL;
> 
> -    virtio_scsi_common_unrealize(dev);
>     vhost_user_cleanup(&s->vhost_user);
> +    virtio_scsi_common_unrealize(dev);
> }
> 
> static Property vhost_user_scsi_properties[] = {
> diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
> index 18f115527c..c5d2c09455 100644
> --- a/include/hw/virtio/vhost-scsi-common.h
> +++ b/include/hw/virtio/vhost-scsi-common.h
> @@ -39,7 +39,7 @@ struct VHostSCSICommon {
>     struct vhost_inflight *inflight;
> };
> 
> -int vhost_scsi_common_start(VHostSCSICommon *vsc);
> +int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp);
> void vhost_scsi_common_stop(VHostSCSICommon *vsc);
> char *vhost_scsi_common_get_fw_dev_path(FWPathProvider *p, BusState *bus,
>                                         DeviceState *dev);
> diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> index 521b08e559..b405ec952a 100644
> --- a/include/hw/virtio/vhost-user-scsi.h
> +++ b/include/hw/virtio/vhost-user-scsi.h
> @@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
> struct VHostUserSCSI {
>     VHostSCSICommon parent_obj;
>     VhostUserState vhost_user;
> +    bool connected;
> +    bool started_vu;
> +
> +    struct vhost_virtqueue *vhost_vqs;
> };
> 
> #endif /* VHOST_USER_SCSI_H */
> -- 
> 2.41.0
> 

Re: [PATCH v6 3/5] vhost-user-scsi: support reconnect to backend

[Li Feng]

Sorry, the reply is late due to being on vacation for half a month.

On Fri, Sep 29, 2023 at 8:55 AM Raphael Norwitz
<raphael.norwitz@nutanix.com> wrote:
>
> One comment on the logging stuff in vhost-scsi. As far as I can tell the logging in vhost-user-scsi looks good.
>
> Markus - does this look better to you? Otherwise do you think we should also fix up the vhost-user-blk realize function?
>
> > On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> >
> > If the backend crashes and restarts, the device is broken.
> > This patch adds reconnect for vhost-user-scsi.
> >
> > This patch also improves the error messages, and reports some silent errors.
> >
> > Tested with spdk backend.
> >
> > Signed-off-by: Li Feng <fengli@smartx.com>
> > ---
> > hw/scsi/vhost-scsi-common.c           |  16 +-
> > hw/scsi/vhost-scsi.c                  |   5 +-
> > hw/scsi/vhost-user-scsi.c             | 204 +++++++++++++++++++++++---
> > include/hw/virtio/vhost-scsi-common.h |   2 +-
> > include/hw/virtio/vhost-user-scsi.h   |   4 +
> > 5 files changed, 201 insertions(+), 30 deletions(-)
> >
> > diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c
> > index a61cd0e907..4c8637045d 100644
> > --- a/hw/scsi/vhost-scsi-common.c
> > +++ b/hw/scsi/vhost-scsi-common.c
> > @@ -16,6 +16,7 @@
> >  */
> >
> > #include "qemu/osdep.h"
> > +#include "qapi/error.h"
> > #include "qemu/error-report.h"
> > #include "qemu/module.h"
> > #include "hw/virtio/vhost.h"
> > @@ -25,7 +26,7 @@
> > #include "hw/virtio/virtio-access.h"
> > #include "hw/fw-path-provider.h"
> >
> > -int vhost_scsi_common_start(VHostSCSICommon *vsc)
> > +int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp)
> > {
> >     int ret, i;
> >     VirtIODevice *vdev = VIRTIO_DEVICE(vsc);
> > @@ -35,18 +36,19 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> >     VirtIOSCSICommon *vs = (VirtIOSCSICommon *)vsc;
> >
> >     if (!k->set_guest_notifiers) {
> > -        error_report("binding does not support guest notifiers");
> > +        error_setg(errp, "binding does not support guest notifiers");
> >         return -ENOSYS;
> >     }
> >
> >     ret = vhost_dev_enable_notifiers(&vsc->dev, vdev);
> >     if (ret < 0) {
> > +        error_setg_errno(errp, -ret, "Error enabling host notifiers");
> >         return ret;
> >     }
> >
> >     ret = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, true);
> >     if (ret < 0) {
> > -        error_report("Error binding guest notifier");
> > +        error_setg_errno(errp, -ret, "Error binding guest notifier");
> >         goto err_host_notifiers;
> >     }
> >
> > @@ -54,7 +56,7 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> >
> >     ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
> >     if (ret < 0) {
> > -        error_report("Error setting inflight format: %d", -ret);
> > +        error_setg_errno(errp, -ret, "Error setting inflight format");
> >         goto err_guest_notifiers;
> >     }
> >
> > @@ -64,21 +66,21 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
> >                                         vs->conf.virtqueue_size,
> >                                         vsc->inflight);
> >             if (ret < 0) {
> > -                error_report("Error getting inflight: %d", -ret);
> > +                error_setg_errno(errp, -ret, "Error getting inflight");
> >                 goto err_guest_notifiers;
> >             }
> >         }
> >
> >         ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
> >         if (ret < 0) {
> > -            error_report("Error setting inflight: %d", -ret);
> > +            error_setg_errno(errp, -ret, "Error setting inflight");
> >             goto err_guest_notifiers;
> >         }
> >     }
> >
> >     ret = vhost_dev_start(&vsc->dev, vdev, true);
> >     if (ret < 0) {
> > -        error_report("Error start vhost dev");
> > +        error_setg_errno(errp, -ret, "Error starting vhost dev");
> >         goto err_guest_notifiers;
> >     }
> >
> > diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
> > index 443f67daa4..01a3ab4277 100644
> > --- a/hw/scsi/vhost-scsi.c
> > +++ b/hw/scsi/vhost-scsi.c
> > @@ -75,6 +75,7 @@ static int vhost_scsi_start(VHostSCSI *s)
> >     int ret, abi_version;
> >     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> >     const VhostOps *vhost_ops = vsc->dev.vhost_ops;
> > +    Error *local_err = NULL;
> >
> >     ret = vhost_ops->vhost_scsi_get_abi_version(&vsc->dev, &abi_version);
> >     if (ret < 0) {
> > @@ -88,14 +89,14 @@ static int vhost_scsi_start(VHostSCSI *s)
> >         return -ENOSYS;
> >     }
> >
> > -    ret = vhost_scsi_common_start(vsc);
> > +    ret = vhost_scsi_common_start(vsc, &local_err);
> >     if (ret < 0) {
>
> Why aren’t you reporting the error here?
I will add reporting the error in the next version.

>
> >         return ret;
> >     }
> >
> >     ret = vhost_scsi_set_endpoint(s);
> >     if (ret < 0) {
> > -        error_report("Error setting vhost-scsi endpoint");
> > +        error_reportf_err(local_err, "Error setting vhost-scsi endpoint");
> >         vhost_scsi_common_stop(vsc);
> >     }
> >
> > diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> > index ee99b19e7a..dc109154ad 100644
> > --- a/hw/scsi/vhost-user-scsi.c
> > +++ b/hw/scsi/vhost-user-scsi.c
> > @@ -43,26 +43,56 @@ enum VhostUserProtocolFeature {
> >     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
> > };
> >
> > +static int vhost_user_scsi_start(VHostUserSCSI *s, Error **errp)
> > +{
> > +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > +    int ret;
> > +
> > +    ret = vhost_scsi_common_start(vsc, errp);
> > +    s->started_vu = (ret < 0 ? false : true);
> > +
> > +    return ret;
> > +}
> > +
> > +static void vhost_user_scsi_stop(VHostUserSCSI *s)
> > +{
> > +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > +
> > +    if (!s->started_vu) {
> > +        return;
> > +    }
> > +    s->started_vu = false;
> > +
> > +    vhost_scsi_common_stop(vsc);
> > +}
> > +
> > static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
> > {
> >     VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> > +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> >     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > -    bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
> > +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> > +    bool should_start = virtio_device_should_start(vdev, status);
> > +    Error *local_err = NULL;
> > +    int ret;
> >
> > -    if (vhost_dev_is_started(&vsc->dev) == start) {
> > +    if (!s->connected) {
> >         return;
> >     }
> >
> > -    if (start) {
> > -        int ret;
> > +    if (vhost_dev_is_started(&vsc->dev) == should_start) {
> > +        return;
> > +    }
> >
> > -        ret = vhost_scsi_common_start(vsc);
> > +    if (should_start) {
> > +        ret = vhost_user_scsi_start(s, &local_err);
> >         if (ret < 0) {
> > -            error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
> > -            exit(1);
> > +            error_reportf_err(local_err, "unable to start vhost-user-scsi: %s",
> > +                              strerror(-ret));
> > +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> >         }
> >     } else {
> > -        vhost_scsi_common_stop(vsc);
> > +        vhost_user_scsi_stop(s);
> >     }
> > }
> >
> > @@ -89,14 +119,127 @@ static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> > {
> > }
> >
> > +static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> > +{
> > +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> > +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> > +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> > +    int ret = 0;
> > +
> > +    if (s->connected) {
> > +        return 0;
> > +    }
> > +    s->connected = true;
> > +
> > +    vsc->dev.num_queues = vs->conf.num_queues;
> > +    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> > +    vsc->dev.vqs = s->vhost_vqs;
> > +    vsc->dev.vq_index = 0;
> > +    vsc->dev.backend_features = 0;
> > +
> > +    ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
> > +                         errp);
> > +    if (ret < 0) {
> > +        return ret;
> > +    }
> > +
> > +    /* restore vhost state */
> > +    if (virtio_device_started(vdev, vdev->status)) {
> > +        ret = vhost_user_scsi_start(s, errp);
> > +        if (ret < 0) {
> > +            return ret;
> > +        }
> > +    }
> > +
> > +    return 0;
> > +}
> > +
> > +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);
> > +
> > +static void vhost_user_scsi_disconnect(DeviceState *dev)
> > +{
> > +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> > +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> > +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> > +
> > +    if (!s->connected) {
> > +        return;
> > +    }
> > +    s->connected = false;
> > +
> > +    vhost_user_scsi_stop(s);
> > +
> > +    vhost_dev_cleanup(&vsc->dev);
> > +
> > +    /* Re-instate the event handler for new connections */
> > +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
> > +                             vhost_user_scsi_event, NULL, dev, NULL, true);
> > +}
> > +
> > +static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
> > +{
> > +    DeviceState *dev = opaque;
> > +    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> > +    VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
> > +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> > +    Error *local_err = NULL;
> > +
> > +    switch (event) {
> > +    case CHR_EVENT_OPENED:
> > +        if (vhost_user_scsi_connect(dev, &local_err) < 0) {
> > +            error_report_err(local_err);
> > +            qemu_chr_fe_disconnect(&vs->conf.chardev);
> > +            return;
> > +        }
> > +        break;
> > +    case CHR_EVENT_CLOSED:
> > +        /* defer close until later to avoid circular close */
> > +        vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> > +                               vhost_user_scsi_disconnect);
> > +        break;
> > +    case CHR_EVENT_BREAK:
> > +    case CHR_EVENT_MUX_IN:
> > +    case CHR_EVENT_MUX_OUT:
> > +        /* Ignore */
> > +        break;
> > +    }
> > +}
> > +
> > +static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
> > +{
> > +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> > +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> > +    int ret;
> > +
> > +    s->connected = false;
> > +
> > +    ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
> > +    if (ret < 0) {
> > +        return ret;
> > +    }
> > +
> > +    ret = vhost_user_scsi_connect(dev, errp);
> > +    if (ret < 0) {
> > +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> > +        return ret;
> > +    }
> > +    assert(s->connected);
> > +
> > +    return 0;
> > +}
> > +
> > static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> > {
> > +    ERRP_GUARD();
> >     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> >     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
> >     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > -    struct vhost_virtqueue *vqs = NULL;
> >     Error *err = NULL;
> >     int ret;
> > +    int retries = VU_REALIZE_CONN_RETRIES;
> >
> >     if (!vs->conf.chardev.chr) {
> >         error_setg(errp, "vhost-user-scsi: missing chardev");
> > @@ -115,18 +258,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> >         goto free_virtio;
> >     }
> >
> > -    vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
> > -    vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
> > -    vsc->dev.vq_index = 0;
> > -    vsc->dev.backend_features = 0;
> > -    vqs = vsc->dev.vqs;
> > +    vsc->inflight = g_new0(struct vhost_inflight, 1);
> > +    s->vhost_vqs = g_new0(struct vhost_virtqueue,
> > +                          VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);
> > +
> > +    assert(!*errp);
> > +    do {
> > +        if (*errp) {
> > +            error_prepend(errp, "Reconnecting after error: ");
> > +            error_report_err(*errp);
> > +            *errp = NULL;
> > +        }
> > +        ret = vhost_user_scsi_realize_connect(s, errp);
> > +    } while (ret < 0 && retries--);
> >
> > -    ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
> > -                         VHOST_BACKEND_TYPE_USER, 0, errp);
> >     if (ret < 0) {
> >         goto free_vhost;
> >     }
> >
> > +    /* we're fully initialized, now we can operate, so add the handler */
> > +    qemu_chr_fe_set_handlers(&vs->conf.chardev,  NULL, NULL,
> > +                             vhost_user_scsi_event, NULL, (void *)dev,
> > +                             NULL, true);
> >     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
> >     vsc->channel = 0;
> >     vsc->lun = 0;
> > @@ -135,8 +288,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
> >     return;
> >
> > free_vhost:
> > +    g_free(s->vhost_vqs);
> > +    s->vhost_vqs = NULL;
> > +    g_free(vsc->inflight);
> > +    vsc->inflight = NULL;
> >     vhost_user_cleanup(&s->vhost_user);
> > -    g_free(vqs);
> > +
> > free_virtio:
> >     virtio_scsi_common_unrealize(dev);
> > }
> > @@ -146,16 +303,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
> >     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
> >     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
> >     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> > -    struct vhost_virtqueue *vqs = vsc->dev.vqs;
> > +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> >
> >     /* This will stop the vhost backend. */
> >     vhost_user_scsi_set_status(vdev, 0);
> > +    qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
> > +                             NULL, false);
> >
> >     vhost_dev_cleanup(&vsc->dev);
> > -    g_free(vqs);
> > +    g_free(s->vhost_vqs);
> > +    s->vhost_vqs = NULL;
> > +
> > +    vhost_dev_free_inflight(vsc->inflight);
> > +    g_free(vsc->inflight);
> > +    vsc->inflight = NULL;
> >
> > -    virtio_scsi_common_unrealize(dev);
> >     vhost_user_cleanup(&s->vhost_user);
> > +    virtio_scsi_common_unrealize(dev);
> > }
> >
> > static Property vhost_user_scsi_properties[] = {
> > diff --git a/include/hw/virtio/vhost-scsi-common.h b/include/hw/virtio/vhost-scsi-common.h
> > index 18f115527c..c5d2c09455 100644
> > --- a/include/hw/virtio/vhost-scsi-common.h
> > +++ b/include/hw/virtio/vhost-scsi-common.h
> > @@ -39,7 +39,7 @@ struct VHostSCSICommon {
> >     struct vhost_inflight *inflight;
> > };
> >
> > -int vhost_scsi_common_start(VHostSCSICommon *vsc);
> > +int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp);
> > void vhost_scsi_common_stop(VHostSCSICommon *vsc);
> > char *vhost_scsi_common_get_fw_dev_path(FWPathProvider *p, BusState *bus,
> >                                         DeviceState *dev);
> > diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
> > index 521b08e559..b405ec952a 100644
> > --- a/include/hw/virtio/vhost-user-scsi.h
> > +++ b/include/hw/virtio/vhost-user-scsi.h
> > @@ -29,6 +29,10 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)
> > struct VHostUserSCSI {
> >     VHostSCSICommon parent_obj;
> >     VhostUserState vhost_user;
> > +    bool connected;
> > +    bool started_vu;
> > +
> > +    struct vhost_virtqueue *vhost_vqs;
> > };
> >
> > #endif /* VHOST_USER_SCSI_H */
> > --
> > 2.41.0
> >
>

[PATCH v6 4/5] vhost-user-scsi: start vhost when guest kicks

[Li Feng]

Let's keep the same behavior as vhost-user-blk.

Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
 1 file changed, 44 insertions(+), 4 deletions(-)

diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index dc109154ad..53a62c3170 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -115,8 +115,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
     }
 }
 
-static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
+    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
+    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
+    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
+
+    Error *local_err = NULL;
+    int i, ret;
+
+    if (!vdev->start_on_kick) {
+        return;
+    }
+
+    if (!s->connected) {
+        return;
+    }
+
+    if (vhost_dev_is_started(&vsc->dev)) {
+        return;
+    }
+
+    /*
+     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * vhost here instead of waiting for .set_status().
+     */
+    ret = vhost_user_scsi_start(s, &local_err);
+    if (ret < 0) {
+        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
+        qemu_chr_fe_disconnect(&vs->conf.chardev);
+        return;
+    }
+
+    /* Kick right away to begin processing requests already in vring */
+    for (i = 0; i < vsc->dev.nvqs; i++) {
+        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
+
+        if (!virtio_queue_get_desc_addr(vdev, i)) {
+            continue;
+        }
+        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
+    }
 }
 
 static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
@@ -246,9 +286,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
-                               vhost_dummy_handle_output,
-                               vhost_dummy_handle_output, &err);
+    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output,
+                               vhost_user_scsi_handle_output, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         return;
-- 
2.41.0

Re: [PATCH v6 4/5] vhost-user-scsi: start vhost when guest kicks

[Raphael Norwitz]

> On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> 
> Let's keep the same behavior as vhost-user-blk.
> 
> Some old guests kick virtqueue before setting VIRTIO_CONFIG_S_DRIVER_OK.
> 

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/scsi/vhost-user-scsi.c | 48 +++++++++++++++++++++++++++++++++++----
> 1 file changed, 44 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index dc109154ad..53a62c3170 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -115,8 +115,48 @@ static void vhost_user_scsi_reset(VirtIODevice *vdev)
>     }
> }
> 
> -static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> +static void vhost_user_scsi_handle_output(VirtIODevice *vdev, VirtQueue *vq)
> {
> +    VHostUserSCSI *s = (VHostUserSCSI *)vdev;
> +    DeviceState *dev = &s->parent_obj.parent_obj.parent_obj.parent_obj;
> +    VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
> +    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
> +
> +    Error *local_err = NULL;
> +    int i, ret;
> +
> +    if (!vdev->start_on_kick) {
> +        return;
> +    }
> +
> +    if (!s->connected) {
> +        return;
> +    }
> +
> +    if (vhost_dev_is_started(&vsc->dev)) {
> +        return;
> +    }
> +
> +    /*
> +     * Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
> +     * vhost here instead of waiting for .set_status().
> +     */
> +    ret = vhost_user_scsi_start(s, &local_err);
> +    if (ret < 0) {
> +        error_reportf_err(local_err, "vhost-user-scsi: vhost start failed: ");
> +        qemu_chr_fe_disconnect(&vs->conf.chardev);
> +        return;
> +    }
> +
> +    /* Kick right away to begin processing requests already in vring */
> +    for (i = 0; i < vsc->dev.nvqs; i++) {
> +        VirtQueue *kick_vq = virtio_get_queue(vdev, i);
> +
> +        if (!virtio_queue_get_desc_addr(vdev, i)) {
> +            continue;
> +        }
> +        event_notifier_set(virtio_queue_get_host_notifier(kick_vq));
> +    }
> }
> 
> static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
> @@ -246,9 +286,9 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
>         return;
>     }
> 
> -    virtio_scsi_common_realize(dev, vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output,
> -                               vhost_dummy_handle_output, &err);
> +    virtio_scsi_common_realize(dev, vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output,
> +                               vhost_user_scsi_handle_output, &err);
>     if (err != NULL) {
>         error_propagate(errp, err);
>         return;
> -- 
> 2.41.0
> 

[PATCH v6 5/5] vhost-user: fix lost reconnect

[Li Feng]

When the vhost-user is reconnecting to the backend, and if the vhost-user fails
at the get_features in vhost_dev_init(), then the reconnect will fail
and it will not be retriggered forever.

The reason is:
When the vhost-user fails at get_features, the vhost_dev_cleanup will be called
immediately.

vhost_dev_cleanup calls 'memset(hdev, 0, sizeof(struct vhost_dev))'.

The reconnect path is:
vhost_user_blk_event
   vhost_user_async_close(.. vhost_user_blk_disconnect ..)
     qemu_chr_fe_set_handlers <----- clear the notifier callback
       schedule vhost_user_async_close_bh

The vhost->vdev is null, so the vhost_user_blk_disconnect will not be
called, then the event fd callback will not be reinstalled.

All vhost-user devices have this issue, including vhost-user-blk/scsi.

With this patch, if the vdev->vdev is null, the fd callback will still
be reinstalled.

Fixes: 71e076a07d ("hw/virtio: generalise CHR_EVENT_CLOSED handling")

Signed-off-by: Li Feng <fengli@smartx.com>
---
 hw/block/vhost-user-blk.c      |  2 +-
 hw/scsi/vhost-user-scsi.c      |  3 ++-
 hw/virtio/vhost-user-gpio.c    |  2 +-
 hw/virtio/vhost-user.c         | 10 ++++++++--
 include/hw/virtio/vhost-user.h |  3 ++-
 5 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index 3c69fa47d5..95c758200d 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -391,7 +391,7 @@ static void vhost_user_blk_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &s->chardev, &s->dev,
-                               vhost_user_blk_disconnect);
+                               vhost_user_blk_disconnect, vhost_user_blk_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index 53a62c3170..0effbb4787 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -238,7 +238,8 @@ static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
-                               vhost_user_scsi_disconnect);
+                               vhost_user_scsi_disconnect,
+                               vhost_user_scsi_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
index d9979aa5db..04c2cc79f4 100644
--- a/hw/virtio/vhost-user-gpio.c
+++ b/hw/virtio/vhost-user-gpio.c
@@ -283,7 +283,7 @@ static void vu_gpio_event(void *opaque, QEMUChrEvent event)
     case CHR_EVENT_CLOSED:
         /* defer close until later to avoid circular close */
         vhost_user_async_close(dev, &gpio->chardev, &gpio->vhost_dev,
-                               vu_gpio_disconnect);
+                               vu_gpio_disconnect, vu_gpio_event);
         break;
     case CHR_EVENT_BREAK:
     case CHR_EVENT_MUX_IN:
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 8dcf049d42..7344f57ba7 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -2643,6 +2643,7 @@ typedef struct {
     DeviceState *dev;
     CharBackend *cd;
     struct vhost_dev *vhost;
+    IOEventHandler *event_cb;
 } VhostAsyncCallback;
 
 static void vhost_user_async_close_bh(void *opaque)
@@ -2657,7 +2658,10 @@ static void vhost_user_async_close_bh(void *opaque)
      */
     if (vhost->vdev) {
         data->cb(data->dev);
-    }
+    } else if (data->event_cb) {
+        qemu_chr_fe_set_handlers(data->cd, NULL, NULL, data->event_cb,
+                                 NULL, data->dev, NULL, true);
+   }
 
     g_free(data);
 }
@@ -2669,7 +2673,8 @@ static void vhost_user_async_close_bh(void *opaque)
  */
 void vhost_user_async_close(DeviceState *d,
                             CharBackend *chardev, struct vhost_dev *vhost,
-                            vu_async_close_fn cb)
+                            vu_async_close_fn cb,
+                            IOEventHandler *event_cb)
 {
     if (!runstate_check(RUN_STATE_SHUTDOWN)) {
         /*
@@ -2685,6 +2690,7 @@ void vhost_user_async_close(DeviceState *d,
         data->dev = d;
         data->cd = chardev;
         data->vhost = vhost;
+        data->event_cb = event_cb;
 
         /* Disable any further notifications on the chardev */
         qemu_chr_fe_set_handlers(chardev,
diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
index 191216a74f..649e9dd54f 100644
--- a/include/hw/virtio/vhost-user.h
+++ b/include/hw/virtio/vhost-user.h
@@ -84,6 +84,7 @@ typedef void (*vu_async_close_fn)(DeviceState *cb);
 
 void vhost_user_async_close(DeviceState *d,
                             CharBackend *chardev, struct vhost_dev *vhost,
-                            vu_async_close_fn cb);
+                            vu_async_close_fn cb,
+                            IOEventHandler *event_cb);
 
 #endif
-- 
2.41.0

Re: [PATCH v6 5/5] vhost-user: fix lost reconnect

[Raphael Norwitz]

> On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> 
> When the vhost-user is reconnecting to the backend, and if the vhost-user fails
> at the get_features in vhost_dev_init(), then the reconnect will fail
> and it will not be retriggered forever.
> 
> The reason is:
> When the vhost-user fails at get_features, the vhost_dev_cleanup will be called
> immediately.
> 
> vhost_dev_cleanup calls 'memset(hdev, 0, sizeof(struct vhost_dev))'.
> 
> The reconnect path is:
> vhost_user_blk_event
>   vhost_user_async_close(.. vhost_user_blk_disconnect ..)
>     qemu_chr_fe_set_handlers <----- clear the notifier callback
>       schedule vhost_user_async_close_bh
> 
> The vhost->vdev is null, so the vhost_user_blk_disconnect will not be
> called, then the event fd callback will not be reinstalled.
> 
> All vhost-user devices have this issue, including vhost-user-blk/scsi.
> 
> With this patch, if the vdev->vdev is null, the fd callback will still
> be reinstalled.
> 
> Fixes: 71e076a07d ("hw/virtio: generalise CHR_EVENT_CLOSED handling")
> 

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/block/vhost-user-blk.c      |  2 +-
> hw/scsi/vhost-user-scsi.c      |  3 ++-
> hw/virtio/vhost-user-gpio.c    |  2 +-
> hw/virtio/vhost-user.c         | 10 ++++++++--
> include/hw/virtio/vhost-user.h |  3 ++-
> 5 files changed, 14 insertions(+), 6 deletions(-)
> 
> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
> index 3c69fa47d5..95c758200d 100644
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -391,7 +391,7 @@ static void vhost_user_blk_event(void *opaque, QEMUChrEvent event)
>     case CHR_EVENT_CLOSED:
>         /* defer close until later to avoid circular close */
>         vhost_user_async_close(dev, &s->chardev, &s->dev,
> -                               vhost_user_blk_disconnect);
> +                               vhost_user_blk_disconnect, vhost_user_blk_event);
>         break;
>     case CHR_EVENT_BREAK:
>     case CHR_EVENT_MUX_IN:
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index 53a62c3170..0effbb4787 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -238,7 +238,8 @@ static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>     case CHR_EVENT_CLOSED:
>         /* defer close until later to avoid circular close */
>         vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> -                               vhost_user_scsi_disconnect);
> +                               vhost_user_scsi_disconnect,
> +                               vhost_user_scsi_event);
>         break;
>     case CHR_EVENT_BREAK:
>     case CHR_EVENT_MUX_IN:
> diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
> index d9979aa5db..04c2cc79f4 100644
> --- a/hw/virtio/vhost-user-gpio.c
> +++ b/hw/virtio/vhost-user-gpio.c
> @@ -283,7 +283,7 @@ static void vu_gpio_event(void *opaque, QEMUChrEvent event)
>     case CHR_EVENT_CLOSED:
>         /* defer close until later to avoid circular close */
>         vhost_user_async_close(dev, &gpio->chardev, &gpio->vhost_dev,
> -                               vu_gpio_disconnect);
> +                               vu_gpio_disconnect, vu_gpio_event);
>         break;
>     case CHR_EVENT_BREAK:
>     case CHR_EVENT_MUX_IN:
> diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
> index 8dcf049d42..7344f57ba7 100644
> --- a/hw/virtio/vhost-user.c
> +++ b/hw/virtio/vhost-user.c
> @@ -2643,6 +2643,7 @@ typedef struct {
>     DeviceState *dev;
>     CharBackend *cd;
>     struct vhost_dev *vhost;
> +    IOEventHandler *event_cb;
> } VhostAsyncCallback;
> 
> static void vhost_user_async_close_bh(void *opaque)
> @@ -2657,7 +2658,10 @@ static void vhost_user_async_close_bh(void *opaque)
>      */
>     if (vhost->vdev) {
>         data->cb(data->dev);
> -    }
> +    } else if (data->event_cb) {
> +        qemu_chr_fe_set_handlers(data->cd, NULL, NULL, data->event_cb,
> +                                 NULL, data->dev, NULL, true);
> +   }
> 
>     g_free(data);
> }
> @@ -2669,7 +2673,8 @@ static void vhost_user_async_close_bh(void *opaque)
>  */
> void vhost_user_async_close(DeviceState *d,
>                             CharBackend *chardev, struct vhost_dev *vhost,
> -                            vu_async_close_fn cb)
> +                            vu_async_close_fn cb,
> +                            IOEventHandler *event_cb)
> {
>     if (!runstate_check(RUN_STATE_SHUTDOWN)) {
>         /*
> @@ -2685,6 +2690,7 @@ void vhost_user_async_close(DeviceState *d,
>         data->dev = d;
>         data->cd = chardev;
>         data->vhost = vhost;
> +        data->event_cb = event_cb;
> 
>         /* Disable any further notifications on the chardev */
>         qemu_chr_fe_set_handlers(chardev,
> diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
> index 191216a74f..649e9dd54f 100644
> --- a/include/hw/virtio/vhost-user.h
> +++ b/include/hw/virtio/vhost-user.h
> @@ -84,6 +84,7 @@ typedef void (*vu_async_close_fn)(DeviceState *cb);
> 
> void vhost_user_async_close(DeviceState *d,
>                             CharBackend *chardev, struct vhost_dev *vhost,
> -                            vu_async_close_fn cb);
> +                            vu_async_close_fn cb,
> +                            IOEventHandler *event_cb);
> 
> #endif
> -- 
> 2.41.0
> 

Re: [PATCH v6 5/5] vhost-user: fix lost reconnect

[Raphael Norwitz]

> On Sep 22, 2023, at 7:46 AM, Li Feng <fengli@smartx.com> wrote:
> 
> When the vhost-user is reconnecting to the backend, and if the vhost-user fails
> at the get_features in vhost_dev_init(), then the reconnect will fail
> and it will not be retriggered forever.
> 
> The reason is:
> When the vhost-user fails at get_features, the vhost_dev_cleanup will be called
> immediately.
> 
> vhost_dev_cleanup calls 'memset(hdev, 0, sizeof(struct vhost_dev))'.
> 
> The reconnect path is:
> vhost_user_blk_event
>   vhost_user_async_close(.. vhost_user_blk_disconnect ..)
>     qemu_chr_fe_set_handlers <----- clear the notifier callback
>       schedule vhost_user_async_close_bh
> 
> The vhost->vdev is null, so the vhost_user_blk_disconnect will not be
> called, then the event fd callback will not be reinstalled.
> 
> All vhost-user devices have this issue, including vhost-user-blk/scsi.
> 
> With this patch, if the vdev->vdev is null, the fd callback will still
> be reinstalled.
> 
> Fixes: 71e076a07d ("hw/virtio: generalise CHR_EVENT_CLOSED handling")
> 

Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>

> Signed-off-by: Li Feng <fengli@smartx.com>
> ---
> hw/block/vhost-user-blk.c      |  2 +-
> hw/scsi/vhost-user-scsi.c      |  3 ++-
> hw/virtio/vhost-user-gpio.c    |  2 +-
> hw/virtio/vhost-user.c         | 10 ++++++++--
> include/hw/virtio/vhost-user.h |  3 ++-
> 5 files changed, 14 insertions(+), 6 deletions(-)
> 
> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
> index 3c69fa47d5..95c758200d 100644
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -391,7 +391,7 @@ static void vhost_user_blk_event(void *opaque, QEMUChrEvent event)
>     case CHR_EVENT_CLOSED:
>         /* defer close until later to avoid circular close */
>         vhost_user_async_close(dev, &s->chardev, &s->dev,
> -                               vhost_user_blk_disconnect);
> +                               vhost_user_blk_disconnect, vhost_user_blk_event);
>         break;
>     case CHR_EVENT_BREAK:
>     case CHR_EVENT_MUX_IN:
> diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
> index 53a62c3170..0effbb4787 100644
> --- a/hw/scsi/vhost-user-scsi.c
> +++ b/hw/scsi/vhost-user-scsi.c
> @@ -238,7 +238,8 @@ static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
>     case CHR_EVENT_CLOSED:
>         /* defer close until later to avoid circular close */
>         vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
> -                               vhost_user_scsi_disconnect);
> +                               vhost_user_scsi_disconnect,
> +                               vhost_user_scsi_event);
>         break;
>     case CHR_EVENT_BREAK:
>     case CHR_EVENT_MUX_IN:
> diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c
> index d9979aa5db..04c2cc79f4 100644
> --- a/hw/virtio/vhost-user-gpio.c
> +++ b/hw/virtio/vhost-user-gpio.c
> @@ -283,7 +283,7 @@ static void vu_gpio_event(void *opaque, QEMUChrEvent event)
>     case CHR_EVENT_CLOSED:
>         /* defer close until later to avoid circular close */
>         vhost_user_async_close(dev, &gpio->chardev, &gpio->vhost_dev,
> -                               vu_gpio_disconnect);
> +                               vu_gpio_disconnect, vu_gpio_event);
>         break;
>     case CHR_EVENT_BREAK:
>     case CHR_EVENT_MUX_IN:
> diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
> index 8dcf049d42..7344f57ba7 100644
> --- a/hw/virtio/vhost-user.c
> +++ b/hw/virtio/vhost-user.c
> @@ -2643,6 +2643,7 @@ typedef struct {
>     DeviceState *dev;
>     CharBackend *cd;
>     struct vhost_dev *vhost;
> +    IOEventHandler *event_cb;
> } VhostAsyncCallback;
> 
> static void vhost_user_async_close_bh(void *opaque)
> @@ -2657,7 +2658,10 @@ static void vhost_user_async_close_bh(void *opaque)
>      */
>     if (vhost->vdev) {
>         data->cb(data->dev);
> -    }
> +    } else if (data->event_cb) {
> +        qemu_chr_fe_set_handlers(data->cd, NULL, NULL, data->event_cb,
> +                                 NULL, data->dev, NULL, true);
> +   }
> 
>     g_free(data);
> }
> @@ -2669,7 +2673,8 @@ static void vhost_user_async_close_bh(void *opaque)
>  */
> void vhost_user_async_close(DeviceState *d,
>                             CharBackend *chardev, struct vhost_dev *vhost,
> -                            vu_async_close_fn cb)
> +                            vu_async_close_fn cb,
> +                            IOEventHandler *event_cb)
> {
>     if (!runstate_check(RUN_STATE_SHUTDOWN)) {
>         /*
> @@ -2685,6 +2690,7 @@ void vhost_user_async_close(DeviceState *d,
>         data->dev = d;
>         data->cd = chardev;
>         data->vhost = vhost;
> +        data->event_cb = event_cb;
> 
>         /* Disable any further notifications on the chardev */
>         qemu_chr_fe_set_handlers(chardev,
> diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
> index 191216a74f..649e9dd54f 100644
> --- a/include/hw/virtio/vhost-user.h
> +++ b/include/hw/virtio/vhost-user.h
> @@ -84,6 +84,7 @@ typedef void (*vu_async_close_fn)(DeviceState *cb);
> 
> void vhost_user_async_close(DeviceState *d,
>                             CharBackend *chardev, struct vhost_dev *vhost,
> -                            vu_async_close_fn cb);
> +                            vu_async_close_fn cb,
> +                            IOEventHandler *event_cb);
> 
> #endif
> -- 
> 2.41.0
> 

Re: [PATCH v6 0/5] Implement reconnect for vhost-user-scsi

[Michael S. Tsirkin]

On Fri, Sep 22, 2023 at 07:46:10PM +0800, Li Feng wrote:
> Changes for v6:
> - [PATCH] vhost-user: fix lost reconnect
>   - Fix missing assign event_cb.


Pls don't make vN+1 a reply to vN - start a new thread
with each version please.

> Changes for v5:
> - No logic has been changed, just move part of the code from patch 4 to patch 5.
> 
> Changes for v4:
> - Merge
>   https://lore.kernel.org/all/20230830045722.611224-1-fengli@smartx.com/ to
>   this series.
> - Add ERRP_GUARD in vhost_user_scsi_realize;
> - Reword the commit messages.
> 
> Changes for v3:
> - Split the vhost_user_scsi_handle_output to a separate patch;
> - Move the started_vu from vhost scsi common header to vhost-user-scsi header;
> - Fix a log print error;
> 
> Changes for v2:
> - Split the v1 patch to small separate patchset;
> - New patch for fixing fd leak, which has sent to reviewers in another
>   mail;
> - Implement the `vhost_user_scsi_handle_output`;
> - Add the started_vu safe check;
> - Fix error handler;
> - Check the inflight before set/get inflight fd.
> 
> Li Feng (5):
>   vhost-user-common: send get_inflight_fd once
>   vhost: move and rename the conn retry times
>   vhost-user-scsi: support reconnect to backend
>   vhost-user-scsi: start vhost when guest kicks
>   vhost-user: fix lost reconnect
> 
>  hw/block/vhost-user-blk.c             |   6 +-
>  hw/scsi/vhost-scsi-common.c           |  47 ++---
>  hw/scsi/vhost-scsi.c                  |   5 +-
>  hw/scsi/vhost-user-scsi.c             | 253 +++++++++++++++++++++++---
>  hw/virtio/vhost-user-gpio.c           |   5 +-
>  hw/virtio/vhost-user.c                |  10 +-
>  include/hw/virtio/vhost-scsi-common.h |   2 +-
>  include/hw/virtio/vhost-user-scsi.h   |   4 +
>  include/hw/virtio/vhost-user.h        |   3 +-
>  include/hw/virtio/vhost.h             |   2 +
>  10 files changed, 277 insertions(+), 60 deletions(-)
> 
> -- 
> 2.41.0

Re: [PATCH v6 0/5] Implement reconnect for vhost-user-scsi

[Li Feng]

On Sun, Oct 8, 2023 at 4:49 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Fri, Sep 22, 2023 at 07:46:10PM +0800, Li Feng wrote:
> > Changes for v6:
> > - [PATCH] vhost-user: fix lost reconnect
> >   - Fix missing assign event_cb.
>
>
> Pls don't make vN+1 a reply to vN - start a new thread
> with each version please.
OK.

>
> > Changes for v5:
> > - No logic has been changed, just move part of the code from patch 4 to patch 5.
> >
> > Changes for v4:
> > - Merge
> >   https://lore.kernel.org/all/20230830045722.611224-1-fengli@smartx.com/ to
> >   this series.
> > - Add ERRP_GUARD in vhost_user_scsi_realize;
> > - Reword the commit messages.
> >
> > Changes for v3:
> > - Split the vhost_user_scsi_handle_output to a separate patch;
> > - Move the started_vu from vhost scsi common header to vhost-user-scsi header;
> > - Fix a log print error;
> >
> > Changes for v2:
> > - Split the v1 patch to small separate patchset;
> > - New patch for fixing fd leak, which has sent to reviewers in another
> >   mail;
> > - Implement the `vhost_user_scsi_handle_output`;
> > - Add the started_vu safe check;
> > - Fix error handler;
> > - Check the inflight before set/get inflight fd.
> >
> > Li Feng (5):
> >   vhost-user-common: send get_inflight_fd once
> >   vhost: move and rename the conn retry times
> >   vhost-user-scsi: support reconnect to backend
> >   vhost-user-scsi: start vhost when guest kicks
> >   vhost-user: fix lost reconnect
> >
> >  hw/block/vhost-user-blk.c             |   6 +-
> >  hw/scsi/vhost-scsi-common.c           |  47 ++---
> >  hw/scsi/vhost-scsi.c                  |   5 +-
> >  hw/scsi/vhost-user-scsi.c             | 253 +++++++++++++++++++++++---
> >  hw/virtio/vhost-user-gpio.c           |   5 +-
> >  hw/virtio/vhost-user.c                |  10 +-
> >  include/hw/virtio/vhost-scsi-common.h |   2 +-
> >  include/hw/virtio/vhost-user-scsi.h   |   4 +
> >  include/hw/virtio/vhost-user.h        |   3 +-
> >  include/hw/virtio/vhost.h             |   2 +
> >  10 files changed, 277 insertions(+), 60 deletions(-)
> >
> > --
> > 2.41.0
>