[PATCH 1/2] tcm_vhost: Use vq->private_data to indicate if the endpoint is setup

[PATCH 1/2] tcm_vhost: Use vq->private_data to indicate if the endpoint is setup

[Asias He]

Currently, vs->vs_endpoint is used indicate if the endpoint is setup or
not. It is set or cleared in vhost_scsi_set_endpoint() or
vhost_scsi_clear_endpoint() under the vs->dev.mutex lock. However, when
we check it in vhost_scsi_handle_vq(), we ignored the lock.

Instead of using the vs->vs_endpoint and the vs->dev.mutex lock to
indicate the status of the endpoint, we use per virtqueue
vq->private_data to indicate it. In this way, we can only take the
vq->mutex lock which is per queue and make the concurrent multiqueue
process having less lock contention. Further, in the read side of
vq->private_data, we can even do not take only lock if it is accessed in
the vhost worker thread, because it is protected by "vhost rcu".

Signed-off-by: Asias He <asias@redhat.com>
---
 drivers/vhost/tcm_vhost.c | 46 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 6 deletions(-)

diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
index 43fb11e..099feef 100644
--- a/drivers/vhost/tcm_vhost.c
+++ b/drivers/vhost/tcm_vhost.c
@@ -67,7 +67,6 @@ struct vhost_scsi {
 	/* Protected by vhost_scsi->dev.mutex */
 	struct tcm_vhost_tpg *vs_tpg[VHOST_SCSI_MAX_TARGET];
 	char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
-	bool vs_endpoint;
 
 	struct vhost_dev dev;
 	struct vhost_virtqueue vqs[VHOST_SCSI_MAX_VQ];
@@ -91,6 +90,24 @@ static int iov_num_pages(struct iovec *iov)
 	       ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT;
 }
 
+static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)
+{
+	bool ret = false;
+
+	/*
+	 * We can handle the vq only after the endpoint is setup by calling the
+	 * VHOST_SCSI_SET_ENDPOINT ioctl.
+	 *
+	 * TODO: Check that we are running from vhost_worker which acts
+	 * as read-side critical section for vhost kind of RCU.
+	 * See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h
+	 */
+	if (rcu_dereference_check(vq->private_data, 1))
+		ret = true;
+
+	return ret;
+}
+
 static int tcm_vhost_check_true(struct se_portal_group *se_tpg)
 {
 	return 1;
@@ -581,8 +598,7 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs,
 	int head, ret;
 	u8 target;
 
-	/* Must use ioctl VHOST_SCSI_SET_ENDPOINT */
-	if (unlikely(!vs->vs_endpoint))
+	if (!tcm_vhost_check_endpoint(vq))
 		return;
 
 	mutex_lock(&vq->mutex);
@@ -781,8 +797,9 @@ static int vhost_scsi_set_endpoint(
 {
 	struct tcm_vhost_tport *tv_tport;
 	struct tcm_vhost_tpg *tv_tpg;
+	struct vhost_virtqueue *vq;
 	bool match = false;
-	int index, ret;
+	int index, ret, i;
 
 	mutex_lock(&vs->dev.mutex);
 	/* Verify that ring has been setup correctly. */
@@ -826,7 +843,13 @@ static int vhost_scsi_set_endpoint(
 	if (match) {
 		memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
 		       sizeof(vs->vs_vhost_wwpn));
-		vs->vs_endpoint = true;
+		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+			vq = &vs->vqs[i];
+			/* Flushing the vhost_work acts as synchronize_rcu */
+			mutex_lock(&vq->mutex);
+			rcu_assign_pointer(vq->private_data, vs);
+			mutex_unlock(&vq->mutex);
+		}
 		ret = 0;
 	} else {
 		ret = -EEXIST;
@@ -842,6 +865,8 @@ static int vhost_scsi_clear_endpoint(
 {
 	struct tcm_vhost_tport *tv_tport;
 	struct tcm_vhost_tpg *tv_tpg;
+	struct vhost_virtqueue *vq;
+	bool match = false;
 	int index, ret, i;
 	u8 target;
 
@@ -877,9 +902,18 @@ static int vhost_scsi_clear_endpoint(
 		}
 		tv_tpg->tv_tpg_vhost_count--;
 		vs->vs_tpg[target] = NULL;
-		vs->vs_endpoint = false;
+		match = true;
 		mutex_unlock(&tv_tpg->tv_tpg_mutex);
 	}
+	if (match) {
+		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+			vq = &vs->vqs[i];
+			/* Flushing the vhost_work acts as synchronize_rcu */
+			mutex_lock(&vq->mutex);
+			rcu_assign_pointer(vq->private_data, NULL);
+			mutex_unlock(&vq->mutex);
+		}
+	}
 	mutex_unlock(&vs->dev.mutex);
 	return 0;
 
-- 
1.8.1.4

Re: [PATCH 1/2] tcm_vhost: Use vq->private_data to indicate if the endpoint is setup

[Michael S. Tsirkin]

On Fri, Mar 22, 2013 at 10:52:27AM +0800, Asias He wrote:
> Currently, vs->vs_endpoint is used indicate if the endpoint is setup or
> not. It is set or cleared in vhost_scsi_set_endpoint() or
> vhost_scsi_clear_endpoint() under the vs->dev.mutex lock. However, when
> we check it in vhost_scsi_handle_vq(), we ignored the lock.
> 
> Instead of using the vs->vs_endpoint and the vs->dev.mutex lock to
> indicate the status of the endpoint, we use per virtqueue
> vq->private_data to indicate it. In this way, we can only take the
> vq->mutex lock which is per queue and make the concurrent multiqueue
> process having less lock contention. Further, in the read side of
> vq->private_data, we can even do not take only lock if it is accessed in
> the vhost worker thread, because it is protected by "vhost rcu".
> 
> Signed-off-by: Asias He <asias@redhat.com>

Need to think some more about this, in any case,
not 3.9 material IMO.

> ---
>  drivers/vhost/tcm_vhost.c | 46 ++++++++++++++++++++++++++++++++++++++++------
>  1 file changed, 40 insertions(+), 6 deletions(-)
> 
> diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> index 43fb11e..099feef 100644
> --- a/drivers/vhost/tcm_vhost.c
> +++ b/drivers/vhost/tcm_vhost.c
> @@ -67,7 +67,6 @@ struct vhost_scsi {
>  	/* Protected by vhost_scsi->dev.mutex */
>  	struct tcm_vhost_tpg *vs_tpg[VHOST_SCSI_MAX_TARGET];
>  	char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
> -	bool vs_endpoint;
>  
>  	struct vhost_dev dev;
>  	struct vhost_virtqueue vqs[VHOST_SCSI_MAX_VQ];
> @@ -91,6 +90,24 @@ static int iov_num_pages(struct iovec *iov)
>  	       ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT;
>  }
>  
> +static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)
> +{
> +	bool ret = false;
> +
> +	/*
> +	 * We can handle the vq only after the endpoint is setup by calling the
> +	 * VHOST_SCSI_SET_ENDPOINT ioctl.
> +	 *
> +	 * TODO: Check that we are running from vhost_worker which acts
> +	 * as read-side critical section for vhost kind of RCU.
> +	 * See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h
> +	 */
> +	if (rcu_dereference_check(vq->private_data, 1))
> +		ret = true;
> +
> +	return ret;
> +}
> +
>  static int tcm_vhost_check_true(struct se_portal_group *se_tpg)
>  {
>  	return 1;
> @@ -581,8 +598,7 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs,
>  	int head, ret;
>  	u8 target;
>  
> -	/* Must use ioctl VHOST_SCSI_SET_ENDPOINT */
> -	if (unlikely(!vs->vs_endpoint))
> +	if (!tcm_vhost_check_endpoint(vq))
>  		return;
>  
>  	mutex_lock(&vq->mutex);
> @@ -781,8 +797,9 @@ static int vhost_scsi_set_endpoint(
>  {
>  	struct tcm_vhost_tport *tv_tport;
>  	struct tcm_vhost_tpg *tv_tpg;
> +	struct vhost_virtqueue *vq;
>  	bool match = false;
> -	int index, ret;
> +	int index, ret, i;
>  
>  	mutex_lock(&vs->dev.mutex);
>  	/* Verify that ring has been setup correctly. */
> @@ -826,7 +843,13 @@ static int vhost_scsi_set_endpoint(
>  	if (match) {
>  		memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
>  		       sizeof(vs->vs_vhost_wwpn));
> -		vs->vs_endpoint = true;
> +		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
> +			vq = &vs->vqs[i];
> +			/* Flushing the vhost_work acts as synchronize_rcu */
> +			mutex_lock(&vq->mutex);
> +			rcu_assign_pointer(vq->private_data, vs);
> +			mutex_unlock(&vq->mutex);
> +		}
>  		ret = 0;
>  	} else {
>  		ret = -EEXIST;
> @@ -842,6 +865,8 @@ static int vhost_scsi_clear_endpoint(
>  {
>  	struct tcm_vhost_tport *tv_tport;
>  	struct tcm_vhost_tpg *tv_tpg;
> +	struct vhost_virtqueue *vq;
> +	bool match = false;
>  	int index, ret, i;
>  	u8 target;
>  
> @@ -877,9 +902,18 @@ static int vhost_scsi_clear_endpoint(
>  		}
>  		tv_tpg->tv_tpg_vhost_count--;
>  		vs->vs_tpg[target] = NULL;
> -		vs->vs_endpoint = false;
> +		match = true;
>  		mutex_unlock(&tv_tpg->tv_tpg_mutex);
>  	}
> +	if (match) {
> +		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
> +			vq = &vs->vqs[i];
> +			/* Flushing the vhost_work acts as synchronize_rcu */
> +			mutex_lock(&vq->mutex);
> +			rcu_assign_pointer(vq->private_data, NULL);
> +			mutex_unlock(&vq->mutex);
> +		}
> +	}
>  	mutex_unlock(&vs->dev.mutex);
>  	return 0;
>  
> -- 
> 1.8.1.4

[PATCH 2/2] tcm_vhost: Initialize vq->last_used_idx when set endpoint

[Asias He]

This patch fixes guest hang when booting seabios and guest.

  [    0.576238] scsi0 : Virtio SCSI HBA
  [    0.616754] virtio_scsi virtio1: request:id 0 is not a head!

vq->last_used_idx is initialized only when /dev/vhost-scsi is
opened or closed.

   vhost_scsi_open -> vhost_dev_init() -> vhost_vq_reset()
   vhost_scsi_release() -> vhost_dev_cleanup -> vhost_vq_reset()

So, when guest talks to tcm_vhost after seabios does, vq->last_used_idx
still contains the old valule for seabios. This confuses guest.

Fix this by calling vhost_init_used() to init vq->last_used_idx when
we set endpoint.

Signed-off-by: Asias He <asias@redhat.com>
---
 drivers/vhost/tcm_vhost.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
index 099feef..0524267 100644
--- a/drivers/vhost/tcm_vhost.c
+++ b/drivers/vhost/tcm_vhost.c
@@ -848,6 +848,7 @@ static int vhost_scsi_set_endpoint(
 			/* Flushing the vhost_work acts as synchronize_rcu */
 			mutex_lock(&vq->mutex);
 			rcu_assign_pointer(vq->private_data, vs);
+			vhost_init_used(vq);
 			mutex_unlock(&vq->mutex);
 		}
 		ret = 0;
-- 
1.8.1.4

Re: [PATCH 2/2] tcm_vhost: Initialize vq->last_used_idx when set endpoint

[Michael S. Tsirkin]

On Fri, Mar 22, 2013 at 10:52:28AM +0800, Asias He wrote:
> This patch fixes guest hang when booting seabios and guest.
> 
>   [    0.576238] scsi0 : Virtio SCSI HBA
>   [    0.616754] virtio_scsi virtio1: request:id 0 is not a head!
> 
> vq->last_used_idx is initialized only when /dev/vhost-scsi is
> opened or closed.
> 
>    vhost_scsi_open -> vhost_dev_init() -> vhost_vq_reset()
>    vhost_scsi_release() -> vhost_dev_cleanup -> vhost_vq_reset()
> 
> So, when guest talks to tcm_vhost after seabios does, vq->last_used_idx
> still contains the old valule for seabios. This confuses guest.
> 
> Fix this by calling vhost_init_used() to init vq->last_used_idx when
> we set endpoint.
> 
> Signed-off-by: Asias He <asias@redhat.com>

Good catch, thanks.

Acked-by: Michael S. Tsirkin <mst@redhat.com>

Nicholas can you pick this one up for 3.9 please?

> ---
>  drivers/vhost/tcm_vhost.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> index 099feef..0524267 100644
> --- a/drivers/vhost/tcm_vhost.c
> +++ b/drivers/vhost/tcm_vhost.c
> @@ -848,6 +848,7 @@ static int vhost_scsi_set_endpoint(
>  			/* Flushing the vhost_work acts as synchronize_rcu */
>  			mutex_lock(&vq->mutex);
>  			rcu_assign_pointer(vq->private_data, vs);
> +			vhost_init_used(vq);
>  			mutex_unlock(&vq->mutex);
>  		}
>  		ret = 0;
> -- 
> 1.8.1.4

Re: [PATCH 2/2] tcm_vhost: Initialize vq->last_used_idx when set endpoint

[Nicholas A. Bellinger]

On Mon, 2013-03-25 at 13:16 +0200, Michael S. Tsirkin wrote:
> On Fri, Mar 22, 2013 at 10:52:28AM +0800, Asias He wrote:
> > This patch fixes guest hang when booting seabios and guest.
> > 
> >   [    0.576238] scsi0 : Virtio SCSI HBA
> >   [    0.616754] virtio_scsi virtio1: request:id 0 is not a head!
> > 
> > vq->last_used_idx is initialized only when /dev/vhost-scsi is
> > opened or closed.
> > 
> >    vhost_scsi_open -> vhost_dev_init() -> vhost_vq_reset()
> >    vhost_scsi_release() -> vhost_dev_cleanup -> vhost_vq_reset()
> > 
> > So, when guest talks to tcm_vhost after seabios does, vq->last_used_idx
> > still contains the old valule for seabios. This confuses guest.
> > 
> > Fix this by calling vhost_init_used() to init vq->last_used_idx when
> > we set endpoint.
> > 
> > Signed-off-by: Asias He <asias@redhat.com>
> 
> Good catch, thanks.
> 
> Acked-by: Michael S. Tsirkin <mst@redhat.com>
> 
> Nicholas can you pick this one up for 3.9 please?
> 

Just a heads up that this needs PATCH 1/2 as vhost_init_used() expects
vq->private_data to already have been set..

Can you take a another look at the first patch as a v3.9 item..?  Both
are required in order for Asias's seabios changes to work.

Thanks,

--nab

> > ---
> >  drivers/vhost/tcm_vhost.c | 1 +
> >  1 file changed, 1 insertion(+)
> > 
> > diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> > index 099feef..0524267 100644
> > --- a/drivers/vhost/tcm_vhost.c
> > +++ b/drivers/vhost/tcm_vhost.c
> > @@ -848,6 +848,7 @@ static int vhost_scsi_set_endpoint(
> >  			/* Flushing the vhost_work acts as synchronize_rcu */
> >  			mutex_lock(&vq->mutex);
> >  			rcu_assign_pointer(vq->private_data, vs);
> > +			vhost_init_used(vq);
> >  			mutex_unlock(&vq->mutex);
> >  		}
> >  		ret = 0;
> > -- 
> > 1.8.1.4
> --
> To unsubscribe from this list: send the line "unsubscribe target-devel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/2] tcm_vhost: Initialize vq->last_used_idx when set endpoint

[Michael S. Tsirkin]

On Wed, Mar 27, 2013 at 01:29:15PM -0700, Nicholas A. Bellinger wrote:
> On Mon, 2013-03-25 at 13:16 +0200, Michael S. Tsirkin wrote:
> > On Fri, Mar 22, 2013 at 10:52:28AM +0800, Asias He wrote:
> > > This patch fixes guest hang when booting seabios and guest.
> > > 
> > >   [    0.576238] scsi0 : Virtio SCSI HBA
> > >   [    0.616754] virtio_scsi virtio1: request:id 0 is not a head!
> > > 
> > > vq->last_used_idx is initialized only when /dev/vhost-scsi is
> > > opened or closed.
> > > 
> > >    vhost_scsi_open -> vhost_dev_init() -> vhost_vq_reset()
> > >    vhost_scsi_release() -> vhost_dev_cleanup -> vhost_vq_reset()
> > > 
> > > So, when guest talks to tcm_vhost after seabios does, vq->last_used_idx
> > > still contains the old valule for seabios. This confuses guest.
> > > 
> > > Fix this by calling vhost_init_used() to init vq->last_used_idx when
> > > we set endpoint.
> > > 
> > > Signed-off-by: Asias He <asias@redhat.com>
> > 
> > Good catch, thanks.
> > 
> > Acked-by: Michael S. Tsirkin <mst@redhat.com>
> > 
> > Nicholas can you pick this one up for 3.9 please?
> > 
> 
> Just a heads up that this needs PATCH 1/2 as vhost_init_used() expects
> vq->private_data to already have been set..
> 
> Can you take a another look at the first patch as a v3.9 item..?  Both
> are required in order for Asias's seabios changes to work.
> 
> Thanks,
> 
> --nab
> 

Both me and Asias agreed it's not 3.9 material.
Asias, can you please rework 2/2 to avoid dependency on 1/2?
Should be trivial enough ...

> > >  drivers/vhost/tcm_vhost.c | 1 +
> > >  1 file changed, 1 insertion(+)
> > > 
> > > diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> > > index 099feef..0524267 100644
> > > --- a/drivers/vhost/tcm_vhost.c
> > > +++ b/drivers/vhost/tcm_vhost.c
> > > @@ -848,6 +848,7 @@ static int vhost_scsi_set_endpoint(
> > >  			/* Flushing the vhost_work acts as synchronize_rcu */
> > >  			mutex_lock(&vq->mutex);
> > >  			rcu_assign_pointer(vq->private_data, vs);
> > > +			vhost_init_used(vq);
> > >  			mutex_unlock(&vq->mutex);
> > >  		}
> > >  		ret = 0;
> > > -- 
> > > 1.8.1.4
> > --
> > To unsubscribe from this list: send the line "unsubscribe target-devel" in
> > the body of a message to majordomo@vger.kernel.org
> > More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

Re: [PATCH 2/2] tcm_vhost: Initialize vq->last_used_idx when set endpoint

[Asias He]

On Wed, Mar 27, 2013 at 11:45:59PM +0200, Michael S. Tsirkin wrote:
> On Wed, Mar 27, 2013 at 01:29:15PM -0700, Nicholas A. Bellinger wrote:
> > On Mon, 2013-03-25 at 13:16 +0200, Michael S. Tsirkin wrote:
> > > On Fri, Mar 22, 2013 at 10:52:28AM +0800, Asias He wrote:
> > > > This patch fixes guest hang when booting seabios and guest.
> > > > 
> > > >   [    0.576238] scsi0 : Virtio SCSI HBA
> > > >   [    0.616754] virtio_scsi virtio1: request:id 0 is not a head!
> > > > 
> > > > vq->last_used_idx is initialized only when /dev/vhost-scsi is
> > > > opened or closed.
> > > > 
> > > >    vhost_scsi_open -> vhost_dev_init() -> vhost_vq_reset()
> > > >    vhost_scsi_release() -> vhost_dev_cleanup -> vhost_vq_reset()
> > > > 
> > > > So, when guest talks to tcm_vhost after seabios does, vq->last_used_idx
> > > > still contains the old valule for seabios. This confuses guest.
> > > > 
> > > > Fix this by calling vhost_init_used() to init vq->last_used_idx when
> > > > we set endpoint.
> > > > 
> > > > Signed-off-by: Asias He <asias@redhat.com>
> > > 
> > > Good catch, thanks.
> > > 
> > > Acked-by: Michael S. Tsirkin <mst@redhat.com>
> > > 
> > > Nicholas can you pick this one up for 3.9 please?
> > > 
> > 
> > Just a heads up that this needs PATCH 1/2 as vhost_init_used() expects
> > vq->private_data to already have been set..
> > 
> > Can you take a another look at the first patch as a v3.9 item..?  Both
> > are required in order for Asias's seabios changes to work.
> > 
> > Thanks,
> > 
> > --nab
> > 
> 
> Both me and Asias agreed it's not 3.9 material.
> Asias, can you please rework 2/2 to avoid dependency on 1/2?
> Should be trivial enough ...

Nicholas, please take the one from '[PATCH V2 0/2] tcm_vhost endpoint' for
3.9.

> > > >  drivers/vhost/tcm_vhost.c | 1 +
> > > >  1 file changed, 1 insertion(+)
> > > > 
> > > > diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> > > > index 099feef..0524267 100644
> > > > --- a/drivers/vhost/tcm_vhost.c
> > > > +++ b/drivers/vhost/tcm_vhost.c
> > > > @@ -848,6 +848,7 @@ static int vhost_scsi_set_endpoint(
> > > >  			/* Flushing the vhost_work acts as synchronize_rcu */
> > > >  			mutex_lock(&vq->mutex);
> > > >  			rcu_assign_pointer(vq->private_data, vs);
> > > > +			vhost_init_used(vq);
> > > >  			mutex_unlock(&vq->mutex);
> > > >  		}
> > > >  		ret = 0;
> > > > -- 
> > > > 1.8.1.4
> > > --
> > > To unsubscribe from this list: send the line "unsubscribe target-devel" in
> > > the body of a message to majordomo@vger.kernel.org
> > > More majordomo info at  http://vger.kernel.org/majordomo-info.html
> > 

-- 
Asias