[PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

A SCSI LLD may start cleaning up host resources as soon as
scsi_remove_host() returns. These host resources may be needed by
the LLD in an implementation of one of the eh_* functions. So if
one of the eh_* functions is in progress when scsi_remove_host()
is invoked, wait until the eh_* function has finished. Also, do
not invoke any of the eh_* functions after scsi_remove_host() has
started. Remove Scsi_Host.tmf_in_progress because it is now
superfluous.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Cc: Hannes Reinecke <hare@suse.de>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Tejun Heo <tj@kernel.org>
---
 drivers/scsi/hosts.c      |    6 ++++
 drivers/scsi/scsi_error.c |   86 ++++++++++++++++++++++++++++++++++++++-------
 include/scsi/scsi_host.h  |    6 ++--
 3 files changed, 81 insertions(+), 17 deletions(-)

diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index 034a567..17e2ccb 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -176,6 +176,12 @@ void scsi_remove_host(struct Scsi_Host *shost)
 		BUG_ON(scsi_host_set_state(shost, SHOST_DEL_RECOVERY));
 	spin_unlock_irq(shost->host_lock);
 
+	/*
+	 * Wait until the error handler has finished invoking LLD callbacks
+	 * before allowing the LLD to proceed.
+	 */
+	wait_event(shost->host_wait, shost->eh_active == 0);
+
 	transport_unregister_device(&shost->shost_gendev);
 	device_unregister(&shost->shost_dev);
 	device_del(&shost->shost_gendev);
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index f43de1e..1f2f593 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -537,8 +537,53 @@ static void scsi_eh_done(struct scsi_cmnd *scmd)
 }
 
 /**
+ * scsi_begin_eh - start host-related error handling
+ *
+ * Must be called before invoking an LLD callback function to avoid that
+ * scsi_remove_host() returns while one of these callback functions is in
+ * progress.
+ *
+ * Returns 0 if invoking an eh_* function is allowed and a negative value if
+ * not. If this function returns 0 then scsi_end_eh() must be called
+ * eventually.
+ */
+static int scsi_begin_eh(struct Scsi_Host *host)
+{
+	int res;
+
+	spin_lock_irq(host->host_lock);
+	switch (host->shost_state) {
+	case SHOST_DEL:
+	case SHOST_DEL_RECOVERY:
+		res = -ENODEV;
+		break;
+	default:
+		WARN_ON_ONCE(host->eh_active < 0);
+		host->eh_active++;
+		res = 0;
+		break;
+	}
+	spin_unlock_irq(host->host_lock);
+
+	return res;
+}
+
+/**
+ * scsi_end_eh - finish host-related error handling
+ */
+static void scsi_end_eh(struct Scsi_Host *host)
+{
+	spin_lock_irq(host->host_lock);
+	host->eh_active--;
+	WARN_ON_ONCE(host->eh_active < 0);
+	if (host->eh_active == 0)
+		wake_up(&host->host_wait);
+	spin_unlock_irq(host->host_lock);
+}
+
+/**
  * scsi_try_host_reset - ask host adapter to reset itself
- * @scmd:	SCSI cmd to send hsot reset.
+ * @scmd:	SCSI cmd to send host reset.
  */
 static int scsi_try_host_reset(struct scsi_cmnd *scmd)
 {
@@ -553,6 +598,9 @@ static int scsi_try_host_reset(struct scsi_cmnd *scmd)
 	if (!hostt->eh_host_reset_handler)
 		return FAILED;
 
+	if (scsi_begin_eh(host))
+		return FAST_IO_FAIL;
+
 	rtn = hostt->eh_host_reset_handler(scmd);
 
 	if (rtn == SUCCESS) {
@@ -562,6 +610,7 @@ static int scsi_try_host_reset(struct scsi_cmnd *scmd)
 		scsi_report_bus_reset(host, scmd_channel(scmd));
 		spin_unlock_irqrestore(host->host_lock, flags);
 	}
+	scsi_end_eh(host);
 
 	return rtn;
 }
@@ -583,6 +632,9 @@ static int scsi_try_bus_reset(struct scsi_cmnd *scmd)
 	if (!hostt->eh_bus_reset_handler)
 		return FAILED;
 
+	if (scsi_begin_eh(host))
+		return FAST_IO_FAIL;
+
 	rtn = hostt->eh_bus_reset_handler(scmd);
 
 	if (rtn == SUCCESS) {
@@ -592,6 +644,7 @@ static int scsi_try_bus_reset(struct scsi_cmnd *scmd)
 		scsi_report_bus_reset(host, scmd_channel(scmd));
 		spin_unlock_irqrestore(host->host_lock, flags);
 	}
+	scsi_end_eh(host);
 
 	return rtn;
 }
@@ -622,6 +675,9 @@ static int scsi_try_target_reset(struct scsi_cmnd *scmd)
 	if (!hostt->eh_target_reset_handler)
 		return FAILED;
 
+	if (scsi_begin_eh(host))
+		return FAST_IO_FAIL;
+
 	rtn = hostt->eh_target_reset_handler(scmd);
 	if (rtn == SUCCESS) {
 		spin_lock_irqsave(host->host_lock, flags);
@@ -629,6 +685,7 @@ static int scsi_try_target_reset(struct scsi_cmnd *scmd)
 					  __scsi_report_device_reset);
 		spin_unlock_irqrestore(host->host_lock, flags);
 	}
+	scsi_end_eh(host);
 
 	return rtn;
 }
@@ -646,14 +703,20 @@ static int scsi_try_target_reset(struct scsi_cmnd *scmd)
 static int scsi_try_bus_device_reset(struct scsi_cmnd *scmd)
 {
 	int rtn;
-	struct scsi_host_template *hostt = scmd->device->host->hostt;
+	struct Scsi_Host *host = scmd->device->host;
+	struct scsi_host_template *hostt = host->hostt;
 
 	if (!hostt->eh_device_reset_handler)
 		return FAILED;
 
+	if (scsi_begin_eh(host))
+		return FAST_IO_FAIL;
+
 	rtn = hostt->eh_device_reset_handler(scmd);
 	if (rtn == SUCCESS)
 		__scsi_report_device_reset(scmd->device, NULL);
+	scsi_end_eh(host);
+
 	return rtn;
 }
 
@@ -797,6 +860,9 @@ static int scsi_send_eh_cmnd(struct scsi_cmnd *scmd, unsigned char *cmnd,
 	const unsigned long stall_for = msecs_to_jiffies(100);
 	int rtn;
 
+	if (scsi_begin_eh(shost))
+		return FAILED;
+
 retry:
 	scsi_eh_prep_cmnd(scmd, &ses, cmnd, cmnd_size, sense_bytes);
 	shost->eh_action = &done;
@@ -867,6 +933,8 @@ retry:
 			rtn = sdrv->eh_action(scmd, cmnd, cmnd_size, rtn);
 	}
 
+	scsi_end_eh(shost);
+
 	return rtn;
 }
 
@@ -1894,6 +1962,9 @@ int scsi_error_handler(void *data)
 	}
 	__set_current_state(TASK_RUNNING);
 
+	WARN_ONCE(shost->eh_active, "scsi_eh_%d: eh_active = %d\n",
+		  shost->host_no, shost->eh_active);
+
 	SCSI_LOG_ERROR_RECOVERY(1,
 		printk("Error handler scsi_eh_%d exiting\n", shost->host_no));
 	shost->ehandler = NULL;
@@ -1990,7 +2061,6 @@ scsi_reset_provider(struct scsi_device *dev, int flag)
 	struct scsi_cmnd *scmd;
 	struct Scsi_Host *shost = dev->host;
 	struct request req;
-	unsigned long flags;
 	int rtn;
 
 	if (scsi_autopm_get_host(shost) < 0)
@@ -2009,10 +2079,6 @@ scsi_reset_provider(struct scsi_device *dev, int flag)
 
 	scmd->sc_data_direction		= DMA_BIDIRECTIONAL;
 
-	spin_lock_irqsave(shost->host_lock, flags);
-	shost->tmf_in_progress = 1;
-	spin_unlock_irqrestore(shost->host_lock, flags);
-
 	switch (flag) {
 	case SCSI_TRY_RESET_DEVICE:
 		rtn = scsi_try_bus_device_reset(scmd);
@@ -2036,10 +2102,6 @@ scsi_reset_provider(struct scsi_device *dev, int flag)
 		rtn = FAILED;
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
-	shost->tmf_in_progress = 0;
-	spin_unlock_irqrestore(shost->host_lock, flags);
-
 	/*
 	 * be sure to wake up anyone who was sleeping or had their queue
 	 * suspended while we performed the TMF.
@@ -2048,8 +2110,6 @@ scsi_reset_provider(struct scsi_device *dev, int flag)
 		printk("%s: waking up host to restart after TMF\n",
 		__func__));
 
-	wake_up(&shost->host_wait);
-
 	scsi_run_host_queues(shost);
 
 	scsi_next_command(scmd);
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 7552435..9785e51 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -578,6 +578,7 @@ struct Scsi_Host {
 	struct task_struct    * ehandler;  /* Error recovery thread. */
 	struct completion     * eh_action; /* Wait for specific actions on the
 					      host. */
+	int			eh_active;
 	wait_queue_head_t       host_wait;
 	struct scsi_host_template *hostt;
 	struct scsi_transport_template *transportt;
@@ -665,9 +666,6 @@ struct Scsi_Host {
 	 */
 	unsigned ordered_tag:1;
 
-	/* Task mgmt function in progress */
-	unsigned tmf_in_progress:1;
-
 	/* Asynchronous scan in progress */
 	unsigned async_scan:1;
 
@@ -771,7 +769,7 @@ static inline int scsi_host_in_recovery(struct Scsi_Host *shost)
 	return shost->shost_state == SHOST_RECOVERY ||
 		shost->shost_state == SHOST_CANCEL_RECOVERY ||
 		shost->shost_state == SHOST_DEL_RECOVERY ||
-		shost->tmf_in_progress;
+		shost->eh_active;
 }
 
 extern int scsi_queue_work(struct Scsi_Host *, struct work_struct *);
-- 
1.7.10.4

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Mike Christie]

On 6/12/13 7:55 AM, Bart Van Assche wrote:
> A SCSI LLD may start cleaning up host resources as soon as
> scsi_remove_host() returns. These host resources may be needed by
> the LLD in an implementation of one of the eh_* functions. So if
> one of the eh_* functions is in progress when scsi_remove_host()
> is invoked, wait until the eh_* function has finished. Also, do
> not invoke any of the eh_* functions after scsi_remove_host() has
> started. Remove Scsi_Host.tmf_in_progress because it is now
> superfluous.
>

I think the patch looks ok for drivers that do not implement their own
eh_strategy_handler, but what about SAS? If you added a scsi_begin_eh in 
scsi_error_handler before the eh_strategy_handler is called and then add 
a scsi_end_eh after it is called, I think it would cover them too.

> @@ -1894,6 +1962,9 @@ int scsi_error_handler(void *data)
>   	}
>   	__set_current_state(TASK_RUNNING);
>
> +	WARN_ONCE(shost->eh_active, "scsi_eh_%d: eh_active = %d\n",
> +		  shost->host_no, shost->eh_active);
> +
>   	SCSI_LOG_ERROR_RECOVERY(1,
>   		printk("Error handler scsi_eh_%d exiting\n", shost->host_no));
>   	shost->ehandler = NULL;

What is the warn for? Is there a chance this can happen with some non 
upstream driver or are you just adding it just in case?

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/24/13 03:15, Mike Christie wrote:
> On 6/12/13 7:55 AM, Bart Van Assche wrote:
>> A SCSI LLD may start cleaning up host resources as soon as
>> scsi_remove_host() returns. These host resources may be needed by
>> the LLD in an implementation of one of the eh_* functions. So if
>> one of the eh_* functions is in progress when scsi_remove_host()
>> is invoked, wait until the eh_* function has finished. Also, do
>> not invoke any of the eh_* functions after scsi_remove_host() has
>> started. Remove Scsi_Host.tmf_in_progress because it is now
>> superfluous.
> 
> I think the patch looks ok for drivers that do not implement their own
> eh_strategy_handler, but what about SAS? If you added a scsi_begin_eh in 
> scsi_error_handler before the eh_strategy_handler is called and then add 
> a scsi_end_eh after it is called, I think it would cover them too.

I will start testing the modification below for the patch at the start of
this thread:

--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -1950,10 +1950,14 @@ int scsi_error_handler(void *data)
                        continue;
                }
 
-               if (shost->transportt->eh_strategy_handler)
-                       shost->transportt->eh_strategy_handler(shost);
-               else
+               if (shost->transportt->eh_strategy_handler) {
+                       if (scsi_begin_eh(shost) == 0) {
+                               shost->transportt->eh_strategy_handler(shost);
+                               scsi_end_eh(shost);
+                       }
+               } else {
                        scsi_unjam_host(shost);
+               }
 
                /*
                 * Note - if the above fails completely, the action is to take


>> @@ -1894,6 +1962,9 @@ int scsi_error_handler(void *data)
>>       }
>>       __set_current_state(TASK_RUNNING);
>>
>> +    WARN_ONCE(shost->eh_active, "scsi_eh_%d: eh_active = %d\n",
>> +          shost->host_no, shost->eh_active);
>> +
>>       SCSI_LOG_ERROR_RECOVERY(1,
>>           printk("Error handler scsi_eh_%d exiting\n", shost->host_no));
>>       shost->ehandler = NULL;
> 
> What is the warn for? Is there a chance this can happen with some non 
> upstream driver or are you just adding it just in case?

This is code that helped me to test this patch. I can leave it out if
you prefer so.

Bart.

RE:[PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Jack Wang]

> @@ -646,14 +703,20 @@ static int scsi_try_target_reset(struct scsi_cmnd *scmd)
>  static int scsi_try_bus_device_reset(struct scsi_cmnd *scmd)
>  {
>         int rtn;
> -       struct scsi_host_template *hostt = scmd->device->host->hostt;
> +       struct Scsi_Host *host = scmd->device->host;
> +       struct scsi_host_template *hostt = host->hostt;
> 
>         if (!hostt->eh_device_reset_handler)
>                 return FAILED;
> 
> +       if (scsi_begin_eh(host))
> +               return FAST_IO_FAIL;
> +
>         rtn = hostt->eh_device_reset_handler(scmd);
>         if (rtn == SUCCESS)
>                 __scsi_report_device_reset(scmd->device, NULL);
> +       scsi_end_eh(host);
> +
>         return rtn;
>  }

As the new eh from Hannes haven't make it into mainline, maybe we still
need also check scsi_try_to_abort_cmd?

Jack

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/24/13 12:17, Jack Wang wrote:
>> @@ -646,14 +703,20 @@ static int scsi_try_target_reset(struct scsi_cmnd *scmd)
>>   static int scsi_try_bus_device_reset(struct scsi_cmnd *scmd)
>>   {
>>          int rtn;
>> -       struct scsi_host_template *hostt = scmd->device->host->hostt;
>> +       struct Scsi_Host *host = scmd->device->host;
>> +       struct scsi_host_template *hostt = host->hostt;
>>
>>          if (!hostt->eh_device_reset_handler)
>>                  return FAILED;
>>
>> +       if (scsi_begin_eh(host))
>> +               return FAST_IO_FAIL;
>> +
>>          rtn = hostt->eh_device_reset_handler(scmd);
>>          if (rtn == SUCCESS)
>>                  __scsi_report_device_reset(scmd->device, NULL);
>> +       scsi_end_eh(host);
>> +
>>          return rtn;
>>   }
>
> As the new eh from Hannes haven't make it into mainline, maybe we still
> need also check scsi_try_to_abort_cmd?

I don't think such checks are necessary in scsi_try_to_abort_cmd(). 
scsi_remove_host() already waits until outstanding SCSI commands have 
finished. scsi_try_to_abort_cmd() only gets invoked if there are still 
one or more unfinished commands so scsi_try_to_abort_cmd() won't be 
invoked anymore after scsi_remove_host() has finished.

Bart.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[James Bottomley]

On Wed, 2013-06-12 at 14:55 +0200, Bart Van Assche wrote:
> A SCSI LLD may start cleaning up host resources as soon as
> scsi_remove_host() returns. These host resources may be needed by
> the LLD in an implementation of one of the eh_* functions. So if
> one of the eh_* functions is in progress when scsi_remove_host()
> is invoked, wait until the eh_* function has finished. Also, do
> not invoke any of the eh_* functions after scsi_remove_host() has
> started.

We already have state guards for this, don't we?  That's the
SHOST_*_RECOVERY ones.  When eh functions are active, the host
transitions to a recovery state, so the wait could just wait on that
state rather than implement an open coded counting semaphore.

However, what's the reasoning behind wanting to do this?  In theory all
necessary resources for the eh thread should only be freed in the
release callback.  That means they aren't freed until all error recovery
completes.

James

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Mike Christie]

On 06/24/2013 02:19 PM, James Bottomley wrote:
> On Wed, 2013-06-12 at 14:55 +0200, Bart Van Assche wrote:
>> A SCSI LLD may start cleaning up host resources as soon as
>> scsi_remove_host() returns. These host resources may be needed by
>> the LLD in an implementation of one of the eh_* functions. So if
>> one of the eh_* functions is in progress when scsi_remove_host()
>> is invoked, wait until the eh_* function has finished. Also, do
>> not invoke any of the eh_* functions after scsi_remove_host() has
>> started.
> 
> We already have state guards for this, don't we?  That's the
> SHOST_*_RECOVERY ones.  When eh functions are active, the host
> transitions to a recovery state, so the wait could just wait on that
> state rather than implement an open coded counting semaphore.
> 

That seems better. For the sg_reset_provider case we just would have to
also wait on the tmf_in_progress bit.

I think I used all my credits messing up reviewing this patchset.

> However, what's the reasoning behind wanting to do this?  In theory all
> necessary resources for the eh thread should only be freed in the
> release callback.  That means they aren't freed until all error recovery
> completes.
> 

I think it makes it easier to handle cleanup of driver resources needed
for aborts/resets for some drivers. If after host removal, the scsi eh
can call into the driver after scsi_remove_host is called then we have
to set some internal bits to fail future eh callback calls and handle
waiting on or flushing running eh operations. If we know that after
scsi_host_remove is called the eh callbacks will not be running and will
not be called we can just free the driver resources.

For iscsi and I think drivers that do scsi_remove_target it would be
helpful to have something similar at the target level.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[James Bottomley]

On Mon, 2013-06-24 at 15:04 -0500, Mike Christie wrote:
> On 06/24/2013 02:19 PM, James Bottomley wrote:
> > On Wed, 2013-06-12 at 14:55 +0200, Bart Van Assche wrote:
> >> A SCSI LLD may start cleaning up host resources as soon as
> >> scsi_remove_host() returns. These host resources may be needed by
> >> the LLD in an implementation of one of the eh_* functions. So if
> >> one of the eh_* functions is in progress when scsi_remove_host()
> >> is invoked, wait until the eh_* function has finished. Also, do
> >> not invoke any of the eh_* functions after scsi_remove_host() has
> >> started.
> > 
> > We already have state guards for this, don't we?  That's the
> > SHOST_*_RECOVERY ones.  When eh functions are active, the host
> > transitions to a recovery state, so the wait could just wait on that
> > state rather than implement an open coded counting semaphore.
> > 
> 
> That seems better. For the sg_reset_provider case we just would have to
> also wait on the tmf_in_progress bit.

The simplest way is may just be to move the kthread_stop() from release
to remove.  That synchronously waits for the outstanding error handling
to complete and the eh thread to stop.  Perhaps the eh thread should
also wait for tmf in progress before it dies?

> I think I used all my credits messing up reviewing this patchset.

Hey, you're one of my best reviewers and this doesn't change that.

For a variety of reasons this patch set is incredibly hard to review:
Almost every patch touches pieces in the mid layer where you have to be
sure in minute detail you know what's going on (and what should be going
on), so usually it's a couple of hours with the source code just making
sure you do know this.  Plus it's code where the underlying usage model
has evolved over the years meaning the original guarantees might have
been violated silently somewhere and the ramifications spread out like
tentacles everywhere.  Finally, it's not clear from the change logs why
the changes are actually being made: for instance sentence one of this
change log says "A SCSI LLD may start cleaning up host resources as soon
as scsi_remove_host() returns." which causes my sanity checker to go off
immediately because in a refcounted model, like we use for dev, target
and host, nothing essential is supposed to be freed until the last put
which may or may not happen in the remove function.

> > However, what's the reasoning behind wanting to do this?  In theory all
> > necessary resources for the eh thread should only be freed in the
> > release callback.  That means they aren't freed until all error recovery
> > completes.
> 
> I think it makes it easier to handle cleanup of driver resources
> needed
> for aborts/resets for some drivers. If after host removal, the scsi eh
> can call into the driver after scsi_remove_host is called then we have
> to set some internal bits to fail future eh callback calls and handle
> waiting on or flushing running eh operations. If we know that after
> scsi_host_remove is called the eh callbacks will not be running and
> will
> not be called we can just free the driver resources.
> 
> For iscsi and I think drivers that do scsi_remove_target it would be
> helpful to have something similar at the target level.

I'm wary of this because it combines two models: a definite state model
(where we move from state to state waiting for the completions) with an
event driven one (in theory the current one); such combinations rarely
lead to good things because you get a mixture of actions causing state
transitions some of which are waited for and some of which have an async
transition and that ends up confusing the heck out of everybody no
matter how carefully documented.  Can you give me some use cases of what
you're trying to achieve?  Could it be as simple as an event that fires
on release?

James

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Mike Christie]

On 06/24/2013 05:27 PM, James Bottomley wrote:
>>> However, what's the reasoning behind wanting to do this?  In theory all
>>> necessary resources for the eh thread should only be freed in the
>>> release callback.  That means they aren't freed until all error recovery
>>> completes.
>>
>> I think it makes it easier to handle cleanup of driver resources
>> needed
>> for aborts/resets for some drivers. If after host removal, the scsi eh
>> can call into the driver after scsi_remove_host is called then we have
>> to set some internal bits to fail future eh callback calls and handle
>> waiting on or flushing running eh operations. If we know that after
>> scsi_host_remove is called the eh callbacks will not be running and
>> will
>> not be called we can just free the driver resources.
>>
>> For iscsi and I think drivers that do scsi_remove_target it would be
>> helpful to have something similar at the target level.
> 
> I'm wary of this because it combines two models: a definite state model
> (where we move from state to state waiting for the completions) with an
> event driven one (in theory the current one); such combinations rarely
> lead to good things because you get a mixture of actions causing state
> transitions some of which are waited for and some of which have an async
> transition and that ends up confusing the heck out of everybody no
> matter how carefully documented.  Can you give me some use cases of what
> you're trying to achieve?  Could it be as simple as an event that fires
> on release?
> 


The problem that we hit in iscsi is that we will call scsi_remove_target
(we used to call scsi_remove_host when we always did a host per target
so we hit the problem at that level before). That will complete, but the
scsi eh might still be trying to abort/reset devices accessed through
that target. To avoid freeing resources that the iscsi scsi eh might be
using, we set internal state bits and wait on host_busy to go to zero
before we tear down the iscsi session, conn and task structs.

I think Bart was hitting a similar issue but a level up in the host
removal case, because srp always does a host per target and so it just
does a scsi_remove_host.

I think some driver/scsi_host_template callbacks that are called when
the host or target is released would work. At least it will for iscsi.
Will let Bart comment on srp, and we can make patches from there.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Michael Christie]

On Jun 24, 2013, at 9:26 PM, Mike Christie <michaelc@cs.wisc.edu> wrote:

> On 06/24/2013 05:27 PM, James Bottomley wrote:
>>>> However, what's the reasoning behind wanting to do this?  In theory all
>>>> necessary resources for the eh thread should only be freed in the
>>>> release callback.  That means they aren't freed until all error recovery
>>>> completes.
>>> 
>>> I think it makes it easier to handle cleanup of driver resources
>>> needed
>>> for aborts/resets for some drivers. If after host removal, the scsi eh
>>> can call into the driver after scsi_remove_host is called then we have
>>> to set some internal bits to fail future eh callback calls and handle
>>> waiting on or flushing running eh operations. If we know that after
>>> scsi_host_remove is called the eh callbacks will not be running and
>>> will
>>> not be called we can just free the driver resources.
>>> 
>>> For iscsi and I think drivers that do scsi_remove_target it would be
>>> helpful to have something similar at the target level.
>> 
>> I'm wary of this because it combines two models: a definite state model
>> (where we move from state to state waiting for the completions) with an
>> event driven one (in theory the current one); such combinations rarely
>> lead to good things because you get a mixture of actions causing state
>> transitions some of which are waited for and some of which have an async
>> transition and that ends up confusing the heck out of everybody no
>> matter how carefully documented.  Can you give me some use cases of what
>> you're trying to achieve?  Could it be as simple as an event that fires
>> on release?
> 
> 
> The problem that we hit in iscsi is that we will call scsi_remove_target
> (we used to call scsi_remove_host when we always did a host per target
> so we hit the problem at that level before). That will complete, but the
> scsi eh might still be trying to abort/reset devices accessed through
> that target. To avoid freeing resources that the iscsi scsi eh might be
> using, we set internal state bits and wait on host_busy to go to zero
> before we tear down the iscsi session, conn and task structs.
> 
> I think Bart was hitting a similar issue but a level up in the host
> removal case, because srp always does a host per target and so it just
> does a scsi_remove_host..

I take this back. I don't think it is a issue anymore and I think I can remove the iscsi hack. With the blk_cleanup_queue/blk_drain_queue code I think the target and the removal of its devices will not complete until the scsi eh is completed. The blk_drain_queue code will now wait for the eh to complete because the rq counters will be incremented.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/25/13 00:27, James Bottomley wrote:
> On Mon, 2013-06-24 at 15:04 -0500, Mike Christie wrote:
>> On 06/24/2013 02:19 PM, James Bottomley wrote:
>>> On Wed, 2013-06-12 at 14:55 +0200, Bart Van Assche wrote:
>>>> A SCSI LLD may start cleaning up host resources as soon as
>>>> scsi_remove_host() returns. These host resources may be needed by
>>>> the LLD in an implementation of one of the eh_* functions. So if
>>>> one of the eh_* functions is in progress when scsi_remove_host()
>>>> is invoked, wait until the eh_* function has finished. Also, do
>>>> not invoke any of the eh_* functions after scsi_remove_host() has
>>>> started.
>>>
>>> We already have state guards for this, don't we?  That's the
>>> SHOST_*_RECOVERY ones.  When eh functions are active, the host
>>> transitions to a recovery state, so the wait could just wait on that
>>> state rather than implement an open coded counting semaphore.
>>
>> That seems better. For the sg_reset_provider case we just would have to
>> also wait on the tmf_in_progress bit.
>
> The simplest way is may just be to move the kthread_stop() from release
> to remove.  That synchronously waits for the outstanding error handling
> to complete and the eh thread to stop.  Perhaps the eh thread should
> also wait for tmf in progress before it dies?

Regarding TMF that are in progress: my preference is to leave it to the 
LLD to wait for any TMF in progress if necessary. At least with SRP over 
RDMA it is possible to prevent receiving further TMF completion 
notifications by closing the connection over which these TMF were sent.

There is a difference though between moving the EH kthread_stop() call 
and the patch at the start of this thread: moving the EH kthread_stop() 
call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_* 
callback after scsi_remove_host() has finished. However, the 
scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can 
cause an eh_* callback to be invoked after scsi_remove_device() finished.

Bart.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/25/13 00:27, James Bottomley wrote:
> For a variety of reasons this patch set is incredibly hard to review:
> Almost every patch touches pieces in the mid layer where you have to be
> sure in minute detail you know what's going on (and what should be going
> on), so usually it's a couple of hours with the source code just making
> sure you do know this.  Plus it's code where the underlying usage model
> has evolved over the years meaning the original guarantees might have
> been violated silently somewhere and the ramifications spread out like
> tentacles everywhere.  Finally, it's not clear from the change logs why
> the changes are actually being made: for instance sentence one of this
> change log says "A SCSI LLD may start cleaning up host resources as soon
> as scsi_remove_host() returns." which causes my sanity checker to go off
> immediately because in a refcounted model, like we use for dev, target
> and host, nothing essential is supposed to be freed until the last put
> which may or may not happen in the remove function.

If the invocations of the eh_* callback functions would be visible to 
the block layer then blk_cleanup_queue() would wait until any such eh_* 
invocations have finished. Such an approach could simplify device 
removal in the SCSI mid-layer significantly. It also would avoid that an 
eh_* callback can be invoked via an ioctl after scsi_remove_device() has 
finished.

Bart.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[James Bottomley]

On Tue, 2013-06-25 at 11:01 +0200, Bart Van Assche wrote:
> On 06/25/13 00:27, James Bottomley wrote:
> > On Mon, 2013-06-24 at 15:04 -0500, Mike Christie wrote:
> >> On 06/24/2013 02:19 PM, James Bottomley wrote:
> >>> On Wed, 2013-06-12 at 14:55 +0200, Bart Van Assche wrote:
> >>>> A SCSI LLD may start cleaning up host resources as soon as
> >>>> scsi_remove_host() returns. These host resources may be needed by
> >>>> the LLD in an implementation of one of the eh_* functions. So if
> >>>> one of the eh_* functions is in progress when scsi_remove_host()
> >>>> is invoked, wait until the eh_* function has finished. Also, do
> >>>> not invoke any of the eh_* functions after scsi_remove_host() has
> >>>> started.
> >>>
> >>> We already have state guards for this, don't we?  That's the
> >>> SHOST_*_RECOVERY ones.  When eh functions are active, the host
> >>> transitions to a recovery state, so the wait could just wait on that
> >>> state rather than implement an open coded counting semaphore.
> >>
> >> That seems better. For the sg_reset_provider case we just would have to
> >> also wait on the tmf_in_progress bit.
> >
> > The simplest way is may just be to move the kthread_stop() from release
> > to remove.  That synchronously waits for the outstanding error handling
> > to complete and the eh thread to stop.  Perhaps the eh thread should
> > also wait for tmf in progress before it dies?
> 
> Regarding TMF that are in progress: my preference is to leave it to the 
> LLD to wait for any TMF in progress if necessary. At least with SRP over 
> RDMA it is possible to prevent receiving further TMF completion 
> notifications by closing the connection over which these TMF were sent.
> 
> There is a difference though between moving the EH kthread_stop() call 
> and the patch at the start of this thread: moving the EH kthread_stop() 
> call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_* 
> callback after scsi_remove_host() has finished. However, the 
> scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can 
> cause an eh_* callback to be invoked after scsi_remove_device() finished.

OK, but this doesn't tell me what you're trying to achieve.

An eh function is allowable as long as the host hadn't had the release
callback executed.  That means you must have to have a reference to the
device/host to execute the eh function, which is currently guaranteed
for all invocations.

James

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/25/13 15:45, James Bottomley wrote:
> On Tue, 2013-06-25 at 11:01 +0200, Bart Van Assche wrote:
>> There is a difference though between moving the EH kthread_stop() call
>> and the patch at the start of this thread: moving the EH kthread_stop()
>> call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_*
>> callback after scsi_remove_host() has finished. However, the
>> scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can
>> cause an eh_* callback to be invoked after scsi_remove_device() finished.
>
> OK, but this doesn't tell me what you're trying to achieve.
>
> An eh function is allowable as long as the host hadn't had the release
> callback executed.  That means you must have to have a reference to the
> device/host to execute the eh function, which is currently guaranteed
> for all invocations.

That raises a new question: how is an LLD expected to clean up resources 
without triggering a race condition ? What you wrote means that it's not 
safe for an LLD to start cleaning up the resources needed by the eh_* 
callbacks immediately after scsi_remove_device() returns since it it not 
guaranteed that at that time all references to the device have already 
been dropped.

Bart.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Michael Christie]

On Jun 25, 2013, at 10:31 AM, Bart Van Assche <bvanassche@acm.org> wrote:

> On 06/25/13 15:45, James Bottomley wrote:
>> On Tue, 2013-06-25 at 11:01 +0200, Bart Van Assche wrote:
>>> There is a difference though between moving the EH kthread_stop() call
>>> and the patch at the start of this thread: moving the EH kthread_stop()
>>> call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_*
>>> callback after scsi_remove_host() has finished. However, the
>>> scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can
>>> cause an eh_* callback to be invoked after scsi_remove_device() finished.
>> 
>> OK, but this doesn't tell me what you're trying to achieve.
>> 
>> An eh function is allowable as long as the host hadn't had the release
>> callback executed.  That means you must have to have a reference to the
>> device/host to execute the eh function, which is currently guaranteed
>> for all invocations.
> 
> That raises a new question: how is an LLD expected to clean up resources without triggering a race condition ? What you wrote means that it's not safe for an LLD to start cleaning up the resources needed by the eh_* callbacks immediately after scsi_remove_device() returns since it it not guaranteed that at that time all references to the device have already been dropped.
> 


A callback in the device/target/host (whatever is needed) release function would do this right? If I understand James right, I think he suggested something like this in another mail.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[James Bottomley]

On Tue, 2013-06-25 at 11:13 -0500, Michael Christie wrote:
> On Jun 25, 2013, at 10:31 AM, Bart Van Assche <bvanassche@acm.org> wrote:
> 
> > On 06/25/13 15:45, James Bottomley wrote:
> >> On Tue, 2013-06-25 at 11:01 +0200, Bart Van Assche wrote:
> >>> There is a difference though between moving the EH kthread_stop() call
> >>> and the patch at the start of this thread: moving the EH kthread_stop()
> >>> call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_*
> >>> callback after scsi_remove_host() has finished. However, the
> >>> scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can
> >>> cause an eh_* callback to be invoked after scsi_remove_device() finished.
> >> 
> >> OK, but this doesn't tell me what you're trying to achieve.
> >> 
> >> An eh function is allowable as long as the host hadn't had the release
> >> callback executed.  That means you must have to have a reference to the
> >> device/host to execute the eh function, which is currently guaranteed
> >> for all invocations.
> > 
> > That raises a new question: how is an LLD expected to clean up resources without triggering a race condition ? What you wrote means that it's not safe for an LLD to start cleaning up the resources needed by the eh_* callbacks immediately after scsi_remove_device() returns since it it not guaranteed that at that time all references to the device have already been dropped.
> > 
> 
> 
> A callback in the device/target/host (whatever is needed) release
> function would do this right? If I understand James right, I think he
> suggested something like this in another mail.

Exactly ... at least that's what we should do.

If I look at what we actually do: all the HBAs treat scsi_remove_host as
a waited for transition.  The reason this works is the loop over
__scsi_remove_device() in scsi_forget_host().  By the time that loop
returns, every scsi_device is gone (and so is every target).  Because
blk_cleanup_queue() induces a synchronous wait for the queue to die in
__scsi_remove_device(), there can be no outstanding I/O and no eh
activity for the device when it returns (and no possibility of starting
any).  Thus at the end of scsi_forget_host, we have no devices to start
I/O and no eh activity, so the final put will be the last.

James

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/25/13 19:40, James Bottomley wrote:
> If I look at what we actually do: all the HBAs treat scsi_remove_host as
> a waited for transition.  The reason this works is the loop over
> __scsi_remove_device() in scsi_forget_host().  By the time that loop
> returns, every scsi_device is gone (and so is every target).  Because
> blk_cleanup_queue() induces a synchronous wait for the queue to die in
> __scsi_remove_device(), there can be no outstanding I/O and no eh
> activity for the device when it returns (and no possibility of starting
> any).  Thus at the end of scsi_forget_host, we have no devices to start
> I/O and no eh activity, so the final put will be the last.

With the patch at the start of this thread everything works like you 
described above. However, without that patch EH activity can continue 
after scsi_remove_device() has returned. I have seen that occurring 
several times while testing SCSI LLD patches.

Bart.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 06/25/13 18:13, Michael Christie wrote:
> On Jun 25, 2013, at 10:31 AM, Bart Van Assche <bvanassche@acm.org> wrote:
>> On 06/25/13 15:45, James Bottomley wrote:
>>> On Tue, 2013-06-25 at 11:01 +0200, Bart Van Assche wrote:
>>>> There is a difference though between moving the EH kthread_stop() call
>>>> and the patch at the start of this thread: moving the EH kthread_stop()
>>>> call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_*
>>>> callback after scsi_remove_host() has finished. However, the
>>>> scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can
>>>> cause an eh_* callback to be invoked after scsi_remove_device() finished.
>>>
>>> OK, but this doesn't tell me what you're trying to achieve.
>>>
>>> An eh function is allowable as long as the host hadn't had the release
>>> callback executed.  That means you must have to have a reference to the
>>> device/host to execute the eh function, which is currently guaranteed
>>> for all invocations.
>>
>> That raises a new question: how is an LLD expected to clean up resources
>> without triggering a race condition ? What you wrote means that it's not
>> safe for an LLD to start cleaning up the resources needed by the eh_*
>> callbacks immediately after scsi_remove_device() returns since it it not
>> guaranteed that at that time all references to the device have already
>> been dropped.
>
> A callback in the device/target/host (whatever is needed) release function
> would do this right? If I understand James right, I think he suggested
> something like this in another mail.

(replying to an e-mail of seven months ago - see also
http://thread.gmane.org/gmane.linux.scsi/82572/focus=82822)

Hello Mike,

Sorry but I'm afraid that making the SCSI core invoke a callback
function from a device, target or host release function would create a
new challenge, namely making sure that all these callback functions have
finished before the module is unloaded that contains the SCSI host
template and the code implementing such a callback function. That
challenge is not specific to the SCSI infrastructure but is a general
question that has not yet been solved in the Linux kernel (see e.g.
"[PATCH] kobject: provide kobject_put_wait to fix module unload race"
for a more general discussion about how to ensure that kobject release
functions are invoked before the module is unloaded that owns the
release function - http://thread.gmane.org/gmane.linux.kernel/1622885).

Or maybe this just means that I misunderstood you ?

In case it is not clear why I'm reviving this discussion: now that the
"improved eh timeout handler" patch is upstream (commit
e494f6a728394ab0df194342549ee20e6f0752df) there is an additional way in
which the SCSI core can invoke an EH function concurrently with or after
scsi_remove_host() has finished, namely from the TMF work queue
(tmf_work_q).

Thanks,

Bart.

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[James Bottomley]

On Thu, 2014-01-30 at 20:46 +0100, Bart Van Assche wrote:
> On 06/25/13 18:13, Michael Christie wrote:
> > On Jun 25, 2013, at 10:31 AM, Bart Van Assche <bvanassche@acm.org> wrote:
> >> On 06/25/13 15:45, James Bottomley wrote:
> >>> On Tue, 2013-06-25 at 11:01 +0200, Bart Van Assche wrote:
> >>>> There is a difference though between moving the EH kthread_stop() call
> >>>> and the patch at the start of this thread: moving the EH kthread_stop()
> >>>> call does not prevent that an ioctl like SG_SCSI_RESET triggers an eh_*
> >>>> callback after scsi_remove_host() has finished. However, the
> >>>> scsi_begin_eh() / scsi_end_eh() functions do prevent that an ioctl can
> >>>> cause an eh_* callback to be invoked after scsi_remove_device() finished.
> >>>
> >>> OK, but this doesn't tell me what you're trying to achieve.
> >>>
> >>> An eh function is allowable as long as the host hadn't had the release
> >>> callback executed.  That means you must have to have a reference to the
> >>> device/host to execute the eh function, which is currently guaranteed
> >>> for all invocations.
> >>
> >> That raises a new question: how is an LLD expected to clean up resources
> >> without triggering a race condition ? What you wrote means that it's not
> >> safe for an LLD to start cleaning up the resources needed by the eh_*
> >> callbacks immediately after scsi_remove_device() returns since it it not
> >> guaranteed that at that time all references to the device have already
> >> been dropped.
> >
> > A callback in the device/target/host (whatever is needed) release function
> > would do this right? If I understand James right, I think he suggested
> > something like this in another mail.
> 
> (replying to an e-mail of seven months ago - see also
> http://thread.gmane.org/gmane.linux.scsi/82572/focus=82822)
> 
> Hello Mike,
> 
> Sorry but I'm afraid that making the SCSI core invoke a callback
> function from a device, target or host release function would create a
> new challenge, namely making sure that all these callback functions have
> finished before the module is unloaded that contains the SCSI host
> template and the code implementing such a callback function. That
> challenge is not specific to the SCSI infrastructure but is a general
> question that has not yet been solved in the Linux kernel (see e.g.
> "[PATCH] kobject: provide kobject_put_wait to fix module unload race"
> for a more general discussion about how to ensure that kobject release
> functions are invoked before the module is unloaded that owns the
> release function - http://thread.gmane.org/gmane.linux.kernel/1622885).

For callbacks, that's easy: it's module_get/module_put

> Or maybe this just means that I misunderstood you ?
> 
> In case it is not clear why I'm reviving this discussion: now that the
> "improved eh timeout handler" patch is upstream (commit
> e494f6a728394ab0df194342549ee20e6f0752df) there is an additional way in
> which the SCSI core can invoke an EH function concurrently with or after
> scsi_remove_host() has finished, namely from the TMF work queue
> (tmf_work_q).

But the fundamental guarantee is that the eh thread for the host (the eh
context if you will) has to be dead before the host can be removed and
the module unloaded.  The thread doesn't die until all the work is done.

James

Re: [PATCH v11 6/9] Make scsi_remove_host() wait until error handling finished

[Bart Van Assche]

On 01/31/14 06:58, James Bottomley wrote:
> On Thu, 2014-01-30 at 20:46 +0100, Bart Van Assche wrote:
>> On 06/25/13 18:13, Michael Christie wrote:
>> Sorry but I'm afraid that making the SCSI core invoke a callback
>> function from a device, target or host release function would create a
>> new challenge, namely making sure that all these callback functions have
>> finished before the module is unloaded that contains the SCSI host
>> template and the code implementing such a callback function. That
>> challenge is not specific to the SCSI infrastructure but is a general
>> question that has not yet been solved in the Linux kernel (see e.g.
>> "[PATCH] kobject: provide kobject_put_wait to fix module unload race"
>> for a more general discussion about how to ensure that kobject release
>> functions are invoked before the module is unloaded that owns the
>> release function - http://thread.gmane.org/gmane.linux.kernel/1622885).
> 
> For callbacks, that's easy: it's module_get/module_put

Adding a module_get() call in scsi_add_host() and a module_put() call in
scsi_host_dev_release() would cause a behavior change that would be
reported by end users as "system hangs during shutdown". Today it is
possible to unload a kernel module via rmmod that created one or more
SCSI hosts. Since user space does not remove SCSI hosts during system
shutdown, trying to unload a SCSI LLD kernel module that had created one
or more SCSI hosts would cause the module unload to fail because of a
non-zero module reference count.

>> In case it is not clear why I'm reviving this discussion: now that the
>> "improved eh timeout handler" patch is upstream (commit
>> e494f6a728394ab0df194342549ee20e6f0752df) there is an additional way in
>> which the SCSI core can invoke an EH function concurrently with or after
>> scsi_remove_host() has finished, namely from the TMF work queue
>> (tmf_work_q).
> 
> But the fundamental guarantee is that the eh thread for the host (the eh
> context if you will) has to be dead before the host can be removed and
> the module unloaded.  The thread doesn't die until all the work is done.

Maybe I'm misunderstanding something, but as far as I can see
kthread_stop(shost->ehandler) is invoked from scsi_host_dev_release().
That last function is called by the last scsi_host_put(). And LLD's
invoke scsi_host_put() after scsi_remove_host(). In other words, the
SCSI error handler thread and TMF work queue are still active when
scsi_remove_host() returns.

Should I repost the patch at the start of this thread ?

Thanks,

Bart.